|
|
|
|
@ -2,12 +2,11 @@ package org.gcube.data.transfer.plugins.sis;
|
|
|
|
|
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.FileInputStream;
|
|
|
|
|
import java.io.FileNotFoundException;
|
|
|
|
|
import java.io.FileOutputStream;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.io.InputStream;
|
|
|
|
|
import java.io.OutputStream;
|
|
|
|
|
import java.net.MalformedURLException;
|
|
|
|
|
import java.net.URL;
|
|
|
|
|
import java.nio.file.Files;
|
|
|
|
|
import java.nio.file.StandardCopyOption;
|
|
|
|
|
import java.security.MessageDigest;
|
|
|
|
|
@ -15,54 +14,117 @@ import java.security.NoSuchAlgorithmException;
|
|
|
|
|
|
|
|
|
|
import org.apache.sis.storage.DataStoreException;
|
|
|
|
|
import org.apache.sis.storage.UnsupportedStorageException;
|
|
|
|
|
import org.gcube.data.transfer.model.TransferTicket.Status;
|
|
|
|
|
import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin;
|
|
|
|
|
|
|
|
|
|
public class TestGetMetadata {
|
|
|
|
|
|
|
|
|
|
public static void main(String[] args) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException {
|
|
|
|
|
String[] toCheckFiles=new String[] {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// ********** HDF - NETCDF 4
|
|
|
|
|
"/Users/FabioISTI/Downloads/NASA_Precipitations_1950_2100_rcp45.nc",
|
|
|
|
|
|
|
|
|
|
"https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp45.nc",
|
|
|
|
|
"https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp85.nc",
|
|
|
|
|
"https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Surface_Air_Temperature_1950_2100_rcp45.nc",
|
|
|
|
|
"https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Surface_Air_Temperature_1950_2100_rcp85.nc",
|
|
|
|
|
"https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/hcaf_v6_1950_2100.nc",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// *********** NETCDF 3
|
|
|
|
|
|
|
|
|
|
"/Users/FabioISTI/Downloads/Aaptos_aaptos.nc"
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
File temp=File.createTempFile("temp", ".temp");
|
|
|
|
|
File original=new File("/home/fabio/Downloads/oscar_vel2011_180.nc");
|
|
|
|
|
File copied=File.createTempFile("copied", ".nc");
|
|
|
|
|
transferStream(new FileInputStream(original),new FileOutputStream(temp));
|
|
|
|
|
System.out.println("copied. Moving..");
|
|
|
|
|
|
|
|
|
|
System.out.println("Checksum original : "+getChecksum(original));
|
|
|
|
|
System.out.println("Checksum temp : "+getChecksum(temp));
|
|
|
|
|
for(String f:toCheckFiles) {
|
|
|
|
|
System.out.println("checking "+f);
|
|
|
|
|
try{
|
|
|
|
|
check(false,f);
|
|
|
|
|
}catch(UnsupportedStorageException e) {
|
|
|
|
|
System.err.println(e.getMessage());
|
|
|
|
|
// System.out.println("Trying opening "+f+" manually");
|
|
|
|
|
// open(f);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
System.out.println("Done");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException {
|
|
|
|
|
// System.out.println("Opening "+path);
|
|
|
|
|
// NetcdfFile f=NetcdfFiles.open(path);
|
|
|
|
|
// System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId());
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// NetcdfDataset ncDs=NetcdfDatasets.openDataset(path);
|
|
|
|
|
//// NcMLReaderNew.
|
|
|
|
|
////
|
|
|
|
|
// System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId());
|
|
|
|
|
//
|
|
|
|
|
//// NetcdfDatasets.openFile(path,null);
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException {
|
|
|
|
|
|
|
|
|
|
File original=new File(src);
|
|
|
|
|
|
|
|
|
|
File dataset=original;
|
|
|
|
|
|
|
|
|
|
if(checkCopy) {
|
|
|
|
|
File temp=File.createTempFile("temp", ".temp");
|
|
|
|
|
File copied=File.createTempFile("copied", ".nc");
|
|
|
|
|
transferStream(new FileInputStream(original),new FileOutputStream(temp));
|
|
|
|
|
System.out.println("copied. Moving..");
|
|
|
|
|
|
|
|
|
|
System.out.println("Checksum original : "+getChecksum(original));
|
|
|
|
|
System.out.println("Checksum temp : "+getChecksum(temp));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Files.move(temp.toPath(), copied.toPath(),StandardCopyOption.ATOMIC_MOVE,StandardCopyOption.REPLACE_EXISTING);
|
|
|
|
|
if(copied.length()<original.length()) throw new RuntimeException("Different size after moving");
|
|
|
|
|
|
|
|
|
|
dataset=copied;
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(dataset.exists())
|
|
|
|
|
System.out.println(SisPlugin.getMetaFromFile(dataset));
|
|
|
|
|
else {
|
|
|
|
|
System.out.println(SisPlugin.getMetaFromFile(new URL(src)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Files.move(temp.toPath(), copied.toPath(),StandardCopyOption.ATOMIC_MOVE,StandardCopyOption.REPLACE_EXISTING);
|
|
|
|
|
if(copied.length()<original.length()) throw new RuntimeException("Different size after moving");
|
|
|
|
|
System.out.println(SisPlugin.getMetaFromFile(copied));
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static String getChecksum(File datafile) throws NoSuchAlgorithmException, IOException {
|
|
|
|
|
MessageDigest md = MessageDigest.getInstance("SHA1");
|
|
|
|
|
FileInputStream fis = new FileInputStream(datafile);
|
|
|
|
|
byte[] dataBytes = new byte[1024];
|
|
|
|
|
FileInputStream fis = new FileInputStream(datafile);
|
|
|
|
|
byte[] dataBytes = new byte[1024];
|
|
|
|
|
|
|
|
|
|
int nread = 0;
|
|
|
|
|
int nread = 0;
|
|
|
|
|
|
|
|
|
|
while ((nread = fis.read(dataBytes)) != -1) {
|
|
|
|
|
md.update(dataBytes, 0, nread);
|
|
|
|
|
};
|
|
|
|
|
while ((nread = fis.read(dataBytes)) != -1) {
|
|
|
|
|
md.update(dataBytes, 0, nread);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
byte[] mdbytes = md.digest();
|
|
|
|
|
byte[] mdbytes = md.digest();
|
|
|
|
|
|
|
|
|
|
//convert the byte to hex format
|
|
|
|
|
StringBuffer sb = new StringBuffer("");
|
|
|
|
|
for (int i = 0; i < mdbytes.length; i++) {
|
|
|
|
|
sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
|
|
|
|
|
}
|
|
|
|
|
//convert the byte to hex format
|
|
|
|
|
StringBuffer sb = new StringBuffer("");
|
|
|
|
|
for (int i = 0; i < mdbytes.length; i++) {
|
|
|
|
|
sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return sb.toString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static void transferStream(InputStream in, OutputStream out){
|
|
|
|
|
|
|
|
|
|
@ -74,12 +136,12 @@ public class TestGetMetadata {
|
|
|
|
|
while ((received=in.read(internalBuf))!=-1){
|
|
|
|
|
out.write(internalBuf,0,received);
|
|
|
|
|
receivedTotal+=received;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
out.flush();
|
|
|
|
|
}catch(IOException e){
|
|
|
|
|
throw new RuntimeException("Unable to read from source.");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
