Latest version with correction by G.Coro

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/SeaDataNetConnector@120149 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2015-11-01 18:24:23 +00:00
parent 874dee3b71
commit 61f3df67e4
3 changed files with 188 additions and 216 deletions

View File

@ -91,7 +91,7 @@ public class DivaHTTPClient {
responseStream.close();
httpUrlConnection.disconnect();
AnalysisLogger.getLogger().debug("Response "+response);
return response;
} catch (IOException e) {
@ -159,7 +159,7 @@ public class DivaHTTPClient {
String response = stringBuilder.toString();
//System.out.println("RESPONSE STRING: "+response);
AnalysisLogger.getLogger().debug("Response from Diva: "+response);
AnalysisLogger.getLogger().debug("Response from DIVA Analysis: \n"+response);
Document doc = loadXMLFromString(response);
Node idNode = doc.getElementsByTagName("result").item(0);
@ -171,8 +171,6 @@ public class DivaHTTPClient {
responseStream.close();
httpUrlConnection.disconnect();
AnalysisLogger.getLogger().debug("Response "+response);
//return response;
return new DivaAnalysisGetResponse(identifier, vmin, vmax, stat_obs_count_used, stat_posteriori_stn);
@ -204,14 +202,17 @@ public class DivaHTTPClient {
//Create Document object by XMLResponse
Document doc = loadXMLFromString(result_xml);
//Print XML Response
AnalysisLogger.getLogger().debug(result_xml);
AnalysisLogger.getLogger().debug("DIVA XML Response: \n"+result_xml);
Node root = doc.getDocumentElement();
if(root== null) throw new SAXException("Error in HTTP response no root element found!");
Integer obs_count=Integer.parseInt(root.getChildNodes().item(1).getTextContent());
Integer obs_count= null;
try{
obs_count=Integer.parseInt(root.getChildNodes().item(1).getTextContent());
}catch(NullPointerException pe ){
throw new Exception("Error in uploading file to DIVA!");
}
Double obs_x0=Double.parseDouble(root.getChildNodes().item(3).getTextContent());
Double obs_x1=Double.parseDouble(root.getChildNodes().item(5).getTextContent());
Double obs_y0=Double.parseDouble(root.getChildNodes().item(7).getTextContent());
@ -225,12 +226,13 @@ public class DivaHTTPClient {
}
public static void downloadFileDiva(String fileurl, String destinationFile) throws Exception{
public static void downloadFileDiva(String fileid, String destinationFile) throws Exception{
try {
String fileurl = DivaHTTPClient.WEB_HTTP + DivaHTTPClient.DOWN_FILE_NC + fileid;
URL smpFile = new URL(fileurl);
URLConnection uc = (URLConnection) smpFile.openConnection();
InputStream is = uc.getInputStream();
AnalysisLogger.getLogger().debug("Retrieving file from " + fileurl + " to :" + destinationFile);
AnalysisLogger.getLogger().debug("Retrieving DIVA file from " + fileurl + " to :" + destinationFile);
inputStreamToFile(is, destinationFile);
is.close();
} catch (Exception e) {

View File

@ -7,7 +7,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
@ -23,208 +22,195 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
import org.hibernate.SessionFactory;
import weka.classifiers.trees.m5.Impurity;
public class SeaDataNetConnector extends StandardLocalExternalAlgorithm {
//Statistical result by DIVA
// Statistical result by DIVA
LinkedHashMap<String, String> statResultMap = new LinkedHashMap<String, String>();
//HashMap for
// HashMap for
LinkedHashMap<String, StatisticalType> outputDivaMap = new LinkedHashMap<String, StatisticalType>();
SessionFactory dbconnection;
@Override
public void init() throws Exception {
// TODO Auto-generated method stub
}
@Override
public String getDescription() {
// TODO Auto-generated method stub
return "Connector for the SeaDataNet infrastructure";
return "A connector for the SeaDataNet infrastructure. This algorithms invokes the Data-Interpolating Variational Analysis (DIVA) SeaDataNet service to interpolate spatial data. " +
"The model uses GEBCO bathymetry data and requires an estimate of the maximum spatial span of the correlation between points and the signal-to-noise ratio, among the other parameters. " +
"It can interpolate up to 100,000 points randomly taken from the input table. " +
"As output, it produces a NetCDF file with a uniform grid of values. " +
"This powerful interpolation model is described in Troupin et al. 2012, 'Generation of analysis and consistent error fields using the Data Interpolating Variational Analysis (Diva)', Ocean Modelling, 52-53, 90-101.";
}
File outputfile;
@Override
protected void process() throws Exception {
String outpath = config.getPersistencePath();
File neofile = new File(outpath,"seadn_diva_"+UUID.randomUUID()+".nc");
/*
BufferedWriter fileWriter = new BufferedWriter(new FileWriter(neofile));
fileWriter.write("test test");
fileWriter.close();
*/
//outputfile=neofile;
AnalysisLogger.getLogger().debug("Input Table: "+config.getParam("InputTable"));
AnalysisLogger.getLogger().debug("Input Long: "+config.getParam("Longitude"));
AnalysisLogger.getLogger().debug("Input Lat: "+config.getParam("Latitude"));
AnalysisLogger.getLogger().debug("Input Qt: "+config.getParam("Quantity"));
AnalysisLogger.getLogger().debug("Longitude min X: "+config.getParam("LongitudeMinValue"));
AnalysisLogger.getLogger().debug("Longitude max X: "+config.getParam("LongitudeMaxValue"));
AnalysisLogger.getLogger().debug("Longitude resolution: "+config.getParam("LongitudeResolution"));
AnalysisLogger.getLogger().debug("Latitude min Y: "+config.getParam("LatitudeMinValue"));
AnalysisLogger.getLogger().debug("Latitude max Y: "+config.getParam("LatitudeMaxValue"));
AnalysisLogger.getLogger().debug("Latitude resolution: "+config.getParam("LatitudeResolution"));
AnalysisLogger.getLogger().debug("Correlation length: "+config.getParam("CorrelationLength"));
AnalysisLogger.getLogger().debug("Signal noise value: "+config.getParam("SignalNoise"));
AnalysisLogger.getLogger().debug("Depth Level: "+config.getParam("DepthLevel"));
File neofile = null;
File fileForDiva = null;
try {
String outpath = config.getPersistencePath();
neofile = new File(outpath, "seadn_diva_" + UUID.randomUUID() + ".nc");
AnalysisLogger.getLogger().debug("Input Parameters");
AnalysisLogger.getLogger().debug("Input Table: " + config.getParam("InputTable"));
AnalysisLogger.getLogger().debug("Input Long: " + config.getParam("Longitude"));
AnalysisLogger.getLogger().debug("Input Lat: " + config.getParam("Latitude"));
AnalysisLogger.getLogger().debug("Input Qt: " + config.getParam("Quantity"));
try{
double longitudeMinValue = Double.parseDouble(config.getParam("LongitudeMinValue"));
}catch(Exception e){
new Exception ("Bad value for LongitudeMinValue");
}
dbconnection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().debug("Longitude min X: " + config.getParam("LongitudeMinValue"));
AnalysisLogger.getLogger().debug("Longitude max X: " + config.getParam("LongitudeMaxValue"));
AnalysisLogger.getLogger().debug("Longitude resolution: " + config.getParam("LongitudeResolution"));
String query = "select "+config.getParam("Longitude")+","+
config.getParam("Latitude")+","+config.getParam("Quantity")
+ " From " + getInputParameter("InputTable")+" ORDER BY RANDOM() limit 1000";
AnalysisLogger.getLogger().debug("Query: "+query);
List<Object> dataList = DatabaseFactory.executeSQLQuery(query , dbconnection);
File fileForDiva= new File(outpath,"file_for_diva_"+UUID.randomUUID()+".txt");
BufferedWriter fileWriterDiva = new BufferedWriter(new FileWriter(fileForDiva));
AnalysisLogger.getLogger().debug("writing input file in: "+fileForDiva.getAbsolutePath() );
for(Object o : dataList){
Object[] oarray = (Object[]) o;
fileWriterDiva.write(" "+oarray[0]+" "+oarray[1]+" "+oarray[2]+"\n");
}
fileWriterDiva.close();
//integration DivaHttpClient
//UPLOADFILE for DIVA
DivaHTTPClient neo= new DivaHTTPClient();
DivaFilePostResponse response=neo.uploadFile(fileForDiva);
AnalysisLogger.getLogger().debug("Server Response: "+response.getSessionid());
try{
//ANALYSIS
Double correlationVal = Double.parseDouble(config.getParam("CorrelationLength"));
Double signalNoiseVal = Double.parseDouble(config.getParam("SignalNoise"));
Double longMinVal = Double.parseDouble(config.getParam("LongitudeMinValue"));
if(longMinVal<-180)
throw new Exception("Longitudine minumum value is less than -180.");
Double longMaxVal = Double.parseDouble(config.getParam("LongitudeMaxValue"));
if(longMaxVal>180)
throw new Exception("Longitudine maximum value is more than 180.");
Double longResolutionVal = Double.parseDouble(config.getParam("LongitudeResolution"));
Double latMinVal = Double.parseDouble(config.getParam("LatitudeMinValue"));
if(latMinVal<-90)
throw new Exception("Latitude minumum value is less than -90.");
Double latMaxVal = Double.parseDouble(config.getParam("LatitudeMaxValue"));
if(latMaxVal>90)
throw new Exception("Latitude maximum value is more than 90.");
AnalysisLogger.getLogger().debug("Latitude min Y: " + config.getParam("LatitudeMinValue"));
AnalysisLogger.getLogger().debug("Latitude max Y: " + config.getParam("LatitudeMaxValue"));
AnalysisLogger.getLogger().debug("Latitude resolution: " + config.getParam("LatitudeResolution"));
Double latResolutionVal = Double.parseDouble(config.getParam("LatitudeResolution"));
Double depthLevelVal = Double.parseDouble(config.getParam("DepthLevel"));
if((depthLevelVal<-10)||(depthLevelVal>10))
throw new Exception("Depth Level is out of range [-10,10]");
AnalysisLogger.getLogger().debug("Correlation length: " + config.getParam("CorrelationLength"));
AnalysisLogger.getLogger().debug("Signal noise value: " + config.getParam("SignalNoise"));
AnalysisLogger.getLogger().debug("Depth Level: " + config.getParam("DepthLevel"));
DivaAnalysisGetResponse respAnalysis = neo.getAnalysis(response.getSessionid(),
correlationVal, signalNoiseVal, longMinVal, longMaxVal, longResolutionVal,
latMinVal, latMaxVal, latResolutionVal, depthLevelVal);
statResultMap.put("Minimum value of the analysis",""+respAnalysis.getVmin());
statResultMap.put("Maximum value of the analysis", ""+respAnalysis.getVmax());
statResultMap.put("Number of observations used", ""+respAnalysis.getStat_obs_count_used());
statResultMap.put("Posterior probability of the model", ""+respAnalysis.getStat_posteriori_stn());
//DOWNLOAD FILE
neo.downloadFileDiva(neo.WEB_HTTP+neo.DOWN_FILE_NC+respAnalysis.getIdentifier(),
neofile.getAbsolutePath());
}
catch(Exception e){
throw e;
}
finally{
neofile.delete();
AnalysisLogger.getLogger().debug("file deleted");;
Double correlationVal = null;
Double signalNoiseVal = null;
Double longMinVal = null;
Double longMaxVal = null;
Double longResolutionVal = null;
Double latMinVal = null;
Double latMaxVal = null;
Double latResolutionVal = null;
Double depthLevelVal = null;
AnalysisLogger.getLogger().debug("Checking parameters");
try {
// ANALYSIS
correlationVal = Double.parseDouble(config.getParam("CorrelationLength"));
if (correlationVal <0 )
throw new Exception("Correlation span cannot be negative.");
signalNoiseVal = Double.parseDouble(config.getParam("SignalNoise"));
if (signalNoiseVal <0 )
throw new Exception("Signal-to-noise ratio cannot be negative.");
longMinVal = Double.parseDouble(config.getParam("LongitudeMinValue"));
if (longMinVal < -180)
throw new Exception("Longitudine minumum value is less than -180.");
longMaxVal = Double.parseDouble(config.getParam("LongitudeMaxValue"));
if (longMaxVal > 180)
throw new Exception("Longitudine maximum value is more than 180.");
longResolutionVal = Double.parseDouble(config.getParam("LongitudeResolution"));
latMinVal = Double.parseDouble(config.getParam("LatitudeMinValue"));
if (latMinVal < -85)
throw new Exception("Latitude minumum value is less than -85.");
latMaxVal = Double.parseDouble(config.getParam("LatitudeMaxValue"));
if (latMaxVal > 85)
throw new Exception("Latitude maximum value is more than 85.");
latResolutionVal = Double.parseDouble(config.getParam("LatitudeResolution"));
depthLevelVal = Double.parseDouble(config.getParam("DepthLevel"));
if (depthLevelVal < 0)
throw new Exception("Depth Level cannot be negative");
} catch (NumberFormatException e) {
throw new Exception("Parameter values are incomplete");
}
AnalysisLogger.getLogger().debug("Parameters are OK");
AnalysisLogger.getLogger().debug("Initializing DB connection");
dbconnection = DatabaseUtils.initDBSession(config);
String query = "select "
+ config.getParam("Longitude") + "," + config.getParam("Latitude") + "," + config.getParam("Quantity") + " From " + getInputParameter("InputTable") +
" ORDER BY RANDOM() limit 100000";
AnalysisLogger.getLogger().debug("Query for extracting data from the DB: " + query);
List<Object> dataList = DatabaseFactory.executeSQLQuery(query, dbconnection);
int ndata = dataList.size();
fileForDiva = new File(outpath, "file_for_diva_" + UUID.randomUUID() + ".txt");
BufferedWriter fileWriterDiva = new BufferedWriter(new FileWriter(fileForDiva));
AnalysisLogger.getLogger().debug("Writing input file in: " + fileForDiva.getAbsolutePath());
for (Object o : dataList) {
Object[] oarray = (Object[]) o;
fileWriterDiva.write(" " + oarray[0] + " " + oarray[1] + " " + oarray[2] + "\n");
}
fileWriterDiva.close();
AnalysisLogger.getLogger().debug("Sending data to DIVA: Uploading "+ndata+" records");
// integration DivaHttpClient
// UPLOADFILE for DIVA
DivaFilePostResponse response = DivaHTTPClient.uploadFile(fileForDiva);
AnalysisLogger.getLogger().debug("DIVA Server Response for the Upload:\n" + response.getSessionid());
AnalysisLogger.getLogger().debug("Requiring analysis to DIVA...");
long t0 = System.currentTimeMillis();
DivaAnalysisGetResponse respAnalysis = DivaHTTPClient.getAnalysis(response.getSessionid(), correlationVal, signalNoiseVal, longMinVal, longMaxVal, longResolutionVal, latMinVal, latMaxVal, latResolutionVal, depthLevelVal);
long t1 = System.currentTimeMillis();
AnalysisLogger.getLogger().debug("Analysis finished in "+(t1-t0)+" ms");
statResultMap.put("Minimum value of the analysis", "" + respAnalysis.getVmin());
statResultMap.put("Maximum value of the analysis", "" + respAnalysis.getVmax());
statResultMap.put("Number of observations used", "" + respAnalysis.getStat_obs_count_used());
statResultMap.put("Posterior probability of the model", "" + respAnalysis.getStat_posteriori_stn());
AnalysisLogger.getLogger().debug("Map of results to be returned: "+statResultMap);
AnalysisLogger.getLogger().debug("Downloading result file in "+neofile.getAbsolutePath());
// DOWNLOAD FILE
DivaHTTPClient.downloadFileDiva(respAnalysis.getIdentifier(), neofile.getAbsolutePath());
AnalysisLogger.getLogger().debug("Downloading finished");
} catch (Exception e) {
if (neofile.exists()){
neofile.delete();
AnalysisLogger.getLogger().debug("Output file "+neofile.getAbsolutePath()+" deleted!");
}
throw e;
} finally {
if (fileForDiva.exists()){
fileForDiva.delete();
AnalysisLogger.getLogger().debug("Input file "+fileForDiva.getAbsolutePath()+" deleted!");
}
}
}
@Override
protected void setInputParameters() {
List<TableTemplates> templates = new ArrayList<TableTemplates>();
templates.add(TableTemplates.GENERIC);
InputTable tinput = new InputTable(templates, "InputTable", "Input tabular resource");
InputTable tinput = new InputTable(templates, "InputTable", "Input tabular resource. Up to 100,000 points will be randomly taken from this table.");
inputs.add(tinput);
ColumnType p1 = new ColumnType("InputTable", "Longitude", "The column containing longitude decimal values", "longitude", false);
ColumnType p2 = new ColumnType("InputTable", "Latitude", "The column containing latitude decimal values", "latitude", false);
ColumnType p3 = new ColumnType("InputTable", "Quantity", "The column containing quantity values", "quantity", false);
inputs.add(p1);
inputs.add(p2);
inputs.add(p3);
PrimitiveType p4 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LongitudeMinValue",
"Minimum value of Longitude Range","-180");
PrimitiveType p5 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LongitudeMaxValue",
"Maximum value of Longitude Range","180");
PrimitiveType p6 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LongitudeResolution",
"Longitude resolution (minimum 0.1 - maximum 10)", "1");
PrimitiveType p7 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LatitudeMinValue",
"Minimum value of Latitude Range","-85");
PrimitiveType p8 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LatitudeMaxValue",
"Maximum value of Latitude Range","85");
PrimitiveType p9 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "LatitudeResolution",
"Latitude resolution (minimum 0.1 - maximum 10)", "1");
PrimitiveType p10 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "CorrelationLength",
"Correlation length (arc degrees)", "10.35");
PrimitiveType p11 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "SignalNoise",
"Signal to noise ratio (non-dimensional)", "1.08");
PrimitiveType p12 = new PrimitiveType(Double.class.getName(),
null, PrimitiveTypes.NUMBER, "DepthLevel",
"Depth level", "0");
PrimitiveType p4 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LongitudeMinValue", "Minimum deg. value of the longitude range (min -180)", "-180");
PrimitiveType p5 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LongitudeMaxValue", "Maximum deg. value of the longitude range (max 180)", "180");
PrimitiveType p6 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LongitudeResolution", "Longitude resolution (minimum 0.1 - maximum 10)", "1");
PrimitiveType p7 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LatitudeMinValue", "Minimum deg value of Latitude Range (min -85)", "-85");
PrimitiveType p8 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LatitudeMaxValue", "Maximum value of Latitude Range (max 90)", "85");
PrimitiveType p9 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "LatitudeResolution", "Latitude resolution (minimum 0.1 - maximum 10)", "1");
PrimitiveType p10 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "CorrelationLength", "Correlation length (arc degrees)", "10.35");
PrimitiveType p11 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "SignalNoise", "Signal to noise ratio", "1.08");
PrimitiveType p12 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, "DepthLevel", "Depth level (meters)", "0");
inputs.add(p4);
inputs.add(p5);
inputs.add(p6);
@ -234,50 +220,34 @@ public class SeaDataNetConnector extends StandardLocalExternalAlgorithm {
inputs.add(p10);
inputs.add(p11);
inputs.add(p12);
DatabaseType.addDefaultDBPars(inputs);
}
@Override
public void shutdown() {
if (dbconnection!=null) dbconnection.close();
if (dbconnection != null)
dbconnection.close();
}
public StatisticalType getOutput(){
PrimitiveType file = new PrimitiveType(File.class.getName(),
outputfile,
PrimitiveTypes.FILE,
"NetCDFOutputFile",
"Output file in NetCDF format");
for (String key:statResultMap.keySet()){
String value = statResultMap.get(key);
PrimitiveType val = new PrimitiveType(String.class.getName(),value,
PrimitiveTypes.STRING, key, key);
outputDivaMap.put(key, val);
}
public StatisticalType getOutput() {
PrimitiveType file = new PrimitiveType(File.class.getName(), outputfile, PrimitiveTypes.FILE, "NetCDFOutputFile", "Output file in NetCDF format");
for (String key : statResultMap.keySet()) {
String value = statResultMap.get(key);
PrimitiveType val = new PrimitiveType(String.class.getName(), value, PrimitiveTypes.STRING, key, key);
outputDivaMap.put(key, val);
}
outputDivaMap.put("Netcdf output file", file);
PrimitiveType hashma = new PrimitiveType(HashMap.class.getName(), outputDivaMap
, PrimitiveTypes.MAP,
"Diva results","Output of DIVA fit");
return hashma;
}
PrimitiveType hashma = new PrimitiveType(HashMap.class.getName(), outputDivaMap, PrimitiveTypes.MAP, "Diva results", "Output of DIVA fit");
return hashma;
}
}

View File

@ -48,12 +48,12 @@ public class TestSeaDataNetConnector {
config.setParam("Latitude", "centerlat");
config.setParam("Quantity", "landdist");
config.setParam("LongitudeMinValue", "-200");
config.setParam("LongitudeMinValue", "-180");
config.setParam("LongitudeMaxValue", "180");
config.setParam("LongitudeResolution", "3");
config.setParam("LatitudeMinValue", "-90");
config.setParam("LatitudeMaxValue", "90");
config.setParam("LatitudeMinValue", "-80");
config.setParam("LatitudeMaxValue", "80");
config.setParam("LatitudeResolution", "3");
config.setParam("CorrelationLength", "10");