From 8005d1ef99eb739b4e3db8ce6fe1bb3572762d44 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Thu, 7 Sep 2017 11:02:44 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/SeaDataNetConnector@152852 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../SeaDataNetConnector_AutoCorrelation.java | 329 ------------------ 1 file changed, 329 deletions(-) delete mode 100644 src/org/gcube/dataanalysis/seadatanet/SeaDataNetConnector_AutoCorrelation.java diff --git a/src/org/gcube/dataanalysis/seadatanet/SeaDataNetConnector_AutoCorrelation.java b/src/org/gcube/dataanalysis/seadatanet/SeaDataNetConnector_AutoCorrelation.java deleted file mode 100644 index b730581..0000000 --- a/src/org/gcube/dataanalysis/seadatanet/SeaDataNetConnector_AutoCorrelation.java +++ /dev/null @@ -1,329 +0,0 @@ -package org.gcube.dataanalysis.seadatanet; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.UUID; - -import org.gcube.contentmanagement.graphtools.utils.HttpRequest; -import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; -import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; -import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; -import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; -import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; -import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm; -import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; -import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; -import org.geotoolkit.geometry.jts.JTS; -import org.geotoolkit.referencing.CRS; -import org.geotoolkit.referencing.crs.DefaultGeographicCRS; -import org.hibernate.SessionFactory; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.operation.MathTransform; - -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.Geometry; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.LinearRing; -import com.vividsolutions.jts.geom.Polygon; -import com.vividsolutions.jts.geom.PrecisionModel; -import com.vividsolutions.jts.geom.impl.CoordinateArraySequence; -import com.vividsolutions.jts.io.WKTReader; - -public class SeaDataNetConnector_AutoCorrelation extends StandardLocalExternalAlgorithm { - - // Statistical result by DIVA - LinkedHashMap statResultMap = new LinkedHashMap(); - - // HashMap for - LinkedHashMap outputDivaMap = new LinkedHashMap(); - SessionFactory dbconnection; - - @Override - public void init() throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public String getDescription() { - //->adjusted description with proper citation - return "An algorithm to geographically interpolate punctual observation data. Based on a connector to the SeaDataNet infrastructure, this version automatically estimates the correlation between the data, and signal to noise ratio. " + - "This algorithms invokes the Data-Interpolating Variational Analysis (DIVA) SeaDataNet service to interpolate spatial data. " + - "The model uses GEBCO bathymetry data. " + - "It can interpolate up to 10,000 points randomly taken from the input table. " + - "As output, it produces a NetCDF file with a uniform grid of values. " + - "The underlying model is described in Troupin et al. 2012, 'Generation of analysis and consistent error fields using the Data Interpolating Variational Analysis (Diva)', Ocean Modelling, 52-53, 90-101."; - } - - File outputfile; - - public static String InputTableP = "InputTable"; - public static String LongitudeP = "Longitude"; - public static String LatitudeP = "Latitude"; - public static String QuantityP = "Quantity"; - public static String ExtensionP = "ProjectionExtension"; - public static String ResolutionP = "ProjectionResolution"; - public static String DepthHeightP = "DepthOrHeightLevel"; - public static String UseCurrentsP = "UseCurrentsInformation"; - public static String CurrentsWeightP = "CurrentsVelocityWeight"; - - Double longMinVal = null; - Double longMaxVal = null; - Double latMinVal = null; - Double latMaxVal = null; - - public void parseExtension(String WKTSquare) throws Exception{ - WKTReader wktr = new WKTReader(); - Polygon g = (Polygon) wktr.read(WKTSquare); - - Coordinate[] coordinates = g.getCoordinates(); - - double lat1 = coordinates[0].y; - double lat2 = coordinates[2].y; - - latMaxVal = Math.max(lat1, lat2); - latMinVal = Math.min(lat1, lat2); - - double lon1 = coordinates[0].x; - double lon2 = coordinates[2].x; - - longMaxVal = Math.max(lon1, lon2); - longMinVal = Math.min(lat1, lat2); - - if (longMaxVal>180) - longMaxVal=180d; - if (longMaxVal<-180) - longMaxVal=-180d; - if (longMinVal>180) - longMaxVal=180d; - if (longMinVal<-180) - longMinVal=-180d; - - if (latMaxVal>85) - latMaxVal=85d; - if (latMaxVal<-85) - longMaxVal=-85d; - if (latMinVal>85) - latMinVal=85d; - if (latMinVal<-85) - latMinVal=-85d; - - } - - @Override - protected void process() throws Exception { - - File neofile = null; - File fileForDiva = null; - try { - String outpath = config.getPersistencePath(); - neofile = new File(outpath, "seadn_diva_" + UUID.randomUUID() + ".nc"); - AnalysisLogger.getLogger().debug("Input Parameters"); - AnalysisLogger.getLogger().debug("Input Table: " + config.getParam(InputTableP)); - AnalysisLogger.getLogger().debug("Input Long: " + config.getParam(LongitudeP)); - AnalysisLogger.getLogger().debug("Input Lat: " + config.getParam(LatitudeP)); - AnalysisLogger.getLogger().debug("Input Qt: " + config.getParam(QuantityP)); - AnalysisLogger.getLogger().debug("Extension: " + config.getParam(ExtensionP)); - AnalysisLogger.getLogger().debug("Resolution: " + config.getParam(ResolutionP)); - AnalysisLogger.getLogger().debug("Depth-Heigh Level: " + config.getParam(DepthHeightP)); - AnalysisLogger.getLogger().debug("Use currents: " + config.getParam(UseCurrentsP)); - AnalysisLogger.getLogger().debug("Currents weight: " + config.getParam(CurrentsWeightP)); - - AnalysisLogger.getLogger().debug("Converting weights"); - - int currentsWeight = (int) Math.round(Double.parseDouble(""+config.getParam(CurrentsWeightP))); - if (currentsWeight<0) - currentsWeight=0; - else - currentsWeight = currentsWeight*2000/100; - - AnalysisLogger.getLogger().debug("Recalculated currents weight: " + currentsWeight); - - AnalysisLogger.getLogger().debug("Checking parameters consistency"); - //->try - catch to manage case of NULL values - //->the check on the values has been put before the initialization of the connection - parseExtension(config.getParam(ExtensionP)); - Double resolutionVal = null; - Double depthLevelVal = null; - - AnalysisLogger.getLogger().debug("Parse extension:"); - AnalysisLogger.getLogger().debug("X: ["+longMinVal+","+longMaxVal+"]"); - AnalysisLogger.getLogger().debug("Y: ["+latMinVal+","+latMaxVal+"]"); - - AnalysisLogger.getLogger().debug("Parsing use currents choice"); - - Boolean useCurrents = Boolean.valueOf(config.getParam(UseCurrentsP)); - - try { - - resolutionVal = Double.parseDouble(config.getParam(ResolutionP)); - depthLevelVal = Double.parseDouble(config.getParam(DepthHeightP)); - if (depthLevelVal < 0) - throw new Exception("Depth-Heigh Level cannot be negative"); - - } catch (NumberFormatException e) { - throw new Exception("Parameter values are incomplete"); - } - - AnalysisLogger.getLogger().debug("Parameters are OK"); - AnalysisLogger.getLogger().debug("Initializing DB connection"); - dbconnection = DatabaseUtils.initDBSession(config); - //->set limit to 100 000 - maximum allowed by DIVA - String query = "select " - + config.getParam(LongitudeP) + "," + config.getParam(LatitudeP) + "," + config.getParam(QuantityP) + " From " + getInputParameter(InputTableP) + - " ORDER BY RANDOM() limit 10000"; - //->indicate the status of the computation - status = 10; - AnalysisLogger.getLogger().debug("Query for extracting data from the DB: " + query); - List dataList = DatabaseFactory.executeSQLQuery(query, dbconnection); - int ndata = dataList.size(); - fileForDiva = new File(outpath, "file_for_diva_" + UUID.randomUUID() + ".txt"); - BufferedWriter fileWriterDiva = new BufferedWriter(new FileWriter(fileForDiva)); - AnalysisLogger.getLogger().debug("Writing input file in: " + fileForDiva.getAbsolutePath()); - for (Object o : dataList) { - Object[] oarray = (Object[]) o; - fileWriterDiva.write(" " + oarray[0] + " " + oarray[1] + " " + oarray[2] + "\n"); - } - fileWriterDiva.close(); - - AnalysisLogger.getLogger().debug("Sending data to DIVA: Uploading "+ndata+" records"); - // integration DivaHttpClient - // UPLOADFILE for DIVA - //->*use the HTTPClient Class methods - DivaFilePostResponse response = DivaHTTPClient.uploadFile(fileForDiva); - AnalysisLogger.getLogger().debug("DIVA Server Response for the Upload:\n" + response.getSessionid()); - status = 30; - - int velocityuploadingOK = 200; - if (useCurrents && currentsWeight>0){ - String currentsFile = "http://data.d4science.org/QW43dXd0RDRIYUJUOENpNXF1ekoxNnFLeUdSTjZQWGZHbWJQNStIS0N6Yz0"; //under VRE Folders->BiodiversityLab - String tempFile = "currents"+UUID.randomUUID()+".nc"; - AnalysisLogger.getLogger().debug("Downloading oceans currents file from "+currentsFile); - HttpRequest.downloadFile(currentsFile, tempFile); - File tempcurrentsfile = new File(tempFile); - AnalysisLogger.getLogger().debug("Sending ocean currents to DIVA - local File exists "+tempcurrentsfile.exists()); - velocityuploadingOK = DivaHTTPClient.postVelocityFiles(response.getSessionid(), "u", "v", tempcurrentsfile); - AnalysisLogger.getLogger().debug("Done - Sending ocean currents to DIVA"); - tempcurrentsfile.delete(); - } - //velocityuploadingOK = DivaHTTPClient.postVelocityFiles(response.getSessionid(), "u", "v", new File ("./example.nc")); - if (velocityuploadingOK!=200) - throw new Exception("Could not upload currents files. Response "+velocityuploadingOK); - - AnalysisLogger.getLogger().debug("Requesting analysis for data correlation to DIVA..."); - DivaFitResponse fitResponse = DivaHTTPClient.getFit(response.getSessionid()); - AnalysisLogger.getLogger().debug("Response from DIVA FIT: "+fitResponse); - - status = 50; - - AnalysisLogger.getLogger().debug("Requesting analysis to DIVA..."); - long t0 = System.currentTimeMillis(); - DivaAnalysisGetResponse respAnalysis = DivaHTTPClient.getAnalysis(response.getSessionid(), fitResponse.getLength_scale(), fitResponse.getSignal_to_noise(), longMinVal, longMaxVal, resolutionVal, latMinVal, latMaxVal, resolutionVal, depthLevelVal, useCurrents, currentsWeight); - AnalysisLogger.getLogger().debug("Response from DIVA analysis: "+respAnalysis); - long t1 = System.currentTimeMillis(); - //->Record the time of the analysis - AnalysisLogger.getLogger().debug("Analysis finished in "+(t1-t0)+" ms"); - status = 80; - //->the number of observations is now an integer - statResultMap.put("Minimum value estimated by the model", "" + respAnalysis.getVmin()); - statResultMap.put("Maximum value estimated by the model", "" + respAnalysis.getVmax()); - statResultMap.put("Number of observations used", "" + respAnalysis.getStat_obs_count_used()); - statResultMap.put("Signal-to-noise ratio", "" + fitResponse.getSignal_to_noise()); - statResultMap.put("Geographical length of the correlation (deg)", "" + fitResponse.getLength_scale()); - statResultMap.put("Quality of the automatic signal-to-noise ratio estimate", "" + fitResponse.getFit_quality()); - //statResultMap.put("A posteriori estimate of signal-to-noise ratio", "" + respAnalysis.getStat_posteriori_stn()); - - AnalysisLogger.getLogger().debug("Map of results to be returned: "+statResultMap); - AnalysisLogger.getLogger().debug("Downloading result file in "+neofile.getAbsolutePath()); - // DOWNLOAD FILE - DivaHTTPClient.downloadFileDiva(respAnalysis.getIdentifier(), neofile.getAbsolutePath()); - //->*put the output file in the output object - outputfile=neofile; - AnalysisLogger.getLogger().debug("Downloading finished"); - - AnalysisLogger.getLogger().debug("FINAL RESULT MAP: "+statResultMap.toString()); - - } catch (Exception e) { - //->*ONLY in case of errors, delete the output file - if (neofile.exists()){ - neofile.delete(); - AnalysisLogger.getLogger().debug("Output file "+neofile.getAbsolutePath()+" deleted!"); - } - throw e; - } finally { - //->*in any case, delete the input file because we don't need it anymore - if (fileForDiva.exists()){ - fileForDiva.delete(); - AnalysisLogger.getLogger().debug("Input file "+fileForDiva.getAbsolutePath()+" deleted!"); - } - AnalysisLogger.getLogger().debug("DIVA process finished"); - } - } - - @Override - protected void setInputParameters() { - - List templates = new ArrayList(); - templates.add(TableTemplates.GENERIC); - InputTable tinput = new InputTable(templates, InputTableP, "Input tabular resource. Up to 10,000 points will be randomly taken from this table."); - - inputs.add(tinput); - - ColumnType p1 = new ColumnType(InputTableP, LongitudeP, "The column containing longitude decimal values", "longitude", false); - ColumnType p2 = new ColumnType(InputTableP, LatitudeP, "The column containing latitude decimal values", "latitude", false); - ColumnType p3 = new ColumnType(InputTableP, QuantityP, "The column containing the values to geographycally interpolate", "quantity", false); - - PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, ExtensionP, "Bounding Box of the projection. Values outside the (-180,+80;-90,+90) range will be automatically adjusted to this range. [WKT_BOX]", "POLYGON((-206.71875 133.59375,-206.71875 -147.65625,212.34375 -147.65625,212.34375 133.59375,-206.71875 133.59375))"); - PrimitiveType p5 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, ResolutionP, "Resolution of the projection (minimum 0.1deg. - maximum 10deg.)", "1"); - PrimitiveType p6 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, DepthHeightP, "Depth (a negative value) or height (a positive value) of the observations (in meters).", "0"); - PrimitiveType p7 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, UseCurrentsP, "Involve information about average annual global currents velocity.", "true"); - PrimitiveType p8 = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, CurrentsWeightP, "The weight of the currents velocity in the analysis in percentace (0-100)", "50"); - - inputs.add(p1); - inputs.add(p2); - inputs.add(p3); - inputs.add(p4); - inputs.add(p5); - inputs.add(p6); - inputs.add(p7); - inputs.add(p8); - - DatabaseType.addDefaultDBPars(inputs); - - } - - @Override - public void shutdown() { - - if (dbconnection != null) - dbconnection.close(); - - } - - public StatisticalType getOutput() { - - PrimitiveType file = new PrimitiveType(File.class.getName(), outputfile, PrimitiveTypes.FILE, "NetCDFOutputFile", "Output file in NetCDF format"); - - for (String key : statResultMap.keySet()) { - String value = statResultMap.get(key); - PrimitiveType val = new PrimitiveType(String.class.getName(), value, PrimitiveTypes.STRING, key, key); - outputDivaMap.put(key, val); - } - - outputDivaMap.put("Netcdf output file", file); - - PrimitiveType hashma = new PrimitiveType(HashMap.class.getName(), outputDivaMap, PrimitiveTypes.MAP, "Diva results", "Output of DIVA fit"); - - return hashma; - } - -}