From 168d78ca64007a54ff369d92935b5e59a3f79513 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Thu, 16 Oct 2014 15:10:50 +0000 Subject: [PATCH] Adjustment for ASC raster mining and geo-table dumps Corrections in the MaxEnt parameters descriptions git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@100760 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../MaxEnt4NicheModellingTransducer.java | 16 +- .../geo/connectors/asc/AscDataExplorer.java | 18 +- .../geo/connectors/asc/AscRaster.java | 17 +- .../geo/connectors/asc/AscRasterWriter.java | 43 +++++ .../geo/connectors/table/Table.java | 3 +- .../geo/connectors/wfs/WFSDataExplorer.java | 29 ++- .../geo/matrixmodel/ASCConverter.java | 2 +- .../geo/matrixmodel/RasterTable.java | 42 ++-- .../dataanalysis/geo/test/TestPointslice.java | 2 +- .../geo/test/infra/TestMaxEnt.java | 10 +- .../test/projections/GeolocateCountry.java | 148 ++++++++++++++ .../geo/test/projections/ProduceASCFile.java | 143 ++++++++++++++ .../test/projections/ProduceCentroids.java | 48 +++++ .../TestXYExtractionAlgorithm.java | 18 +- .../TestXYExtractionConnectors.java | 182 +++++++++--------- .../geo/utils/VectorOperations.java | 29 ++- 16 files changed, 579 insertions(+), 171 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/geo/test/projections/GeolocateCountry.java create mode 100644 src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceASCFile.java create mode 100644 src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceCentroids.java diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/MaxEnt4NicheModellingTransducer.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/MaxEnt4NicheModellingTransducer.java index 9b7c195..3574f4c 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/algorithms/MaxEnt4NicheModellingTransducer.java +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/MaxEnt4NicheModellingTransducer.java @@ -109,10 +109,10 @@ public class MaxEnt4NicheModellingTransducer implements Transducerer { @Override public String getDescription() { return "A Maximum-Entropy model for species habitat modeling, based on the implementation by Shapire et al. v 3.3.3k, Princeton University, http://www.cs.princeton.edu/schapire/maxent/. " + - "In this adaptation for the D4Science infrastructure, the software can accept a table produced by the Species Product Discovery service and a set of environmental layers in various formats (NetCDF, WFS, WCS, ASC, GeoTiff) via direct links or GeoExplorer UUIDs. " + - "The user can also establish the bounding box and the spatial resolution (in deg) of the training and the projection. The application will adapt the layers to that resolution if this is higher than the native one." + + "In this adaptation for the D4Science infrastructure, the software accepts a table produced by the Species Product Discovery service and a set of environmental layers in various formats (NetCDF, WFS, WCS, ASC, GeoTiff) via direct links or GeoExplorer UUIDs. " + + "The user can also establish the bounding box and the spatial resolution (in decimal deg.) of the training and the projection. The application will adapt the layers to that resolution if this is higher than the native one." + "The output contains: a thumbnail map of the projected model, the ROC curve, the Omission/Commission chart, a table containing the raw assigned values, a threshold to transform the table into a 0-1 probability distribution, a report of the importance of the used layers in the model, ASCII representations of the input layers to check their alignment." + - "Other processes can be later applied to the raw values to produce a GIS map (e.g. the Statistical Manager Points to Map process) and results can be shared. Demo video: http://goo.gl/TYYnTO"; + "Other processes can be later applied to the raw values to produce a GIS map (e.g. the Statistical Manager Points-to-Map process) and results can be shared. Demo video: http://goo.gl/TYYnTO and instructions http://wiki.i-marine.eu/index.php/MaxEnt"; } @Override @@ -134,16 +134,16 @@ public class MaxEnt4NicheModellingTransducer implements Transducerer { ColumnType p2 = new ColumnType(OccurrencesTableNameParameter, LatitudeColumn, "The column containing latitude values", "decimallatitude", false); inputs.add(p2); - IOHelper.addDoubleInput(inputs, z, "Value of Z. Default is 0, that means processing will be at surface level or at the first avaliable Z value in the layer", "0"); - IOHelper.addIntegerInput(inputs, t, "Time Index. The default is the first time indexed dataset", "0"); - - IOHelper.addDoubleInput(inputs, xRes, "Projection resolution on the X axis in degrees", "1"); - IOHelper.addDoubleInput(inputs, yRes, "Projection resolution on the Y axis in degrees", "1"); + IOHelper.addDoubleInput(inputs, xRes, "Model projection resolution on the X axis in decimal degrees", "1"); + IOHelper.addDoubleInput(inputs, yRes, "Model projection resolution on the Y axis in decimal degrees", "1"); // layers to use in the model PrimitiveTypesList listEnvLayers = new PrimitiveTypesList(String.class.getName(), PrimitiveTypes.STRING, Layers, "The list of environmental layers to use for enriching the points. Each entry is a layer Title or UUID or HTTP link. E.g. the title or the UUID (preferred) of a layer indexed in the e-Infrastructure on GeoNetwork - You can retrieve it from GeoExplorer. Otherwise you can supply the direct HTTP link of the layer. The format will be guessed from the link. The default is GeoTiff. Supports several standards (NETCDF, WFS, WCS, ASC, GeoTiff ). E.g. https://dl.dropboxusercontent.com/u/12809149/wind1.tif", false); inputs.add(listEnvLayers); + IOHelper.addDoubleInput(inputs, z, "Value of Z. Default is 0, that means environmental layers processing will be at surface level or at the first avaliable Z value in the layer", "0"); + IOHelper.addIntegerInput(inputs, t, "Time Index. The default is the first time indexed in the input environmental datasets", "0"); + DatabaseType.addDefaultDBPars(inputs); return inputs; } diff --git a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscDataExplorer.java b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscDataExplorer.java index 841ae70..d439700 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscDataExplorer.java +++ b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscDataExplorer.java @@ -54,20 +54,6 @@ public class AscDataExplorer { } - public int longitude2Index (double longitude){ - if (dx>0) - return (int)Math.round((longitude-xOrigin)/dx); - else - return (int)Math.round((longitude-xOrigin)/cellsize); - } - - public int latitude2Index (double latitude){ - if (dy>0) - return (nrows-1) - (int) Math.round((latitude-yOrigin)/dy); - else - return (nrows-1) - (int) Math.round((latitude-yOrigin)/cellsize); - } - public List retrieveDataFromAsc( List> triplets, int time) throws Exception{ if (time>0) throw new Exception("No Time Dimension For ASC Files!"); @@ -76,8 +62,8 @@ public class AscDataExplorer { for (Tuple triplet:triplets){ double x = triplet.getElements().get(0); double y = triplet.getElements().get(1); - int j = longitude2Index(x); - int i = latitude2Index(y); + int j = ascFile.longitude2Index(x); + int i = ascFile.latitude2Index(y); if ((j>ncolumns) || (j<0) || (i>nrows) || (i<0)){ values.add(Double.NaN); diff --git a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRaster.java b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRaster.java index fcbd1aa..d57ab20 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRaster.java +++ b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRaster.java @@ -158,7 +158,11 @@ public class AscRaster { public double getValue(int row, int column) { if (row < rows && column < cols) - return data[row][column]; + if (data[row][column]!= Double.parseDouble(NDATA)) + return data[row][column]; + else + return Double.NaN; + return Double.NaN; } @@ -259,10 +263,17 @@ public class AscRaster { } public int longitude2Index(double longitude) { - return (int) Math.round((longitude - xll) / cellsize); + double factor = dx; + if (dx<0) + factor = cellsize; + return (int) Math.round((longitude - xll) / factor); } public int latitude2Index(double latitude) { - return (rows - 1) - (int) Math.round((latitude - yll) / cellsize); + double factor = dy; + if (dy<0) + factor = cellsize; + return (rows-1) - (int) Math.round((latitude - yll) / factor); + } } diff --git a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRasterWriter.java b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRasterWriter.java index 2f40e84..09b6e33 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRasterWriter.java +++ b/src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRasterWriter.java @@ -45,8 +45,46 @@ public class AscRasterWriter { } o.println( "NODATA_value " + r.getNDATA() ); + double[][] values = r.getData(); + //for(int k=values.length-1;k>=0;k-- ) + for(int k=0;k0) + o.println( "cellsize " + r.getCellsize() ); + else + { + o.println( "dx " + r.getdx() ); + o.println( "dy " + r.getdy() ); + } + o.println( "NODATA_value " + r.getNDATA() ); + double[][] values = r.getData(); for(int k=values.length-1;k>=0;k-- ) +// for(int k=0;k v = VectorOperations.assignPointsValuesToGrid(coordinates3d, time, tuples, tolerance); // AnalysisLogger.getLogger().debug("VALUES "+v); diff --git a/src/main/java/org/gcube/dataanalysis/geo/connectors/wfs/WFSDataExplorer.java b/src/main/java/org/gcube/dataanalysis/geo/connectors/wfs/WFSDataExplorer.java index 93ac236..271404c 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/connectors/wfs/WFSDataExplorer.java +++ b/src/main/java/org/gcube/dataanalysis/geo/connectors/wfs/WFSDataExplorer.java @@ -47,7 +47,11 @@ public class WFSDataExplorer { // String wfsURL = OGCFormatter.getWfsUrl(geoServer, layer, OGCFormatter.buildBoundingBox(xL, yL, xR, yR), 0, "json"); // there is a bug in WFS in the retrieval according to a bounding box: y must be in the range -180;180. then I preferred to take all the features - String wfsURL = OGCFormatter.getWfsUrl(geoServer, layer, null, 0, "json"); + //"51,-120,57,-106" + String bbox = null; +// bbox = "60,0,90,180"; + + String wfsURL = OGCFormatter.getWfsUrl(geoServer, layer, bbox, 0, "json"); AnalysisLogger.getLogger().debug("WFSDataExplorer-> Requesting URL: " + wfsURL); String returned = null; try { @@ -102,12 +106,12 @@ public class WFSDataExplorer { if (poly == null) poly = new FeaturedPolygon(); - LinkedHashMap props = (LinkedHashMap) propertiesMap.get(properties); + LinkedHashMap props = (LinkedHashMap) propertiesMap.get(properties); // fill the properties of the fpolygon for (String keyprop : props.keySet()) { try { // fulfill the FeaturedPolygon - String value = props.get(keyprop); + String value = (""+props.get(keyprop)).replace("{", "").replace("}", ""); try { String lowcaseprop = keyprop.toLowerCase(); // System.out.println(poly.p.getCentroid()+" -> "+value); @@ -122,6 +126,7 @@ public class WFSDataExplorer { poly.addFeature(keyprop, value); } } catch (Exception e) { + e.printStackTrace(); } } } else if (properties.contains("geometry") && !properties.contains("geometry_")) { @@ -216,7 +221,8 @@ public class WFSDataExplorer { boolean found = false; int h = 0; - for (Polygon polnh : polysnoholes) { + while(h polysnoholes, Polygon polnh, Polygon pp){ + private static int addDifference(int h , List polysnoholes, Polygon polnh, Polygon pp){ Geometry mp = polnh.difference(pp); if (mp instanceof com.vividsolutions.jts.geom.Polygon) @@ -272,9 +278,16 @@ public class WFSDataExplorer { int innerpolygons = mup.getNumGeometries(); for (int k = 0; k < innerpolygons; k++) { Polygon ip = (Polygon) mup.getGeometryN(k); - polysnoholes.set(h, ip); + if (k==0) + polysnoholes.set(h, ip); + else + polysnoholes.add(h, ip); + h++; } + if (innerpolygons>0) + h--;//set the cursor on the last element } + return h; } diff --git a/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/ASCConverter.java b/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/ASCConverter.java index 32a26d3..169e4a4 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/ASCConverter.java +++ b/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/ASCConverter.java @@ -38,7 +38,7 @@ public class ASCConverter { String outputFile =new File(outFilePath).getAbsolutePath(); AscRasterWriter writer = new AscRasterWriter(); - writer.writeRaster(outputFile, raster); + writer.writeRasterInvertYAxis(outputFile, raster); return outputFile; } catch (Exception e) { diff --git a/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/RasterTable.java b/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/RasterTable.java index 0751d20..d1b6a5f 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/RasterTable.java +++ b/src/main/java/org/gcube/dataanalysis/geo/matrixmodel/RasterTable.java @@ -82,7 +82,7 @@ public class RasterTable { public void init(double x1, double x2, double y1, double y2, double z, double time, double xResolution, double yResolution, double[][] values, HashMap> valuesPropertiesMap, AlgorithmConfiguration configuration) { this.valuesMatrix = values; - if (valuesPropertiesMap!=null && valuesPropertiesMap.size()>0) + if (valuesPropertiesMap != null && valuesPropertiesMap.size() > 0) this.valuesPropertiesMap = valuesPropertiesMap; this.configuration = configuration; this.x1 = x1; @@ -104,18 +104,17 @@ public class RasterTable { // create a table String columnNames = columnsnamesStandard; String emptycolumns = ""; - if (valuesPropertiesMap == null){ + if (valuesPropertiesMap == null) { AnalysisLogger.getLogger().debug("Rasterization->No properties to associate"); DatabaseFactory.executeSQLUpdate(String.format(createTableStatementStandard, tablename), dbconnection); - } - else { + } else { AnalysisLogger.getLogger().debug("Managing Table with Custom Fields"); Map valuesMap = valuesPropertiesMap.values().iterator().next(); - AnalysisLogger.getLogger().debug("Rasterization->Sample of properties: "+valuesMap); + AnalysisLogger.getLogger().debug("Rasterization->Sample of properties: " + valuesMap); emptycolumns = generateEmptyValues(valuesMap.size()); DatabaseFactory.executeSQLUpdate(String.format(createTableStatementWithFields, tablename, propertiesMapToColumnString(valuesMap, true)), dbconnection); columnNames = String.format(columnsnamesWithFields, propertiesMapToColumnString(valuesMap, false)); - AnalysisLogger.getLogger().debug("Column names: "+columnNames); + AnalysisLogger.getLogger().debug("Column names: " + columnNames); } AnalysisLogger.getLogger().debug("Table " + tablename + " created"); if (coordinates == null) @@ -127,7 +126,7 @@ public class RasterTable { AnalysisLogger.getLogger().debug("Association to values completed - fulfilling buffer"); // for each element in the matrix, build the corresponding csquare code StringBuffer sb = new StringBuffer(); - int rowcounter = 1; + int rowcounter = 1; for (int i = 0; i < triplets; i++) { // save the string in a buffer Tuple cset = coordinates.get(i); @@ -140,12 +139,11 @@ public class RasterTable { if (valuesPropertiesMap == null) { // we do not use NaNs in this case every value will be filled - if (value.isNaN()){ + if (value.isNaN()) { value = 0d; valueForTable = null; - } - else - valueForTable = "'"+value+"'"; + } else + valueForTable = "'" + value + "'"; } else { // we do not use NaNs in this case every value will be filled if (value.isNaN()) @@ -171,29 +169,28 @@ public class RasterTable { if (valuesPropertiesMap == null) sb.append("('" + csquare + "'," + x + "," + y + "," + zVal + "," + tVal + "," + valueForTable + ")"); else - sb.append("('" + csquare + "',"+ x + "," + y + "," + zVal + "," + tVal + "," + valueForTable + ")"); + sb.append("('" + csquare + "'," + x + "," + y + "," + zVal + "," + tVal + "," + valueForTable + ")"); } if (rowcounter % 5000 == 0) { // AnalysisLogger.getLogger().debug("Partial Inserting Buffer of " + sb.length() + " Values"); - + if (sb.length() > 0) { String insertStatement = DatabaseUtils.insertFromBuffer(tablename, columnNames, sb); -// AnalysisLogger.getLogger().debug("Inserting Buffer " + insertStatement); + // AnalysisLogger.getLogger().debug("Inserting Buffer " + insertStatement); DatabaseFactory.executeSQLUpdate(insertStatement, dbconnection); - + } // AnalysisLogger.getLogger().debug("Partial Insertion completed with Success!"); sb = new StringBuffer(); - } - else if (valueForTable != null) + } else if (valueForTable != null) sb.append(","); - + } AnalysisLogger.getLogger().debug("Inserting Final Buffer of " + sb.length() + " Values"); // AnalysisLogger.getLogger().debug("Inserting Final Buffer " + sb); // save all the strings on the table if (sb.length() > 0) { - String insertStatement = DatabaseUtils.insertFromString(tablename, columnNames, sb.substring(0, sb.length()-1)); -// AnalysisLogger.getLogger().debug("Inserting Buffer " + insertStatement); + String insertStatement = DatabaseUtils.insertFromString(tablename, columnNames, sb.substring(0, sb.length() - 1)); + AnalysisLogger.getLogger().debug("Inserting Buffer " + insertStatement); DatabaseFactory.executeSQLUpdate(insertStatement, dbconnection); AnalysisLogger.getLogger().debug("Insertion completed with Success!"); } @@ -211,11 +208,12 @@ public class RasterTable { StringBuffer sb = new StringBuffer(); int m = valuesMap.size(); int i = 0; + for (String value : valuesMap.values()) { if (value.equals("NULL")) sb.append(value); else - sb.append("'" + value .replace("'", ""+(char)96)+ "'"); + sb.append("'" + value.replace("'", "" + (char) 96) + "'"); if (i < m - 1) sb.append(","); @@ -230,7 +228,7 @@ public class RasterTable { int m = valuesMap.size(); int i = 0; for (String keys : valuesMap.keySet()) { - sb.append("f_"+keys); + sb.append("f_" + keys); if (withtype) sb.append(" character varying"); if (i < m - 1) diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/TestPointslice.java b/src/main/java/org/gcube/dataanalysis/geo/test/TestPointslice.java index e75e8d5..c3ee9dd 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/test/TestPointslice.java +++ b/src/main/java/org/gcube/dataanalysis/geo/test/TestPointslice.java @@ -87,7 +87,7 @@ public static void main3(String[] args) throws Exception{ List currentTimeValues = extractor.currentTimeValues; AscRasterWriter writer = new AscRasterWriter(); - writer.writeRaster("testwritten.asc", new AscRaster(values, 0.08333333, -1, -1, x1, y1)); + writer.writeRasterInvertYAxis("testwritten.asc", new AscRaster(values, 0.08333333, -1, -1, x1, y1)); AscDataExplorer ade2 = new AscDataExplorer("./maxentCompleteLayers/layer1.asc"); Listfeatures2 = ade2.retrieveDataFromAsc(extractor.currentTuples,0); diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/infra/TestMaxEnt.java b/src/main/java/org/gcube/dataanalysis/geo/test/infra/TestMaxEnt.java index 316916e..d255cda 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/test/infra/TestMaxEnt.java +++ b/src/main/java/org/gcube/dataanalysis/geo/test/infra/TestMaxEnt.java @@ -55,7 +55,11 @@ public class TestMaxEnt { String sep=AlgorithmConfiguration.getListSeparator(); // config.setParam("Layers","dfd1bad2-ab00-42ac-8bb2-46a17162f509"+sep+"23646f93-23a8-4be4-974e-aee6bebe1707"); //config.setParam("Layers","94ea5767-ae76-41dc-be87-f9a0bdc96419");//temperature 99-09 2D - config.setParam("Layers","23646f93-23a8-4be4-974e-aee6bebe1707");//ph +// config.setParam("Layers","23646f93-23a8-4be4-974e-aee6bebe1707");//ph +// config.setParam("Layers","fc9ac2f4-a2bd-43d1-a361-ac67c5ceac31");//temperature + + config.setParam("Layers","http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=aquamaps:worldborders"); + config.setParam("MaxIterations","10000"); config.setParam("DefaultPrevalence","1"); @@ -74,8 +78,12 @@ public class TestMaxEnt { config.setParam("BBox_UpperRightLat","10"); */ +// config.setParam("XResolution","0.5"); +// config.setParam("YResolution","0.5"); + config.setParam("XResolution","1"); config.setParam("YResolution","1"); + return config; } diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/projections/GeolocateCountry.java b/src/main/java/org/gcube/dataanalysis/geo/test/projections/GeolocateCountry.java new file mode 100644 index 0000000..74e81ae --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/projections/GeolocateCountry.java @@ -0,0 +1,148 @@ +package org.gcube.dataanalysis.geo.test.projections; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.contentmanagement.lexicalmatcher.utils.DistanceCalculator; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; + +public class GeolocateCountry { + + static String faoreport = "FAO data.csv"; + + // static String faoreport = "C:\\Users\\coro\\Desktop\\allCountries.txt"; + + public static void main1(String[] args) throws Exception { + BufferedReader fr = new BufferedReader(new FileReader(new File(faoreport))); + String line = fr.readLine(); + long counter = 0; + while (line != null) { + // System.out.println(line); + String[] split = line.split("\t"); + String country = split[17]; + String x = split[5]; + String y = split[4]; + if (country.contains("Russia")) + break; + // else + // System.out.println("Country:"+country+" "+x+","+y); + + counter++; + if (counter % 500000 == 0) + System.out.println("Country:" + country + " " + x + "," + y); + line = fr.readLine(); + } + + fr.close(); + } + + public static void main(String[] args) throws Exception { + BufferedReader fr = new BufferedReader(new FileReader(new File(faoreport))); + String line = fr.readLine(); + parseCentroidsFile(); + parseWorldCapitalsFile(); + line = fr.readLine(); + HashMap yetDone = new HashMap(); + while (line != null) { + List p = Transformations.parseCVSString(line, ","); + String country = p.get(1); + //TO DO rebuild the original CSV file + String suggestion = yetDone.get(country); + if (suggestion==null){ + suggestion = getCentroid(country,capitals,0.6); + if (suggestion.length()==0) + suggestion = getCentroid(country,centroids,0.3); + + yetDone.put(country, suggestion); + } + + System.out.println(line+","+suggestion); + + line = fr.readLine(); + } + + fr.close(); + } + + static HashMap centroids = new HashMap(); + static HashMap capitals = new HashMap(); + + public static void parseCentroidsFile() throws Exception { + BufferedReader fr = new BufferedReader(new FileReader(new File("countriescentroids.txt"))); + String line = fr.readLine(); + + while (line != null) { + String[] elems = line.split(","); + String x = elems[0]; + String y = elems[1]; + String cntry_name = elems[2]; + centroids.put(cntry_name, x + "," + y); + line = fr.readLine(); + } + + fr.close(); + } + + public static void parseWorldCapitalsFile() throws Exception { + BufferedReader fr = new BufferedReader(new FileReader(new File("country-capitals.csv"))); + String line = fr.readLine(); + + while (line != null) { + String[] elems = line.split(","); + String x = elems[3]; + String y = elems[2]; + String cntry_name = elems[0]; + capitals.put(cntry_name, x + "," + y); + line = fr.readLine(); + } + + fr.close(); + } + + public static String getCentroid(String country, HashMap centroids, double threshold) { + + String c = centroids.get(country); + List sb = new ArrayList(); + List scores = new ArrayList(); + DistanceCalculator dc = new DistanceCalculator(); + if (c == null) { + for (String key : centroids.keySet()) { + if (key.length() > 0) { +/* + if (key.contains(country) || country.contains(key)) { + if (sb.length() > 0) + sb.append("/"); + + sb.append(key + "," + centroids.get(key) + "("+0.8+")"+" "); + } else { + */ + double score = dc.CD(false, country, key,true,false); + if (score > threshold) { + int i = 0; + for (Double cscore : scores){ + if (cscore0) + return sb.get(0).toString(); + else + return ""; + } else + return country+","+c+ ","+1; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceASCFile.java b/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceASCFile.java new file mode 100644 index 0000000..dec058a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceASCFile.java @@ -0,0 +1,143 @@ +package org.gcube.dataanalysis.geo.test.projections; + +import java.io.File; +import java.io.FileWriter; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; +import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import org.gcube.dataanalysis.geo.connectors.asc.AscRasterWriter; +import org.gcube.dataanalysis.geo.connectors.table.Table; +import org.gcube.dataanalysis.geo.connectors.table.TableMatrixRepresentation; +import org.gcube.dataanalysis.geo.utils.MapUtils; +import org.gcube.dataanalysis.geo.utils.VectorOperations; + +public class ProduceASCFile { + + static String layer = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=aquamaps:worldborders"; + +// static String layer = "ed8f77bd-2423-4036-b34d-2f1cb5fcaffc"; +// static String layer = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=aquamaps:eezall"; +// static String layer = "http://geo.vliz.be/geoserver/MarineRegions/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&typename=MarineRegions:eez"; +// static String layer = "aeabfdb5-9ddb-495e-b628-5b7d2cf1d8a2"; + + static String field = "f_cat"; +// static String field = "eez_id"; +// static String field = "f_eezall"; +// static String field = "f_eez_id"; +// static String field = "f_zone"; + + static double res = 0.3; + static String table = "testextraction4"; + static String scope = "/gcube/devsec/devVRE"; + static String databaseUser = "gcube"; + static String databasePwd = "d4science2"; + static String databaseURL = "jdbc:postgresql://localhost/testdb"; + static String databaseDriver = "org.postgresql.Driver"; + static double xll = -180; + static double yll=-90; + static double xur=180; + static double yur=90; + + static String outASCIIMAP = "producedmap.txt"; + static String outASCFile = "produced.asc"; + + private static AlgorithmConfiguration XYExtractionConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setAgent("XYEXTRACTOR"); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", databaseUser); + config.setParam("DatabasePassword", databasePwd); + config.setParam("DatabaseURL", databaseURL); + config.setParam("DatabaseDriver", databaseDriver); + config.setGcubeScope(scope); + + config.setParam("Layer", layer); + + config.setParam("Z", "0"); + config.setParam("TimeIndex", "0"); + config.setParam("BBox_LowerLeftLat", ""+yll); + config.setParam("BBox_LowerLeftLong", ""+xll); + config.setParam("BBox_UpperRightLat", ""+yur); + config.setParam("BBox_UpperRightLong", ""+xur); + config.setParam("XResolution", ""+res); + config.setParam("YResolution", ""+res); + config.setParam("OutputTableName", table); + config.setParam("OutputTableLabel", table); + + return config; + } + + private static AlgorithmConfiguration TableExtractionConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", databaseUser); + config.setParam("DatabasePassword", databasePwd); + config.setParam("DatabaseURL", databaseURL); + config.setParam("DatabaseDriver", databaseDriver); + config.setGcubeScope(scope); + config.setParam("BBox_LowerLeftLat", ""+yll); + config.setParam("BBox_LowerLeftLong", ""+xll); + config.setParam("BBox_UpperRightLat", ""+xur); + config.setParam("BBox_UpperRightLong", ""+yur); + config.setParam("XResolution", ""+res); + config.setParam("YResolution", ""+res); + config.setParam("OutputTableName", table); + config.setParam("OutputTableLabel", table); + + config.setParam(TableMatrixRepresentation.tableNameParameter, table); + config.setParam(TableMatrixRepresentation.xDimensionColumnParameter, "approx_x"); + config.setParam(TableMatrixRepresentation.yDimensionColumnParameter, "approx_y"); + config.setParam(TableMatrixRepresentation.timeDimensionColumnParameter, "time"); + config.setParam(TableMatrixRepresentation.valueDimensionColumnParameter, field); + config.setParam(TableMatrixRepresentation.filterParameter, " "); + + return config; + } + + public static void main(String[] args) throws Exception{ + +// produce(XYExtractionConfig()); + AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile); + List> tuples = VectorOperations.generateCoordinateTripletsInBoundingBox(xll,xur,yll,yur, 0, res, res); + Table connector = new Table(TableExtractionConfig(), res); + List values = connector.getFeaturesInTimeInstantAndArea(null, null, 0, tuples, xll,xur,yll,yur); + double[][] matrix = VectorOperations.vectorToMatix(values, xll,xur,yll,yur,res, res); + System.out.println(MapUtils.globalASCIIMap(matrix)); + FileWriter fw = new FileWriter(new File(outASCIIMAP)); + fw.write(MapUtils.globalASCIIMap(matrix)); + fw.close(); + + AscRasterWriter writer = new AscRasterWriter(); + writer.writeRasterInvertYAxis(outASCFile, matrix, xll,yll, res, "-9999"); + } + + + public static void produce(AlgorithmConfiguration config) throws Exception { + + System.out.println("TEST 1"); + + + AnalysisLogger.getLogger().debug("Executing: "+config.getAgent()); + List trans = null; + trans = TransducerersFactory.getTransducerers(config); + trans.get(0).init(); + Regressor.process(trans.get(0)); + StatisticalType st = trans.get(0).getOutput(); + AnalysisLogger.getLogger().debug("ST:" + st); + trans = null; + + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceCentroids.java b/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceCentroids.java new file mode 100644 index 0000000..7b30a2a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/projections/ProduceCentroids.java @@ -0,0 +1,48 @@ +package org.gcube.dataanalysis.geo.test.projections; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.geo.connectors.wfs.FeaturedPolygon; +import org.gcube.dataanalysis.geo.connectors.wfs.WFSDataExplorer; + +import com.vividsolutions.jts.geom.Geometry; +import com.vividsolutions.jts.geom.Point; +import com.vividsolutions.jts.geom.Polygon; + +public class ProduceCentroids { + static String layer = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=aquamaps:worldborders"; + static String layername = "aquamaps:worldborders"; + + public static void main(String[] args) throws Exception{ + + List featuresInTime = new ArrayList(); + AnalysisLogger.getLogger().debug("taking WFS features from layer: "+layer); + featuresInTime = WFSDataExplorer.getFeatures(layer, layername, -180, -90, 180, 90); + HashMap centroidsmap = new HashMap(); + HashMap polymap = new HashMap(); + for (FeaturedPolygon fpoly:featuresInTime){ +// Point centroid = fpoly.p.getCentroid(); + Geometry prevPoly = polymap.get(fpoly.features.get("cntry_name")); + + if (prevPoly!=null){ + prevPoly = prevPoly.union(fpoly.p); + } + else + prevPoly = fpoly.p; + +// if ((""+fpoly.features).contains("United States")) +// System.out.println("centroid:"+fpoly.p.getCentroid()+" now "+prevPoly.getCentroid()); + + polymap.put(fpoly.features.get("cntry_name"),prevPoly); + } + + for (String key:polymap.keySet()){ + Point centroid = polymap.get(key).getCentroid(); + System.out.println(centroid.getX()+","+centroid.getY()+","+key); + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionAlgorithm.java b/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionAlgorithm.java index 0091211..2639886 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionAlgorithm.java +++ b/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionAlgorithm.java @@ -193,19 +193,23 @@ public class TestXYExtractionAlgorithm { config.setParam("DatabaseDriver","org.postgresql.Driver"); config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/EGIP"); - config.setParam("Layer","http://repoigg.services.iit.cnr.it/geoserver/IGG/ows?service=WFS&version=1.1.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=IGG:HeatFlowUnit"); + //config.setParam("Layer","http://repoigg.services.iit.cnr.it/geoserver/IGG/ows?service=WFS&version=1.1.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=IGG:HeatFlowUnit"); // config.setParam("Layer","http://egip.brgm-rec.fr/wxs/?service=WFS&version=1.1.0&request=GetFeature&typeName=TemperatureUnit&srsName=EPSG:4326"); - + config.setParam("Layer","http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/ows?service=wfs&version=1.0.0&request=GetFeature&srsName=urn:x-ogc:def:crs:EPSG:4326&TYPENAME=aquamaps:worldborders"); + + https://issue.imarine.research-infrastructures.eu/raw-attachment/ticket/3082/gifgeomap.gif config.setParam("Z","0"); config.setParam("TimeIndex","0"); - config.setParam("BBox_LowerLeftLat","0"); + config.setParam("BBox_LowerLeftLat","-90"); config.setParam("BBox_LowerLeftLong","-180"); config.setParam("BBox_UpperRightLat","90"); config.setParam("BBox_UpperRightLong","180"); - config.setParam("XResolution","0.3"); - config.setParam("YResolution","0.3"); - config.setParam("OutputTableName","testextraction3"); - config.setParam("OutputTableLabel","testextraction3"); +// config.setParam("XResolution","0.3"); +// config.setParam("YResolution","0.3"); + config.setParam("XResolution","0.5"); + config.setParam("YResolution","0.5"); + config.setParam("OutputTableName","testextraction4"); + config.setParam("OutputTableLabel","testextraction4"); return config; } diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionConnectors.java b/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionConnectors.java index 65ed708..9ebfa1f 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionConnectors.java +++ b/src/main/java/org/gcube/dataanalysis/geo/test/projections/TestXYExtractionConnectors.java @@ -8,8 +8,11 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.utils.Tuple; import org.gcube.dataanalysis.geo.connectors.asc.AscDataExplorer; +import org.gcube.dataanalysis.geo.connectors.asc.AscRasterWriter; import org.gcube.dataanalysis.geo.connectors.geotiff.GeoTiff; import org.gcube.dataanalysis.geo.connectors.netcdf.NetCDF; +import org.gcube.dataanalysis.geo.connectors.table.Table; +import org.gcube.dataanalysis.geo.connectors.table.TableMatrixRepresentation; import org.gcube.dataanalysis.geo.connectors.wcs.WCS; import org.gcube.dataanalysis.geo.connectors.wfs.WFS; import org.gcube.dataanalysis.geo.utils.MapUtils; @@ -18,80 +21,46 @@ import org.gcube.dataanalysis.geo.utils.VectorOperations; public class TestXYExtractionConnectors { static String[] urlToTest3 = { -// "http://geoserver3.d4science.research-infrastructures.eu/geoserver" -// "http://geoserver2.d4science.research-infrastructures.eu/geoserver" - "http://www.fao.org/figis/geoserver/species/ows" - }; + // "http://geoserver3.d4science.research-infrastructures.eu/geoserver" + // "http://geoserver2.d4science.research-infrastructures.eu/geoserver" + "http://www.fao.org/figis/geoserver/species/ows" }; static String[] layernamesTest3 = { -// "lsoleasolea20121217184934494cet" -// "lcarcharodoncarcharias20121217173706733cet" -// "lxiphiasgladius20130410182141778cest" -// "SPECIES_DIST_BIB" - - "SPECIES_DIST_SWO" - }; + // "lsoleasolea20121217184934494cet" + // "lcarcharodoncarcharias20121217173706733cet" + // "lxiphiasgladius20130410182141778cest" + // "SPECIES_DIST_BIB" - static String[] urlToTest1 = { - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-gridv_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211498692.nc", - }; + "SPECIES_DIST_SWO" }; - static String[] layernamesTest1= { - "vomecrty" - }; + static String[] urlToTest1 = { "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-gridv_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211498692.nc", }; - - - static String[] urlToTest2 = { - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/WOA2005TemperatureAnnual_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc" - }; + static String[] layernamesTest1 = { "vomecrty" }; - - static String[] layernamesTest2 = { - "t00an1" - }; + static String[] urlToTest2 = { "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/WOA2005TemperatureAnnual_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc" }; - static String[] urlToTest = { - "https://dl.dropboxusercontent.com/u/12809149/layer1.asc", - "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/ph.asc", - "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/calcite.asc", - "https://dl.dropboxusercontent.com/u/12809149/wind1.tif", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/WOA2005TemperatureAnnual_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/salinity_annual_1deg_ENVIRONMENT_OCEANS_.nc", - "http://thredds.d4science.org/thredds/fileServer/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-icemod_ENVIRONMENT_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211441189.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/CERSAT-GLO-CLIM_WIND_L4-OBS_FULL_TIME_SERIE_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366217956317.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/phosphate_seasonal_5deg_ENVIRONMENT_BIOTA_.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-analysis-bio-001-008-_a_BIOTA_ENVIRONMENT_1366217546908.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/dissolved_oxygen_annual_1deg_ENVIRONMENT_BIOTA_.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-gridv_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211498692.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/nitrate_seasonal_5deg_ENVIRONMENT_BIOTA_.nc", - "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-analysis-bio-001-008-a_BIOTA_ENVIRONMENT_1366217608283.nc", - "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/cloudmean.asc", - "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/wcs/wcs?service=wcs&version=1.0.0&request=GetCoverage&coverage=aquamaps:WorldClimBio2&CRS=EPSG:4326&bbox=-180,0,180,90&width=1&height=1&format=geotiff&RESPONSE_CRS=EPSG:4326", - "http://geoserver2.d4science.research-infrastructures.eu/geoserver" - - }; - - static String[] layernamesTest = { - "layer1", - "ph", - "calcite", - "wind", - "t00an1", - "s_sd", - "iicevelu", - "wind_speed", - "p_mn", - "CHL", - "o_mn", - "vomecrty", - "n_mn", - "PHYC", - "cloud", - "aquamaps:WorldClimBio2", - "lxiphiasgladius20130410182141778cest" - }; + static String[] layernamesTest2 = { "t00an1" }; + + static String[] urlToTest5 = { "./maxent3719990c-7998-4859-9dca-4b0a792f9d2f/layer1.asc" }; + + static String[] layernamesTest5 = { "layer1" }; + + static String[] urlToTest6 = { "table" }; + + static String[] layernamesTest6 = { "table" }; + + static String[] urlToTest = { "tableeez" }; + + static String[] layernamesTest = { "tableeez" }; + static String[] urlToTest_ = { "https://dl.dropboxusercontent.com/u/12809149/layer1.asc", "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/ph.asc", "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/calcite.asc", + "https://dl.dropboxusercontent.com/u/12809149/wind1.tif", + "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/WOA2005TemperatureAnnual_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/salinity_annual_1deg_ENVIRONMENT_OCEANS_.nc", "http://thredds.d4science.org/thredds/fileServer/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-icemod_ENVIRONMENT_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211441189.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/CERSAT-GLO-CLIM_WIND_L4-OBS_FULL_TIME_SERIE_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366217956317.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/phosphate_seasonal_5deg_ENVIRONMENT_BIOTA_.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-analysis-bio-001-008-_a_BIOTA_ENVIRONMENT_1366217546908.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/dissolved_oxygen_annual_1deg_ENVIRONMENT_BIOTA_.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-gridv_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366211498692.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/nitrate_seasonal_5deg_ENVIRONMENT_BIOTA_.nc", "http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-analysis-bio-001-008-a_BIOTA_ENVIRONMENT_1366217608283.nc", "http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/cloudmean.asc", "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver/wcs/wcs?service=wcs&version=1.0.0&request=GetCoverage&coverage=aquamaps:WorldClimBio2&CRS=EPSG:4326&bbox=-180,0,180,90&width=1&height=1&format=geotiff&RESPONSE_CRS=EPSG:4326", "http://geoserver2.d4science.research-infrastructures.eu/geoserver" + + }; + + static String[] layernamesTest_ = { "layer1", "ph", "calcite", "wind", "t00an1", "s_sd", "iicevelu", "wind_speed", "p_mn", "CHL", "o_mn", "vomecrty", "n_mn", "PHYC", "cloud", "aquamaps:WorldClimBio2", "lxiphiasgladius20130410182141778cest" }; + public static void main(String[] args) throws Exception { AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); AlgorithmConfiguration config = new AlgorithmConfiguration(); @@ -103,48 +72,69 @@ public class TestXYExtractionConnectors { config.setParam("DatabaseURL", "jdbc:postgresql://localhost/testdb"); config.setParam("DatabaseDriver", "org.postgresql.Driver"); config.setGcubeScope("/gcube/devsec/devVRE"); + FileWriter fw = new FileWriter(new File("mapsconnectors.txt")); - for (int t=0;t values = null; - double step = 2d; - List> tuples = VectorOperations.generateCoordinateTripletsInBoundingBox(-180,180,-90,90, 0, step, step); - - if (layerURL.endsWith(".nc")){ - NetCDF geotiff = new NetCDF(layerURL,layerName); - values = geotiff.getFeaturesInTimeInstantAndArea(layerURL,layerName, 0, tuples, -180, 180, -90, 90); - } - else if (layerURL.endsWith(".asc")){ + double res = 0.5d; + List> tuples = VectorOperations.generateCoordinateTripletsInBoundingBox(-180, 180, -90, 90, 0, res, res); + + if (layerURL.endsWith(".nc")) { + NetCDF geotiff = new NetCDF(layerURL, layerName); + values = geotiff.getFeaturesInTimeInstantAndArea(layerURL, layerName, 0, tuples, -180, 180, -90, 90); + } else if (layerURL.endsWith(".asc")) { AscDataExplorer asc = new AscDataExplorer(layerURL); values = asc.retrieveDataFromAsc(tuples, 0); - } - else if (layerURL.endsWith("tif")){ + } else if (layerURL.endsWith("tif")) { GeoTiff geotiff = new GeoTiff(config); - values = geotiff.getFeaturesInTimeInstantAndArea(layerURL, layerName, 0, tuples, -180, 180, -90, 90); - } - else if (layerURL.contains("wcs")){ - WCS wcs = new WCS(config,layerURL); - values=wcs.getFeaturesInTimeInstantAndArea(layerURL,layerName, 0, tuples, -180, 180, -90, 90); - } - else if (layerURL.contains("geoserver")){ + values = geotiff.getFeaturesInTimeInstantAndArea(layerURL, layerName, 0, tuples, -180, 180, -90, 90); + } else if (layerURL.contains("wcs")) { + WCS wcs = new WCS(config, layerURL); + values = wcs.getFeaturesInTimeInstantAndArea(layerURL, layerName, 0, tuples, -180, 180, -90, 90); + } else if (layerURL.contains("geoserver")) { WFS wfs = new WFS(); - values=wfs.getFeaturesInTimeInstantAndArea(layerURL,layerName, 0, tuples, -180, 180, -90, 90); + values = wfs.getFeaturesInTimeInstantAndArea(layerURL, layerName, 0, tuples, -180, 180, -90, 90); + } else if (layerURL.equals("table")) { + config.setParam(TableMatrixRepresentation.tableNameParameter, "testextraction4"); + config.setParam(TableMatrixRepresentation.xDimensionColumnParameter, "approx_x"); + config.setParam(TableMatrixRepresentation.yDimensionColumnParameter, "approx_y"); + config.setParam(TableMatrixRepresentation.timeDimensionColumnParameter, "time"); + config.setParam(TableMatrixRepresentation.valueDimensionColumnParameter, "f_cat"); + config.setParam(TableMatrixRepresentation.filterParameter, " "); + + Table connector = new Table(config, res); + values = connector.getFeaturesInTimeInstantAndArea(null, null, 0, tuples, -180, 180, -90, 90); + } else if (layerURL.contains("tableeez")) { + config.setParam("DatabaseUserName", "postgres"); + config.setParam("DatabasePassword", "d4science2"); + config.setParam("DatabaseURL", "jdbc:postgresql://geoserver-dev.d4science-ii.research-infrastructures.eu/aquamapsdb"); + config.setParam(TableMatrixRepresentation.tableNameParameter, "\"WorldEEZv72012HR\""); + config.setParam(TableMatrixRepresentation.xDimensionColumnParameter, "longitude"); + config.setParam(TableMatrixRepresentation.yDimensionColumnParameter, "latitude"); + config.setParam(TableMatrixRepresentation.timeDimensionColumnParameter, ""); + config.setParam(TableMatrixRepresentation.valueDimensionColumnParameter, "eez_id"); + config.setParam(TableMatrixRepresentation.filterParameter, " "); + + Table connector = new Table(config, res); + values = connector.getFeaturesInTimeInstantAndArea(null, null, 0, tuples, -180, 180, -90, 90); } - - - double[][] matrix = VectorOperations.vectorToMatix(values, -180,180,-90,90,step, step); - -// System.out.println(MapUtils.globalASCIIMap(values,step,step)); + double[][] matrix = VectorOperations.vectorToMatix(values, -180, 180, -90, 90, res, res); + + // System.out.println(MapUtils.globalASCIIMap(values,step,step)); System.out.println(MapUtils.globalASCIIMap(matrix)); fw.write(MapUtils.globalASCIIMap(matrix)); - + + AscRasterWriter writer = new AscRasterWriter(); + writer.writeRasterInvertYAxis("testraster.asc", matrix, -180, -90, res, "-9999"); + } - + fw.close(); - + } - + } diff --git a/src/main/java/org/gcube/dataanalysis/geo/utils/VectorOperations.java b/src/main/java/org/gcube/dataanalysis/geo/utils/VectorOperations.java index 90a9611..934f5c5 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/utils/VectorOperations.java +++ b/src/main/java/org/gcube/dataanalysis/geo/utils/VectorOperations.java @@ -6,6 +6,7 @@ import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import scala.actors.threadpool.Arrays; import ucar.ma2.ArrayByte; import ucar.ma2.ArrayDouble; import ucar.ma2.IndexIterator; @@ -56,15 +57,20 @@ public class VectorOperations { **/ // suggestion: given a resolution R, give Math.sqrt(2)*R/2=0.7*R as tolerance public static List assignPointsValuesToGrid(List> grid3d, int gridTimeInstant, List> coordinates5d, double tolerance) { - List valuesForGrid = new ArrayList(); +// List valuesForGrid = new ArrayList(); int gridSize = grid3d.size(); + Double [] valuesForGridd = new Double[gridSize]; + for (int i = 0; i < gridSize; i++) { - valuesForGrid.add(Double.NaN); +// valuesForGrid.add(Double.NaN); + valuesForGridd [i] = Double.NaN; } // AnalysisLogger.getLogger().debug("Grid contains: "+grid3d.size()+" values"); // AnalysisLogger.getLogger().debug("Dataset contains: "+coordinates5d.size()+" values"); int foundmatches = 0; + long count = 0; + AnalysisLogger.getLogger().debug("Assigning : "+coordinates5d.size()+" elements"); for (Tuple coord5d : coordinates5d) { double rx = coord5d.getElements().get(0); double ry = coord5d.getElements().get(1); @@ -81,23 +87,31 @@ public class VectorOperations { if (d <= tolerance) { // AnalysisLogger.getLogger().debug("Association: distance between grid:("+x+","+y+","+z+","+gridTimeInstant+") and point:("+rx+","+ry+","+rz+","+rt+") is "+d); - valuesForGrid.set(gridIdx, rvalue); +// valuesForGrid.set(gridIdx, rvalue); + valuesForGridd[gridIdx]=rvalue; foundmatches++; } gridIdx++; } + count++; + if (count%50000==0) + AnalysisLogger.getLogger().debug("Vector Operations Assigned: "+count+" elements"); } AnalysisLogger.getLogger().debug("Association: Found "+foundmatches+" matches between the grid of points and the coordinates"); + + List valuesForGrid = Arrays.asList(valuesForGridd); return valuesForGrid; } public static List assignGridValuesToPoints2D(List> grid3d, List gridValues, List> coordinates4d, double tolerance) { - List valuesForPoints = new ArrayList(); +// List valuesForPoints = new ArrayList(); int gridSize = coordinates4d.size(); + Double [] valuesForPoints = new Double[gridSize]; + for (int i = 0; i < gridSize; i++) { - valuesForPoints.add(Double.NaN); + valuesForPoints[i]=Double.NaN; } @@ -115,7 +129,7 @@ public class VectorOperations { double d = distance(x, y, 0, 0, rx, ry, 0, 0); if (d <= tolerance) { // AnalysisLogger.getLogger().debug("Association: distance between grid:("+x+","+y+","+z+","+gridTimeInstant+") and point:("+rx+","+ry+","+rz+","+rt+") is "+d); - valuesForPoints.set(points, gridValues.get(gridIdx)); + valuesForPoints[points] = gridValues.get(gridIdx); foundmatches++; break; } @@ -125,7 +139,8 @@ public class VectorOperations { } AnalysisLogger.getLogger().debug("Association: Found "+foundmatches+" matches between the points and the grid"); - return valuesForPoints; + + return Arrays.asList(valuesForPoints); } public static List assignGridValuesToPoints(List> grid3d, int gridTimeInstant, List gridValues, List> coordinates4d, double tolerance) {