From 6ed47d5a451643ab1890fe55dd32897c37262cdb Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Fri, 19 Jul 2013 09:57:05 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@79433 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../geo/algorithms/MapsComparator.java | 38 +++- .../geo/batch/EnvriMetadataInsertDev.java | 132 +++++++++++++ .../batch/ThreddsMetadataBatchInserter.java | 2 +- .../geo/batch/WorldClimMetadataInsertDev.java | 49 +++++ .../geo/insertion/ThreddsFetcher.java | 4 +- .../geo/meta/GenericLayerMetadata.java | 178 +++++++++++++++++- .../dataanalysis/geo/meta/NetCDFMetadata.java | 4 +- .../geo/meta/features/FeaturesManager.java | 15 +- .../geo/retrieval/GeoIntersector.java | 1 + .../maps/RegressionTestMapsComparison.java | 42 ++++- .../geo/utils/EnvDataExplorer.java | 74 +++++++- .../geo/utils/ThreddsDataExplorer.java | 5 + 12 files changed, 523 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java create mode 100644 src/main/java/org/gcube/dataanalysis/geo/batch/WorldClimMetadataInsertDev.java diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/MapsComparator.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/MapsComparator.java index df1f958..5a92021 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/algorithms/MapsComparator.java +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/MapsComparator.java @@ -27,14 +27,19 @@ import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.IOHelper; import org.gcube.dataanalysis.geo.insertion.RasterTable; +import org.gcube.dataanalysis.geo.meta.OGCFormatter; import org.gcube.dataanalysis.geo.meta.features.FeaturesManager; import org.gcube.dataanalysis.geo.retrieval.GeoIntersector; +import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer; import org.jfree.chart.JFreeChart; import org.jfree.data.function.NormalDistributionFunction2D; import org.jfree.data.general.DatasetUtilities; import org.jfree.data.xy.XYSeriesCollection; import org.opengis.metadata.Metadata; +import ucar.nc2.dt.GridDatatype; +import ucar.nc2.dt.grid.GridDataset; + public class MapsComparator extends DataAnalysis { static String layer1 = "Layer_1"; @@ -94,18 +99,21 @@ public class MapsComparator extends DataAnalysis { try { // delete this force -// String scope = config.getGcubeScope(); - String scope = ScopeProvider.instance.get(); -// String scope = "/gcube"; + String scope = config.getGcubeScope(); + if (scope == null) + scope = ScopeProvider.instance.get(); + +// scope = "/gcube"; // String scope = null; AnalysisLogger.getLogger().debug("MapsComparator: Using Scope: " + scope + " Z: " + z + " Values Threshold: " + valuesthreshold + " Layer1: " + layerT1 + " vs " + layerT2); GeoIntersector intersector = new GeoIntersector(scope, config.getConfigPath()); AnalysisLogger.getLogger().debug("MapsComparator: GeoIntersector initialized"); - + double x1 = -180; double x2 = 180; double y1 = -90; double y2 = 90; + status = 10; FeaturesManager fm = intersector.getFeaturer(); AnalysisLogger.getLogger().debug("MapsComparator: Taking info for the layer: " + layerT1); @@ -118,6 +126,21 @@ public class MapsComparator extends DataAnalysis { AnalysisLogger.getLogger().debug("MapsComparator: Undefined resolution"); } AnalysisLogger.getLogger().debug("MapsComparator: Resolution: " + resolution1); + + if (fm.isThreddsFile(meta1)){ + AnalysisLogger.getLogger().debug("MapsComparator: recalculating the spatial extent of the comparison"); + String fileurl = fm.getOpenDapLink(meta1); + GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(fileurl); + List gridTypes = gds.getGrids(); + GridDatatype gdt = gridTypes.get(0); + x1 = ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()); + x2 = ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()); + y1 = ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()); + y2 = ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()); + } + + AnalysisLogger.getLogger().debug("MapsComparator: Spatial extent of the comparison: x1: "+x1+" x2: "+x2+" y1: "+y1+" y2: "+y2); + status = 15; AnalysisLogger.getLogger().debug("MapsComparator: Taking info for the layer: " + layerT2); AnalysisLogger.getLogger().debug("MapsComparator: Trying with UUID..." + layerT2); @@ -134,9 +157,14 @@ public class MapsComparator extends DataAnalysis { status = 20; // take the lowest resolution to perform the comparison double resolution = Math.max(resolution1, resolution2); + AnalysisLogger.getLogger().debug("MapsComparator: Theoretical Resolution: " + resolution); if (resolution == 0) resolution = 0.5d; - AnalysisLogger.getLogger().debug("MapsComparator: Evaluation Resolution: " + resolution); + // I added the following control to limit the amount of calculations + if (resolution<0.01) + resolution = 0.01d; + + AnalysisLogger.getLogger().debug("MapsComparator: Evaluation Indeed at Resolution: " + resolution); AnalysisLogger.getLogger().debug("MapsComparator: ****Rasterizing map 1****"); double[][] slice1 = intersector.takeTimeSlice(layerT1, time1, x1, x2, y1, y2, z, resolution, resolution); diff --git a/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java b/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java new file mode 100644 index 0000000..5ab375e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java @@ -0,0 +1,132 @@ +package org.gcube.dataanalysis.geo.batch; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; + +import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata; +import org.gcube.dataanalysis.geo.meta.OGCFormatter; +import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer; +import org.opengis.metadata.identification.TopicCategory; + +import ucar.nc2.dataset.CoordinateAxis; +import ucar.nc2.dt.GridDatatype; +import ucar.nc2.dt.grid.GridDataset; + +public class EnvriMetadataInsertDev { + +// static String geonetworkurl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/"; + static String geonetworkurl = "http://geoserver-last.d4science-ii.research-infrastructures.eu/geonetwork/"; + + /* + static String geonetworkurl = "http://geonetwork.d4science.org/geonetwork/"; + static String geoserverurl = "http://geoserver.d4science-ii.research-infrastructures.eu/geoserver"; + */ + //static String geoserverurl = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver"; + + static String user = "admin"; + static String password = "admin"; + static String[] envrilayers = { + "geo_filt_070620-070725-sim_HDR_4rlks.nc", + "geo_filt_070620-070829-sim_HDR_4rlks.nc", + "geo_filt_070620-071003-sim_HDR_4rlks.nc", + "geo_filt_070620-071107-sim_HDR_4rlks.nc", + "geo_filt_070620-071212-sim_HDR_4rlks.nc", + "geo_filt_070620-080326-sim_HDR_4rlks.nc", + "geo_filt_070620-080604-sim_HDR_4rlks.nc", + "geo_filt_070620-080709-sim_HDR_4rlks.nc", + "geo_filt_070620-080813-sim_HDR_4rlks.nc", + "geo_filt_070620-080917-sim_HDR_4rlks.nc", + "geo_filt_070620-081022-sim_HDR_4rlks.nc", + "geo_filt_070620-081231-sim_HDR_4rlks.nc"}; + + static String threddsbaseURL= "thredds.d4science.org"; + static String threddsURL = "http://"+threddsbaseURL+"/thredds/catalog/public/netcdf/catalog.xml"; + + public static void main(String[] args) throws Exception{ + + for (int i=1;i gridTypes = gds.getGrids(); + GridDatatype gdt = gridTypes.get(0); + + double minX = ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()); + double maxX = ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()); + double minY = ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()); + double maxY = ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()); + + System.out.println("minX: "+minX+" minY: "+minY+" maxX:"+maxX+" maxY:"+maxY); + + String wms = OGCFormatter.getWmsNetCDFUrl(url, layername, OGCFormatter.buildBoundingBox(minX, minY, maxX, maxY)).replace("width=676", "width=640").replace("height=330", "height=480"); + + System.out.println("WMS "+wms); + String wcs = OGCFormatter.getWcsNetCDFUrl(url, layername, OGCFormatter.buildBoundingBox(minX, minY, maxX, maxY)).replace("width=676", "width=640").replace("height=330", "height=480"); + System.out.println("WCS "+wcs); + + String [] urls = {"http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/"+filename,wms,wcs,url}; + + String [] protocols = {"HTTP","WMS","WCS","OPeNDAP"}; + + DateFormat formatter = new SimpleDateFormat("yyMMdd"); + Date datestart = formatter.parse(datastart); + Date dateend = formatter.parse(dataend); + + metadataInserter.setStartDate(datestart); + metadataInserter.setEndDate(dateend); + + metadataInserter.setXLeftLow(minX); + metadataInserter.setYLeftLow(minY); + metadataInserter.setXRightUpper(maxX); + metadataInserter.setYRightUpper(maxY); + + CoordinateAxis xAxis = gdt.getCoordinateSystem().getXHorizAxis(); + CoordinateAxis yAxis = gdt.getCoordinateSystem().getYHorizAxis(); + + double resolutionX = Math.abs((double) (xAxis.getMaxValue() - xAxis.getMinValue()) / (double) xAxis.getShape()[0]); + double resolutionY = Math.abs((double) (yAxis.getMaxValue() - yAxis.getMinValue()) / (double) yAxis.getShape()[0]); + + metadataInserter.setResolution(Math.max(resolutionX, resolutionY)); + + System.out.println("Resolution: "+Math.max(resolutionX, resolutionY)); + + metadataInserter.customMetaDataInsert(urls,protocols); + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/batch/ThreddsMetadataBatchInserter.java b/src/main/java/org/gcube/dataanalysis/geo/batch/ThreddsMetadataBatchInserter.java index 59af307..aeccced 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/batch/ThreddsMetadataBatchInserter.java +++ b/src/main/java/org/gcube/dataanalysis/geo/batch/ThreddsMetadataBatchInserter.java @@ -8,7 +8,7 @@ public class ThreddsMetadataBatchInserter { public static void main(String[] args) throws Exception{ AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile); - ThreddsFetcher tf = new ThreddsFetcher(null); + ThreddsFetcher tf = new ThreddsFetcher("/gcube/devsec"); tf.fetch("http://thredds.research-infrastructures.eu/thredds/catalog/public/netcdf/catalog.xml"); } } diff --git a/src/main/java/org/gcube/dataanalysis/geo/batch/WorldClimMetadataInsertDev.java b/src/main/java/org/gcube/dataanalysis/geo/batch/WorldClimMetadataInsertDev.java new file mode 100644 index 0000000..d41904f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/batch/WorldClimMetadataInsertDev.java @@ -0,0 +1,49 @@ +package org.gcube.dataanalysis.geo.batch; + +import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata; +import org.gcube.dataanalysis.geo.meta.NetCDFMetadata; +import org.opengis.metadata.identification.TopicCategory; + +public class WorldClimMetadataInsertDev { + + static String geonetworkurl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/"; + /* + static String geonetworkurl = "http://geonetwork.d4science.org/geonetwork/"; + static String geoserverurl = "http://geoserver.d4science-ii.research-infrastructures.eu/geoserver"; + */ + //static String geoserverurl = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geoserver"; + + static String user = "admin"; + static String password = "admin"; + + public static void main(String[] args) throws Exception{ + + for (int i=1;i<20;i++) { + worldclim(i); +// break; + } + } + + + + //gebco + private static void worldclim(int index) throws Exception{ + GenericLayerMetadata metadataInserter = new GenericLayerMetadata(); + metadataInserter.setGeonetworkUrl(geonetworkurl); + metadataInserter.setGeonetworkUser(user); + metadataInserter.setGeonetworkPwd(password); + + metadataInserter.setTitle("WorldClimBio"+index); + + metadataInserter.setCategoryTypes("_"+TopicCategory.ENVIRONMENT.name()+"_"); + metadataInserter.setResolution(0.0083); + metadataInserter.setAbstractField("WorldClim is a set of global climate layers (climate grids) with a spatial resolution of about 1 square kilometer. The data can be used for mapping and spatial modeling in a GIS or with other computer programs. Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978. Hosted on the D4Science Thredds Catalog: http://thredds.research-infrastructures.eu/thredds/catalog/public/netcdf/catalog.xml"); + metadataInserter.setCustomTopics("D4Science","EUBrazilOpenBio","WorldClim","WorldClimBio"+index+".tiff","Thredds"); + metadataInserter.setAuthor("D4Science"); + String [] urls = {"http://thredds.research-infrastructures.eu/thredds/fileServer/public/netcdf/WorldClimBio"+index+".tiff"}; + String [] protocols = {"HTTP"}; + metadataInserter.customMetaDataInsert(urls,protocols); + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/insertion/ThreddsFetcher.java b/src/main/java/org/gcube/dataanalysis/geo/insertion/ThreddsFetcher.java index 4fc0492..7873bc4 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/insertion/ThreddsFetcher.java +++ b/src/main/java/org/gcube/dataanalysis/geo/insertion/ThreddsFetcher.java @@ -33,7 +33,9 @@ public class ThreddsFetcher { List filesURL = ThreddsDataExplorer.getFiles(threddsCatalogURL); for (String filename : filesURL) { -// if (!filename.equalsIgnoreCase("cami_0000-09-01_64x128_L26_c030918.nc")) + if (!filename.endsWith(".nc")) + continue; +// if (!filename.equalsIgnoreCase("geo_filt_070620-070725-sim_HDR_4rlks_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc")) // continue; String url = OGCFormatter.getOpenDapURL(threddsCatalogURL, filename); diff --git a/src/main/java/org/gcube/dataanalysis/geo/meta/GenericLayerMetadata.java b/src/main/java/org/gcube/dataanalysis/geo/meta/GenericLayerMetadata.java index 1fc18e8..695966b 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/meta/GenericLayerMetadata.java +++ b/src/main/java/org/gcube/dataanalysis/geo/meta/GenericLayerMetadata.java @@ -78,7 +78,7 @@ public class GenericLayerMetadata { private String usageLimitations = "Not for commercial scopes"; private Date sourceGenerationDate = new Date(System.currentTimeMillis()); private String categoryTypes = "_BIOTA_"; - private String contactInfo = "support@d4science.research-infrastructures.eu"; + private String contactInfo = "support@d4science.org"; private String abstractField = ""; private String purpose = "Maps publication"; private String author = "i-Marine"; @@ -462,6 +462,182 @@ public class GenericLayerMetadata { } + + public void customMetaDataInsert(String[] urls, String[] protocols) throws Exception { + + // layer uri: wms, wfs wcs + List layerUris = new ArrayList(); + for (int i=0;i> descriptiveKeyWords = new HashMap>(); + HashSet keySet = new HashSet(); + + if (customTopics!=null) + keySet.addAll(customTopics); + + descriptiveKeyWords.put(KeywordType.THEME, keySet); + + if (startDate!=null){ + HashSet temporalkeySet = new HashSet(); + temporalkeySet.add(startDate.toString()); + if (!endDate.equals(startDate)) + temporalkeySet.add(endDate.toString()); + descriptiveKeyWords.put(KeywordType.TEMPORAL, temporalkeySet); + } + + // author: + DefaultResponsibleParty party = new DefaultResponsibleParty(); + party.setIndividualName(author); + DefaultContact contact = new DefaultContact(); + contact.setContactInstructions(new DefaultInternationalString(contactInfo)); + party.setContactInfo(contact); + party.setRole(Role.ORIGINATOR); + + // citation: + DefaultCitation citation = new DefaultCitation(); + citation.setTitle(new DefaultInternationalString(title)); + ArrayList citDates = new ArrayList(); + citDates.add(new DefaultCitationDate(sourceGenerationDate, DateType.CREATION)); + citDates.add(new DefaultCitationDate(sourceGenerationDate, DateType.PUBLICATION)); + citDates.add(new DefaultCitationDate(sourceGenerationDate, DateType.REVISION)); + citation.setDates(citDates); + ArrayList citAltTitle = new ArrayList(); + citAltTitle.add(new DefaultInternationalString(title)); + citation.setAlternateTitles(citAltTitle); + citation.setEditionDate(sourceGenerationDate); + citation.getPresentationForms().add(PresentationForm.MAP_DIGITAL); + ArrayList keywordslist = new ArrayList(); + for (Entry> entry : descriptiveKeyWords.entrySet()) { + DefaultKeywords keywords = new DefaultKeywords(); + for (String key : entry.getValue()) + keywords.getKeywords().add(new DefaultInternationalString(key)); + + keywords.setType(entry.getKey()); + DefaultCitation thesaurus = new DefaultCitation(); + thesaurus.setTitle(new DefaultInternationalString("General")); + thesaurus.setDates(citDates); + keywords.setThesaurusName(thesaurus); + keywordslist.add(keywords); + } + + // usage: + DefaultUsage usage = new DefaultUsage(); + usage.setSpecificUsage(new DefaultInternationalString(usageField)); + usage.setUsageDate(sourceGenerationDate); + usage.setUserDeterminedLimitations(new DefaultInternationalString(usageLimitations)); + usage.setUserContactInfo(new ArrayList(Arrays.asList(party))); + ArrayList usages = new ArrayList(Arrays.asList(usage)); + //build categories by guessing on the filename + List categories = guessTopicCategory(categoryTypes); + AnalysisLogger.getLogger().debug("Guessed Topics: "+categories); + // Spatial Rapresentation Info + DefaultGeometricObjects geoObjs = new DefaultGeometricObjects(); + geoObjs.setGeometricObjectType(GeometricObjectType.COMPLEX); + DefaultVectorSpatialRepresentation spatial = new DefaultVectorSpatialRepresentation(); + spatial.setTopologyLevel(TopologyLevel.GEOMETRY_ONLY); + spatial.getGeometricObjects().add(geoObjs); + + // Extent: + DefaultExtent extent = new DefaultExtent(); + extent.setGeographicElements(Collections.singleton(new DefaultGeographicBoundingBox(xLL, xRU, yLL, yRU))); + extent.setDescription(new DefaultInternationalString("Bounding box")); + + /*Only with Geotoolkit 4.x + DefaultTemporalExtent stext = new DefaultTemporalExtent(startDate,endDate); + stext.setStartTime(startDate); + stext.setEndTime(endDate); + extent.setTemporalElements(Arrays.asList(stext)); + */ + extent.freeze(); + + //resolution + DefaultNominalResolution resolution = new DefaultNominalResolution(); + resolution.setGroundResolution(res); + resolution.setScanningResolution(res); + DefaultResolution dres = new DefaultResolution(); + dres.setDistance(res); + + // layers access: + DefaultDistribution distribution = new DefaultDistribution(); + DefaultDigitalTransferOptions transferOptions = new DefaultDigitalTransferOptions(); + for (String uri : layerUris) + transferOptions.getOnLines().add(new DefaultOnlineResource(new URI(uri))); + distribution.getTransferOptions().add(transferOptions); + + DefaultFormat [] formats = new DefaultFormat[protocols.length]; + for (int i=0;i(Arrays.asList(formats))); + + // legal constraints + DefaultLegalConstraints constraints = new DefaultLegalConstraints(); + constraints.getUseLimitations().add(new DefaultInternationalString("Licensed")); + constraints.getAccessConstraints().add(Restriction.LICENSE); + constraints.getUseConstraints().add(Restriction.LICENSE); + + // quality declaration: + DefaultDataQuality processQuality = new DefaultDataQuality(); + + //citation + DefaultCitation sourceCitation = new DefaultCitation(); + sourceCitation.setTitle(new DefaultInternationalString(title)); + sourceCitation.getDates().add(new DefaultCitationDate(sourceGenerationDate, DateType.CREATION)); + sourceCitation.getIdentifiers().add(new DefaultIdentifier(categoryTypes)); + + //source + DefaultSource source = new DefaultSource(); + source.setResolution(resolution); + source.setDescription(new DefaultInternationalString(title)); + source.setSourceCitation(sourceCitation); + + // provenance + DefaultProcessStep preprocessStep = new DefaultProcessStep(); + DefaultProcessStep processStep = new DefaultProcessStep(preprocessStep); + DefaultProcessing processing = new DefaultProcessing(); + processing.setSoftwareReferences(new ArrayList(Arrays.asList(sourceCitation))); + processStep.setDescription(new DefaultInternationalString(processdescription)); + DefaultLineage processLineage = new DefaultLineage(); + processLineage.setProcessSteps(new ArrayList(Arrays.asList(processStep))); + processQuality.setLineage(processLineage); + processQuality.setScope(new DefaultScope(ScopeCode.DATASET)); + + // fulfill identification + DefaultDataIdentification ident = new DefaultDataIdentification(); + ident.setCitation(citation); + ident.setAbstract(new DefaultInternationalString(abstractField)); + ident.setPurpose(new DefaultInternationalString(purpose)); + ident.getResourceMaintenances().add(new DefaultMaintenanceInformation(MaintenanceFrequency.AS_NEEDED)); + ident.setDescriptiveKeywords(keywordslist); + ident.setTopicCategories(categories); + ident.setResourceSpecificUsages(usages); + ident.setExtents(new ArrayList(Arrays.asList(extent))); + ident.setSpatialRepresentationTypes(new ArrayList(Arrays.asList(SpatialRepresentationType.GRID))); + ident.setSpatialResolutions(new ArrayList(Arrays.asList(dres))); + ident.setLanguages(new ArrayList(Arrays.asList(Locale.ENGLISH))); + + // Metadata Obj: + DefaultMetadata meta = new DefaultMetadata(party, sourceGenerationDate, ident); + meta.getSpatialRepresentationInfo().add(spatial); + meta.setDistributionInfo(distribution); + meta.getMetadataConstraints().add(constraints); + meta.getDataQualityInfo().add(processQuality); + meta.setLanguage(Locale.ENGLISH); +// System.out.println(meta); + GNClient client = new GNClient(geonetworkUrl); + client.login(geonetworkUser, geonetworkPwd); + File tmetafile = meta2File(meta); + client.insertMetadata(new GNInsertConfiguration("3", "datasets", "_none_", true), tmetafile); + tmetafile.delete(); + + } + public static List guessTopicCategory(String refString){ String searcher = refString.toLowerCase(); List categories = new ArrayList(); diff --git a/src/main/java/org/gcube/dataanalysis/geo/meta/NetCDFMetadata.java b/src/main/java/org/gcube/dataanalysis/geo/meta/NetCDFMetadata.java index dd35549..4a7b719 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/meta/NetCDFMetadata.java +++ b/src/main/java/org/gcube/dataanalysis/geo/meta/NetCDFMetadata.java @@ -81,7 +81,7 @@ public class NetCDFMetadata { private String contactInfo = "support@d4science.research-infrastructures.eu"; private String abstractField = "T: temperature (degK) from 04091217ruc.nc resident on the THREDDS instance " + threddsCatalogUrl; private String purpose = "Maps publication"; - private String author = "i-Marine"; + private String author = "D4Science"; private double res = 0.5d; private double xLL = -180; private double xRU = 180; @@ -311,7 +311,7 @@ public class NetCDFMetadata { HashMap> descriptiveKeyWords = new HashMap>(); HashSet keySet = new HashSet(); keySet.add("THREDDS"); - keySet.add("i-Marine"); + keySet.add("D4Science"); keySet.add("NetCDF"); if (customTopics!=null) keySet.addAll(customTopics); diff --git a/src/main/java/org/gcube/dataanalysis/geo/meta/features/FeaturesManager.java b/src/main/java/org/gcube/dataanalysis/geo/meta/features/FeaturesManager.java index aa71e16..11e4b9e 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/meta/features/FeaturesManager.java +++ b/src/main/java/org/gcube/dataanalysis/geo/meta/features/FeaturesManager.java @@ -20,8 +20,8 @@ import org.opengis.metadata.identification.Identification; import org.opengis.metadata.identification.Resolution; public class FeaturesManager { -// private String geonetworkUrl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/"; - private String geonetworkUrl = "http://geoserver-last.d4science-ii.research-infrastructures.eu/geonetwork/"; + private String geonetworkUrl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/"; +// private String geonetworkUrl = "http://geoserver-last.d4science-ii.research-infrastructures.eu/geonetwork/"; // private String geonetworkUrl = "http://geoserver.d4science-ii.research-infrastructures.eu/geonetwork/"; private String geonetworkUser = "admin"; private String geonetworkPwd = "admin"; @@ -107,7 +107,9 @@ public class FeaturesManager { String link = null; for (DigitalTransferOptions option : meta.getDistributionInfo().getTransferOptions()) { for (OnlineResource resource : option.getOnLines()) { - String tlink = resource.getLinkage().toString(); + String tlink = ""; + + try{tlink = resource.getLinkage().toString();}catch(Exception e){} if (tlink.toLowerCase().contains(criterion.toLowerCase())) { link = tlink; break; @@ -126,7 +128,11 @@ public class FeaturesManager { for (DigitalTransferOptions option : meta.getDistributionInfo().getTransferOptions()) { for (OnlineResource resource : option.getOnLines()) { String layername = resource.getName(); - if (layername!=null) { + String link = ""; + try{ + link = resource.getLinkage().toString().toLowerCase(); + }catch(Exception e){} + if ((layername!=null)&&link.contains("wms")) { innerlayername = layername; break; } @@ -439,6 +445,7 @@ public class FeaturesManager { public static String treatTitleForGN(String origLayerTitle) { String layerTitle = origLayerTitle.toLowerCase(); int idx = layerTitle.indexOf(" from ["); +// int idx = layerTitle.indexOf(" "); //let's search among many layers String layerTitle2 = layerTitle; if (idx>0) layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim(); diff --git a/src/main/java/org/gcube/dataanalysis/geo/retrieval/GeoIntersector.java b/src/main/java/org/gcube/dataanalysis/geo/retrieval/GeoIntersector.java index 22fccea..2d73a99 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/retrieval/GeoIntersector.java +++ b/src/main/java/org/gcube/dataanalysis/geo/retrieval/GeoIntersector.java @@ -246,6 +246,7 @@ public class GeoIntersector { double[][] slice = new double[ysteps + 1][xsteps + 1]; List> tuples = new ArrayList>(); AnalysisLogger.getLogger().debug("Building the points grid according to YRes:" + yResolution + " and XRes:" + xResolution); + AnalysisLogger.getLogger().debug("Points to reassign:" + (ysteps*xsteps)); // build the tuples according to the desired resolution for (int i = 0; i < ysteps + 1; i++) { double y = (i * yResolution) + y1; diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/maps/RegressionTestMapsComparison.java b/src/main/java/org/gcube/dataanalysis/geo/test/maps/RegressionTestMapsComparison.java index 457e31d..8962b66 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/test/maps/RegressionTestMapsComparison.java +++ b/src/main/java/org/gcube/dataanalysis/geo/test/maps/RegressionTestMapsComparison.java @@ -35,12 +35,46 @@ public class RegressionTestMapsComparison { config.setParam("DatabaseDriver","org.postgresql.Driver"); // config.setParam("Layer_1","86a7ac79-866a-49c6-b5d5-602fc2d87ddd"); // config.setParam("Layer_2","86a7ac79-866a-49c6-b5d5-602fc2d87ddd"); - config.setParam("Layer_1","aeabfdb5-9ddb-495e-b628-5b7d2cf1d8a2"); - config.setParam("Layer_2","aeabfdb5-9ddb-495e-b628-5b7d2cf1d8a2"); + + //World seas : IHO vs Marine regions +// config.setParam("Layer_2","70a6d757-e607-46f7-b643-e21749f45a42"); +// config.setParam("Layer_1","a2a8c130-124f-45b5-973f-c9358028a2a6"); + + //FAO vs FAO: +// config.setParam("Layer_1","b040894b-c5db-47fc-ba9c-d4fafcdcf620"); //goblin shark +// config.setParam("Layer_2","c9a31223-cc00-4acd-bc5b-a0c76a7f79c7"); //humbolt squid + + //FAO vs AquaMaps +// config.setParam("Layer_1","b040894b-c5db-47fc-ba9c-d4fafcdcf620"); +// config.setParam("Layer_2","c9a31223-cc00-4acd-bc5b-a0c76a7f79c7"); + + //NetCDF vs NETCDF WOA + config.setParam("Layer_1","e0dbbcc0-8364-4087-8bcb-c7d95b2f55c8"); //statistical mean oxygen + config.setParam("Layer_2","49f5a5a1-80ff-4a00-8c84-dac29bda1a23");//statistical mean phosphate + + //NetCDF vs NetCDF MyOcean +// config.setParam("Layer_1","e0dbbcc0-8364-4087-8bcb-c7d95b2f55c8"); //statistical mean oxygen +// config.setParam("Layer_2","fc9ac2f4-a2bd-43d1-a361-ac67c5ceac31");//temperature + + //NetCDF vs IHO +// config.setParam("Layer_1","70a6d757-e607-46f7-b643-e21749f45a42");//IHO +// config.setParam("Layer_2","fc9ac2f4-a2bd-43d1-a361-ac67c5ceac31");//temperature + + //NetCDF vs NetCDF MyOcean only +// config.setParam("Layer_1","fc9ac2f4-a2bd-43d1-a361-ac67c5ceac31"); //statistical mean oxygen +// config.setParam("Layer_2","fc9ac2f4-a2bd-43d1-a361-ac67c5ceac31");//temperature + + //NetCDF vs NetCDF Envri +// config.setParam("Layer_1","Etna Volcano SAR Analysis 1"); // +// config.setParam("Layer_2","Etna Volcano SAR Analysis 7");// + + + config.setParam("ValuesComparisonThreshold",""+0.1); config.setParam("Z","0"); - config.setGcubeScope(null); - + config.setGcubeScope("/gcube"); +// config.setGcubeScope(null); + return config; } } diff --git a/src/main/java/org/gcube/dataanalysis/geo/utils/EnvDataExplorer.java b/src/main/java/org/gcube/dataanalysis/geo/utils/EnvDataExplorer.java index bbff8ec..a89dde2 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/utils/EnvDataExplorer.java +++ b/src/main/java/org/gcube/dataanalysis/geo/utils/EnvDataExplorer.java @@ -10,6 +10,11 @@ import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.geo.meta.OGCFormatter; +import ucar.ma2.ArrayByte; +import ucar.ma2.ArrayDouble; +import ucar.ma2.Index; +import ucar.ma2.IndexIterator; + import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.GeometryFactory; @@ -20,6 +25,7 @@ import com.vividsolutions.jts.geom.Polygon; import com.vividsolutions.jts.geom.PrecisionModel; import com.vividsolutions.jts.geom.impl.CoordinateArraySequence; import com.vividsolutions.jts.geom.util.GeometryTransformer; +import com.vividsolutions.jts.operation.overlay.snap.GeometrySnapper; public class EnvDataExplorer { @@ -206,18 +212,30 @@ public class EnvDataExplorer { // cut the holes List polysnoholes = new ArrayList(); for (Polygon pp : polys) { + boolean found = false; int h = 0; for (Polygon polnh : polysnoholes) { - if (polnh.covers(pp)) { + boolean covers = false; + + try{ + covers = polnh.covers(pp); + }catch(Exception e){ + AnalysisLogger.getLogger().debug("Error in calculating superpositions: Snapping the geometries"); + double snapTol = GeometrySnapper.computeOverlaySnapTolerance(polnh, pp); + pp = (Polygon) selfSnap(pp,snapTol); + polnh = (Polygon) selfSnap(polnh,snapTol); + AnalysisLogger.getLogger().debug("Geometries have been snapped"); + covers = polnh.covers(pp); + } + + if (covers) { // System.out.println("found hole! "+pp+" vs "+polnh); addDifference(h, polysnoholes, polnh, pp); - found = true; } else if (pp.covers(polnh)) { // polysnoholes.set(h, (Polygon) pp.difference(polnh)); addDifference(h, polysnoholes, pp, polnh); - found = true; } h++; @@ -232,6 +250,17 @@ public class EnvDataExplorer { return p; } + + private static Geometry selfSnap(Geometry g, double snapTolerance) + { + GeometrySnapper snapper = new GeometrySnapper(g); + Geometry snapped = snapper.snapTo(g, snapTolerance); + // need to "clean" snapped geometry - use buffer(0) as a simple way to do this + Geometry fix = snapped.buffer(0); + return fix; + } + + private static void addDifference(int h , List polysnoholes, Polygon polnh, Polygon pp){ Geometry mp = polnh.difference(pp); @@ -327,6 +356,45 @@ public class EnvDataExplorer { } } + public static ArrayDouble.D3 arrayByte3DArrayDouble(ArrayByte bytes){ + int[] shapeD = bytes.getShape(); + int zD=shapeD[0]; + int yD=shapeD[1]; + int xD=shapeD[2]; + + AnalysisLogger.getLogger().debug(xD+"X"+yD+"X"+zD+"="+(xD*yD*zD)); + ArrayDouble.D3 doublea = new ArrayDouble.D3(zD, yD,xD); + + IndexIterator iterator = bytes.getIndexIterator(); + for (int x=0;x