git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@73630 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
0fdd3294e5
commit
b44640be23
|
@ -55,6 +55,14 @@ public class ThreddsFetcher {
|
||||||
String duration = null;
|
String duration = null;
|
||||||
String resolution = gdt.getTimeDimension()==null?null:""+gdt.getTimeDimension().getLength();
|
String resolution = gdt.getTimeDimension()==null?null:""+gdt.getTimeDimension().getLength();
|
||||||
|
|
||||||
|
int numberOfDimensions = 2;
|
||||||
|
if ((gdt.getZDimension()!=null)&&(gdt.getZDimension().getLength()>1)){
|
||||||
|
numberOfDimensions = 3;
|
||||||
|
AnalysisLogger.getLogger().debug("Length of Z Dimension: "+gdt.getZDimension().getLength());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
AnalysisLogger.getLogger().debug("Bidimensional Layer ");
|
||||||
|
|
||||||
if (dr != null) {
|
if (dr != null) {
|
||||||
hStartDate = dr.getStart() == null ? null : humanDF.format(dr.getStart().getDate());
|
hStartDate = dr.getStart() == null ? null : humanDF.format(dr.getStart().getDate());
|
||||||
hEndDate = dr.getEnd() == null ? null : humanDF.format(dr.getEnd().getDate());
|
hEndDate = dr.getEnd() == null ? null : humanDF.format(dr.getEnd().getDate());
|
||||||
|
@ -62,9 +70,12 @@ public class ThreddsFetcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
// control if the file is yet on GN
|
// control if the file is yet on GN
|
||||||
String generatedTitle = generateTitle(filename, description, hStartDate, hEndDate);
|
String generatedTitle = generateTitle(filename, description, hStartDate, hEndDate, numberOfDimensions);
|
||||||
|
CoordinateAxis xAxis = gdt.getCoordinateSystem().getXHorizAxis();
|
||||||
|
CoordinateAxis yAxis = gdt.getCoordinateSystem().getYHorizAxis();
|
||||||
|
AnalysisLogger.getLogger().debug("Bounds:"+xAxis.getMinValue()+","+yAxis.getMinValue()+","+xAxis.getMaxValue()+","+yAxis.getMaxValue());
|
||||||
|
|
||||||
Metadata previousmeta = featurer.checkForMetadatabyTitle(FeaturesManager.treatTitleForGN(generatedTitle), generatedTitle);
|
Metadata previousmeta = featurer.checkForMetadatabyTitle(FeaturesManager.treatTitleForGN(generatedTitle), generatedTitle,filename);
|
||||||
if (previousmeta!=null){
|
if (previousmeta!=null){
|
||||||
AnalysisLogger.getLogger().debug("***WARNING: layer yet found on GeoNetwork***");
|
AnalysisLogger.getLogger().debug("***WARNING: layer yet found on GeoNetwork***");
|
||||||
continue;
|
continue;
|
||||||
|
@ -76,17 +87,9 @@ public class ThreddsFetcher {
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
int numberOfDimensions = 2;
|
|
||||||
if ((gdt.getZDimension()!=null)&&(gdt.getZDimension().getLength()>1)){
|
|
||||||
numberOfDimensions = 3;
|
|
||||||
AnalysisLogger.getLogger().debug("Length of Z Dimension: "+gdt.getZDimension().getLength());
|
|
||||||
}
|
|
||||||
else
|
|
||||||
AnalysisLogger.getLogger().debug("Bidimensional Layer ");
|
|
||||||
|
|
||||||
// get resolution - take the maximum regular step
|
// get resolution - take the maximum regular step
|
||||||
CoordinateAxis xAxis = gdt.getCoordinateSystem().getXHorizAxis();
|
|
||||||
CoordinateAxis yAxis = gdt.getCoordinateSystem().getYHorizAxis();
|
|
||||||
double resolutionX = Math.abs((double) (xAxis.getMaxValue() - xAxis.getMinValue()) / (double) xAxis.getShape()[0]);
|
double resolutionX = Math.abs((double) (xAxis.getMaxValue() - xAxis.getMinValue()) / (double) xAxis.getShape()[0]);
|
||||||
double resolutionY = Math.abs((double) (yAxis.getMaxValue() - yAxis.getMinValue()) / (double) yAxis.getShape()[0]);
|
double resolutionY = Math.abs((double) (yAxis.getMaxValue() - yAxis.getMinValue()) / (double) yAxis.getShape()[0]);
|
||||||
|
|
||||||
|
@ -95,6 +98,7 @@ public class ThreddsFetcher {
|
||||||
metadataInserter.setGeonetworkUrl(featurer.getGeonetworkURLFromScope());
|
metadataInserter.setGeonetworkUrl(featurer.getGeonetworkURLFromScope());
|
||||||
metadataInserter.setGeonetworkUser(featurer.getGeonetworkUserFromScope());
|
metadataInserter.setGeonetworkUser(featurer.getGeonetworkUserFromScope());
|
||||||
metadataInserter.setGeonetworkPwd(featurer.getGeonetworkPasswordFromScope());
|
metadataInserter.setGeonetworkPwd(featurer.getGeonetworkPasswordFromScope());
|
||||||
|
|
||||||
// Build standard info:
|
// Build standard info:
|
||||||
metadataInserter.setThreddsCatalogUrl(threddsCatalogURL);
|
metadataInserter.setThreddsCatalogUrl(threddsCatalogURL);
|
||||||
metadataInserter.setLayerUrl(url);
|
metadataInserter.setLayerUrl(url);
|
||||||
|
@ -105,12 +109,14 @@ public class ThreddsFetcher {
|
||||||
metadataInserter.setAbstractField(generateAbstractField(gdt.getFullName(), filename, description, gdt.getUnitsString().trim(), hStartDate, hEndDate, duration, resolution, numberOfDimensions));
|
metadataInserter.setAbstractField(generateAbstractField(gdt.getFullName(), filename, description, gdt.getUnitsString().trim(), hStartDate, hEndDate, duration, resolution, numberOfDimensions));
|
||||||
metadataInserter.setResolution(Math.max(resolutionX, resolutionY));
|
metadataInserter.setResolution(Math.max(resolutionX, resolutionY));
|
||||||
// set Bounding box
|
// set Bounding box
|
||||||
|
|
||||||
metadataInserter.setXLeftLow(ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()));
|
metadataInserter.setXLeftLow(ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()));
|
||||||
metadataInserter.setYLeftLow(ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()));
|
metadataInserter.setYLeftLow(ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()));
|
||||||
metadataInserter.setXRightUpper(ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()));
|
metadataInserter.setXRightUpper(ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()));
|
||||||
metadataInserter.setYRightUpper(ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()));
|
metadataInserter.setYRightUpper(ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()));
|
||||||
|
|
||||||
//set keywords
|
//set keywords
|
||||||
metadataInserter.setCustomTopics(filename, description);
|
metadataInserter.setCustomTopics(filename, description,numberOfDimensions+"D");
|
||||||
//set Temporal Extent
|
//set Temporal Extent
|
||||||
if (hStartDate!=null){
|
if (hStartDate!=null){
|
||||||
metadataInserter.setStartDate(dr.getStart().getDate());
|
metadataInserter.setStartDate(dr.getStart().getDate());
|
||||||
|
@ -121,15 +127,15 @@ public class ThreddsFetcher {
|
||||||
AnalysisLogger.getLogger().debug("abstract: " + metadataInserter.getAbstractField());
|
AnalysisLogger.getLogger().debug("abstract: " + metadataInserter.getAbstractField());
|
||||||
|
|
||||||
metadataInserter.insertMetaData();
|
metadataInserter.insertMetaData();
|
||||||
break;
|
// break;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
break;
|
// break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String generateTitle(String filename, String description, String startDate, String endDate) {
|
public static String generateTitle(String filename, String description, String startDate, String endDate, int numberOfDimensions) {
|
||||||
String dateString = "";
|
String dateString = "";
|
||||||
if (startDate != null){
|
if (startDate != null){
|
||||||
if (startDate.equals(endDate))
|
if (startDate.equals(endDate))
|
||||||
|
@ -137,7 +143,7 @@ public class ThreddsFetcher {
|
||||||
else
|
else
|
||||||
dateString = " from [" + startDate + "] to [" + endDate + "]";
|
dateString = " from [" + startDate + "] to [" + endDate + "]";
|
||||||
}
|
}
|
||||||
return description.replaceAll("( )+", " ") + dateString+" (" + filename + ")";
|
return description.replaceAll("( )+", " ") + dateString+" (" + numberOfDimensions+ "D)";//+" (" + filename + ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String generateAbstractField(String layername, String filename, String description, String unit, String startDate, String endDate, String duration, String timeInstants, int numberOfDimensions) {
|
public static String generateAbstractField(String layername, String filename, String description, String unit, String startDate, String endDate, String duration, String timeInstants, int numberOfDimensions) {
|
||||||
|
@ -167,7 +173,7 @@ public class ThreddsFetcher {
|
||||||
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
|
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
|
||||||
// ThreddsFetcher tf = new ThreddsFetcher("/gcube/devsec");
|
// ThreddsFetcher tf = new ThreddsFetcher("/gcube/devsec");
|
||||||
ThreddsFetcher tf = new ThreddsFetcher(null);
|
ThreddsFetcher tf = new ThreddsFetcher(null);
|
||||||
tf.fetch("http://thredds.research-infrastructures.eu:8080/thredds/catalog/public/netcdf/catalog.xml");
|
tf.fetch("http://thredds.research-infrastructures.eu/thredds/catalog/public/netcdf/catalog.xml");
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@ import java.util.Map.Entry;
|
||||||
|
|
||||||
import javax.xml.bind.JAXBException;
|
import javax.xml.bind.JAXBException;
|
||||||
|
|
||||||
import org.gcube.dataanalysis.geo.utils.NetCDFTemporalPrimitive;
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
import org.geotoolkit.metadata.iso.DefaultIdentifier;
|
import org.geotoolkit.metadata.iso.DefaultIdentifier;
|
||||||
import org.geotoolkit.metadata.iso.DefaultMetadata;
|
import org.geotoolkit.metadata.iso.DefaultMetadata;
|
||||||
import org.geotoolkit.metadata.iso.citation.DefaultCitation;
|
import org.geotoolkit.metadata.iso.citation.DefaultCitation;
|
||||||
|
@ -33,8 +33,6 @@ import org.geotoolkit.metadata.iso.distribution.DefaultDistribution;
|
||||||
import org.geotoolkit.metadata.iso.distribution.DefaultFormat;
|
import org.geotoolkit.metadata.iso.distribution.DefaultFormat;
|
||||||
import org.geotoolkit.metadata.iso.extent.DefaultExtent;
|
import org.geotoolkit.metadata.iso.extent.DefaultExtent;
|
||||||
import org.geotoolkit.metadata.iso.extent.DefaultGeographicBoundingBox;
|
import org.geotoolkit.metadata.iso.extent.DefaultGeographicBoundingBox;
|
||||||
import org.geotoolkit.metadata.iso.extent.DefaultSpatialTemporalExtent;
|
|
||||||
import org.geotoolkit.metadata.iso.extent.DefaultTemporalExtent;
|
|
||||||
import org.geotoolkit.metadata.iso.identification.DefaultDataIdentification;
|
import org.geotoolkit.metadata.iso.identification.DefaultDataIdentification;
|
||||||
import org.geotoolkit.metadata.iso.identification.DefaultKeywords;
|
import org.geotoolkit.metadata.iso.identification.DefaultKeywords;
|
||||||
import org.geotoolkit.metadata.iso.identification.DefaultResolution;
|
import org.geotoolkit.metadata.iso.identification.DefaultResolution;
|
||||||
|
@ -65,7 +63,6 @@ import org.opengis.metadata.maintenance.ScopeCode;
|
||||||
import org.opengis.metadata.spatial.GeometricObjectType;
|
import org.opengis.metadata.spatial.GeometricObjectType;
|
||||||
import org.opengis.metadata.spatial.SpatialRepresentationType;
|
import org.opengis.metadata.spatial.SpatialRepresentationType;
|
||||||
import org.opengis.metadata.spatial.TopologyLevel;
|
import org.opengis.metadata.spatial.TopologyLevel;
|
||||||
import org.opengis.temporal.TemporalPrimitive;
|
|
||||||
import org.opengis.util.InternationalString;
|
import org.opengis.util.InternationalString;
|
||||||
|
|
||||||
public class NetCDFMetadata {
|
public class NetCDFMetadata {
|
||||||
|
@ -368,11 +365,9 @@ public class NetCDFMetadata {
|
||||||
usage.setUserDeterminedLimitations(new DefaultInternationalString(usageLimitations));
|
usage.setUserDeterminedLimitations(new DefaultInternationalString(usageLimitations));
|
||||||
usage.setUserContactInfo(new ArrayList<ResponsibleParty>(Arrays.asList(party)));
|
usage.setUserContactInfo(new ArrayList<ResponsibleParty>(Arrays.asList(party)));
|
||||||
ArrayList<DefaultUsage> usages = new ArrayList<DefaultUsage>(Arrays.asList(usage));
|
ArrayList<DefaultUsage> usages = new ArrayList<DefaultUsage>(Arrays.asList(usage));
|
||||||
ArrayList<TopicCategory> categories = new ArrayList<TopicCategory>();
|
//build categories by guessing on the filename
|
||||||
categories.add(TopicCategory.BIOTA);
|
List<TopicCategory> categories = guessTopicCategory(sourceFileName);
|
||||||
categories.add(TopicCategory.ENVIRONMENT);
|
AnalysisLogger.getLogger().debug("Guessed Topics: "+categories);
|
||||||
categories.add(TopicCategory.OCEANS);
|
|
||||||
|
|
||||||
// Spatial Rapresentation Info
|
// Spatial Rapresentation Info
|
||||||
DefaultGeometricObjects geoObjs = new DefaultGeometricObjects();
|
DefaultGeometricObjects geoObjs = new DefaultGeometricObjects();
|
||||||
geoObjs.setGeometricObjectType(GeometricObjectType.COMPLEX);
|
geoObjs.setGeometricObjectType(GeometricObjectType.COMPLEX);
|
||||||
|
@ -471,12 +466,23 @@ public class NetCDFMetadata {
|
||||||
meta.getMetadataConstraints().add(constraints);
|
meta.getMetadataConstraints().add(constraints);
|
||||||
meta.getDataQualityInfo().add(processQuality);
|
meta.getDataQualityInfo().add(processQuality);
|
||||||
meta.setLanguage(Locale.ENGLISH);
|
meta.setLanguage(Locale.ENGLISH);
|
||||||
System.out.println(meta);
|
// System.out.println(meta);
|
||||||
GNClient client = new GNClient(geonetworkUrl);
|
GNClient client = new GNClient(geonetworkUrl);
|
||||||
client.login(geonetworkUser, geonetworkPwd);
|
client.login(geonetworkUser, geonetworkPwd);
|
||||||
File tmetafile = meta2File(meta);
|
File tmetafile = meta2File(meta);
|
||||||
client.insertMetadata(new GNInsertConfiguration("3", "datasets", "_none_", true), tmetafile);
|
client.insertMetadata(new GNInsertConfiguration("3", "datasets", "_none_", true), tmetafile);
|
||||||
// tmetafile.delete();
|
tmetafile.delete();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static List<TopicCategory> guessTopicCategory(String refString){
|
||||||
|
String searcher = refString.toLowerCase();
|
||||||
|
List<TopicCategory> categories = new ArrayList<TopicCategory>();
|
||||||
|
for (TopicCategory topic:TopicCategory.values()){
|
||||||
|
if (searcher.contains("_"+topic.name().toLowerCase()+"_")){
|
||||||
|
categories.add(topic);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return categories;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,8 @@ import it.geosolutions.geonetwork.util.GNSearchRequest;
|
||||||
import it.geosolutions.geonetwork.util.GNSearchResponse;
|
import it.geosolutions.geonetwork.util.GNSearchResponse;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
@ -12,9 +14,14 @@ import org.gcube.spatial.data.geonetwork.GeoNetwork;
|
||||||
import org.gcube.spatial.data.geonetwork.GeoNetworkReader;
|
import org.gcube.spatial.data.geonetwork.GeoNetworkReader;
|
||||||
import org.gcube.spatial.data.geonetwork.configuration.Configuration;
|
import org.gcube.spatial.data.geonetwork.configuration.Configuration;
|
||||||
import org.gcube.spatial.data.geonetwork.configuration.ConfigurationManager;
|
import org.gcube.spatial.data.geonetwork.configuration.ConfigurationManager;
|
||||||
|
import org.geotoolkit.metadata.iso.identification.DefaultKeywords;
|
||||||
|
import org.geotoolkit.util.DefaultInternationalString;
|
||||||
import org.opengis.metadata.Metadata;
|
import org.opengis.metadata.Metadata;
|
||||||
import org.opengis.metadata.citation.OnlineResource;
|
import org.opengis.metadata.citation.OnlineResource;
|
||||||
import org.opengis.metadata.distribution.DigitalTransferOptions;
|
import org.opengis.metadata.distribution.DigitalTransferOptions;
|
||||||
|
import org.opengis.metadata.identification.Identification;
|
||||||
|
import org.opengis.metadata.identification.Keywords;
|
||||||
|
import org.opengis.util.InternationalString;
|
||||||
|
|
||||||
public class FeaturesManager {
|
public class FeaturesManager {
|
||||||
private String geonetworkUrl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/";
|
private String geonetworkUrl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/";
|
||||||
|
@ -175,16 +182,31 @@ public class FeaturesManager {
|
||||||
return metadatalist;
|
return metadatalist;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Metadata checkForMetadatabyTitle(String searchString, String completetitle) throws Exception {
|
public Metadata checkForMetadatabyTitle(String searchString, String completetitle, String filename) throws Exception {
|
||||||
List<Metadata> mlist = getAllGNInfobyTitle(searchString, "1");
|
|
||||||
AnalysisLogger.getLogger().debug("Searching for: "+searchString);
|
AnalysisLogger.getLogger().debug("Searching for: "+searchString);
|
||||||
|
List<Metadata> mlist = getAllGNInfobyTitle(searchString, "1");
|
||||||
Metadata mfound = null;
|
Metadata mfound = null;
|
||||||
|
DefaultInternationalString intfilename = new DefaultInternationalString(filename);
|
||||||
for (Metadata m : mlist) {
|
for (Metadata m : mlist) {
|
||||||
String title = m.getIdentificationInfo().iterator().next().getCitation().getTitle().toString();
|
Identification id = m.getIdentificationInfo().iterator().next();
|
||||||
|
String title = id.getCitation().getTitle().toString();
|
||||||
if (completetitle.equalsIgnoreCase(title)) {
|
if (completetitle.equalsIgnoreCase(title)) {
|
||||||
mfound = m;
|
Iterator<? extends Keywords> it = id.getDescriptiveKeywords().iterator();
|
||||||
break;
|
while (it.hasNext()){
|
||||||
|
Keywords keys = (Keywords)it.next();
|
||||||
|
for (InternationalString is :keys.getKeywords()){
|
||||||
|
// System.out.println(is);
|
||||||
|
if (is.toString().equals(filename)){
|
||||||
|
mfound = m;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (mfound!=null)
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
if (mfound!=null)
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
return mfound;
|
return mfound;
|
||||||
|
@ -215,13 +237,19 @@ public class FeaturesManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String treatTitleForGN(String layerTitle) {
|
public static String treatTitleForGN(String layerTitle) {
|
||||||
int idx = layerTitle.indexOf("from");
|
int idx = layerTitle.indexOf(" from ");
|
||||||
String layerTitle2 = layerTitle;
|
String layerTitle2 = layerTitle;
|
||||||
|
if (idx>0)
|
||||||
|
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
|
||||||
|
else {
|
||||||
|
idx = layerTitle.indexOf(" in ");
|
||||||
if (idx>0)
|
if (idx>0)
|
||||||
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
|
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
|
||||||
else{
|
else{
|
||||||
idx = layerTitle.indexOf("(");
|
idx = layerTitle.indexOf("(");
|
||||||
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
|
if (idx>0)
|
||||||
|
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
layerTitle2 = layerTitle2.replaceAll("(\\(.*\\))", " ");
|
layerTitle2 = layerTitle2.replaceAll("(\\(.*\\))", " ");
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ import org.gcube.dataanalysis.geo.meta.features.FeaturesManager;
|
||||||
import org.gcube.dataanalysis.geo.utils.EnvDataExplorer;
|
import org.gcube.dataanalysis.geo.utils.EnvDataExplorer;
|
||||||
import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer;
|
import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer;
|
||||||
import org.opengis.metadata.Metadata;
|
import org.opengis.metadata.Metadata;
|
||||||
|
import org.opengis.metadata.identification.Identification;
|
||||||
|
|
||||||
public class GeoIntersector {
|
public class GeoIntersector {
|
||||||
|
|
||||||
|
@ -35,7 +36,10 @@ public class GeoIntersector {
|
||||||
|
|
||||||
// check if it is a NetCDF
|
// check if it is a NetCDF
|
||||||
if (featurer.isThreddsFile(meta)) {
|
if (featurer.isThreddsFile(meta)) {
|
||||||
System.out.println("found a netCDF file with title " + layerTitle+" and layer name "+layer);
|
Identification id = meta.getIdentificationInfo().iterator().next();
|
||||||
|
String title = id.getCitation().getTitle().toString();
|
||||||
|
|
||||||
|
System.out.println("found a netCDF file with title " +title +" and layer name "+layer);
|
||||||
features = getFeaturesFromNetCDF(featurer.getOpenDapLink(meta), layer, x, y, z);
|
features = getFeaturesFromNetCDF(featurer.getOpenDapLink(meta), layer, x, y, z);
|
||||||
} else {
|
} else {
|
||||||
System.out.println("found a Geo Layer with title " + layerTitle+" and layer name "+layer);
|
System.out.println("found a Geo Layer with title " + layerTitle+" and layer name "+layer);
|
||||||
|
@ -62,10 +66,11 @@ public class GeoIntersector {
|
||||||
|
|
||||||
public static void main(String args[] ) throws Exception{
|
public static void main(String args[] ) throws Exception{
|
||||||
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
|
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
|
||||||
GeoIntersector inters = new GeoIntersector("/gcube/devsec");
|
// GeoIntersector inters = new GeoIntersector("/gcube/devsec");
|
||||||
|
GeoIntersector inters = new GeoIntersector(null);
|
||||||
|
|
||||||
// System.out.println(inters.getFeaturesInTime("temperature (04091217ruc.nc)", 0.1, 0.1));
|
// System.out.println(inters.getFeaturesInTime("temperature (04091217ruc.nc)", 0.1, 0.1));
|
||||||
System.out.println(inters.getFeaturesInTime("occpoints_occ50f01212e04247c288b603c93c4325cd", 0.1, 0.1));
|
System.out.println(inters.getFeaturesInTime("Temperature", 0.1, 0.1,3000));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,117 @@
|
||||||
|
package org.gcube.dataanalysis.geo.utils;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.io.Reader;
|
||||||
|
import java.io.Writer;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.net.URLConnection;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class Downloader {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
List<String> hfiles = getfiles("netcdf_data.html", "fileServer","http", ".nc");
|
||||||
|
System.out.println(hfiles);
|
||||||
|
System.out.println("Number of links:"+hfiles.size());
|
||||||
|
List<String> files = enrichfiles(hfiles);
|
||||||
|
System.out.println(files);
|
||||||
|
buildwgetFile("wgetfiles.sh", hfiles, files);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void buildwgetFile(String filename, List<String> hfiles, List<String> files) throws Exception{
|
||||||
|
int size = hfiles.size();
|
||||||
|
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename)));
|
||||||
|
for (int i=0;i<size;i++){
|
||||||
|
bw.write(buildGetterString(hfiles.get(i), files.get(i))+System.getProperty("line.separator"));
|
||||||
|
}
|
||||||
|
|
||||||
|
bw.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<String> enrichfiles(List<String> files) throws Exception{
|
||||||
|
List<String> arrayfile = new ArrayList<String>();
|
||||||
|
for (String fileh:files){
|
||||||
|
String file = fileh.substring(fileh.lastIndexOf("/")+1);
|
||||||
|
if (file.contains("temperature"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
|
||||||
|
else if (file.contains("salinity"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
|
||||||
|
else if (file.contains("oxygen"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
|
||||||
|
else if (file.contains("phosphate"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
|
||||||
|
else if (file.contains("nitrate"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
|
||||||
|
else if (file.contains("silicate"))
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
|
||||||
|
else
|
||||||
|
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
|
||||||
|
arrayfile.add(file);
|
||||||
|
}
|
||||||
|
return arrayfile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<String> getfiles(String filename,String criterion,String initselection, String endselection) throws Exception{
|
||||||
|
List<String> files = new ArrayList<String>();
|
||||||
|
BufferedReader br = new BufferedReader(new FileReader(new File(filename)));
|
||||||
|
String line = br.readLine();
|
||||||
|
while (line!=null){
|
||||||
|
if (line.contains(criterion)){
|
||||||
|
String cut = line.substring(line.indexOf(initselection),line.indexOf(endselection)+endselection.length());
|
||||||
|
files.add(cut);
|
||||||
|
}
|
||||||
|
line = br.readLine();
|
||||||
|
}
|
||||||
|
br.close();
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
public static String buildGetterString(String httpstring,String filename){
|
||||||
|
return String.format("wget --output-document=%1$s %2$s",filename,httpstring);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String buildTopicString(String filename, String... topics){
|
||||||
|
int idx = filename.indexOf(".n");
|
||||||
|
String file = filename.substring(0,idx);
|
||||||
|
String ext = filename.substring(idx);
|
||||||
|
for (String topic:topics){
|
||||||
|
file=file+"_"+topic;
|
||||||
|
}
|
||||||
|
return file+"_"+ext;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void downloadData(String endpoint, String file) throws Exception {
|
||||||
|
// Send data
|
||||||
|
String urlStr = endpoint;
|
||||||
|
URL url = new URL(urlStr);
|
||||||
|
URLConnection conn = url.openConnection();
|
||||||
|
InputStreamReader isr = new InputStreamReader(conn.getInputStream());
|
||||||
|
FileWriter fw = new FileWriter(new File(file));
|
||||||
|
pipe(isr, fw);
|
||||||
|
fw.close();
|
||||||
|
isr.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void pipe(Reader reader, Writer writer) throws IOException {
|
||||||
|
char[] buf = new char[1024];
|
||||||
|
int read = 0;
|
||||||
|
double bytes = 0;
|
||||||
|
long i = 0;
|
||||||
|
while ((read = reader.read(buf)) >= 0) {
|
||||||
|
writer.write(buf, 0, read);
|
||||||
|
bytes=(bytes+read);
|
||||||
|
if (i%1000==0)
|
||||||
|
System.out.println("B:"+bytes);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -86,9 +86,9 @@ public class ThreddsDataExplorer {
|
||||||
GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(filename);
|
GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(filename);
|
||||||
List<GridDatatype> gridTypes = gds.getGrids();
|
List<GridDatatype> gridTypes = gds.getGrids();
|
||||||
for (GridDatatype gdt : gridTypes) {
|
for (GridDatatype gdt : gridTypes) {
|
||||||
AnalysisLogger.getLogger().debug("Inside File - layer name: "+gdt.getFullName());
|
AnalysisLogger.getLogger().debug("Inside File - layer name: " + gdt.getFullName());
|
||||||
if (layer.equalsIgnoreCase(gdt.getFullName())) {
|
if (layer.equalsIgnoreCase(gdt.getFullName())) {
|
||||||
AnalysisLogger.getLogger().debug("Found layer "+layer +" inside file");
|
AnalysisLogger.getLogger().debug("Found layer " + layer + " inside file");
|
||||||
GridDatatype grid = gds.findGridDatatype(gdt.getName());
|
GridDatatype grid = gds.findGridDatatype(gdt.getName());
|
||||||
GridCoordSystem gcs = grid.getCoordinateSystem();
|
GridCoordSystem gcs = grid.getCoordinateSystem();
|
||||||
long timeSteps = 0;
|
long timeSteps = 0;
|
||||||
|
@ -101,20 +101,26 @@ public class ThreddsDataExplorer {
|
||||||
CoordinateAxis tAxis = gcs.getTimeAxis();
|
CoordinateAxis tAxis = gcs.getTimeAxis();
|
||||||
timeSteps = tAxis.getSize();
|
timeSteps = tAxis.getSize();
|
||||||
}
|
}
|
||||||
|
|
||||||
CoordinateAxis zAxis = gdt.getCoordinateSystem().getVerticalAxis();
|
CoordinateAxis zAxis = gdt.getCoordinateSystem().getVerticalAxis();
|
||||||
double resolutionZ = Math.abs((double) (zAxis.getMaxValue() - zAxis.getMinValue()) / (double) zAxis.getShape()[0]);
|
double resolutionZ = Math.abs((double) (zAxis.getMaxValue() - zAxis.getMinValue()) / (double) zAxis.getShape()[0]);
|
||||||
int zint = (int) Math.round(z/resolutionZ);
|
int zint = (int) Math.round(z / resolutionZ);
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Z index to take: " + zint);
|
||||||
|
|
||||||
int[] xy = gcs.findXYindexFromLatLon(y, x, null);
|
int[] xy = gcs.findXYindexFromLatLon(y, x, null);
|
||||||
for (int j = 0; j < timeSteps; j++) {
|
for (int j = 0; j < timeSteps; j++) {
|
||||||
Array data = grid.readDataSlice(j, zint, xy[1], xy[0]); // note order is t, z, y, x
|
try {
|
||||||
Double val = takeFirstDouble(data);
|
Array data = grid.readDataSlice(j, zint, xy[1], xy[0]); // note order is t, z, y, x
|
||||||
if (!val.isNaN()) {
|
Double val = takeFirstDouble(data);
|
||||||
String date = "" + j;
|
if (!val.isNaN()) {
|
||||||
if (dates != null)
|
String date = "" + j;
|
||||||
date = dates[j].toString();
|
if (dates != null)
|
||||||
valuesMap.put(timePrefix + date, Double.parseDouble("" + val));
|
date = dates[j].toString();
|
||||||
|
valuesMap.put(timePrefix + date, Double.parseDouble("" + val));
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
AnalysisLogger.getLogger().debug("Error in getting grid values in (" + x + "," + y + "," + z + "= with zint: " + zint + " resolution: " + resolutionZ + " and shape: " + zAxis.getShape()[0]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
@ -202,7 +208,7 @@ public class ThreddsDataExplorer {
|
||||||
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
|
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
|
||||||
public static boolean isGridDataset(String filename) {
|
public static boolean isGridDataset(String filename) {
|
||||||
try {
|
try {
|
||||||
AnalysisLogger.getLogger().debug("Analyzing file "+filename);
|
AnalysisLogger.getLogger().debug("Analyzing file " + filename);
|
||||||
Formatter errlog = new Formatter();
|
Formatter errlog = new Formatter();
|
||||||
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.GRID, filename, null, errlog);
|
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.GRID, filename, null, errlog);
|
||||||
if (fdataset == null) {
|
if (fdataset == null) {
|
||||||
|
@ -250,41 +256,40 @@ public class ThreddsDataExplorer {
|
||||||
return isdataset;
|
return isdataset;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static double adjX(double x) {
|
||||||
public static double adjX(double x){
|
|
||||||
if (x < -180)
|
if (x < -180)
|
||||||
x = -180;
|
x = -180;
|
||||||
if (x > 180)
|
if (x > 180)
|
||||||
x = 180;
|
x = 180;
|
||||||
|
|
||||||
return x;
|
return x;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static double adjY(double y){
|
public static double adjY(double y) {
|
||||||
if (y < -90)
|
if (y < -90)
|
||||||
y = -90;
|
y = -90;
|
||||||
if (y > 90)
|
if (y > 90)
|
||||||
y = 90;
|
y = 90;
|
||||||
|
|
||||||
return y;
|
return y;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static double getMinX(GridCoordSystem gcs){
|
public static double getMinX(GridCoordSystem gcs) {
|
||||||
CoordinateAxis xAxis = gcs.getXHorizAxis();
|
CoordinateAxis xAxis = gcs.getXHorizAxis();
|
||||||
return adjX(xAxis.getMinValue() - 180);
|
return adjX(xAxis.getMinValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static double getMaxX(GridCoordSystem gcs){
|
public static double getMaxX(GridCoordSystem gcs) {
|
||||||
CoordinateAxis xAxis = gcs.getXHorizAxis();
|
CoordinateAxis xAxis = gcs.getXHorizAxis();
|
||||||
return adjX(xAxis.getMaxValue() - 180);
|
return adjX(xAxis.getMaxValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static double getMinY(GridCoordSystem gcs){
|
public static double getMinY(GridCoordSystem gcs) {
|
||||||
CoordinateAxis yAxis = gcs.getYHorizAxis();
|
CoordinateAxis yAxis = gcs.getYHorizAxis();
|
||||||
return adjY(yAxis.getMinValue());
|
return adjY(yAxis.getMinValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static double getMaxY(GridCoordSystem gcs){
|
public static double getMaxY(GridCoordSystem gcs) {
|
||||||
CoordinateAxis yAxis = gcs.getYHorizAxis();
|
CoordinateAxis yAxis = gcs.getYHorizAxis();
|
||||||
return adjY(yAxis.getMaxValue());
|
return adjY(yAxis.getMaxValue());
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue