gCubeSystem
/
ecological-engine-geospatial-extensions
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@73630 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2013-04-18 17:05:08 +00:00
parent 0fdd3294e5
commit b44640be23
6 changed files with 233 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/java/org/gcube/dataanalysis/geo/insertion/ThreddsFetcher.java
View File

@ -55,6 +55,14 @@ public class ThreddsFetcher {
String duration = null;
String resolution = gdt.getTimeDimension()==null?null:""+gdt.getTimeDimension().getLength();
int numberOfDimensions = 2;
if ((gdt.getZDimension()!=null)&&(gdt.getZDimension().getLength()>1)){
numberOfDimensions = 3;
AnalysisLogger.getLogger().debug("Length of Z Dimension: "+gdt.getZDimension().getLength());
}
else
AnalysisLogger.getLogger().debug("Bidimensional Layer ");
if (dr != null) {
hStartDate = dr.getStart() == null ? null : humanDF.format(dr.getStart().getDate());
hEndDate = dr.getEnd() == null ? null : humanDF.format(dr.getEnd().getDate());
@ -62,9 +70,12 @@ public class ThreddsFetcher {
}
// control if the file is yet on GN
String generatedTitle = generateTitle(filename, description, hStartDate, hEndDate);
String generatedTitle = generateTitle(filename, description, hStartDate, hEndDate, numberOfDimensions);
CoordinateAxis xAxis = gdt.getCoordinateSystem().getXHorizAxis();
CoordinateAxis yAxis = gdt.getCoordinateSystem().getYHorizAxis();
AnalysisLogger.getLogger().debug("Bounds:"+xAxis.getMinValue()+","+yAxis.getMinValue()+","+xAxis.getMaxValue()+","+yAxis.getMaxValue());
Metadata previousmeta = featurer.checkForMetadatabyTitle(FeaturesManager.treatTitleForGN(generatedTitle), generatedTitle);
Metadata previousmeta = featurer.checkForMetadatabyTitle(FeaturesManager.treatTitleForGN(generatedTitle), generatedTitle,filename);
if (previousmeta!=null){
AnalysisLogger.getLogger().debug("***WARNING: layer yet found on GeoNetwork***");
continue;
@ -76,17 +87,9 @@ public class ThreddsFetcher {
}
*/
int numberOfDimensions = 2;
if ((gdt.getZDimension()!=null)&&(gdt.getZDimension().getLength()>1)){
numberOfDimensions = 3;
AnalysisLogger.getLogger().debug("Length of Z Dimension: "+gdt.getZDimension().getLength());
}
else
AnalysisLogger.getLogger().debug("Bidimensional Layer ");
// get resolution - take the maximum regular step
CoordinateAxis xAxis = gdt.getCoordinateSystem().getXHorizAxis();
CoordinateAxis yAxis = gdt.getCoordinateSystem().getYHorizAxis();
double resolutionX = Math.abs((double) (xAxis.getMaxValue() - xAxis.getMinValue()) / (double) xAxis.getShape()[0]);
double resolutionY = Math.abs((double) (yAxis.getMaxValue() - yAxis.getMinValue()) / (double) yAxis.getShape()[0]);
@ -95,6 +98,7 @@ public class ThreddsFetcher {
metadataInserter.setGeonetworkUrl(featurer.getGeonetworkURLFromScope());
metadataInserter.setGeonetworkUser(featurer.getGeonetworkUserFromScope());
metadataInserter.setGeonetworkPwd(featurer.getGeonetworkPasswordFromScope());
// Build standard info:
metadataInserter.setThreddsCatalogUrl(threddsCatalogURL);
metadataInserter.setLayerUrl(url);
@ -105,12 +109,14 @@ public class ThreddsFetcher {
metadataInserter.setAbstractField(generateAbstractField(gdt.getFullName(), filename, description, gdt.getUnitsString().trim(), hStartDate, hEndDate, duration, resolution, numberOfDimensions));
metadataInserter.setResolution(Math.max(resolutionX, resolutionY));
// set Bounding box
metadataInserter.setXLeftLow(ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()));
metadataInserter.setYLeftLow(ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()));
metadataInserter.setXRightUpper(ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()));
metadataInserter.setYRightUpper(ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()));
//set keywords
metadataInserter.setCustomTopics(filename, description);
metadataInserter.setCustomTopics(filename, description,numberOfDimensions+"D");
//set Temporal Extent
if (hStartDate!=null){
metadataInserter.setStartDate(dr.getStart().getDate());
@ -121,15 +127,15 @@ public class ThreddsFetcher {
AnalysisLogger.getLogger().debug("abstract: " + metadataInserter.getAbstractField());
metadataInserter.insertMetaData();
break;
// break;
}
}
break;
// break;
}
}
public static String generateTitle(String filename, String description, String startDate, String endDate) {
public static String generateTitle(String filename, String description, String startDate, String endDate, int numberOfDimensions) {
String dateString = "";
if (startDate != null){
if (startDate.equals(endDate))
@ -137,7 +143,7 @@ public class ThreddsFetcher {
else
dateString = " from [" + startDate + "] to [" + endDate + "]";
}
return description.replaceAll("( )+", " ") + dateString+" (" + filename + ")";
return description.replaceAll("( )+", " ") + dateString+" (" + numberOfDimensions+ "D)";//+" (" + filename + ")";
}
public static String generateAbstractField(String layername, String filename, String description, String unit, String startDate, String endDate, String duration, String timeInstants, int numberOfDimensions) {
@ -167,7 +173,7 @@ public class ThreddsFetcher {
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
// ThreddsFetcher tf = new ThreddsFetcher("/gcube/devsec");
ThreddsFetcher tf = new ThreddsFetcher(null);
tf.fetch("http://thredds.research-infrastructures.eu:8080/thredds/catalog/public/netcdf/catalog.xml");
tf.fetch("http://thredds.research-infrastructures.eu/thredds/catalog/public/netcdf/catalog.xml");
}
}

28
src/main/java/org/gcube/dataanalysis/geo/meta/NetCDFMetadata.java
View File

@ -19,7 +19,7 @@ import java.util.Map.Entry;
import javax.xml.bind.JAXBException;
import org.gcube.dataanalysis.geo.utils.NetCDFTemporalPrimitive;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.geotoolkit.metadata.iso.DefaultIdentifier;
import org.geotoolkit.metadata.iso.DefaultMetadata;
import org.geotoolkit.metadata.iso.citation.DefaultCitation;
@ -33,8 +33,6 @@ import org.geotoolkit.metadata.iso.distribution.DefaultDistribution;
import org.geotoolkit.metadata.iso.distribution.DefaultFormat;
import org.geotoolkit.metadata.iso.extent.DefaultExtent;
import org.geotoolkit.metadata.iso.extent.DefaultGeographicBoundingBox;
import org.geotoolkit.metadata.iso.extent.DefaultSpatialTemporalExtent;
import org.geotoolkit.metadata.iso.extent.DefaultTemporalExtent;
import org.geotoolkit.metadata.iso.identification.DefaultDataIdentification;
import org.geotoolkit.metadata.iso.identification.DefaultKeywords;
import org.geotoolkit.metadata.iso.identification.DefaultResolution;
@ -65,7 +63,6 @@ import org.opengis.metadata.maintenance.ScopeCode;
import org.opengis.metadata.spatial.GeometricObjectType;
import org.opengis.metadata.spatial.SpatialRepresentationType;
import org.opengis.metadata.spatial.TopologyLevel;
import org.opengis.temporal.TemporalPrimitive;
import org.opengis.util.InternationalString;
public class NetCDFMetadata {
@ -368,11 +365,9 @@ public class NetCDFMetadata {
usage.setUserDeterminedLimitations(new DefaultInternationalString(usageLimitations));
usage.setUserContactInfo(new ArrayList<ResponsibleParty>(Arrays.asList(party)));
ArrayList<DefaultUsage> usages = new ArrayList<DefaultUsage>(Arrays.asList(usage));
ArrayList<TopicCategory> categories = new ArrayList<TopicCategory>();
categories.add(TopicCategory.BIOTA);
categories.add(TopicCategory.ENVIRONMENT);
categories.add(TopicCategory.OCEANS);
//build categories by guessing on the filename
List<TopicCategory> categories = guessTopicCategory(sourceFileName);
AnalysisLogger.getLogger().debug("Guessed Topics: "+categories);
// Spatial Rapresentation Info
DefaultGeometricObjects geoObjs = new DefaultGeometricObjects();
geoObjs.setGeometricObjectType(GeometricObjectType.COMPLEX);
@ -471,12 +466,23 @@ public class NetCDFMetadata {
meta.getMetadataConstraints().add(constraints);
meta.getDataQualityInfo().add(processQuality);
meta.setLanguage(Locale.ENGLISH);
System.out.println(meta);
// System.out.println(meta);
GNClient client = new GNClient(geonetworkUrl);
client.login(geonetworkUser, geonetworkPwd);
File tmetafile = meta2File(meta);
client.insertMetadata(new GNInsertConfiguration("3", "datasets", "_none_", true), tmetafile);
// tmetafile.delete();
tmetafile.delete();
}
public static List<TopicCategory> guessTopicCategory(String refString){
String searcher = refString.toLowerCase();
List<TopicCategory> categories = new ArrayList<TopicCategory>();
for (TopicCategory topic:TopicCategory.values()){
if (searcher.contains("_"+topic.name().toLowerCase()+"_")){
categories.add(topic);
}
}
return categories;
}
}

42
src/main/java/org/gcube/dataanalysis/geo/meta/features/FeaturesManager.java
View File

@ -4,6 +4,8 @@ import it.geosolutions.geonetwork.util.GNSearchRequest;
import it.geosolutions.geonetwork.util.GNSearchResponse;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import org.gcube.common.scope.api.ScopeProvider;
@ -12,9 +14,14 @@ import org.gcube.spatial.data.geonetwork.GeoNetwork;
import org.gcube.spatial.data.geonetwork.GeoNetworkReader;
import org.gcube.spatial.data.geonetwork.configuration.Configuration;
import org.gcube.spatial.data.geonetwork.configuration.ConfigurationManager;
import org.geotoolkit.metadata.iso.identification.DefaultKeywords;
import org.geotoolkit.util.DefaultInternationalString;
import org.opengis.metadata.Metadata;
import org.opengis.metadata.citation.OnlineResource;
import org.opengis.metadata.distribution.DigitalTransferOptions;
import org.opengis.metadata.identification.Identification;
import org.opengis.metadata.identification.Keywords;
import org.opengis.util.InternationalString;
public class FeaturesManager {
private String geonetworkUrl = "http://geoserver-dev2.d4science-ii.research-infrastructures.eu/geonetwork/";
@ -175,16 +182,31 @@ public class FeaturesManager {
return metadatalist;
}
public Metadata checkForMetadatabyTitle(String searchString, String completetitle) throws Exception {
List<Metadata> mlist = getAllGNInfobyTitle(searchString, "1");
public Metadata checkForMetadatabyTitle(String searchString, String completetitle, String filename) throws Exception {
AnalysisLogger.getLogger().debug("Searching for: "+searchString);
List<Metadata> mlist = getAllGNInfobyTitle(searchString, "1");
Metadata mfound = null;
DefaultInternationalString intfilename = new DefaultInternationalString(filename);
for (Metadata m : mlist) {
String title = m.getIdentificationInfo().iterator().next().getCitation().getTitle().toString();
Identification id = m.getIdentificationInfo().iterator().next();
String title = id.getCitation().getTitle().toString();
if (completetitle.equalsIgnoreCase(title)) {
mfound = m;
break;
Iterator<? extends Keywords> it = id.getDescriptiveKeywords().iterator();
while (it.hasNext()){
Keywords keys = (Keywords)it.next();
for (InternationalString is :keys.getKeywords()){
// System.out.println(is);
if (is.toString().equals(filename)){
mfound = m;
break;
}
}
if (mfound!=null)
break;
}
}
if (mfound!=null)
break;
}
return mfound;
@ -215,13 +237,19 @@ public class FeaturesManager {
}
public static String treatTitleForGN(String layerTitle) {
int idx = layerTitle.indexOf("from");
int idx = layerTitle.indexOf(" from ");
String layerTitle2 = layerTitle;
if (idx>0)
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
else {
idx = layerTitle.indexOf(" in ");
if (idx>0)
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
else{
idx = layerTitle.indexOf("(");
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
if (idx>0)
layerTitle2 = layerTitle.toLowerCase().substring(0, idx).trim();
}
}
layerTitle2 = layerTitle2.replaceAll("(\\(.*\\))", " ");

11
src/main/java/org/gcube/dataanalysis/geo/retrieval/GeoIntersector.java
View File

@ -8,6 +8,7 @@ import org.gcube.dataanalysis.geo.meta.features.FeaturesManager;
import org.gcube.dataanalysis.geo.utils.EnvDataExplorer;
import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer;
import org.opengis.metadata.Metadata;
import org.opengis.metadata.identification.Identification;
public class GeoIntersector {
@ -35,7 +36,10 @@ public class GeoIntersector {
// check if it is a NetCDF
if (featurer.isThreddsFile(meta)) {
System.out.println("found a netCDF file with title " + layerTitle+" and layer name "+layer);
Identification id = meta.getIdentificationInfo().iterator().next();
String title = id.getCitation().getTitle().toString();
System.out.println("found a netCDF file with title " +title +" and layer name "+layer);
features = getFeaturesFromNetCDF(featurer.getOpenDapLink(meta), layer, x, y, z);
} else {
System.out.println("found a Geo Layer with title " + layerTitle+" and layer name "+layer);
@ -62,10 +66,11 @@ public class GeoIntersector {
public static void main(String args[] ) throws Exception{
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
GeoIntersector inters = new GeoIntersector("/gcube/devsec");
// GeoIntersector inters = new GeoIntersector("/gcube/devsec");
GeoIntersector inters = new GeoIntersector(null);
// System.out.println(inters.getFeaturesInTime("temperature (04091217ruc.nc)", 0.1, 0.1));
System.out.println(inters.getFeaturesInTime("occpoints_occ50f01212e04247c288b603c93c4325cd", 0.1, 0.1));
System.out.println(inters.getFeaturesInTime("Temperature", 0.1, 0.1,3000));
}
}

117
src/main/java/org/gcube/dataanalysis/geo/utils/Downloader.java Normal file
View File

@ -0,0 +1,117 @@
package org.gcube.dataanalysis.geo.utils;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
public class Downloader {
public static void main(String[] args) throws Exception {
List<String> hfiles = getfiles("netcdf_data.html", "fileServer","http", ".nc");
System.out.println(hfiles);
System.out.println("Number of links:"+hfiles.size());
List<String> files = enrichfiles(hfiles);
System.out.println(files);
buildwgetFile("wgetfiles.sh", hfiles, files);
}
public static void buildwgetFile(String filename, List<String> hfiles, List<String> files) throws Exception{
int size = hfiles.size();
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename)));
for (int i=0;i<size;i++){
bw.write(buildGetterString(hfiles.get(i), files.get(i))+System.getProperty("line.separator"));
}
bw.close();
}
public static List<String> enrichfiles(List<String> files) throws Exception{
List<String> arrayfile = new ArrayList<String>();
for (String fileh:files){
String file = fileh.substring(fileh.lastIndexOf("/")+1);
if (file.contains("temperature"))
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
else if (file.contains("salinity"))
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
else if (file.contains("oxygen"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("phosphate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("nitrate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("silicate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
arrayfile.add(file);
}
return arrayfile;
}
public static List<String> getfiles(String filename,String criterion,String initselection, String endselection) throws Exception{
List<String> files = new ArrayList<String>();
BufferedReader br = new BufferedReader(new FileReader(new File(filename)));
String line = br.readLine();
while (line!=null){
if (line.contains(criterion)){
String cut = line.substring(line.indexOf(initselection),line.indexOf(endselection)+endselection.length());
files.add(cut);
}
line = br.readLine();
}
br.close();
return files;
}
public static String buildGetterString(String httpstring,String filename){
return String.format("wget --output-document=%1$s %2$s",filename,httpstring);
}
public static String buildTopicString(String filename, String... topics){
int idx = filename.indexOf(".n");
String file = filename.substring(0,idx);
String ext = filename.substring(idx);
for (String topic:topics){
file=file+"_"+topic;
}
return file+"_"+ext;
}
public static void downloadData(String endpoint, String file) throws Exception {
// Send data
String urlStr = endpoint;
URL url = new URL(urlStr);
URLConnection conn = url.openConnection();
InputStreamReader isr = new InputStreamReader(conn.getInputStream());
FileWriter fw = new FileWriter(new File(file));
pipe(isr, fw);
fw.close();
isr.close();
}
private static void pipe(Reader reader, Writer writer) throws IOException {
char[] buf = new char[1024];
int read = 0;
double bytes = 0;
long i = 0;
while ((read = reader.read(buf)) >= 0) {
writer.write(buf, 0, read);
bytes=(bytes+read);
if (i%1000==0)
System.out.println("B:"+bytes);
i++;
}
writer.flush();
}
}

61
src/main/java/org/gcube/dataanalysis/geo/utils/ThreddsDataExplorer.java
View File

@ -86,9 +86,9 @@ public class ThreddsDataExplorer {
GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(filename);
List<GridDatatype> gridTypes = gds.getGrids();
for (GridDatatype gdt : gridTypes) {
AnalysisLogger.getLogger().debug("Inside File - layer name: "+gdt.getFullName());
AnalysisLogger.getLogger().debug("Inside File - layer name: " + gdt.getFullName());
if (layer.equalsIgnoreCase(gdt.getFullName())) {
AnalysisLogger.getLogger().debug("Found layer "+layer +" inside file");
AnalysisLogger.getLogger().debug("Found layer " + layer + " inside file");
GridDatatype grid = gds.findGridDatatype(gdt.getName());
GridCoordSystem gcs = grid.getCoordinateSystem();
long timeSteps = 0;
@ -101,20 +101,26 @@ public class ThreddsDataExplorer {
CoordinateAxis tAxis = gcs.getTimeAxis();
timeSteps = tAxis.getSize();
}
CoordinateAxis zAxis = gdt.getCoordinateSystem().getVerticalAxis();
double resolutionZ = Math.abs((double) (zAxis.getMaxValue() - zAxis.getMinValue()) / (double) zAxis.getShape()[0]);
int zint = (int) Math.round(z/resolutionZ);
int zint = (int) Math.round(z / resolutionZ);
AnalysisLogger.getLogger().debug("Z index to take: " + zint);
int[] xy = gcs.findXYindexFromLatLon(y, x, null);
for (int j = 0; j < timeSteps; j++) {
Array data = grid.readDataSlice(j, zint, xy[1], xy[0]); // note order is t, z, y, x
Double val = takeFirstDouble(data);
if (!val.isNaN()) {
String date = "" + j;
if (dates != null)
date = dates[j].toString();
valuesMap.put(timePrefix + date, Double.parseDouble("" + val));
try {
Array data = grid.readDataSlice(j, zint, xy[1], xy[0]); // note order is t, z, y, x
Double val = takeFirstDouble(data);
if (!val.isNaN()) {
String date = "" + j;
if (dates != null)
date = dates[j].toString();
valuesMap.put(timePrefix + date, Double.parseDouble("" + val));
}
} catch (Exception e) {
AnalysisLogger.getLogger().debug("Error in getting grid values in (" + x + "," + y + "," + z + "= with zint: " + zint + " resolution: " + resolutionZ + " and shape: " + zAxis.getShape()[0]);
}
}
break;
@ -202,7 +208,7 @@ public class ThreddsDataExplorer {
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
public static boolean isGridDataset(String filename) {
try {
AnalysisLogger.getLogger().debug("Analyzing file "+filename);
AnalysisLogger.getLogger().debug("Analyzing file " + filename);
Formatter errlog = new Formatter();
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.GRID, filename, null, errlog);
if (fdataset == null) {
@ -250,41 +256,40 @@ public class ThreddsDataExplorer {
return isdataset;
}
public static double adjX(double x){
public static double adjX(double x) {
if (x < -180)
x = -180;
if (x > 180)
x = 180;
return x;
}
public static double adjY(double y){
public static double adjY(double y) {
if (y < -90)
y = -90;
if (y > 90)
y = 90;
return y;
}
public static double getMinX(GridCoordSystem gcs){
public static double getMinX(GridCoordSystem gcs) {
CoordinateAxis xAxis = gcs.getXHorizAxis();
return adjX(xAxis.getMinValue() - 180);
return adjX(xAxis.getMinValue());
}
public static double getMaxX(GridCoordSystem gcs){
public static double getMaxX(GridCoordSystem gcs) {
CoordinateAxis xAxis = gcs.getXHorizAxis();
return adjX(xAxis.getMaxValue() - 180);
return adjX(xAxis.getMaxValue());
}
public static double getMinY(GridCoordSystem gcs){
public static double getMinY(GridCoordSystem gcs) {
CoordinateAxis yAxis = gcs.getYHorizAxis();
return adjY(yAxis.getMinValue());
}
public static double getMaxY(GridCoordSystem gcs){
public static double getMaxY(GridCoordSystem gcs) {
CoordinateAxis yAxis = gcs.getYHorizAxis();
return adjY(yAxis.getMaxValue());
}