Gianpaolo Coro 2013-04-15 16:14:01 +00:00
parent be571cdba8
commit e5008ea09c
6 changed files with 577 additions and 0 deletions

View File

@ -0,0 +1,141 @@
package org.gcube.dataanalysis.geo.meta.features;
import it.geosolutions.geonetwork.util.GNSearchRequest;
import it.geosolutions.geonetwork.util.GNSearchResponse;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.spatial.data.geonetwork.GeoNetwork;
import org.gcube.spatial.data.geonetwork.GeoNetworkReader;
import org.opengis.metadata.Metadata;
import org.opengis.metadata.citation.OnlineResource;
import org.opengis.metadata.distribution.DigitalTransferOptions;
public class FeaturesManager {
private String geonetworkUrl = "http://geoserver-dev.d4science-ii.research-infrastructures.eu/geonetwork/";
private String geonetworkUser = "admin";
private String geonetworkPwd = "admin";
private String scope = "/gcube/devsec";
public String getScope() {
return scope;
}
public void setScope(String scope) {
this.scope = scope;
}
private String searchInUrl(Metadata meta, String criterion) {
String link = null;
for (DigitalTransferOptions option : meta.getDistributionInfo().getTransferOptions()) {
for (OnlineResource resource : option.getOnLines()) {
String tlink = resource.getLinkage().toString();
if (tlink.toLowerCase().contains(criterion.toLowerCase())) {
link = tlink;
break;
}
}
}
if (link == null)
System.out.println("NO ONLINE LINK WAS FOUND ACCORDING TO THE CRITERION :" + criterion);
return link;
}
public String getWFSLink(Metadata meta) {
return searchInUrl(meta, "service=wfs");
}
// retrieves the wms link
public String getWMSLink(Metadata meta) {
return searchInUrl(meta, "service=wms");
}
public String getWCSLink(Metadata meta) {
return searchInUrl(meta, "service=wcs");
}
public String getOpenDapLink(Metadata meta) {
return searchInUrl(meta, "/dodsC");
}
public String getThreddsLink(Metadata meta) {
return searchInUrl(meta, "catalog.xml");
}
public String getLayerName(Metadata meta) {
String wmslink = getWMSLink(meta);
String layer = null;
String finder = "layers=";
if (wmslink != null) {
int idxfinder = wmslink.indexOf(finder);
if (idxfinder > 0) {
wmslink = wmslink.substring(idxfinder);
int andIdx = wmslink.indexOf("&");
if (andIdx < 0)
andIdx = wmslink.length();
layer = wmslink.substring(finder.length(), andIdx).trim();
}
}
return layer;
}
public boolean isThreddsFile(Metadata meta) {
return (getOpenDapLink(meta) != null);
}
public Metadata getGNInfobyTitle(String info) throws Exception {
ScopeProvider.instance.set(scope);
GeoNetworkReader gn = GeoNetwork.get();
// Form query object
gn.login();
GNSearchRequest req = new GNSearchRequest();
req.addParam(GNSearchRequest.Param.title, info);
GNSearchResponse resp = gn.query(req);
Metadata meta = null;
if (resp.getCount() != 0)
for (GNSearchResponse.GNMetadata metadata : resp) {
meta = gn.getById(metadata.getUUID());
break;
}
return meta;
}
public String getGeonetworkUrl() {
return geonetworkUrl;
}
public void setGeonetworkUrl(String geonetworkUrl) {
this.geonetworkUrl = geonetworkUrl;
}
public String getGeonetworkUser() {
return geonetworkUser;
}
public void setGeonetworkUser(String geonetworkUser) {
this.geonetworkUser = geonetworkUser;
}
public String getGeonetworkPwd() {
return geonetworkPwd;
}
public void setGeonetworkPwd(String geonetworkPwd) {
this.geonetworkPwd = geonetworkPwd;
}
public static void main(String args[]) throws Exception {
// String title = "temperature (04091217ruc.nc)";
// String title = "Bathymetry";
String title = "FAO aquatic species distribution map of Melanogrammus aeglefinus";
FeaturesManager fm = new FeaturesManager();
Metadata meta = fm.getGNInfobyTitle(title);
System.out.println("is file? " + fm.isThreddsFile(meta));
System.out.println("opendap: " + fm.getOpenDapLink(meta));
System.out.println("wcs:" + fm.getWCSLink(meta));
System.out.println("wms:" + fm.getWMSLink(meta));
System.out.println("thredds:" + fm.getThreddsLink(meta));
}
}

View File

@ -0,0 +1,17 @@
package org.gcube.dataanalysis.geo.meta.test;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.geo.retrieval.GeoIntersector;
public class TestIntersectionNetCDF {
public static void main(String args[] ) throws Exception{
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
GeoIntersector inters = new GeoIntersector("/gcube/devsec");
System.out.println(inters.getFeaturesInTime("temperature (04091217ruc.nc)", 0.1, 0.1));
}
}

View File

@ -0,0 +1,67 @@
package org.gcube.dataanalysis.geo.retrieval;
import java.util.LinkedHashMap;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.geo.meta.features.FeaturesManager;
import org.gcube.dataanalysis.geo.utils.EnvDataExplorer;
import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer;
import org.opengis.metadata.Metadata;
public class GeoIntersector {
private FeaturesManager featurer;
public GeoIntersector(String scope) {
featurer = new FeaturesManager();
featurer.setScope(scope);
}
public LinkedHashMap<String, Double> getFeaturesInTime(String layerTitle, double x, double y) throws Exception {
LinkedHashMap<String, Double> features = new LinkedHashMap<String, Double>();
// get the layer
Metadata meta = featurer.getGNInfobyTitle(layerTitle);
// if the layer is good
if (meta != null) {
String layer = featurer.getLayerName(meta);
if (layer == null)
layer = layerTitle;
// check if it is a NetCDF
if (featurer.isThreddsFile(meta)) {
System.out.println("found a netCDF file with title " + layerTitle+" and layer name "+layer);
features = getFeaturesFromNetCDF(featurer.getOpenDapLink(meta), layer, x, y);
} else {
System.out.println("found a Geo Layer with title " + layerTitle+" and layer name "+layer);
features = getFeaturesFromWFS(featurer.getWFSLink(meta), layer, x, y);
}
}
return features;
}
private LinkedHashMap<String, Double> getFeaturesFromNetCDF(String opendapURL, String layer, double x, double y) {
if (opendapURL == null)
return null;
return ThreddsDataExplorer.retrieveDataFromNetCDF(opendapURL, layer, x, y);
}
private LinkedHashMap<String, Double> getFeaturesFromWFS(String wfsUrl, String layer, double x, double y) {
if (wfsUrl == null)
return null;
return EnvDataExplorer.getFeatures(wfsUrl, layer, x, y);
}
public static void main(String args[] ) throws Exception{
AnalysisLogger.setLogger("./cfg/"+AlgorithmConfiguration.defaultLoggerFile);
GeoIntersector inters = new GeoIntersector("/gcube/devsec");
// System.out.println(inters.getFeaturesInTime("temperature (04091217ruc.nc)", 0.1, 0.1));
System.out.println(inters.getFeaturesInTime("occpoints_occ50f01212e04247c288b603c93c4325cd", 0.1, 0.1));
}
}

View File

@ -0,0 +1,46 @@
package org.gcube.dataanalysis.geo.utils;
import java.util.HashMap;
import java.util.LinkedHashMap;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.geo.meta.OGCFormatter;
public class EnvDataExplorer {
private static String callWFS(String geoServer, String layer, double x, double y) {
float tolerance = 0.25f;
String wfsURL = OGCFormatter.getWfsUrl(geoServer, layer, OGCFormatter.pointToBoundingBox(x, y, tolerance), 1, "json");
AnalysisLogger.getLogger().debug("EnvDataExplorer-> Requesting URL: " + wfsURL);
String returned = null;
try {
returned = HttpRequest.sendGetRequest(wfsURL, null);
} catch (Exception e) {
AnalysisLogger.getLogger().debug("EnvDataExplorer-> ERROR " + e.getLocalizedMessage());
}
if (returned != null)
AnalysisLogger.getLogger().debug("EnvDataExplorer-> Found Intersection: " + returned);
else
AnalysisLogger.getLogger().debug("EnvDataExplorer-> Found Nothing!");
return returned;
}
public static LinkedHashMap<String, Double> getFeatures(String geoserver, String layer, double x, double y) {
try {
String jsonString = callWFS(geoserver, layer, x, y);
LinkedHashMap<String, Object> map = JsonMapper.parse(jsonString);
LinkedHashMap<String, String> mapout = (LinkedHashMap<String, String>) ((HashMap<String, Object>) map.get("features")).get("properties");
LinkedHashMap<String, Double> values = new LinkedHashMap<String, Double>();
for (String key:mapout.keySet()){
values.put(key, Double.parseDouble(mapout.get(key)));
}
return values;
} catch (Exception e) {
AnalysisLogger.getLogger().debug("EnvDataExplorer-> Error in getting properties");
return null;
}
}
}

View File

@ -0,0 +1,59 @@
package org.gcube.dataanalysis.geo.utils;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
public class JsonMapper {
public static LinkedHashMap<String, Object> parse(String json) {
Object genericobject = new com.google.gson.JsonParser().parse(json);
com.google.gson.JsonObject object = null;
LinkedHashMap<String, Object> map = new LinkedHashMap<String, Object>();
if (genericobject instanceof com.google.gson.JsonObject){
object = (JsonObject) genericobject;
parseMap(map,object);
}
else if (genericobject instanceof com.google.gson.JsonArray){
JsonArray ArrObject = (JsonArray) new com.google.gson.JsonParser().parse(json);
Iterator<JsonElement> iterator = ArrObject.iterator();
while (iterator.hasNext()) {
JsonElement element = iterator.next();
if (element instanceof JsonObject){
if (!element.isJsonPrimitive()) {
parseMap(map, (JsonObject) element);
}
}
}
}
return map;
}
private static void parseMap(LinkedHashMap<String, Object> map, JsonObject object ){
Set<Map.Entry<String, JsonElement>> set = object.entrySet();
Iterator<Map.Entry<String, JsonElement>> iterator = set.iterator();
while (iterator.hasNext()) {
Map.Entry<String, JsonElement> entry = iterator.next();
String key = entry.getKey();
JsonElement value = entry.getValue();
if (!value.isJsonPrimitive()) {
map.put(key, parse(value.toString()));
} else {
map.put(key, value.getAsString());
}
}
}
}

View File

@ -0,0 +1,247 @@
package org.gcube.dataanalysis.geo.utils;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.LinkedHashMap;
import java.util.List;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import ucar.ma2.Array;
import ucar.ma2.StructureData;
import ucar.ma2.StructureMembers.Member;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dataset.CoordinateAxis1DTime;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.ft.FeatureCollection;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.ft.FeatureDatasetFactoryManager;
import ucar.nc2.ft.PointFeatureCollection;
import ucar.nc2.ft.PointFeatureIterator;
import ucar.nc2.ft.point.PointDatasetImpl;
import ucar.nc2.ft.point.standard.StandardPointCollectionImpl;
import ucar.unidata.geoloc.LatLonPointImpl;
import ucar.unidata.geoloc.LatLonRect;
public class ThreddsDataExplorer {
// http://thredds.research-infrastructures.eu:8080/thredds/catalog/public/netcdf/catalog.xml
public static String timePrefix = "time:";
public static List<String> getFiles(String catalogURL) throws Exception {
String xml = HttpRequest.sendGetRequest(catalogURL, null);
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression xPathExpression = xpath.compile("//child::*[local-name()='catalog']/child::*[local-name()='dataset']/child::*[local-name()='dataset']");
InputSource inputSource = new InputSource(new ByteArrayInputStream(xml.getBytes("UTF-8")));
NodeList nodes = (NodeList) xPathExpression.evaluate(inputSource, XPathConstants.NODESET);
List<String> fileNames = new ArrayList<String>();
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
String name = node.getAttributes().getNamedItem("name").getNodeValue();
if (name != null)
fileNames.add(name);
}
return fileNames;
}
public static LinkedHashMap<String, Double> retrieveDataFromNetCDF(String openDapLink, String layer, double x, double y) {
try {
LinkedHashMap<String, Double> map = new LinkedHashMap<String, Double>();
if (isGridDataset(openDapLink)) {
AnalysisLogger.getLogger().debug("Managing Grid File");
return manageGridDataset(layer, openDapLink, x, y);
}
/*
* else if (isPointDataset(openDapLink)) { AnalysisLogger.getLogger().debug("Managing Points File"); }
*/
else
AnalysisLogger.getLogger().debug("Warning: the NETCDF file is of an unknown type");
return map;
} catch (Exception e) {
AnalysisLogger.getLogger().debug("ERROR: " + e.getMessage());
AnalysisLogger.getLogger().debug(e);
e.printStackTrace();
return null;
}
}
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
public static LinkedHashMap<String, Double> manageGridDataset(String layer, String filename, double x, double y) throws Exception {
LinkedHashMap<String, Double> valuesMap = new LinkedHashMap<String, Double>();
GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(filename);
List<GridDatatype> gridTypes = gds.getGrids();
for (GridDatatype gdt : gridTypes) {
AnalysisLogger.getLogger().debug("Inside File - layer name: "+gdt.getFullName());
if (layer.equalsIgnoreCase(gdt.getFullName())) {
AnalysisLogger.getLogger().debug("Found layer "+layer +" inside file");
GridDatatype grid = gds.findGridDatatype(gdt.getName());
GridCoordSystem gcs = grid.getCoordinateSystem();
long timeSteps = 0;
java.util.Date[] dates = null;
if (gcs.hasTimeAxis1D()) {
CoordinateAxis1DTime tAxis1D = gcs.getTimeAxis1D();
dates = tAxis1D.getTimeDates();
timeSteps = dates.length;
} else if (gcs.hasTimeAxis()) {
CoordinateAxis tAxis = gcs.getTimeAxis();
timeSteps = tAxis.getSize();
}
int[] xy = gcs.findXYindexFromLatLon(y, x, null);
for (int j = 0; j < timeSteps; j++) {
Array data = grid.readDataSlice(j, 0, xy[1], xy[0]); // note order is t, z, y, x
Double val = takeFirstDouble(data);
if (!val.isNaN()) {
String date = "" + j;
if (dates != null)
date = dates[j].toString();
valuesMap.put(timePrefix + date, Double.parseDouble("" + val));
}
}
break;
}
}
return valuesMap;
}
public static Double takeFirstDouble(Array data) {
long datal = data.getSize();
Double val = Double.NaN;
try {
for (int k = 0; k < datal; k++) {
Double testVal = data.getDouble(k);
if (!testVal.isNaN()) {
val = testVal;
break;
}
}
} catch (Exception ee) {
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> WARNING: Error in getting value: " + ee.getLocalizedMessage());
}
return val;
}
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
public LinkedHashMap<String, String> managePointsDataset(String layer, String filename, double x, double y) throws Exception {
LinkedHashMap<String, String> valuesMap = new LinkedHashMap<String, String>();
float tolerance = 0.25f;
Formatter errlog = new Formatter();
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.POINT, filename, null, errlog);
PointDatasetImpl ds = (PointDatasetImpl) fdataset;
List<FeatureCollection> lfc = ds.getPointFeatureCollectionList();
for (FeatureCollection fc : lfc) {
StandardPointCollectionImpl spf = (StandardPointCollectionImpl) fc;
PointFeatureIterator iter = null;
while ((y - tolerance > -90) && (x - tolerance > -180) && (y + tolerance < 90) && (x + tolerance < 180)) {
LatLonRect rect = new LatLonRect(new LatLonPointImpl(y - tolerance, x - tolerance), new LatLonPointImpl(y + tolerance, x + tolerance));
PointFeatureCollection coll = spf.subset(rect, null);
iter = coll.getPointFeatureIterator(100 * 1000); // 100Kb buffer
if (iter.getCount() == 0)
iter.finish();
else
break;
tolerance = tolerance + 0.25f;
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> tolerance = " + tolerance);
}
if (iter != null) {
try {
while (iter.hasNext()) {
ucar.nc2.ft.PointFeature pf = iter.next();
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> EarthLoc: " + pf.getLocation());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> EarthTime: " + pf.getObservationTime());
StructureData sd = pf.getData();
List<Member> mems = sd.getMembers();
for (Member m : mems) {
String unit = m.getUnitsString();
if ((unit != null) && (unit.length() > 0)) {
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> description: " + m.getDescription());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> data param: " + m.getDataParam());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> name: " + m.getName());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> unit: " + m.getUnitsString());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> type: " + m.getDataType());
Array arr = sd.getArray(m.getName());
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> is Time: " + m.getDataType());
Double val = takeFirstDouble(arr);
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> extracted value: " + val);
}
}
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> EarthTime: ");
}
} finally {
iter.finish();
}
}
break;
}
return valuesMap;
}
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
public static boolean isGridDataset(String filename) {
try {
Formatter errlog = new Formatter();
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.GRID, filename, null, errlog);
if (fdataset == null) {
// System.out.printf("GRID Parse failed --> %s\n", errlog);
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> NOT GRID");
return false;
} else
return true;
} catch (Exception e) {
return false;
}
}
// A GridDatatype is like a specialized Variable that explicitly handles X,Y,Z,T dimensions
public static boolean isPointDataset(String filename) {
try {
Formatter errlog = new Formatter();
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(FeatureType.POINT, filename, null, errlog);
if (fdataset == null) {
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> NOT POINT");
return false;
} else
return true;
} catch (Exception e) {
return false;
}
}
public static boolean isDataset(String filename) throws Exception {
boolean isdataset = false;
try {
Formatter errlog = new Formatter();
FeatureType[] fts = FeatureType.values();
for (int i = 0; i < fts.length; i++) {
FeatureDataset fdataset = FeatureDatasetFactoryManager.open(fts[i], filename, null, errlog);
if (fdataset == null) {
// System.out.printf(fts[i]+": Parse failed --> %s\n",errlog);
} else {
AnalysisLogger.getLogger().debug("ThreddsDataExplorer-> " + fts[i] + " OK!");
isdataset = true;
}
}
} catch (Exception e) {
}
return isdataset;
}
}