ref #18694: SPD - Support the transition to the new OBIS plugin

https://support.d4science.org/issues/#18694

Updated support to OBIS API v3
This commit is contained in:
Giancarlo Panichi 2020-03-05 19:28:55 +01:00
parent 5e62838eef
commit 74d2aa040b
13 changed files with 364 additions and 234 deletions

View File

@ -59,14 +59,17 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>

View File

@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory;
*/
public class ObisPlugin extends AbstractPlugin {
protected static final String LOGO_URL = "http://iobis.org/sites/all/themes/corolla/logo.png";
protected static final String HOME_URL = "http://iobis.org";
protected static final String LOGO_URL = "https://obis.org/images/obis_4.png";
protected static final String HOME_URL = "https://obis.org";
protected static final String DESCRIPTION = "The Ocean Biogeographic information System (OBIS) seeks to absorb, integrate, and assess isolated datasets into a larger, more comprehensive pictures of life in our oceans. " +
"The system hopes to stimulate research about our oceans to generate new hypotheses concerning evolutionary processes, species distributions, and roles of organisms in marine systems on a global scale. " +
"Created by the Census of Marine Life, OBIS is now part of the Intergovernmental Oceanographic Commission (IOC) of UNESCO, under its International Oceanographic Data and Information Exchange (IODE) programme.";
@ -42,7 +42,7 @@ public class ObisPlugin extends AbstractPlugin {
//protected ObisClassification obisClassification;
protected static final SimpleDateFormat sdf = new SimpleDateFormat();
private String baseUrl = "http://api.iobis.org/";
private String baseUrl = "https://api.obis.org/v3/";
@SuppressWarnings("serial")
@Override

View File

@ -19,36 +19,48 @@ public class DataSetRetreiver {
private static Logger log = LoggerFactory.getLogger(DataSetRetreiver.class);
@SuppressWarnings("unchecked")
public static DataSet get(String key, String baseURL) throws Exception{
public static DataSet get(String baseURL, String key) throws Exception {
log.debug("Dataset Get: [ key={}, baseURL={} ]", key, baseURL);
DataSet dataset = new DataSet(key);
long start = System.currentTimeMillis();
QueryByIdentifier datasetQuery = new QueryByIdentifier(baseURL, key, QueryType.Dataset);
Map<String, Object> mapping = MappingUtils.getObjectMapping(datasetQuery.build());
DataSet dataset = new DataSet(key);
dataset.setName(getAsString(mapping,"name"));
Map<String, Object> listMaps = MappingUtils.getObjectMapping(datasetQuery.build());
log.debug("Dataset Retrieved: " + listMaps);
if (listMaps != null) {
List<Map<String, Object>> results = (List<Map<String, Object>>) listMaps.get("results");
if (results != null && !results.isEmpty()) {
Map<String, Object> mapping = results.get(0);
log.debug("Dataset Name: " + getAsString(mapping, "title"));
dataset.setName(getAsString(mapping, "title"));
String citation = getAsString(mapping, "citation");
if (citation == null) {
List<Map<String, Object>> institutionMapping = (List<Map<String, Object>>) mapping
.get("institutes");
if (institutionMapping != null && !institutionMapping.isEmpty()) {
if (getAsString(institutionMapping.get(0), "name") != null)
citation = getAsString(institutionMapping.get(0), "name");
dataset.setCitation(citation);
}
} else {
dataset.setCitation(citation);
}
String citation = getAsString(mapping,"citation");
if (citation ==null){
List<Map<String, Object>> institutionMapping = (List<Map<String, Object>>)mapping.get("institutes");
if (institutionMapping.size()>0){
if (getAsString(institutionMapping.get(0),"parent")!=null)
citation += " - "+getAsString(institutionMapping.get(0),"parent");
dataset.setCitation(citation);
String providerKey = key;
List<Map<String, Object>> providerMapping = (List<Map<String, Object>>) mapping.get("nodes");
DataProvider provider = new DataProvider(providerKey);
if (providerMapping != null && !providerMapping.isEmpty()) {
Map<String, Object> pMapping = (Map<String, Object>) providerMapping.get(0);
provider.setName(getAsString(pMapping, "name"));
} else {
provider.setName(Constants.REPOSITORY_NAME);
}
dataset.setDataProvider(provider);
}
}
String providerKey = key;
Map<String, Object> providerMapping = (Map<String, Object>)mapping.get("provider");
DataProvider provider = new DataProvider(providerKey);
if (providerMapping!=null)
provider.setName(getAsString(providerMapping,"name"));
else
provider.setName(Constants.REPOSITORY_NAME);
dataset.setDataProvider(provider);
log.trace("[Benchmark] time to retrieve dataset is "+(System.currentTimeMillis()-start));
log.trace("[Benchmark] time to retrieve dataset is " + (System.currentTimeMillis() - start));
return dataset;
}

View File

@ -2,7 +2,6 @@ package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsCalendar;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsDouble;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import java.text.SimpleDateFormat;
@ -13,7 +12,6 @@ import java.util.Map;
import org.gcube.data.spd.model.BasisOfRecord;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.obisplugin.Constants;
@ -40,75 +38,63 @@ public class OccurrenceSearch {
this.baseURL = baseURL;
}
public void search(ObjectWriter<OccurrencePoint> writer, String scientificName, int limit, Condition ...conditions) throws Exception{
public void search(ObjectWriter<OccurrencePoint> writer, String scientificName, int limit, Condition... conditions)
throws Exception {
PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit);
List<QueryCondition> queryConditions = Utils.elaborateConditions(conditions);
occurrencesQuery.setConditions(QueryCondition.cond("scientificname",scientificName.replaceAll(" ", "%20")));
occurrencesQuery.setConditions(QueryCondition.cond("scientificname", scientificName.replaceAll(" ", "%20")));
occurrencesQuery.getConditions().addAll(queryConditions);
writeElements(writer, occurrencesQuery);
}
public void searchByKey(ObjectWriter<OccurrencePoint> writer, String key, int limit) throws Exception{
public void searchByKey(ObjectWriter<OccurrencePoint> writer, String key, int limit) throws Exception {
PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit);
ProductKey productKey = Utils.elaborateProductsKey(key);
occurrencesQuery.getConditions().addAll(productKey.getQueryCondition());
writeElements(writer, occurrencesQuery);
}
private void writeElements(ObjectWriter<OccurrencePoint> writer, PagedQueryObject occurrencesQuery){
private void writeElements(ObjectWriter<OccurrencePoint> writer, PagedQueryObject occurrencesQuery) {
PagedQueryIterator<OccurrencePoint> pagedIterator = new PagedQueryIterator<OccurrencePoint>(occurrencesQuery) {
@Override
protected OccurrencePoint getObject(Map<String, Object> mappedObject)
throws Exception {
protected OccurrencePoint getObject(Map<String, Object> mappedObject) throws Exception {
OccurrencePoint op = retrieveElement(mappedObject);
Calendar now = Calendar.getInstance();
String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, "+format.format(now.getTime())+")";
String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, "
+ format.format(now.getTime()) + ")";
op.setCredits(credits);
return op;
}
};
try{
try {
while (pagedIterator.hasNext() && writer.isAlive())
writer.write(pagedIterator.next());
}catch(Exception e){
log.error("error writing occurrences",e);
} catch (Exception e) {
log.error("error writing occurrences", e);
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
}
}
public OccurrencePoint searchById(String id) throws Exception{
public OccurrencePoint searchById(String id) throws Exception {
QueryByIdentifier queryByIdentifier = new QueryByIdentifier(baseURL, id, QueryType.Occurrence);
return retrieveElement(MappingUtils.getObjectMapping(queryByIdentifier.build()));
}
/*
FOSSIL_SPECIMEN
An occurrence record describing a fossilized specimen.
HUMAN_OBSERVATION
An occurrence record describing an observation made by one or more people.
LITERATURE
An occurrence record based on literature alone.
LIVING_SPECIMEN
An occurrence record describing a living specimen, e.g.
MACHINE_OBSERVATION
An occurrence record describing an observation made by a machine.
MATERIAL_SAMPLE
An occurrence record based on samples taken from other specimens or the environment.
OBSERVATION
An occurrence record describing an observation.
PRESERVED_SPECIMEN
An occurrence record describing a preserved specimen.
UNKNOWN
* FOSSIL_SPECIMEN An occurrence record describing a fossilized specimen.
* HUMAN_OBSERVATION An occurrence record describing an observation made by
* one or more people. LITERATURE An occurrence record based on literature
* alone. LIVING_SPECIMEN An occurrence record describing a living specimen,
* e.g. MACHINE_OBSERVATION An occurrence record describing an observation
* made by a machine. MATERIAL_SAMPLE An occurrence record based on samples
* taken from other specimens or the environment. OBSERVATION An occurrence
* record describing an observation. PRESERVED_SPECIMEN An occurrence record
* describing a preserved specimen. UNKNOWN
*/
private OccurrencePoint retrieveElement(Map<String, Object> mappedObj) throws Exception{
/*
@ -124,49 +110,74 @@ public class OccurrenceSearch {
*/
long start = System.currentTimeMillis();
String occurrenceId = getAsInteger(mappedObj, "id").toString();
String occurrenceId = getAsString(mappedObj, "id");
OccurrencePoint occurrence = new OccurrencePoint(occurrenceId);
occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude"));
occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude"));
occurrence.setCoordinateUncertaintyInMeters(getAsString(mappedObj, "coordinatePrecision"));
Calendar eventDate = getAsCalendar(mappedObj, "eventDate");
occurrence.setEventDate(eventDate);
occurrence.setCollectionCode(getAsString(mappedObj, "collectionCode"));
occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode"));
occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber"));
//occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy"));
//occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy"));
//occurrence.setCountry(getAsString(mappedObj, "country"));
//occurrence.setLocality(getAsString(mappedObj, "locality"));
occurrence.setCountry(getAsString(mappedObj, "country"));
occurrence.setLocality(getAsString(mappedObj, "locality"));
occurrence.setBasisOfRecord(BasisOfRecord.Unknown);
/*occurrence.setMinDepth(getAsDouble(mappedObj, "elevation"));
occurrence.setMaxDepth(getAsDouble(mappedObj, "depth"));
*/
occurrence.setBasisOfRecord(getBasisOfRecord(getAsString(mappedObj, "basisOfRecord")));
occurrence.setMinDepth(getAsDouble(mappedObj, "minimumDepthInMeters"));
occurrence.setMaxDepth(getAsDouble(mappedObj, "maximumDepthInMeters"));
occurrence.setKingdom("Animalia");
occurrence.setKingdom(getAsString(mappedObj, "kingdom"));
occurrence.setFamily(getAsString(mappedObj, "family"));
occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship"));
occurrence.setScientificName(getAsString(mappedObj, "scientificName"));
String datasetName = getAsString(mappedObj, "datasetName");
DataSet dataset = new DataSet(datasetName);
dataset.setCitation(datasetName);
dataset.setName(datasetName);
DataProvider dataProvider = new DataProvider("OBIS");
dataProvider.setName("OBIS");
dataset.setDataProvider(dataProvider);
String datasetId = getAsString(mappedObj, "dataset_id");
DataSet dataset = DataSetRetreiver.get(baseURL,datasetId);
occurrence.setProvider(dataset.getDataProvider().getName());
occurrence.setDataSet(dataset);
occurrence.setProvider("OBIS");
//occurrence.setCitation(getAsString(mappedObj, "accordingTo"));
log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start));
return occurrence;
}
private BasisOfRecord getBasisOfRecord(String basisOfRecord) {
if (basisOfRecord == null || basisOfRecord.isEmpty()) {
return BasisOfRecord.Unknown;
}
switch (basisOfRecord) {
case "PreservedSpecimen":
return BasisOfRecord.PreservedSpecimen;
case "FossilSpecimen":
return BasisOfRecord.FossilSpecimen;
case "LivingSpecimen":
return BasisOfRecord.LivingSpecimen;
case "HumanObservation":
return BasisOfRecord.HumanObservation;
case "MachineObservation":
return BasisOfRecord.MachineObservation;
case "Observation":
return BasisOfRecord.Observation;
case "Literature":
return BasisOfRecord.Literature;
case "MaterialSample":
return BasisOfRecord.MaterialSample;
default:
return BasisOfRecord.Unknown;
}
}
}

View File

@ -13,6 +13,7 @@ import java.util.Set;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.Product;
import org.gcube.data.spd.model.products.Product.ProductType;
@ -34,141 +35,165 @@ import org.slf4j.LoggerFactory;
public class ResultItemSearch {
private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
private static Logger log = LoggerFactory.getLogger(ResultItemSearch.class);
private List<QueryCondition> queryConditions = new ArrayList<QueryCondition>();
private String baseURL;
String searchQuery;
public ResultItemSearch(String baseURL, String searchQuery, Condition ... conditions){
public ResultItemSearch(String baseURL, String searchQuery, Condition... conditions) {
this.baseURL = baseURL;
this.searchQuery = searchQuery.replaceAll(" ", "%20").trim();
this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase()+this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase();
try{
this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase()
+ this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase();
try {
this.queryConditions = Utils.elaborateConditions(conditions);
}catch(Exception e){
log.error("error elaborating conditions",e);
} catch (Exception e) {
log.error("error elaborating conditions", e);
}
}
public void search(ObjectWriter<ResultItem> writer, int limit){
PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence,limit);
queryObject.setConditions(QueryCondition.cond("scientificname",searchQuery));
public void search(ObjectWriter<ResultItem> writer, int limit) {
PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence, limit);
queryObject.setConditions(QueryCondition.cond("scientificname", searchQuery));
queryObject.getConditions().addAll(this.queryConditions);
try{
try {
PagedQueryIterator<ResultItem> pagedIterator = new PagedQueryIterator<ResultItem>(queryObject) {
Set<String> alreadyVisited =new HashSet<String>();
Set<String> alreadyVisited = new HashSet<String>();
@Override
protected ResultItem getObject(Map<String, Object> mappedObject) throws Exception {
log.debug("retrieved mapped object");
return buildResult(mappedObject);
ResultItem resultItem = buildResult(mappedObject);
log.debug("ResultItem: " + resultItem);
return resultItem;
}
@Override
protected boolean useIt(Map<String, Object> mappedObject) {
String datasetKey = ((Integer)mappedObject.get("resourceID")).toString();
Integer taxonId = (Integer)mappedObject.get("obisID");
String key = datasetKey+"|"+taxonId;
if (alreadyVisited.contains(key))
String datasetKey = (String) mappedObject.get("dataset_id");// resourceID
Integer taxonId = (Integer) mappedObject.get("aphiaID");// obisID
String key = datasetKey + "|" + taxonId;
if (alreadyVisited.contains(key))
return false;
alreadyVisited.add(key);
return true;
}
};
while (pagedIterator.hasNext() && writer.isAlive())
writer.write(pagedIterator.next());
}catch(Exception e){
log.error("error writing resultItems",e);
} catch (Exception e) {
log.error("error writing resultItems", e);
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
}
}
ResultItem buildResult(Map<String,Object> singleObject) throws Exception{
try{
long start = System.currentTimeMillis();
Integer taxonId = getAsInteger(singleObject,"obisID");
String scientificName = getAsString(singleObject,"scientificName");
ResultItem resItem = new ResultItem(taxonId.toString(), scientificName );
String scientificNameAuthorship = getAsString(singleObject,"scientificNameAuthorship");
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> singleTaxon = MappingUtils.getObjectMapping(query.build());
resItem.setScientificNameAuthorship(scientificNameAuthorship);
resItem.setRank(getAsString(singleTaxon, "rank_name"));
//resItem.setCitation(getAsString(singleTaxon,"tauthor"));
resItem.setParent(retrieveParentTaxon(getAsInteger(singleTaxon,"parent_id")));
DataSet dataset = DataSetRetreiver.get(getAsInteger(singleObject,"resourceID").toString(), baseURL);
resItem.setDataSet(dataset);
ResultItem buildResult(Map<String, Object> singleObject) throws Exception {
try {
long start = System.currentTimeMillis();
// log.debug("Retrieved Occurence: "+singleObject);
Integer taxonId = getAsInteger(singleObject, "aphiaID");// obisID
String scientificName = getAsString(singleObject, "scientificName");
ResultItem resItem = new ResultItem(taxonId.toString(), scientificName);
String scientificNameAuthorship = getAsString(singleObject, "scientificNameAuthorship");
resItem.setScientificNameAuthorship(scientificNameAuthorship);
List<Product> products = retrieveProducts(taxonId.toString(), dataset);
resItem.setProducts(products);
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> taxonData = MappingUtils.getObjectMapping(query.build());
log.debug("Retrieved taxon: " + taxonData);
if (taxonData != null && !taxonData.isEmpty()) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> taxonList = (List<Map<String, Object>>) taxonData.get("results");
if (taxonList != null && !taxonList.isEmpty()) {
Map<String, Object> taxonSingle = taxonList.get(0);
resItem.setRank(getAsString(taxonSingle, "taxonRank"));// rank_name
//resItem.setParent(retrieveParentTaxon(getAsInteger(taxonSingle, "familyid")));// parent_id
}
}
String credits = "Biodiversity occurrence accessed through OBIS WebService, http://api.iobis.org/, "+format.format(Calendar.getInstance().getTime())+")";
resItem.setCredits(credits);
log.trace("[Benchmark] time to retrieve ResultItem is "+(System.currentTimeMillis()-start));
log.debug("found species {} with authorship {}",scientificName, scientificNameAuthorship);
return resItem;
}catch(Exception e){
DataSet dataset = DataSetRetreiver.get(baseURL, getAsString(singleObject, "dataset_id"));
resItem.setProvider(dataset.getDataProvider().getName());
resItem.setDataSet(dataset);
List<Product> products = retrieveProducts(taxonId.toString(), dataset);
resItem.setProducts(products);
String credits = "Biodiversity occurrence accessed through OBIS WebService, https://api.obis.org/v3/, "
+ format.format(Calendar.getInstance().getTime()) + ")";
resItem.setCredits(credits);
log.trace("[Benchmark] time to retrieve ResultItem is " + (System.currentTimeMillis() - start));
log.debug("found species: id={}, name={}", resItem.getId(), resItem.getScientificName());
return resItem;
} catch (Exception e) {
throw e;
}
}
//TODO
private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception {
if (parentTaxonId==0) return null;
if (parentTaxonId == 0)
return null;
long start = System.currentTimeMillis();
Integer taxonId = parentTaxonId;
Integer taxonIdS = 0;
Taxon previousTaxon = null;
Taxon taxonToReturn = null;
do{
do {
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> singleTaxon = MappingUtils.getObjectMapping(query.build());
Taxon taxon = new Taxon(getAsInteger(singleTaxon, "id").toString(), getAsString(singleTaxon, "tname"));
taxon.setScientificNameAuthorship(getAsString(singleTaxon, "tauthor"));
//taxon.setCitation(getAsString(mappedObject, "accordingTo"));
taxon.setRank(getAsString(singleTaxon, "rank_name"));
if (previousTaxon!=null)
previousTaxon.setParent(taxon);
previousTaxon = taxon;
taxonId = getAsInteger(singleTaxon, "parent_id");
if (taxonToReturn==null)
taxonToReturn = taxon;
} while (taxonId>0);
log.trace("[Benchmark] time to retrieve taxon is "+(System.currentTimeMillis()-start));
Map<String, Object> parentTaxonData = MappingUtils.getObjectMapping(query.build());
log.debug("ParentTaxon: " + parentTaxonData);
if (parentTaxonData != null && !parentTaxonData.isEmpty()) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> parentTaxonList = (List<Map<String, Object>>) parentTaxonData.get("results");
if (parentTaxonList != null && !parentTaxonList.isEmpty()) {
Map<String,Object> parentTaxon=parentTaxonList.get(0);
Taxon taxon = new Taxon(getAsInteger(parentTaxon, "taxonID").toString(),
getAsString(parentTaxon, "scientificName"));
taxon.setScientificNameAuthorship(getAsString(parentTaxon, "scientificNameAuthorship"));
taxon.setRank(getAsString(parentTaxon, "taxonRank"));// rank_name
if (previousTaxon != null){
previousTaxon.setParent(taxon);
}
previousTaxon = taxon;
if (taxonToReturn == null) {
taxonToReturn = taxon;
}
taxonIdS = getAsInteger(parentTaxon, "familyid");// parent_id
if(taxonIdS>0&&taxonIdS!=taxonId){
taxonId=taxonIdS;
}else {
taxonId=0;
}
}
}
} while (taxonId > 0);
log.trace("[Benchmark] time to retrieve taxon is " + (System.currentTimeMillis() - start));
return taxonToReturn;
}
private List<Product> retrieveProducts( String taxonId, DataSet dataset) throws Exception{
private List<Product> retrieveProducts(String taxonId, DataSet dataset) throws Exception {
long start = System.currentTimeMillis();
QueryCount occurrencesQuery = new QueryCount(baseURL, ResultType.Occurrence);
occurrencesQuery.setConditions(QueryCondition.cond("obisid",taxonId), QueryCondition.cond("resourceid", dataset.getId()));
occurrencesQuery.setConditions(QueryCondition.cond("taxonid", taxonId),
QueryCondition.cond("datasetid", dataset.getId()));
occurrencesQuery.getConditions().addAll(this.queryConditions);
String productId = Utils.createProductsKey(Utils.getDataSetAsString(dataset), taxonId, this.queryConditions);
Product product = new Product(ProductType.Occurrence, productId);
product.setCount(occurrencesQuery.getCount());
log.trace("[Benchmark] time to retrieve product is "+(System.currentTimeMillis()-start));
log.trace("[Benchmark] time to retrieve product is " + (System.currentTimeMillis() - start));
return Arrays.asList(product);
}
}

View File

@ -38,8 +38,8 @@ public class Utils {
public static ProductKey elaborateProductsKey(String id) {
List<QueryCondition> queryConditions = new ArrayList<QueryCondition>();
String[] splitString = id.split("\\|\\|");
queryConditions.add(cond("resourceid", splitString[0]));
queryConditions.add(cond("obisid", splitString[1]));
queryConditions.add(cond("datasetid", splitString[0]));
queryConditions.add(cond("taxonid", splitString[1]));
if (splitString.length>2)
for (int i = 2; i<splitString.length; i++){
String[] equalSplit = splitString[i].split("=");

View File

@ -31,16 +31,16 @@ public class MappingUtils {
private static Logger log = LoggerFactory.getLogger(MappingUtils.class);
@SuppressWarnings("unchecked")
public static Map<String, Object> getObjectMapping(String query) throws Exception{
public static Map<String, Object> getObjectMapping(String query) throws Exception {
String response = executeQuery(query);
ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally
return mapper.readValue(new StringReader(response), Map.class);
}
public static List<Map<String, Object>> getObjectList(String query) throws Exception{
public static List<Map<String, Object>> getObjectList(String query) throws Exception {
String response = executeQuery(query);
ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally
return mapper.readValue(new StringReader(response), new TypeReference<LinkedList<HashMap<String, Object>>>() {
@ -48,61 +48,91 @@ public class MappingUtils {
}
public static String getAsString(Map<String, Object> map, String key){
if (!map.containsKey(key)) return null;
public static String getAsString(Map<String, Object> map, String key) {
if (!map.containsKey(key))
return null;
return (String) map.get(key);
}
public static Double getAsDouble(Map<String, Object> map, String key){
if (!map.containsKey(key)) return 0d;
return (Double) map.get(key);
public static Double getAsDouble(Map<String, Object> map, String key) {
if (!map.containsKey(key))
return 0d;
if (map.get(key) instanceof Double) {
return (Double) map.get(key);
} else {
if (map.get(key) instanceof Integer) {
Integer value = (Integer) map.get(key);
return value.doubleValue();
} else {
if (map.get(key) instanceof String) {
String value = (String) map.get(key);
return Double.valueOf(value);
} else {
return 0d;
}
}
}
}
public static Integer getAsInteger(Map<String, Object> map, String key){
if (!map.containsKey(key)) return 0;
return (Integer) map.get(key);
public static Integer getAsInteger(Map<String, Object> map, String key) {
if (!map.containsKey(key))
return 0;
if (map.get(key) instanceof Integer) {
Integer value = (Integer) map.get(key);
return value;
} else {
if (map.get(key) instanceof String) {
String value = (String) map.get(key);
return Integer.valueOf(value);
} else {
return 0;
}
}
}
public static Calendar getAsCalendar(Map<String, Object> map, String key){
if (!map.containsKey(key)) return null;
public static Calendar getAsCalendar(Map<String, Object> map, String key) {
if (!map.containsKey(key))
return null;
return parseCalendar((String) map.get(key));
}
public static Calendar parseCalendar(String date){
try{
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar calendar= Calendar.getInstance();
public static Calendar parseCalendar(String date) {
try {
//"2001-11-27T12:00:00Z"
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
Calendar calendar = Calendar.getInstance();
calendar.setTime(df.parse(date));
return calendar;
}catch (ParseException e) {
log.warn("date discarded ("+date+")");
} catch (ParseException e) {
log.warn("date discarded (" + date + ")");
return null;
}
}
private static String executeQuery(String query){
private static String executeQuery(String query) {
DefaultClientConfig clientConfig = new DefaultClientConfig();
Client client = Client.create(clientConfig);
client.setConnectTimeout(TIMEOUT);
client.setReadTimeout(TIMEOUT);
WebResource target = client.resource(query);
//NameUsageWsClient nuws = new NameUsageWsClient(target);
// NameUsageWsClient nuws = new NameUsageWsClient(target);
int tries = 1;
String response = null;
do {
log.debug("try number {} STARTED for query {} ", tries,query);
try{
log.debug("try number {} STARTED for query {} ", tries, query);
try {
response = target.type(MediaType.APPLICATION_JSON).acceptLanguage(Locale.ENGLISH).get(String.class);
}catch (Exception e) {
log.debug("try number {} FAILED for query {} ", tries,query,e);
} catch (Exception e) {
log.debug("try number {} FAILED for query {} ", tries, query, e);
try {
if (tries<retries)Thread.sleep(SLEEP_TIME);
} catch (InterruptedException e1) {}
if (tries < retries)
Thread.sleep(SLEEP_TIME);
} catch (InterruptedException e1) {
}
}
tries++;
}while (response==null && tries<=retries);
} while (response == null && tries <= retries);
return response;
}

View File

@ -34,6 +34,7 @@ public abstract class PagedQueryIterator<T> implements Iterator<T>{
public boolean hasNext() {
try{
if (resultIterator==null){
log.trace("PagedQuery fetch new page");
String query = pagedQuery.buildNext();
start = System.currentTimeMillis();
mapping = MappingUtils.getObjectMapping(query);
@ -45,14 +46,17 @@ public abstract class PagedQueryIterator<T> implements Iterator<T>{
if (!resultIterator.hasNext()){
log.trace("[Benchmark] page retrieved and parsed in "+(System.currentTimeMillis()-start));
if ((Boolean)mapping.get("lastpage")){
log.trace("is end of records, no next element");
Integer endOfRecords=(Integer)mapping.get("total");
if (pagedQuery.getPageCount()>=endOfRecords){
log.debug("End of records, no next element");
return false;
}
resultIterator = null;
} else{
log.trace("Read new record");
actualObject = resultIterator.next();
pagedQuery.setAfter((String)actualObject.get("id"));
if (useIt(actualObject))
return true;
}

View File

@ -9,42 +9,52 @@ import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class PagedQueryObject {
private @NonNull String baseUri;
@Getter
List<QueryCondition> conditions = new ArrayList<QueryCondition>();
private @NonNull ResultType resultType;
private @NonNull Integer resultPerQuery;
private int offset = 0;
public void setConditions(QueryCondition ... conditions){
private String after = null;
private Integer pageCount = 0;
public void setConditions(QueryCondition... conditions) {
this.conditions.addAll(Arrays.asList(conditions));
}
public String buildNext(){
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/")) query.append("/");
query.append(this.resultType.getQueryEntry()).append("/");
query.append("?limit=").append(resultPerQuery);
query.append("&offset=").append(offset);
if (conditions.size()>0)
for (QueryCondition queryCond: conditions)
public void setAfter(String after) {
this.after = after;
}
public Integer getPageCount() {
return pageCount;
}
public String buildNext() {
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/"))
query.append("/");
query.append(this.resultType.getQueryEntry()).append("/");
query.append("?size=").append(resultPerQuery);
if (after != null) {
query.append("&after=").append(after);
}
pageCount += resultPerQuery;
if (conditions.size() > 0)
for (QueryCondition queryCond : conditions)
query.append("&").append(queryCond.getKey()).append("=").append(queryCond.getValue());
offset = offset+resultPerQuery;
log.debug("executed query is "+query.toString());
log.debug("executed query is " + query.toString());
return query.toString();
}
}

View File

@ -30,7 +30,7 @@ public class QueryByIdentifier {
query.append(key);
for (String path : paths)
query.append("/").append(path);
log.trace("query by dentifier is "+query.toString());
log.debug("query by identifier is "+query.toString());
return query.toString();
}

View File

@ -35,8 +35,8 @@ public class QueryCount {
Map<String, Object> mapping;
try {
mapping = MappingUtils.getObjectMapping(this.build());
if (mapping.get("count")==null) return 0;
return (Integer)mapping.get("count");
if (mapping.get("total")==null) return 0;
return (Integer)mapping.get("total");
} catch (Exception e) {
log.error("error computing count, returning 0",e);
return 0;
@ -49,7 +49,7 @@ public class QueryCount {
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/")) query.append("/");
query.append(this.resultType.getQueryEntry()).append("/");
query.append("?limit=0");
query.append("?size=0");
if (conditions.size()>0)
for (QueryCondition queryCond: conditions)

View File

@ -8,7 +8,7 @@ import lombok.NonNull;
public @AllArgsConstructor enum QueryType{
Occurrence("occurrence"),
Taxon("taxon"),
Dataset("resource");
Dataset("dataset");
@Getter
private @NonNull String queryEntry;

View File

@ -2,10 +2,6 @@ package org.gcube.data.obisplugin;
import java.util.Arrays;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Condition.Operator;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.Coordinate;
import org.gcube.data.spd.model.exceptions.StreamException;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.model.products.ResultItem;
@ -18,7 +14,7 @@ import org.junit.Test;
public class ObisTest {
@Test
public void search() throws Exception{
public void searchByKey() throws Exception{
/*ObisPlugin plugin= new ObisPlugin();
plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3"));
plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/
@ -49,13 +45,16 @@ public class ObisTest {
OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("http://api.iobis.org/");
OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("https://api.obis.org/v3/");
//impl.searchByScientificName("Cetacea", writer);
impl.getOccurrencesByProductKeys(writer, Arrays.asList("3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))").iterator() );
//8f843938-1617-4c9e-be03-d1a2af8ebd89||127021
//"3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))"
impl.getOccurrencesByProductKeys(writer, Arrays.asList("8f843938-1617-4c9e-be03-d1a2af8ebd89||127021").iterator() );
}
@Test
public void searchRI() throws Exception{
public void searchByScientificName() throws Exception{
/*ObisPlugin plugin= new ObisPlugin();
plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3"));
plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/
@ -78,10 +77,46 @@ public class ObisTest {
}
};
//, new Condition(Conditions.COORDINATE, new Coordinate(20, 30) , Operator.GT)
ResultItemSearch search = new ResultItemSearch("http://api.iobis.org/", "Gamidae");
//Balaenoptera bonaerensis
ResultItemSearch search = new ResultItemSearch("https://api.obis.org/v3/", "Sarda sarda");
search.search(writer, 50);
}
@Test
public void searchByGeometry() throws Exception{
ClosableWriter<OccurrencePoint> writer = new ClosableWriter<OccurrencePoint>() {
@Override
public boolean isAlive() {
return true;
}
@Override
public boolean write(OccurrencePoint arg0) {
return true;
}
@Override
public boolean write(StreamException arg0) {
return false;
}
@Override
public void close() {
}
};
OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("https://api.obis.org/v3/");
//"3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))"
impl.getOccurrencesByProductKeys(writer, Arrays.asList("9b787218-7138-4ffa-8ba6-36d63d391f8c||127021||geometry=POLYGON((1.6306044624659184%206.903464725745124,1.58434708579508%20-0.943627209502182,8.754369215807587%20-0.8048685858041154,8.569345073542273%206.857540488275561,1.6306044624659184%206.903464725745124))").iterator() );
}
}