From 74d2aa040ba2be83aec3d03986890abcc8d6e92b Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 5 Mar 2020 19:28:55 +0100 Subject: [PATCH] ref #18694: SPD - Support the transition to the new OBIS plugin https://support.d4science.org/issues/#18694 Updated support to OBIS API v3 --- pom.xml | 3 + .../gcube/data/spd/obisplugin/ObisPlugin.java | 6 +- .../obisplugin/search/DataSetRetreiver.java | 58 +++--- .../obisplugin/search/OccurrenceSearch.java | 113 ++++++----- .../obisplugin/search/ResultItemSearch.java | 191 ++++++++++-------- .../data/spd/obisplugin/search/Utils.java | 4 +- .../obisplugin/search/query/MappingUtils.java | 94 ++++++--- .../search/query/PagedQueryIterator.java | 8 +- .../search/query/PagedQueryObject.java | 56 ++--- .../search/query/QueryByIdentifier.java | 2 +- .../obisplugin/search/query/QueryCount.java | 6 +- .../obisplugin/search/query/QueryType.java | 2 +- .../org/gcube/data/obisplugin/ObisTest.java | 55 ++++- 13 files changed, 364 insertions(+), 234 deletions(-) diff --git a/pom.xml b/pom.xml index 770d5c9..17be615 100644 --- a/pom.xml +++ b/pom.xml @@ -59,14 +59,17 @@ com.fasterxml.jackson.core jackson-databind + ${jackson.version} com.fasterxml.jackson.core jackson-annotations + ${jackson.version} com.fasterxml.jackson.core jackson-core + ${jackson.version} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java index eb74696..0edf1bb 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java @@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory; */ public class ObisPlugin extends AbstractPlugin { - protected static final String LOGO_URL = "http://iobis.org/sites/all/themes/corolla/logo.png"; - protected static final String HOME_URL = "http://iobis.org"; + protected static final String LOGO_URL = "https://obis.org/images/obis_4.png"; + protected static final String HOME_URL = "https://obis.org"; protected static final String DESCRIPTION = "The Ocean Biogeographic information System (OBIS) seeks to absorb, integrate, and assess isolated datasets into a larger, more comprehensive pictures of life in our oceans. " + "The system hopes to stimulate research about our oceans to generate new hypotheses concerning evolutionary processes, species distributions, and roles of organisms in marine systems on a global scale. " + "Created by the Census of Marine Life, OBIS is now part of the Intergovernmental Oceanographic Commission (IOC) of UNESCO, under its International Oceanographic Data and Information Exchange (IODE) programme."; @@ -42,7 +42,7 @@ public class ObisPlugin extends AbstractPlugin { //protected ObisClassification obisClassification; protected static final SimpleDateFormat sdf = new SimpleDateFormat(); - private String baseUrl = "http://api.iobis.org/"; + private String baseUrl = "https://api.obis.org/v3/"; @SuppressWarnings("serial") @Override diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java index 4721264..fdeb602 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java @@ -19,36 +19,48 @@ public class DataSetRetreiver { private static Logger log = LoggerFactory.getLogger(DataSetRetreiver.class); @SuppressWarnings("unchecked") - public static DataSet get(String key, String baseURL) throws Exception{ + public static DataSet get(String baseURL, String key) throws Exception { + log.debug("Dataset Get: [ key={}, baseURL={} ]", key, baseURL); + DataSet dataset = new DataSet(key); long start = System.currentTimeMillis(); QueryByIdentifier datasetQuery = new QueryByIdentifier(baseURL, key, QueryType.Dataset); - Map mapping = MappingUtils.getObjectMapping(datasetQuery.build()); - DataSet dataset = new DataSet(key); - dataset.setName(getAsString(mapping,"name")); + Map listMaps = MappingUtils.getObjectMapping(datasetQuery.build()); + log.debug("Dataset Retrieved: " + listMaps); + if (listMaps != null) { + List> results = (List>) listMaps.get("results"); + if (results != null && !results.isEmpty()) { + Map mapping = results.get(0); + log.debug("Dataset Name: " + getAsString(mapping, "title")); + dataset.setName(getAsString(mapping, "title")); + String citation = getAsString(mapping, "citation"); + if (citation == null) { + List> institutionMapping = (List>) mapping + .get("institutes"); + if (institutionMapping != null && !institutionMapping.isEmpty()) { + if (getAsString(institutionMapping.get(0), "name") != null) + citation = getAsString(institutionMapping.get(0), "name"); + dataset.setCitation(citation); + } + } else { + dataset.setCitation(citation); + } - String citation = getAsString(mapping,"citation"); - if (citation ==null){ - List> institutionMapping = (List>)mapping.get("institutes"); - if (institutionMapping.size()>0){ - if (getAsString(institutionMapping.get(0),"parent")!=null) - citation += " - "+getAsString(institutionMapping.get(0),"parent"); - dataset.setCitation(citation); + String providerKey = key; + List> providerMapping = (List>) mapping.get("nodes"); + DataProvider provider = new DataProvider(providerKey); + if (providerMapping != null && !providerMapping.isEmpty()) { + Map pMapping = (Map) providerMapping.get(0); + provider.setName(getAsString(pMapping, "name")); + } else { + provider.setName(Constants.REPOSITORY_NAME); + } + + dataset.setDataProvider(provider); } } - - String providerKey = key; - Map providerMapping = (Map)mapping.get("provider"); - DataProvider provider = new DataProvider(providerKey); - if (providerMapping!=null) - provider.setName(getAsString(providerMapping,"name")); - else - provider.setName(Constants.REPOSITORY_NAME); - - dataset.setDataProvider(provider); - - log.trace("[Benchmark] time to retrieve dataset is "+(System.currentTimeMillis()-start)); + log.trace("[Benchmark] time to retrieve dataset is " + (System.currentTimeMillis() - start)); return dataset; } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java index 43b423f..a4726f6 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java @@ -2,7 +2,6 @@ package org.gcube.data.spd.obisplugin.search; import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsCalendar; import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsDouble; -import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger; import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; import java.text.SimpleDateFormat; @@ -13,7 +12,6 @@ import java.util.Map; import org.gcube.data.spd.model.BasisOfRecord; import org.gcube.data.spd.model.Condition; import org.gcube.data.spd.model.exceptions.StreamBlockingException; -import org.gcube.data.spd.model.products.DataProvider; import org.gcube.data.spd.model.products.DataSet; import org.gcube.data.spd.model.products.OccurrencePoint; import org.gcube.data.spd.obisplugin.Constants; @@ -40,75 +38,63 @@ public class OccurrenceSearch { this.baseURL = baseURL; } - public void search(ObjectWriter writer, String scientificName, int limit, Condition ...conditions) throws Exception{ + public void search(ObjectWriter writer, String scientificName, int limit, Condition... conditions) + throws Exception { PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit); List queryConditions = Utils.elaborateConditions(conditions); - occurrencesQuery.setConditions(QueryCondition.cond("scientificname",scientificName.replaceAll(" ", "%20"))); + occurrencesQuery.setConditions(QueryCondition.cond("scientificname", scientificName.replaceAll(" ", "%20"))); occurrencesQuery.getConditions().addAll(queryConditions); writeElements(writer, occurrencesQuery); } - public void searchByKey(ObjectWriter writer, String key, int limit) throws Exception{ + public void searchByKey(ObjectWriter writer, String key, int limit) throws Exception { PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit); ProductKey productKey = Utils.elaborateProductsKey(key); occurrencesQuery.getConditions().addAll(productKey.getQueryCondition()); writeElements(writer, occurrencesQuery); } - private void writeElements(ObjectWriter writer, PagedQueryObject occurrencesQuery){ + private void writeElements(ObjectWriter writer, PagedQueryObject occurrencesQuery) { PagedQueryIterator pagedIterator = new PagedQueryIterator(occurrencesQuery) { @Override - protected OccurrencePoint getObject(Map mappedObject) - throws Exception { + protected OccurrencePoint getObject(Map mappedObject) throws Exception { OccurrencePoint op = retrieveElement(mappedObject); Calendar now = Calendar.getInstance(); - String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, "+format.format(now.getTime())+")"; + String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, " + + format.format(now.getTime()) + ")"; op.setCredits(credits); return op; } }; - try{ + try { while (pagedIterator.hasNext() && writer.isAlive()) writer.write(pagedIterator.next()); - }catch(Exception e){ - log.error("error writing occurrences",e); + } catch (Exception e) { + log.error("error writing occurrences", e); writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); } } - public OccurrencePoint searchById(String id) throws Exception{ + public OccurrencePoint searchById(String id) throws Exception { QueryByIdentifier queryByIdentifier = new QueryByIdentifier(baseURL, id, QueryType.Occurrence); return retrieveElement(MappingUtils.getObjectMapping(queryByIdentifier.build())); } - /* - FOSSIL_SPECIMEN - An occurrence record describing a fossilized specimen. - HUMAN_OBSERVATION - An occurrence record describing an observation made by one or more people. - LITERATURE - An occurrence record based on literature alone. - LIVING_SPECIMEN - An occurrence record describing a living specimen, e.g. - MACHINE_OBSERVATION - An occurrence record describing an observation made by a machine. - MATERIAL_SAMPLE - An occurrence record based on samples taken from other specimens or the environment. - OBSERVATION - An occurrence record describing an observation. - PRESERVED_SPECIMEN - An occurrence record describing a preserved specimen. - UNKNOWN + * FOSSIL_SPECIMEN An occurrence record describing a fossilized specimen. + * HUMAN_OBSERVATION An occurrence record describing an observation made by + * one or more people. LITERATURE An occurrence record based on literature + * alone. LIVING_SPECIMEN An occurrence record describing a living specimen, + * e.g. MACHINE_OBSERVATION An occurrence record describing an observation + * made by a machine. MATERIAL_SAMPLE An occurrence record based on samples + * taken from other specimens or the environment. OBSERVATION An occurrence + * record describing an observation. PRESERVED_SPECIMEN An occurrence record + * describing a preserved specimen. UNKNOWN */ - - - - private OccurrencePoint retrieveElement(Map mappedObj) throws Exception{ /* @@ -124,49 +110,74 @@ public class OccurrenceSearch { */ long start = System.currentTimeMillis(); - String occurrenceId = getAsInteger(mappedObj, "id").toString(); + String occurrenceId = getAsString(mappedObj, "id"); OccurrencePoint occurrence = new OccurrencePoint(occurrenceId); occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude")); occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude")); + occurrence.setCoordinateUncertaintyInMeters(getAsString(mappedObj, "coordinatePrecision")); Calendar eventDate = getAsCalendar(mappedObj, "eventDate"); occurrence.setEventDate(eventDate); + occurrence.setCollectionCode(getAsString(mappedObj, "collectionCode")); occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode")); occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber")); //occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy")); //occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy")); - //occurrence.setCountry(getAsString(mappedObj, "country")); - //occurrence.setLocality(getAsString(mappedObj, "locality")); + occurrence.setCountry(getAsString(mappedObj, "country")); + occurrence.setLocality(getAsString(mappedObj, "locality")); - occurrence.setBasisOfRecord(BasisOfRecord.Unknown); - - /*occurrence.setMinDepth(getAsDouble(mappedObj, "elevation")); - occurrence.setMaxDepth(getAsDouble(mappedObj, "depth")); - */ + occurrence.setBasisOfRecord(getBasisOfRecord(getAsString(mappedObj, "basisOfRecord"))); + occurrence.setMinDepth(getAsDouble(mappedObj, "minimumDepthInMeters")); + occurrence.setMaxDepth(getAsDouble(mappedObj, "maximumDepthInMeters")); + - occurrence.setKingdom("Animalia"); + occurrence.setKingdom(getAsString(mappedObj, "kingdom")); occurrence.setFamily(getAsString(mappedObj, "family")); occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship")); occurrence.setScientificName(getAsString(mappedObj, "scientificName")); - String datasetName = getAsString(mappedObj, "datasetName"); - DataSet dataset = new DataSet(datasetName); - dataset.setCitation(datasetName); - dataset.setName(datasetName); - DataProvider dataProvider = new DataProvider("OBIS"); - dataProvider.setName("OBIS"); - dataset.setDataProvider(dataProvider); + String datasetId = getAsString(mappedObj, "dataset_id"); + DataSet dataset = DataSetRetreiver.get(baseURL,datasetId); + + occurrence.setProvider(dataset.getDataProvider().getName()); occurrence.setDataSet(dataset); + occurrence.setProvider("OBIS"); + //occurrence.setCitation(getAsString(mappedObj, "accordingTo")); log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start)); return occurrence; } + private BasisOfRecord getBasisOfRecord(String basisOfRecord) { + if (basisOfRecord == null || basisOfRecord.isEmpty()) { + return BasisOfRecord.Unknown; + } + switch (basisOfRecord) { + case "PreservedSpecimen": + return BasisOfRecord.PreservedSpecimen; + case "FossilSpecimen": + return BasisOfRecord.FossilSpecimen; + case "LivingSpecimen": + return BasisOfRecord.LivingSpecimen; + case "HumanObservation": + return BasisOfRecord.HumanObservation; + case "MachineObservation": + return BasisOfRecord.MachineObservation; + case "Observation": + return BasisOfRecord.Observation; + case "Literature": + return BasisOfRecord.Literature; + case "MaterialSample": + return BasisOfRecord.MaterialSample; + default: + return BasisOfRecord.Unknown; + } + } } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java index 3afe96b..dfd50dc 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java @@ -13,6 +13,7 @@ import java.util.Set; import org.gcube.data.spd.model.Condition; import org.gcube.data.spd.model.exceptions.StreamBlockingException; +import org.gcube.data.spd.model.products.DataProvider; import org.gcube.data.spd.model.products.DataSet; import org.gcube.data.spd.model.products.Product; import org.gcube.data.spd.model.products.Product.ProductType; @@ -34,141 +35,165 @@ import org.slf4j.LoggerFactory; public class ResultItemSearch { private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - + private static Logger log = LoggerFactory.getLogger(ResultItemSearch.class); - + private List queryConditions = new ArrayList(); - + private String baseURL; - + String searchQuery; - - public ResultItemSearch(String baseURL, String searchQuery, Condition ... conditions){ + + public ResultItemSearch(String baseURL, String searchQuery, Condition... conditions) { this.baseURL = baseURL; this.searchQuery = searchQuery.replaceAll(" ", "%20").trim(); - this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase()+this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase(); - try{ + this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase() + + this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase(); + try { this.queryConditions = Utils.elaborateConditions(conditions); - }catch(Exception e){ - log.error("error elaborating conditions",e); + } catch (Exception e) { + log.error("error elaborating conditions", e); } } - - public void search(ObjectWriter writer, int limit){ - PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence,limit); - queryObject.setConditions(QueryCondition.cond("scientificname",searchQuery)); + + public void search(ObjectWriter writer, int limit) { + PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence, limit); + queryObject.setConditions(QueryCondition.cond("scientificname", searchQuery)); queryObject.getConditions().addAll(this.queryConditions); - try{ + try { PagedQueryIterator pagedIterator = new PagedQueryIterator(queryObject) { - Set alreadyVisited =new HashSet(); - + Set alreadyVisited = new HashSet(); + @Override protected ResultItem getObject(Map mappedObject) throws Exception { - log.debug("retrieved mapped object"); - return buildResult(mappedObject); + ResultItem resultItem = buildResult(mappedObject); + log.debug("ResultItem: " + resultItem); + return resultItem; } @Override protected boolean useIt(Map mappedObject) { - String datasetKey = ((Integer)mappedObject.get("resourceID")).toString(); - Integer taxonId = (Integer)mappedObject.get("obisID"); - String key = datasetKey+"|"+taxonId; - if (alreadyVisited.contains(key)) + String datasetKey = (String) mappedObject.get("dataset_id");// resourceID + Integer taxonId = (Integer) mappedObject.get("aphiaID");// obisID + String key = datasetKey + "|" + taxonId; + if (alreadyVisited.contains(key)) return false; alreadyVisited.add(key); return true; } - + }; while (pagedIterator.hasNext() && writer.isAlive()) writer.write(pagedIterator.next()); - - }catch(Exception e){ - log.error("error writing resultItems",e); + + } catch (Exception e) { + log.error("error writing resultItems", e); writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); } - } - ResultItem buildResult(Map singleObject) throws Exception{ - try{ - long start = System.currentTimeMillis(); - Integer taxonId = getAsInteger(singleObject,"obisID"); - String scientificName = getAsString(singleObject,"scientificName"); - ResultItem resItem = new ResultItem(taxonId.toString(), scientificName ); - - String scientificNameAuthorship = getAsString(singleObject,"scientificNameAuthorship"); - - QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); - Map singleTaxon = MappingUtils.getObjectMapping(query.build()); - - - - resItem.setScientificNameAuthorship(scientificNameAuthorship); - - resItem.setRank(getAsString(singleTaxon, "rank_name")); - - //resItem.setCitation(getAsString(singleTaxon,"tauthor")); - - resItem.setParent(retrieveParentTaxon(getAsInteger(singleTaxon,"parent_id"))); - - DataSet dataset = DataSetRetreiver.get(getAsInteger(singleObject,"resourceID").toString(), baseURL); - resItem.setDataSet(dataset); + ResultItem buildResult(Map singleObject) throws Exception { + try { + long start = System.currentTimeMillis(); + // log.debug("Retrieved Occurence: "+singleObject); + Integer taxonId = getAsInteger(singleObject, "aphiaID");// obisID + String scientificName = getAsString(singleObject, "scientificName"); + ResultItem resItem = new ResultItem(taxonId.toString(), scientificName); + String scientificNameAuthorship = getAsString(singleObject, "scientificNameAuthorship"); + resItem.setScientificNameAuthorship(scientificNameAuthorship); - List products = retrieveProducts(taxonId.toString(), dataset); - resItem.setProducts(products); + QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); + Map taxonData = MappingUtils.getObjectMapping(query.build()); + log.debug("Retrieved taxon: " + taxonData); + if (taxonData != null && !taxonData.isEmpty()) { + @SuppressWarnings("unchecked") + List> taxonList = (List>) taxonData.get("results"); + if (taxonList != null && !taxonList.isEmpty()) { + Map taxonSingle = taxonList.get(0); + resItem.setRank(getAsString(taxonSingle, "taxonRank"));// rank_name + //resItem.setParent(retrieveParentTaxon(getAsInteger(taxonSingle, "familyid")));// parent_id + } + } - String credits = "Biodiversity occurrence accessed through OBIS WebService, http://api.iobis.org/, "+format.format(Calendar.getInstance().getTime())+")"; - resItem.setCredits(credits); - log.trace("[Benchmark] time to retrieve ResultItem is "+(System.currentTimeMillis()-start)); - log.debug("found species {} with authorship {}",scientificName, scientificNameAuthorship); - return resItem; - }catch(Exception e){ + DataSet dataset = DataSetRetreiver.get(baseURL, getAsString(singleObject, "dataset_id")); + resItem.setProvider(dataset.getDataProvider().getName()); + resItem.setDataSet(dataset); + + List products = retrieveProducts(taxonId.toString(), dataset); + resItem.setProducts(products); + + String credits = "Biodiversity occurrence accessed through OBIS WebService, https://api.obis.org/v3/, " + + format.format(Calendar.getInstance().getTime()) + ")"; + resItem.setCredits(credits); + log.trace("[Benchmark] time to retrieve ResultItem is " + (System.currentTimeMillis() - start)); + log.debug("found species: id={}, name={}", resItem.getId(), resItem.getScientificName()); + return resItem; + } catch (Exception e) { throw e; } } - + + //TODO private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception { - if (parentTaxonId==0) return null; + if (parentTaxonId == 0) + return null; long start = System.currentTimeMillis(); - + Integer taxonId = parentTaxonId; + Integer taxonIdS = 0; Taxon previousTaxon = null; Taxon taxonToReturn = null; - do{ + do { QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); - Map singleTaxon = MappingUtils.getObjectMapping(query.build()); - - Taxon taxon = new Taxon(getAsInteger(singleTaxon, "id").toString(), getAsString(singleTaxon, "tname")); - taxon.setScientificNameAuthorship(getAsString(singleTaxon, "tauthor")); - //taxon.setCitation(getAsString(mappedObject, "accordingTo")); - taxon.setRank(getAsString(singleTaxon, "rank_name")); - if (previousTaxon!=null) - previousTaxon.setParent(taxon); - previousTaxon = taxon; - taxonId = getAsInteger(singleTaxon, "parent_id"); - if (taxonToReturn==null) - taxonToReturn = taxon; - } while (taxonId>0); - - log.trace("[Benchmark] time to retrieve taxon is "+(System.currentTimeMillis()-start)); + Map parentTaxonData = MappingUtils.getObjectMapping(query.build()); + log.debug("ParentTaxon: " + parentTaxonData); + if (parentTaxonData != null && !parentTaxonData.isEmpty()) { + @SuppressWarnings("unchecked") + List> parentTaxonList = (List>) parentTaxonData.get("results"); + if (parentTaxonList != null && !parentTaxonList.isEmpty()) { + Map parentTaxon=parentTaxonList.get(0); + + Taxon taxon = new Taxon(getAsInteger(parentTaxon, "taxonID").toString(), + getAsString(parentTaxon, "scientificName")); + taxon.setScientificNameAuthorship(getAsString(parentTaxon, "scientificNameAuthorship")); + taxon.setRank(getAsString(parentTaxon, "taxonRank"));// rank_name + if (previousTaxon != null){ + previousTaxon.setParent(taxon); + } + previousTaxon = taxon; + if (taxonToReturn == null) { + taxonToReturn = taxon; + } + + taxonIdS = getAsInteger(parentTaxon, "familyid");// parent_id + if(taxonIdS>0&&taxonIdS!=taxonId){ + taxonId=taxonIdS; + }else { + taxonId=0; + } + + } + } + } while (taxonId > 0); + + log.trace("[Benchmark] time to retrieve taxon is " + (System.currentTimeMillis() - start)); return taxonToReturn; } - private List retrieveProducts( String taxonId, DataSet dataset) throws Exception{ + private List retrieveProducts(String taxonId, DataSet dataset) throws Exception { long start = System.currentTimeMillis(); QueryCount occurrencesQuery = new QueryCount(baseURL, ResultType.Occurrence); - occurrencesQuery.setConditions(QueryCondition.cond("obisid",taxonId), QueryCondition.cond("resourceid", dataset.getId())); + occurrencesQuery.setConditions(QueryCondition.cond("taxonid", taxonId), + QueryCondition.cond("datasetid", dataset.getId())); occurrencesQuery.getConditions().addAll(this.queryConditions); String productId = Utils.createProductsKey(Utils.getDataSetAsString(dataset), taxonId, this.queryConditions); Product product = new Product(ProductType.Occurrence, productId); product.setCount(occurrencesQuery.getCount()); - log.trace("[Benchmark] time to retrieve product is "+(System.currentTimeMillis()-start)); + log.trace("[Benchmark] time to retrieve product is " + (System.currentTimeMillis() - start)); return Arrays.asList(product); } - } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java b/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java index ab32bc5..6025e1c 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java @@ -38,8 +38,8 @@ public class Utils { public static ProductKey elaborateProductsKey(String id) { List queryConditions = new ArrayList(); String[] splitString = id.split("\\|\\|"); - queryConditions.add(cond("resourceid", splitString[0])); - queryConditions.add(cond("obisid", splitString[1])); + queryConditions.add(cond("datasetid", splitString[0])); + queryConditions.add(cond("taxonid", splitString[1])); if (splitString.length>2) for (int i = 2; i getObjectMapping(String query) throws Exception{ - + public static Map getObjectMapping(String query) throws Exception { + String response = executeQuery(query); - + ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally return mapper.readValue(new StringReader(response), Map.class); } - public static List> getObjectList(String query) throws Exception{ + public static List> getObjectList(String query) throws Exception { String response = executeQuery(query); ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally return mapper.readValue(new StringReader(response), new TypeReference>>() { @@ -48,61 +48,91 @@ public class MappingUtils { } - public static String getAsString(Map map, String key){ - if (!map.containsKey(key)) return null; + public static String getAsString(Map map, String key) { + if (!map.containsKey(key)) + return null; return (String) map.get(key); } - public static Double getAsDouble(Map map, String key){ - if (!map.containsKey(key)) return 0d; - return (Double) map.get(key); + public static Double getAsDouble(Map map, String key) { + if (!map.containsKey(key)) + return 0d; + if (map.get(key) instanceof Double) { + return (Double) map.get(key); + } else { + if (map.get(key) instanceof Integer) { + Integer value = (Integer) map.get(key); + return value.doubleValue(); + } else { + if (map.get(key) instanceof String) { + String value = (String) map.get(key); + return Double.valueOf(value); + } else { + return 0d; + } + } + } } - public static Integer getAsInteger(Map map, String key){ - if (!map.containsKey(key)) return 0; - return (Integer) map.get(key); + public static Integer getAsInteger(Map map, String key) { + if (!map.containsKey(key)) + return 0; + if (map.get(key) instanceof Integer) { + Integer value = (Integer) map.get(key); + return value; + } else { + if (map.get(key) instanceof String) { + String value = (String) map.get(key); + return Integer.valueOf(value); + } else { + return 0; + } + } } - public static Calendar getAsCalendar(Map map, String key){ - if (!map.containsKey(key)) return null; + public static Calendar getAsCalendar(Map map, String key) { + if (!map.containsKey(key)) + return null; return parseCalendar((String) map.get(key)); } - - public static Calendar parseCalendar(String date){ - try{ - DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Calendar calendar= Calendar.getInstance(); + public static Calendar parseCalendar(String date) { + try { + //"2001-11-27T12:00:00Z" + DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + Calendar calendar = Calendar.getInstance(); calendar.setTime(df.parse(date)); return calendar; - }catch (ParseException e) { - log.warn("date discarded ("+date+")"); + } catch (ParseException e) { + log.warn("date discarded (" + date + ")"); return null; } } - - private static String executeQuery(String query){ + + private static String executeQuery(String query) { DefaultClientConfig clientConfig = new DefaultClientConfig(); Client client = Client.create(clientConfig); client.setConnectTimeout(TIMEOUT); client.setReadTimeout(TIMEOUT); WebResource target = client.resource(query); - //NameUsageWsClient nuws = new NameUsageWsClient(target); + // NameUsageWsClient nuws = new NameUsageWsClient(target); int tries = 1; String response = null; do { - - log.debug("try number {} STARTED for query {} ", tries,query); - try{ + + log.debug("try number {} STARTED for query {} ", tries, query); + try { response = target.type(MediaType.APPLICATION_JSON).acceptLanguage(Locale.ENGLISH).get(String.class); - }catch (Exception e) { - log.debug("try number {} FAILED for query {} ", tries,query,e); + } catch (Exception e) { + log.debug("try number {} FAILED for query {} ", tries, query, e); try { - if (tries implements Iterator{ public boolean hasNext() { try{ if (resultIterator==null){ + log.trace("PagedQuery fetch new page"); String query = pagedQuery.buildNext(); start = System.currentTimeMillis(); mapping = MappingUtils.getObjectMapping(query); @@ -45,14 +46,17 @@ public abstract class PagedQueryIterator implements Iterator{ if (!resultIterator.hasNext()){ log.trace("[Benchmark] page retrieved and parsed in "+(System.currentTimeMillis()-start)); - if ((Boolean)mapping.get("lastpage")){ - log.trace("is end of records, no next element"); + Integer endOfRecords=(Integer)mapping.get("total"); + if (pagedQuery.getPageCount()>=endOfRecords){ + log.debug("End of records, no next element"); return false; } resultIterator = null; } else{ + log.trace("Read new record"); actualObject = resultIterator.next(); + pagedQuery.setAfter((String)actualObject.get("id")); if (useIt(actualObject)) return true; } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java index 63ca787..459ab47 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java @@ -9,42 +9,52 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - @RequiredArgsConstructor @Slf4j public class PagedQueryObject { private @NonNull String baseUri; - + @Getter List conditions = new ArrayList(); - + private @NonNull ResultType resultType; - + private @NonNull Integer resultPerQuery; - - private int offset = 0; - - public void setConditions(QueryCondition ... conditions){ + + private String after = null; + + private Integer pageCount = 0; + + public void setConditions(QueryCondition... conditions) { this.conditions.addAll(Arrays.asList(conditions)); } - - - - public String buildNext(){ - StringBuilder query = new StringBuilder(baseUri); - if (!baseUri.endsWith("/")) query.append("/"); - query.append(this.resultType.getQueryEntry()).append("/"); - query.append("?limit=").append(resultPerQuery); - query.append("&offset=").append(offset); - if (conditions.size()>0) - for (QueryCondition queryCond: conditions) + public void setAfter(String after) { + this.after = after; + } + + public Integer getPageCount() { + return pageCount; + } + + public String buildNext() { + StringBuilder query = new StringBuilder(baseUri); + if (!baseUri.endsWith("/")) + query.append("/"); + query.append(this.resultType.getQueryEntry()).append("/"); + query.append("?size=").append(resultPerQuery); + if (after != null) { + query.append("&after=").append(after); + } + + pageCount += resultPerQuery; + + if (conditions.size() > 0) + for (QueryCondition queryCond : conditions) query.append("&").append(queryCond.getKey()).append("=").append(queryCond.getValue()); - offset = offset+resultPerQuery; - log.debug("executed query is "+query.toString()); + log.debug("executed query is " + query.toString()); return query.toString(); } - - + } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java index 6dc280f..9c281db 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java @@ -30,7 +30,7 @@ public class QueryByIdentifier { query.append(key); for (String path : paths) query.append("/").append(path); - log.trace("query by dentifier is "+query.toString()); + log.debug("query by identifier is "+query.toString()); return query.toString(); } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java index ef6fea6..1eb7a09 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java @@ -35,8 +35,8 @@ public class QueryCount { Map mapping; try { mapping = MappingUtils.getObjectMapping(this.build()); - if (mapping.get("count")==null) return 0; - return (Integer)mapping.get("count"); + if (mapping.get("total")==null) return 0; + return (Integer)mapping.get("total"); } catch (Exception e) { log.error("error computing count, returning 0",e); return 0; @@ -49,7 +49,7 @@ public class QueryCount { StringBuilder query = new StringBuilder(baseUri); if (!baseUri.endsWith("/")) query.append("/"); query.append(this.resultType.getQueryEntry()).append("/"); - query.append("?limit=0"); + query.append("?size=0"); if (conditions.size()>0) for (QueryCondition queryCond: conditions) diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java index a7b1b53..a4f944f 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java @@ -8,7 +8,7 @@ import lombok.NonNull; public @AllArgsConstructor enum QueryType{ Occurrence("occurrence"), Taxon("taxon"), - Dataset("resource"); + Dataset("dataset"); @Getter private @NonNull String queryEntry; diff --git a/src/test/java/org/gcube/data/obisplugin/ObisTest.java b/src/test/java/org/gcube/data/obisplugin/ObisTest.java index 458cc45..36089af 100644 --- a/src/test/java/org/gcube/data/obisplugin/ObisTest.java +++ b/src/test/java/org/gcube/data/obisplugin/ObisTest.java @@ -2,10 +2,6 @@ package org.gcube.data.obisplugin; import java.util.Arrays; -import org.gcube.data.spd.model.Condition; -import org.gcube.data.spd.model.Condition.Operator; -import org.gcube.data.spd.model.Conditions; -import org.gcube.data.spd.model.Coordinate; import org.gcube.data.spd.model.exceptions.StreamException; import org.gcube.data.spd.model.products.OccurrencePoint; import org.gcube.data.spd.model.products.ResultItem; @@ -18,7 +14,7 @@ import org.junit.Test; public class ObisTest { @Test - public void search() throws Exception{ + public void searchByKey() throws Exception{ /*ObisPlugin plugin= new ObisPlugin(); plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3")); plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/ @@ -49,13 +45,16 @@ public class ObisTest { - OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("http://api.iobis.org/"); + OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("https://api.obis.org/v3/"); //impl.searchByScientificName("Cetacea", writer); - impl.getOccurrencesByProductKeys(writer, Arrays.asList("3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))").iterator() ); + //8f843938-1617-4c9e-be03-d1a2af8ebd89||127021 + //"3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))" + + impl.getOccurrencesByProductKeys(writer, Arrays.asList("8f843938-1617-4c9e-be03-d1a2af8ebd89||127021").iterator() ); } @Test - public void searchRI() throws Exception{ + public void searchByScientificName() throws Exception{ /*ObisPlugin plugin= new ObisPlugin(); plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3")); plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/ @@ -78,10 +77,46 @@ public class ObisTest { } }; - //, new Condition(Conditions.COORDINATE, new Coordinate(20, 30) , Operator.GT) - ResultItemSearch search = new ResultItemSearch("http://api.iobis.org/", "Gamidae"); + //Balaenoptera bonaerensis + ResultItemSearch search = new ResultItemSearch("https://api.obis.org/v3/", "Sarda sarda"); search.search(writer, 50); } + + @Test + public void searchByGeometry() throws Exception{ + ClosableWriter writer = new ClosableWriter() { + + @Override + public boolean isAlive() { + return true; + } + + @Override + public boolean write(OccurrencePoint arg0) { + return true; + } + + @Override + public boolean write(StreamException arg0) { + return false; + } + + @Override + public void close() { + + } + + }; + + + + OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("https://api.obis.org/v3/"); + //"3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))" + impl.getOccurrencesByProductKeys(writer, Arrays.asList("9b787218-7138-4ffa-8ba6-36d63d391f8c||127021||geometry=POLYGON((1.6306044624659184%206.903464725745124,1.58434708579508%20-0.943627209502182,8.754369215807587%20-0.8048685858041154,8.569345073542273%206.857540488275561,1.6306044624659184%206.903464725745124))").iterator() ); + + + } + }