From 60341d7ec896c7d528c0d13decd668b1398e98ae Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Mon, 9 Mar 2020 16:33:31 +0100 Subject: [PATCH] ref #18694: SPD - Support the transition to the new OBIS plugin https://support.d4science.org/issues/#18694 Updated support to Taxon by OBIS API v3 --- .../obisplugin/search/DataSetRetreiver.java | 33 +++-- .../obisplugin/search/OccurrenceSearch.java | 16 +-- .../obisplugin/search/ResultItemSearch.java | 126 ++++++++++++------ .../obisplugin/search/TaxonCategories.java | 76 +++++++++++ .../obisplugin/search/query/MappingUtils.java | 62 +++++++-- 5 files changed, 241 insertions(+), 72 deletions(-) create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/TaxonCategories.java diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java index fdeb602..1f93993 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java @@ -1,7 +1,5 @@ package org.gcube.data.spd.obisplugin.search; -import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; - import java.util.List; import java.util.Map; @@ -30,33 +28,42 @@ public class DataSetRetreiver { List> results = (List>) listMaps.get("results"); if (results != null && !results.isEmpty()) { Map mapping = results.get(0); - log.debug("Dataset Name: " + getAsString(mapping, "title")); - dataset.setName(getAsString(mapping, "title")); + log.debug("Dataset Name: " + MappingUtils.getAsString(mapping, "title")); + dataset.setName(MappingUtils.getAsString(mapping, "title")); - String citation = getAsString(mapping, "citation"); + String citation = MappingUtils.getAsString(mapping, "citation"); if (citation == null) { List> institutionMapping = (List>) mapping .get("institutes"); if (institutionMapping != null && !institutionMapping.isEmpty()) { - if (getAsString(institutionMapping.get(0), "name") != null) - citation = getAsString(institutionMapping.get(0), "name"); + if (MappingUtils.getAsString(institutionMapping.get(0), "name") != null) + citation = MappingUtils.getAsString(institutionMapping.get(0), "name"); dataset.setCitation(citation); } } else { dataset.setCitation(citation); } - String providerKey = key; - List> providerMapping = (List>) mapping.get("nodes"); - DataProvider provider = new DataProvider(providerKey); + List> providerMapping = (List>) mapping.get("contacts"); + DataProvider provider = null; if (providerMapping != null && !providerMapping.isEmpty()) { - Map pMapping = (Map) providerMapping.get(0); - provider.setName(getAsString(pMapping, "name")); + for (Map contact : providerMapping) { + String contactType = MappingUtils.getAsString(contact, "type"); + if (contactType != null && !contactType.isEmpty() + && contactType.compareToIgnoreCase("creator") == 0) { + Integer providerKey = MappingUtils.getAsInteger(contact, "organization_oceanexpert_id"); + String providerName = MappingUtils.getAsString(contact, "organization"); + provider = new DataProvider(providerKey.toString()); + provider.setName(providerName); + break; + } + + } } else { + provider=new DataProvider(Constants.REPOSITORY_NAME); provider.setName(Constants.REPOSITORY_NAME); } - dataset.setDataProvider(provider); } } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java index a4726f6..97b9dde 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java @@ -113,6 +113,10 @@ public class OccurrenceSearch { String occurrenceId = getAsString(mappedObj, "id"); OccurrencePoint occurrence = new OccurrencePoint(occurrenceId); + occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship")); + occurrence.setScientificName(getAsString(mappedObj, "scientificName")); + occurrence.setIdentifiedBy(getAsString(mappedObj,"scientificNameID")); + occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude")); occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude")); occurrence.setCoordinateUncertaintyInMeters(getAsString(mappedObj, "coordinatePrecision")); @@ -124,8 +128,9 @@ public class OccurrenceSearch { occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode")); occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber")); - //occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy")); - //occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy")); + occurrence.setRecordedBy(getAsString(mappedObj,"recordedBy")); + occurrence.setCredits(getAsString(mappedObj,"credits")); + occurrence.setCountry(getAsString(mappedObj, "country")); occurrence.setLocality(getAsString(mappedObj, "locality")); @@ -137,17 +142,12 @@ public class OccurrenceSearch { occurrence.setKingdom(getAsString(mappedObj, "kingdom")); occurrence.setFamily(getAsString(mappedObj, "family")); - occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship")); - occurrence.setScientificName(getAsString(mappedObj, "scientificName")); String datasetId = getAsString(mappedObj, "dataset_id"); DataSet dataset = DataSetRetreiver.get(baseURL,datasetId); - + occurrence.setDataSet(dataset); occurrence.setProvider(dataset.getDataProvider().getName()); - occurrence.setDataSet(dataset); - - occurrence.setProvider("OBIS"); //occurrence.setCitation(getAsString(mappedObj, "accordingTo")); log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start)); diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java index dfd50dc..c3709a5 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java @@ -1,19 +1,18 @@ package org.gcube.data.spd.obisplugin.search; -import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; -import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger; + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import org.gcube.data.spd.model.Condition; import org.gcube.data.spd.model.exceptions.StreamBlockingException; -import org.gcube.data.spd.model.products.DataProvider; import org.gcube.data.spd.model.products.DataSet; import org.gcube.data.spd.model.products.Product; import org.gcube.data.spd.model.products.Product.ProductType; @@ -99,12 +98,13 @@ public class ResultItemSearch { try { long start = System.currentTimeMillis(); // log.debug("Retrieved Occurence: "+singleObject); - Integer taxonId = getAsInteger(singleObject, "aphiaID");// obisID - String scientificName = getAsString(singleObject, "scientificName"); + Integer taxonId = MappingUtils.getAsInteger(singleObject, "aphiaID");// obisID + String scientificName = MappingUtils.getAsString(singleObject, "scientificName"); ResultItem resItem = new ResultItem(taxonId.toString(), scientificName); - String scientificNameAuthorship = getAsString(singleObject, "scientificNameAuthorship"); + String scientificNameAuthorship = MappingUtils.getAsString(singleObject, "scientificNameAuthorship"); resItem.setScientificNameAuthorship(scientificNameAuthorship); - + resItem.setLsid(MappingUtils.getAsString(singleObject,"scientificNameID")); + QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); Map taxonData = MappingUtils.getObjectMapping(query.build()); log.debug("Retrieved taxon: " + taxonData); @@ -113,12 +113,15 @@ public class ResultItemSearch { List> taxonList = (List>) taxonData.get("results"); if (taxonList != null && !taxonList.isEmpty()) { Map taxonSingle = taxonList.get(0); - resItem.setRank(getAsString(taxonSingle, "taxonRank"));// rank_name - //resItem.setParent(retrieveParentTaxon(getAsInteger(taxonSingle, "familyid")));// parent_id + String taxonRank = MappingUtils.getAsString(taxonSingle, "taxonRank"); + resItem.setRank(taxonRank);// rank_name + resItem.setParent(retrieveParentTaxon(taxonId, taxonRank, taxonSingle));// parent_id } } - - DataSet dataset = DataSetRetreiver.get(baseURL, getAsString(singleObject, "dataset_id")); + + + + DataSet dataset = DataSetRetreiver.get(baseURL, MappingUtils.getAsString(singleObject, "dataset_id")); resItem.setProvider(dataset.getDataProvider().getName()); resItem.setDataSet(dataset); @@ -135,50 +138,87 @@ public class ResultItemSearch { throw e; } } - - //TODO - private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception { - if (parentTaxonId == 0) + + private Taxon retrieveParentTaxon(Integer taxonId, String taxonRank, Map taxonMap) + throws Exception { + log.debug("Call retrieve parentTaxon: [taxonId={}, taxonRank={}]",taxonId,taxonRank); + if (taxonId == 0) return null; long start = System.currentTimeMillis(); - Integer taxonId = parentTaxonId; - Integer taxonIdS = 0; + LinkedList taxons = new LinkedList<>(); + + String searchTaxon = taxonRank; + Map searchMap = taxonMap; Taxon previousTaxon = null; Taxon taxonToReturn = null; - do { - QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); + TaxonCategories currentTaxon=null; + boolean end = false; + + while (!end) { + + currentTaxon = TaxonCategories.getTaxonCategory(searchTaxon); + if (currentTaxon == null) { + break; + } + boolean foundParentId = false; + Integer parentId=null; + while (!foundParentId&&!end) { + TaxonCategories parentTaxon = TaxonCategories.getParent(currentTaxon); + if (parentTaxon == null) { + end=true; + break; + } + String parentIdKey = parentTaxon.name().toLowerCase() + "id"; + if (searchMap.containsKey(parentIdKey)) { + parentId = MappingUtils.getAsInteger(searchMap, parentIdKey); + currentTaxon=parentTaxon; + foundParentId=true; + }else { + currentTaxon=parentTaxon; + } + } + + if(end||!foundParentId){ + break; + } + log.debug("Found parentId: "+parentId); + + QueryByIdentifier query = new QueryByIdentifier(baseURL, parentId.toString(), QueryType.Taxon); Map parentTaxonData = MappingUtils.getObjectMapping(query.build()); log.debug("ParentTaxon: " + parentTaxonData); if (parentTaxonData != null && !parentTaxonData.isEmpty()) { @SuppressWarnings("unchecked") List> parentTaxonList = (List>) parentTaxonData.get("results"); if (parentTaxonList != null && !parentTaxonList.isEmpty()) { - Map parentTaxon=parentTaxonList.get(0); - - Taxon taxon = new Taxon(getAsInteger(parentTaxon, "taxonID").toString(), - getAsString(parentTaxon, "scientificName")); - taxon.setScientificNameAuthorship(getAsString(parentTaxon, "scientificNameAuthorship")); - taxon.setRank(getAsString(parentTaxon, "taxonRank"));// rank_name - if (previousTaxon != null){ - previousTaxon.setParent(taxon); - } - previousTaxon = taxon; - if (taxonToReturn == null) { - taxonToReturn = taxon; - } - - taxonIdS = getAsInteger(parentTaxon, "familyid");// parent_id - if(taxonIdS>0&&taxonIdS!=taxonId){ - taxonId=taxonIdS; - }else { - taxonId=0; - } - - } - } - } while (taxonId > 0); + searchMap = parentTaxonList.get(0); + Taxon taxon = new Taxon(MappingUtils.getAsInteger(searchMap, "taxonID").toString(), + MappingUtils.getAsString(searchMap, "scientificName")); + taxon.setScientificNameAuthorship(MappingUtils.getAsString(searchMap, "scientificNameAuthorship")); + searchTaxon=MappingUtils.getAsString(searchMap, "taxonRank"); + taxon.setRank(searchTaxon);// rank_name + if (previousTaxon == null) { + previousTaxon = taxon; + } else { + previousTaxon.setParent(taxon); + taxons.add(previousTaxon); + previousTaxon = taxon; + } + + } else { + end = true; + } + } else { + end = true; + } + } + + if (taxons.isEmpty()) { + taxonToReturn = previousTaxon; + } else { + taxonToReturn = taxons.getFirst(); + } log.trace("[Benchmark] time to retrieve taxon is " + (System.currentTimeMillis() - start)); return taxonToReturn; } diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/TaxonCategories.java b/src/main/java/org/gcube/data/spd/obisplugin/search/TaxonCategories.java new file mode 100644 index 0000000..407f814 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/TaxonCategories.java @@ -0,0 +1,76 @@ +package org.gcube.data.spd.obisplugin.search; + +/** + * + * @author Giancarlo Panichi + * + */ +public enum TaxonCategories implements Comparable { + // Taxonomic ranks. In capital the most used. + // DOMAIN, + // KINGDOM, + // PHYLUM OR DIVISION + // Subphylum or Subdivision + // CLASS + // Subclass + // Superorder + // ORDER + // Suborder + // FAMILY + // Subfamily + // GENUS + // SPECIES + + DOMAIN(1), KINGDOM(2), PHYLUM(3), DIVISION(4), SUBPHYLUM(5), SUBDIVISION(6), CLASS(7), SUBCLASS(8), SUPERORDER( + 9), ORDER(10), SUBORDER(11), FAMILY(12), SUBFAMILY(13), GENUS(14), SPECIES(15); + + private int order; + + TaxonCategories(int order) { + this.order = order; + } + + public int getOrder() { + return order; + } + + public static TaxonCategories getByOrder(int position) { + if (position < 1 || position > 15) { + return null; + } + for (TaxonCategories taxon : TaxonCategories.values()) { + if (taxon.order == position) { + return taxon; + } + } + return null; + } + + public static TaxonCategories getParent(TaxonCategories taxon) { + if (taxon == null) { + return null; + } + if (taxon.order <= 1) { + return null; + } else { + int position = taxon.order; + position--; + return getByOrder(position); + } + + } + + public static TaxonCategories getTaxonCategory(String taxonRank) { + if (taxonRank == null || taxonRank.isEmpty()) { + return null; + } + String taxonR = taxonRank.toUpperCase(); + for (TaxonCategories taxon : TaxonCategories.values()) { + if (taxon.name().compareTo(taxonR) == 0) { + return taxon; + } + } + return null; + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java index ec715ab..4d02a45 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java @@ -1,10 +1,15 @@ package org.gcube.data.spd.obisplugin.search.query; import java.io.StringReader; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; import java.util.Calendar; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -98,12 +103,53 @@ public class MappingUtils { public static Calendar parseCalendar(String date) { try { - //"2001-11-27T12:00:00Z" - DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(df.parse(date)); + + Calendar calendar = null; + try { + // "2007-12-03T10:15:30+01:00[Europe/Rome]" + ZonedDateTime zdt = ZonedDateTime.parse(date); + calendar = GregorianCalendar.from(zdt); + return calendar; + } catch (DateTimeParseException e) { + log.warn("date discarded not in ZoneDateTime format (" + date + ")"); + } + + try { + // "2001-11-27T12:00:00Z" + Instant instant = Instant.parse(date); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of(ZoneOffset.UTC.getId())); + calendar = GregorianCalendar.from(zdt); + return calendar; + } catch (DateTimeParseException e) { + log.warn("date discarded not in ZoneDateTime UTC format (" + date + ")"); + + } + + try { + // "2001-11-27T12:00:00" + LocalDateTime localDateTime = LocalDateTime.parse(date); + calendar = Calendar.getInstance(); + calendar.clear(); + calendar.set(localDateTime.getYear(), localDateTime.getMonthValue()-1, localDateTime.getDayOfMonth(), + localDateTime.getHour(), localDateTime.getMinute(), localDateTime.getSecond()); + return calendar; + } catch (DateTimeParseException e) { + log.warn("date discarded not in LocalDateTime format (" + date + ")"); + } + + try { + // "2001-11-27" + LocalDate localDate = LocalDate.parse(date); + calendar = Calendar.getInstance(); + calendar.clear(); + calendar.set(localDate.getYear(), localDate.getMonthValue()-1, localDate.getDayOfMonth()); + return calendar; + } catch (DateTimeParseException e) { + log.warn("date discarded not in LocalDate format (" + date + ")"); + } + return calendar; - } catch (ParseException e) { + } catch (Exception e) { log.warn("date discarded (" + date + ")"); return null; }