gCubeSystem
/
obis-spd-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ref #18694: SPD - Support the transition to the new OBIS plugin 

https://support.d4science.org/issues/#18694

Updated support to Taxon by OBIS API v3
This commit is contained in:
Giancarlo Panichi 2020-03-09 16:33:31 +01:00
parent 74d2aa040b
commit 60341d7ec8
5 changed files with 241 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java
View File

@ -1,7 +1,5 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import java.util.List;
import java.util.Map;
@ -30,33 +28,42 @@ public class DataSetRetreiver {
List<Map<String, Object>> results = (List<Map<String, Object>>) listMaps.get("results");
if (results != null && !results.isEmpty()) {
Map<String, Object> mapping = results.get(0);
log.debug("Dataset Name: " + getAsString(mapping, "title"));
dataset.setName(getAsString(mapping, "title"));
log.debug("Dataset Name: " + MappingUtils.getAsString(mapping, "title"));
dataset.setName(MappingUtils.getAsString(mapping, "title"));
String citation = getAsString(mapping, "citation");
String citation = MappingUtils.getAsString(mapping, "citation");
if (citation == null) {
List<Map<String, Object>> institutionMapping = (List<Map<String, Object>>) mapping
.get("institutes");
if (institutionMapping != null && !institutionMapping.isEmpty()) {
if (getAsString(institutionMapping.get(0), "name") != null)
citation = getAsString(institutionMapping.get(0), "name");
if (MappingUtils.getAsString(institutionMapping.get(0), "name") != null)
citation = MappingUtils.getAsString(institutionMapping.get(0), "name");
dataset.setCitation(citation);
}
} else {
dataset.setCitation(citation);
}
String providerKey = key;
List<Map<String, Object>> providerMapping = (List<Map<String, Object>>) mapping.get("nodes");
DataProvider provider = new DataProvider(providerKey);
List<Map<String, Object>> providerMapping = (List<Map<String, Object>>) mapping.get("contacts");
DataProvider provider = null;
if (providerMapping != null && !providerMapping.isEmpty()) {
Map<String, Object> pMapping = (Map<String, Object>) providerMapping.get(0);
provider.setName(getAsString(pMapping, "name"));
for (Map<String, Object> contact : providerMapping) {
String contactType = MappingUtils.getAsString(contact, "type");
if (contactType != null && !contactType.isEmpty()
&& contactType.compareToIgnoreCase("creator") == 0) {
Integer providerKey = MappingUtils.getAsInteger(contact, "organization_oceanexpert_id");
String providerName = MappingUtils.getAsString(contact, "organization");
provider = new DataProvider(providerKey.toString());
provider.setName(providerName);
break;
}
}
} else {
provider=new DataProvider(Constants.REPOSITORY_NAME);
provider.setName(Constants.REPOSITORY_NAME);
}
dataset.setDataProvider(provider);
}
}

16
src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java
View File

@ -113,6 +113,10 @@ public class OccurrenceSearch {
String occurrenceId = getAsString(mappedObj, "id");
OccurrencePoint occurrence = new OccurrencePoint(occurrenceId);
occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship"));
occurrence.setScientificName(getAsString(mappedObj, "scientificName"));
occurrence.setIdentifiedBy(getAsString(mappedObj,"scientificNameID"));
occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude"));
occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude"));
occurrence.setCoordinateUncertaintyInMeters(getAsString(mappedObj, "coordinatePrecision"));
@ -124,8 +128,9 @@ public class OccurrenceSearch {
occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode"));
occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber"));
//occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy"));
//occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy"));
occurrence.setRecordedBy(getAsString(mappedObj,"recordedBy"));
occurrence.setCredits(getAsString(mappedObj,"credits"));
occurrence.setCountry(getAsString(mappedObj, "country"));
occurrence.setLocality(getAsString(mappedObj, "locality"));
@ -137,17 +142,12 @@ public class OccurrenceSearch {
occurrence.setKingdom(getAsString(mappedObj, "kingdom"));
occurrence.setFamily(getAsString(mappedObj, "family"));
occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship"));
occurrence.setScientificName(getAsString(mappedObj, "scientificName"));
String datasetId = getAsString(mappedObj, "dataset_id");
DataSet dataset = DataSetRetreiver.get(baseURL,datasetId);
occurrence.setDataSet(dataset);
occurrence.setProvider(dataset.getDataProvider().getName());
occurrence.setDataSet(dataset);
occurrence.setProvider("OBIS");
//occurrence.setCitation(getAsString(mappedObj, "accordingTo"));
log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start));

126
src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java
View File

@ -1,19 +1,18 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.Product;
import org.gcube.data.spd.model.products.Product.ProductType;
@ -99,12 +98,13 @@ public class ResultItemSearch {
try {
long start = System.currentTimeMillis();
// log.debug("Retrieved Occurence: "+singleObject);
Integer taxonId = getAsInteger(singleObject, "aphiaID");// obisID
String scientificName = getAsString(singleObject, "scientificName");
Integer taxonId = MappingUtils.getAsInteger(singleObject, "aphiaID");// obisID
String scientificName = MappingUtils.getAsString(singleObject, "scientificName");
ResultItem resItem = new ResultItem(taxonId.toString(), scientificName);
String scientificNameAuthorship = getAsString(singleObject, "scientificNameAuthorship");
String scientificNameAuthorship = MappingUtils.getAsString(singleObject, "scientificNameAuthorship");
resItem.setScientificNameAuthorship(scientificNameAuthorship);
resItem.setLsid(MappingUtils.getAsString(singleObject,"scientificNameID"));
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> taxonData = MappingUtils.getObjectMapping(query.build());
log.debug("Retrieved taxon: " + taxonData);
@ -113,12 +113,15 @@ public class ResultItemSearch {
List<Map<String, Object>> taxonList = (List<Map<String, Object>>) taxonData.get("results");
if (taxonList != null && !taxonList.isEmpty()) {
Map<String, Object> taxonSingle = taxonList.get(0);
resItem.setRank(getAsString(taxonSingle, "taxonRank"));// rank_name
//resItem.setParent(retrieveParentTaxon(getAsInteger(taxonSingle, "familyid")));// parent_id
String taxonRank = MappingUtils.getAsString(taxonSingle, "taxonRank");
resItem.setRank(taxonRank);// rank_name
resItem.setParent(retrieveParentTaxon(taxonId, taxonRank, taxonSingle));// parent_id
}
}
DataSet dataset = DataSetRetreiver.get(baseURL, getAsString(singleObject, "dataset_id"));
DataSet dataset = DataSetRetreiver.get(baseURL, MappingUtils.getAsString(singleObject, "dataset_id"));
resItem.setProvider(dataset.getDataProvider().getName());
resItem.setDataSet(dataset);
@ -135,50 +138,87 @@ public class ResultItemSearch {
throw e;
}
}
//TODO
private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception {
if (parentTaxonId == 0)
private Taxon retrieveParentTaxon(Integer taxonId, String taxonRank, Map<String, Object> taxonMap)
throws Exception {
log.debug("Call retrieve parentTaxon: [taxonId={}, taxonRank={}]",taxonId,taxonRank);
if (taxonId == 0)
return null;
long start = System.currentTimeMillis();
Integer taxonId = parentTaxonId;
Integer taxonIdS = 0;
LinkedList<Taxon> taxons = new LinkedList<>();
String searchTaxon = taxonRank;
Map<String, Object> searchMap = taxonMap;
Taxon previousTaxon = null;
Taxon taxonToReturn = null;
do {
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
TaxonCategories currentTaxon=null;
boolean end = false;
while (!end) {
currentTaxon = TaxonCategories.getTaxonCategory(searchTaxon);
if (currentTaxon == null) {
break;
}
boolean foundParentId = false;
Integer parentId=null;
while (!foundParentId&&!end) {
TaxonCategories parentTaxon = TaxonCategories.getParent(currentTaxon);
if (parentTaxon == null) {
end=true;
break;
}
String parentIdKey = parentTaxon.name().toLowerCase() + "id";
if (searchMap.containsKey(parentIdKey)) {
parentId = MappingUtils.getAsInteger(searchMap, parentIdKey);
currentTaxon=parentTaxon;
foundParentId=true;
}else {
currentTaxon=parentTaxon;
}
}
if(end||!foundParentId){
break;
}
log.debug("Found parentId: "+parentId);
QueryByIdentifier query = new QueryByIdentifier(baseURL, parentId.toString(), QueryType.Taxon);
Map<String, Object> parentTaxonData = MappingUtils.getObjectMapping(query.build());
log.debug("ParentTaxon: " + parentTaxonData);
if (parentTaxonData != null && !parentTaxonData.isEmpty()) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> parentTaxonList = (List<Map<String, Object>>) parentTaxonData.get("results");
if (parentTaxonList != null && !parentTaxonList.isEmpty()) {
Map<String,Object> parentTaxon=parentTaxonList.get(0);
Taxon taxon = new Taxon(getAsInteger(parentTaxon, "taxonID").toString(),
getAsString(parentTaxon, "scientificName"));
taxon.setScientificNameAuthorship(getAsString(parentTaxon, "scientificNameAuthorship"));
taxon.setRank(getAsString(parentTaxon, "taxonRank"));// rank_name
if (previousTaxon != null){
previousTaxon.setParent(taxon);
}
previousTaxon = taxon;
if (taxonToReturn == null) {
taxonToReturn = taxon;
}
taxonIdS = getAsInteger(parentTaxon, "familyid");// parent_id
if(taxonIdS>0&&taxonIdS!=taxonId){
taxonId=taxonIdS;
}else {
taxonId=0;
}
}
}
} while (taxonId > 0);
searchMap = parentTaxonList.get(0);
Taxon taxon = new Taxon(MappingUtils.getAsInteger(searchMap, "taxonID").toString(),
MappingUtils.getAsString(searchMap, "scientificName"));
taxon.setScientificNameAuthorship(MappingUtils.getAsString(searchMap, "scientificNameAuthorship"));
searchTaxon=MappingUtils.getAsString(searchMap, "taxonRank");
taxon.setRank(searchTaxon);// rank_name
if (previousTaxon == null) {
previousTaxon = taxon;
} else {
previousTaxon.setParent(taxon);
taxons.add(previousTaxon);
previousTaxon = taxon;
}
} else {
end = true;
}
} else {
end = true;
}
}
if (taxons.isEmpty()) {
taxonToReturn = previousTaxon;
} else {
taxonToReturn = taxons.getFirst();
}
log.trace("[Benchmark] time to retrieve taxon is " + (System.currentTimeMillis() - start));
return taxonToReturn;
}

76
src/main/java/org/gcube/data/spd/obisplugin/search/TaxonCategories.java Normal file
View File

@ -0,0 +1,76 @@
package org.gcube.data.spd.obisplugin.search;
/**
*
* @author Giancarlo Panichi
*
*/
public enum TaxonCategories implements Comparable<TaxonCategories> {
// Taxonomic ranks. In capital the most used.
// DOMAIN,
// KINGDOM,
// PHYLUM OR DIVISION
// Subphylum or Subdivision
// CLASS
// Subclass
// Superorder
// ORDER
// Suborder
// FAMILY
// Subfamily
// GENUS
// SPECIES
DOMAIN(1), KINGDOM(2), PHYLUM(3), DIVISION(4), SUBPHYLUM(5), SUBDIVISION(6), CLASS(7), SUBCLASS(8), SUPERORDER(
9), ORDER(10), SUBORDER(11), FAMILY(12), SUBFAMILY(13), GENUS(14), SPECIES(15);
private int order;
TaxonCategories(int order) {
this.order = order;
}
public int getOrder() {
return order;
}
public static TaxonCategories getByOrder(int position) {
if (position < 1 || position > 15) {
return null;
}
for (TaxonCategories taxon : TaxonCategories.values()) {
if (taxon.order == position) {
return taxon;
}
}
return null;
}
public static TaxonCategories getParent(TaxonCategories taxon) {
if (taxon == null) {
return null;
}
if (taxon.order <= 1) {
return null;
} else {
int position = taxon.order;
position--;
return getByOrder(position);
}
}
public static TaxonCategories getTaxonCategory(String taxonRank) {
if (taxonRank == null || taxonRank.isEmpty()) {
return null;
}
String taxonR = taxonRank.toUpperCase();
for (TaxonCategories taxon : TaxonCategories.values()) {
if (taxon.name().compareTo(taxonR) == 0) {
return taxon;
}
}
return null;
}
}

62
src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java
View File

@ -1,10 +1,15 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.io.StringReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -98,12 +103,53 @@ public class MappingUtils {
public static Calendar parseCalendar(String date) {
try {
//"2001-11-27T12:00:00Z"
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
Calendar calendar = Calendar.getInstance();
calendar.setTime(df.parse(date));
Calendar calendar = null;
try {
// "2007-12-03T10:15:30+01:00[Europe/Rome]"
ZonedDateTime zdt = ZonedDateTime.parse(date);
calendar = GregorianCalendar.from(zdt);
return calendar;
} catch (DateTimeParseException e) {
log.warn("date discarded not in ZoneDateTime format (" + date + ")");
}
try {
// "2001-11-27T12:00:00Z"
Instant instant = Instant.parse(date);
ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of(ZoneOffset.UTC.getId()));
calendar = GregorianCalendar.from(zdt);
return calendar;
} catch (DateTimeParseException e) {
log.warn("date discarded not in ZoneDateTime UTC format (" + date + ")");
}
try {
// "2001-11-27T12:00:00"
LocalDateTime localDateTime = LocalDateTime.parse(date);
calendar = Calendar.getInstance();
calendar.clear();
calendar.set(localDateTime.getYear(), localDateTime.getMonthValue()-1, localDateTime.getDayOfMonth(),
localDateTime.getHour(), localDateTime.getMinute(), localDateTime.getSecond());
return calendar;
} catch (DateTimeParseException e) {
log.warn("date discarded not in LocalDateTime format (" + date + ")");
}
try {
// "2001-11-27"
LocalDate localDate = LocalDate.parse(date);
calendar = Calendar.getInstance();
calendar.clear();
calendar.set(localDate.getYear(), localDate.getMonthValue()-1, localDate.getDayOfMonth());
return calendar;
} catch (DateTimeParseException e) {
log.warn("date discarded not in LocalDate format (" + date + ")");
}
return calendar;
} catch (ParseException e) {
} catch (Exception e) {
log.warn("date discarded (" + date + ")");
return null;
}