From cecdb72930771b6954920fc79275308298045bab Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Wed, 1 Feb 2017 15:23:25 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/branches/data-access/obis-spd-plugin/1.8@141980 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 66 +- .../gcube/data/spd/obisplugin/Constants.java | 21 + .../spd/obisplugin/ObisClassification.java | 144 ---- .../gcube/data/spd/obisplugin/ObisClient.java | 682 ------------------ .../data/spd/obisplugin/ObisNameMapping.java | 55 -- .../obisplugin/ObisOccurrencesInterface.java | 133 ---- .../gcube/data/spd/obisplugin/ObisPlugin.java | 96 +-- .../data/spd/obisplugin/PluginSession.java | 429 ----------- .../OccurrencesCapabilityImpl.java | 110 +++ .../data/spd/obisplugin/data/ProductKey.java | 88 --- .../spd/obisplugin/data/SearchFilters.java | 50 -- .../obisplugin/pool/DatabaseCredential.java | 90 --- .../obisplugin/pool/JDBCConnectionPool.java | 61 -- .../data/spd/obisplugin/pool/ObjectPool.java | 126 ---- .../obisplugin/pool/PluginSessionPool.java | 83 --- .../obisplugin/search/DataSetRetreiver.java | 55 ++ .../obisplugin/search/OccurrenceSearch.java | 172 +++++ .../spd/obisplugin/search/ProductKey.java | 16 + .../obisplugin/search/ResultItemSearch.java | 174 +++++ .../data/spd/obisplugin/search/Utils.java | 132 ++++ .../obisplugin/search/query/MappingUtils.java | 109 +++ .../search/query/PagedQueryIterator.java | 86 +++ .../search/query/PagedQueryObject.java | 50 ++ .../search/query/QueryByIdentifier.java | 38 + .../search/query/QueryCondition.java | 20 + .../obisplugin/search/query/QueryCount.java | 63 ++ .../obisplugin/search/query/QueryType.java | 16 + .../obisplugin/search/query/ResultType.java | 15 + .../gcube/data/spd/obisplugin/util/Cache.java | 114 --- .../data/spd/obisplugin/util/DateUtil.java | 84 --- .../spd/obisplugin/util/RegExpDateFormat.java | 38 - .../gcube/data/spd/obisplugin/util/Util.java | 89 --- .../org/gcube/data/obisplugin/ObisTest.java | 64 +- src/test/resources/log4j.properties | 10 + 34 files changed, 1213 insertions(+), 2366 deletions(-) create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/Constants.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/capabilities/OccurrencesCapabilityImpl.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/ProductKey.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryIterator.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCondition.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java create mode 100644 src/main/java/org/gcube/data/spd/obisplugin/search/query/ResultType.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java delete mode 100644 src/main/java/org/gcube/data/spd/obisplugin/util/Util.java create mode 100644 src/test/resources/log4j.properties diff --git a/pom.xml b/pom.xml index 206e8fd..947fcfe 100644 --- a/pom.xml +++ b/pom.xml @@ -17,36 +17,56 @@ ${project.basedir}/distro + 2.4.4 - - - org.gcube.core - gcf - LATEST - + + + + org.gcube.distribution + gcube-bom + LATEST + pom + import + + + + org.gcube.data.spd spd-plugin-framework [3.0.0-SNAPSHOT, 4.0.0-SNAPSHOT) + + + + org.projectlombok + lombok + 1.16.2 + provided + - postgresql - postgresql - 8.4-702.jdbc4 - compile + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} - joda-time - joda-time - 2.1 + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} - - org.gcube.core - common-encryption - [1.0.2-SNAPSHOT,2.0.0-SNAPSHOT) + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + com.sun.jersey + jersey-client + 1.19 @@ -56,6 +76,18 @@ test + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + 1.7.4 + test + + diff --git a/src/main/java/org/gcube/data/spd/obisplugin/Constants.java b/src/main/java/org/gcube/data/spd/obisplugin/Constants.java new file mode 100644 index 0000000..8d9781f --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/Constants.java @@ -0,0 +1,21 @@ +package org.gcube.data.spd.obisplugin; + + +public class Constants { + + //public static final String BASE_URL = "http://api.gbif.org/v1"; + + public static final int QUERY_LIMIT = 50; +/* + public static final QName GBIFKEY_ATTR= new QName("gbifKey"); + public static final QName ABOUT_ATTR= new QName("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "about"); + public static final QName RESOURCE_ATTR= new QName("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "resource"); + public static final QName TOTAL_MATCHED_ATTR = new QName("totalMatched"); + + + + public static final String CHILD_RELATIONSHIP_VALUE = "http://rs.tdwg.org/ontology/voc/TaxonConcept#IsChildTaxonOf"; +*/ + public static final String REPOSITORY_NAME="OBIS"; + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java deleted file mode 100644 index 9c85678..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin; - -import java.sql.SQLException; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.log4j.Logger; -import org.gcube.data.spd.model.Conditions; -import org.gcube.data.spd.model.Condition; -import org.gcube.data.spd.model.exceptions.ExternalRepositoryException; -import org.gcube.data.spd.model.exceptions.IdNotValidException; -import org.gcube.data.spd.model.products.TaxonomyItem; -import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; -import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability; -import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; -import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class ObisClassification extends ClassificationCapability { - - protected Logger logger = Logger.getLogger(ObisClassification.class); - protected PluginSessionPool sessionPool; - - /** - * @param sessionPool - */ - public ObisClassification(PluginSessionPool sessionPool) { - this.sessionPool = sessionPool; - } - - /** - * {@inheritDoc} - */ - @Override - public Set getSupportedProperties() { - return Collections.emptySet(); - } - - - /** - * {@inheritDoc} - */ - @Override - public void searchByScientificName(String word, final ObjectWriter writer, Condition... properties) { - PluginSession session = sessionPool.checkOut(); - - try { - ObisClient.getTaxonByScientificNames(session, word, new Writer() { - - @Override - public boolean write(TaxonomyItem item) { - writer.write(item); - return writer.isAlive(); - } - }); - } catch (Exception e) { - logger.error("Error retrieving taxon with word \""+word+"\"", e); - } finally { - sessionPool.checkIn(session); - } - - } - - /** - * {@inheritDoc} - */ - @Override - public List retrieveTaxonChildrenByTaxonId(String taxonId) throws IdNotValidException, ExternalRepositoryException { - PluginSession session = sessionPool.checkOut(); - - try { - int id = converId(taxonId); - return ObisClient.getChildrenTaxon(session, id); - } catch (SQLException e) { - logger.error("Error retrieving TaxonChildsByTaxonId", e); - throw new ExternalRepositoryException(e); - } finally { - sessionPool.checkIn(session); - } - } - - /** - * {@inheritDoc} writer.put(new StreamException()); - */ - @Override - public void retrieveTaxonByIds(Iterator reader, ClosableWriter writer) { - PluginSession session = sessionPool.checkOut(); - - try { - while(reader.hasNext() && writer.isAlive()) { - try { - String taxonId = reader.next(); - int id = converId(taxonId); - TaxonomyItem item = ObisClient.getTaxonById(session, id); - writer.write(item); - } catch (Exception e) { - logger.error("Error retrieving TaxonById", e); - } - } - } finally { - sessionPool.checkIn(session); - } - - } - - @Override - public TaxonomyItem retrieveTaxonById(String taxonId) throws IdNotValidException { - PluginSession session = sessionPool.checkOut(); - int id = converId(taxonId); - try { - TaxonomyItem item = ObisClient.getTaxonById(session, id); - return item; - } catch (IdNotValidException inve) - { - logger.error("Error retrieving TaxonById", inve); - throw inve; - } catch (Exception e) { - logger.error("Error retrieving TaxonById", e); - return null; - } finally { - sessionPool.checkIn(session); - } - } - - protected int converId(String taxonId) throws IdNotValidException - { - try { - return Integer.parseInt(taxonId); - } catch(NumberFormatException nfe) - { - logger.error("Invalid id "+taxonId, nfe); - throw new IdNotValidException(); - } - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java deleted file mode 100644 index a849531..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java +++ /dev/null @@ -1,682 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import org.gcube.common.core.utils.logging.GCUBELog; -import org.gcube.data.spd.model.BasisOfRecord; -import org.gcube.data.spd.model.CommonName; -import org.gcube.data.spd.model.exceptions.IdNotValidException; -import org.gcube.data.spd.model.products.DataProvider; -import org.gcube.data.spd.model.products.DataSet; -import org.gcube.data.spd.model.products.OccurrencePoint; -import org.gcube.data.spd.model.products.Product; -import org.gcube.data.spd.model.products.ResultItem; -import org.gcube.data.spd.model.products.Taxon; -import org.gcube.data.spd.model.products.TaxonomyItem; -import org.gcube.data.spd.model.products.TaxonomyStatus; -import org.gcube.data.spd.model.products.Product.ProductType; -import org.gcube.data.spd.model.products.TaxonomyStatus.Status; -import org.gcube.data.spd.model.util.ElementProperty; -import org.gcube.data.spd.obisplugin.data.ProductKey; -import org.gcube.data.spd.obisplugin.data.SearchFilters; -import org.gcube.data.spd.obisplugin.util.Cache; -import org.gcube.data.spd.obisplugin.util.DateUtil; -import org.gcube.data.spd.obisplugin.util.Util; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class ObisClient { - - protected static GCUBELog logger = new GCUBELog(ObisClient.class); - //"2009-12-11 11:30:00-07" - protected static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("y-M-d"); - protected static final DateUtil DATE_UTIL = DateUtil.getInstance(); - protected static final SimpleDateFormat sdf = new SimpleDateFormat(); - - public static final Cache taxonomyItemCache = new Cache(1000); - - public static int getOccurrencesCount(PluginSession session, int taxonId, int datasetId, SearchFilters filters) throws SQLException - { - PreparedStatement statement = session.getOccurrencesCountPreparedStatement(taxonId, datasetId, filters); - - ResultSet rs = statement.executeQuery(); - - if (!rs.next()) return 0; - int occurrences = rs.getInt("occurrences"); - rs.close(); - - return occurrences; - } - - public static void getOccurrences(PluginSession session, String key, Writer writer) throws Exception - { - - ProductKey productKey = ProductKey.deserialize(key); - - PreparedStatement statement = session.getOccurrencesPreparedStatement(productKey.getTaxonId(), productKey.getDataSetId(), productKey.getFilters()); - ResultSet rs = statement.executeQuery(); - - String credits = generateCredits(); - //String citation = generateCitation(); - - DataSet dataset = getDataSetById(session, productKey.getDataSetId()); - - boolean continueWrite = true; - while(rs.next() && continueWrite) { - OccurrencePoint occurrence = generateOccurrencePoint(rs, credits, dataset); - continueWrite = writer.write(occurrence); - - }; - - rs.close(); - } - - private static DataSet getDataSetById(PluginSession session, int datasetId){ - logger.debug("retrieving dataset for id "+datasetId); - DataSet dataset = null; - ResultSet datasetRs = null; - try{ - PreparedStatement datasetStatement = session.getDatasetPerDatasetIdPreparedStatement(datasetId); - datasetRs = datasetStatement.executeQuery(); - if (!datasetRs.next()) return null; - dataset = retrieveDatasetInformation(datasetRs); - }catch(Exception e){ - logger.warn("cannot retrieve dataset for occurrence point with id "+datasetId,e); - }finally{ - try{ - if (datasetRs!=null) - datasetRs.close(); - }catch(Exception e) {} - } - logger.debug("dataset returned "+dataset); - return dataset; - } - - protected static OccurrencePoint generateOccurrencePoint(ResultSet rs, String credits, DataSet dataSet) throws SQLException - { - int id = rs.getInt("id"); - OccurrencePoint occurrence = new OccurrencePoint(String.valueOf(id)); - - //drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, - occurrence.setDecimalLatitude(rs.getDouble("latitude")); - occurrence.setDecimalLongitude(rs.getDouble("longitude")); - - Timestamp dateValue = rs.getTimestamp("datecollected"); - if (dateValue!=null) { - Calendar dateCollected = Calendar.getInstance(); - dateCollected.setTimeInMillis(dateValue.getTime()); - occurrence.setEventDate(dateCollected); - } else { - //dxs.yearcollected, dxs.monthcollected, dxs.daycollected - try { - Calendar dateCollected = Calendar.getInstance(); - int year = Integer.parseInt(rs.getString("yearcollected")); - int month = Integer.parseInt(rs.getString("monthcollected")); - int date = Integer.parseInt(rs.getString("daycollected")); - dateCollected.set(year, month, date); - occurrence.setEventDate(dateCollected); - } catch(NumberFormatException nfe){} - } - - String basisOfRecord = rs.getString("basisofrecord"); - occurrence.setBasisOfRecord(getBasisOfRecord(basisOfRecord)); - - //dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector - occurrence.setCitation(rs.getString("citation")); - occurrence.setCredits(credits); - occurrence.setInstitutionCode(rs.getString("institutioncode")); - occurrence.setCollectionCode(rs.getString("collectioncode")); - occurrence.setCatalogueNumber(rs.getString("catalognumber")); - occurrence.setRecordedBy(rs.getString("collector")); - - //, dxs.datelastmodified, - - String datelastmodified = rs.getString("datelastmodified"); - if (datelastmodified!=null) { - - try { - java.util.Date date = DATE_UTIL.parse(datelastmodified); - Calendar lastmodified = Calendar.getInstance(); - lastmodified.setTimeInMillis(date.getTime()); - occurrence.setModified(lastmodified); - } catch (Exception e) { - //logger.warn("Unknow date format "+datelastmodified); - } - - } - - //dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated - occurrence.setCountry(rs.getString("country")); - occurrence.setLocality(rs.getString("locality")); - occurrence.setMinDepth(rs.getDouble("minimumdepth")); - occurrence.setMaxDepth(rs.getDouble("maximumdepth")); - occurrence.setCoordinateUncertaintyInMeters(rs.getString("coordinateprecision")); - //"Animalia|Chordata|Chondrichthyes|Lamniformes|Lamnidae|Carcharodon||carcharias||Carcharodon carcharias|Linnaeus" - - /* - * Kingdom: Animalia - Phylum: Chordata - Class: Chondrichthyes - Subclass: Elasmobranchii - Order: Lamniformes - Family: Lamnidae - Genus: Carcharodon - A. Smith, 1838 - Species: C. carcharias - */ - - String concatenated = rs.getString("concatenated"); - if (concatenated!=null) { - int authorStartIndex = concatenated.lastIndexOf('|'); - if (authorStartIndex>0) { - String snPart = concatenated.substring(0, authorStartIndex); - int scientificNameStartIndex = snPart.lastIndexOf('|'); - if (scientificNameStartIndex>0) { - String author = (authorStartIndex+10) occurrence.setKingdom(taxon[0]); - if (taxon.length>4) occurrence.setFamily(taxon[4]); - if (taxon.length>11) { - String scientific = taxon[9] +"("+taxon[10]+")"; - occurrence.setScientificName(scientific); - }*/ - } - - occurrence.setScientificNameAuthorship(rs.getString("snAuthor")); - - occurrence.setIdentifiedBy(rs.getString("identifiedBy")); - - occurrence.setDataSet(dataSet); - - //retrieving dataset - return occurrence; - } - - public static OccurrencePoint getOccurrenceById(PluginSession session, String id) throws Exception - { - PreparedStatement statement = session.getOccurrenceByIdPreparedStatement(Integer.parseInt(id)); - ResultSet rs = statement.executeQuery(); - - OccurrencePoint occurrence = null; - String credits = generateCredits(); - if(rs.next()){ - int objId = rs.getInt("valid_id"); - occurrence = generateOccurrencePoint(rs, credits, getDataSetById(session, objId)); - } - rs.close(); - - return occurrence; - } - - - public static BasisOfRecord getBasisOfRecord(String basis) - { - if (basis==null) return BasisOfRecord.Unknown; - if (basis.equalsIgnoreCase("HumanObservation") || basis.equalsIgnoreCase("O")) return BasisOfRecord.HumanObservation; - if (basis.equalsIgnoreCase("L")) return BasisOfRecord.LivingSpecimen; - if (basis.equalsIgnoreCase("P")) return BasisOfRecord.Literature; - if (basis.equalsIgnoreCase("PreservedSpecimen") || basis.equalsIgnoreCase("S")) return BasisOfRecord.PreservedSpecimen; - return BasisOfRecord.Unknown; - - } - - public static void searchByCommonName(PluginSession session, String searchTerm, SearchFilters filters, Writer writer) throws Exception - { - - PreparedStatement statement = session.getSearchCommonNamePreparedStatement(searchTerm); - - ResultSet rs = statement.executeQuery(); - - generateResultItems(session, rs, filters, writer); - } - - protected static void fillProducts(PluginSession session, int speciesId, int datasetId, String key, SearchFilters filters, ResultItem item) throws SQLException - { - - List products = new LinkedList(); - - //OCCURRENCES - Product occurences = new Product(ProductType.Occurrence, key); - - int occurencesCount = getOccurrencesCount(session, speciesId, datasetId, filters); - occurences.setCount(occurencesCount); - products.add(occurences); - - logger.trace("product is "+occurences); - - item.setProducts(products); - } - - public static void searchByScientificName(PluginSession session, String searchTerm, SearchFilters filters, Writer writer) throws Exception - { - - PreparedStatement statement = session.getSearchScientificNamePreparedStatement(searchTerm); - - ResultSet rs = statement.executeQuery(); - - generateResultItems(session, rs, filters, writer); - } - - protected static void generateResultItems(PluginSession session, ResultSet rs, SearchFilters filters, Writer writer) throws Exception - { - //System.out.println("generating records"); - boolean continueWrite = true; - String credits = generateCredits(); - String citation = generateCitation(); - while(rs.next() && continueWrite) { - - int id = rs.getInt("id"); - //System.out.println("id "+id); - - ResultItem baseItem = new ResultItem(String.valueOf(id), ""); - - fillTaxon(session, id, baseItem, credits, citation); - - fillCommonNames(session, id, baseItem); - - - PreparedStatement datasetStatement = session.getDatasetPreparedStatement(id); - ResultSet dataSetrs = datasetStatement.executeQuery(); - - while(dataSetrs.next()) { - - ResultItem item = Util.cloneResultItem(baseItem); - - DataSet dataset = retrieveDatasetInformation(dataSetrs); - - item.setDataSet(dataset); - - int dataSetId = Integer.parseInt(dataset.getId()); - - ProductKey key = new ProductKey(id, dataSetId, filters); - - logger.trace("datasetid is "+dataSetId+" and product key created is "+key); - - fillProducts(session, id, dataSetId, key.serialize(), filters, item); - - continueWrite = writer.write(item); - } - } - rs.close(); - } - - /** - * Fills the node with the taxon information. Also information about parent are retrieved. - * @param connection the db connection. - * @param id the taxon id. - * @param taxonNode the node to fill. - * @throws SQLException - */ - protected static void fillTaxon(PluginSession session, int id, Taxon taxon, String credits, String citation) throws SQLException - { - - PreparedStatement statement = session.getTaxonPreparedStatement(id); - - ResultSet rs = statement.executeQuery(); - - if (rs.next()) { - taxon.setCitation(citation); - taxon.setCredits(credits); - - //taxon informations - taxon.setScientificName(rs.getString("tname")); - taxon.setScientificNameAuthorship(rs.getString("tauthor")); - - String rank = rs.getString("rank_name"); - taxon.setRank((rank!=null)?rank:""); - - int parentId = rs.getInt("parent_id"); - rs.close(); - - //check for parent - if (parentId!=id) { - - //create and fill the parent - Taxon parent = new Taxon(String.valueOf(parentId)); - fillTaxon(session, parentId, parent, credits, citation); - taxon.setParent(parent); - } - } - } - - protected static void fillTaxonomyItem(PluginSession session, int id, TaxonomyItem item, String credits, String citation) throws Exception - { - PreparedStatement statement = session.getTaxonPreparedStatement(id); - - ResultSet rs = statement.executeQuery(); - - if (rs.next()) { - - //taxon informations - item.setScientificName(rs.getString("tname")); - - String author = Util.stripNotValidXMLCharacters(rs.getString("tauthor")); - item.setScientificNameAuthorship(author); - - //properties - item.addProperty(new ElementProperty("worms_id", rs.getString("worms_id"))); - item.addProperty(new ElementProperty("col_id", rs.getString("col_id"))); - item.addProperty(new ElementProperty("irmng_id", rs.getString("irmng_id"))); - item.addProperty(new ElementProperty("itis_id", rs.getString("itis_id"))); - - item.setCredits(credits); - item.setCitation(citation); - - String rank = rs.getString("rank_name"); - item.setRank((rank!=null)?rank:""); - - item.setStatus(new TaxonomyStatus("", Status.ACCEPTED)); - - boolean parentNull = rs.getObject("parent_id")==null; - int parentId = rs.getInt("parent_id"); - rs.close(); - - //fill common names - fillCommonNames(session, id, item); - - //check for parent - if (!parentNull && parentId!=id) { - - //create and fill the parent - TaxonomyItem parent = taxonomyItemCache.get(parentId); - if (parent == null) { - parent = new TaxonomyItem(String.valueOf(parentId)); - fillTaxonomyItem(session, parentId, parent, credits, citation); - } - item.setParent(parent); - } - } else throw new IdNotValidException("Taxon with id "+id+" not found"); - } - - protected static void fillCommonNames(PluginSession session, int taxonNameId, TaxonomyItem item) throws SQLException - { - PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId); - ResultSet rs = statement.executeQuery(); - - List commonNames = new ArrayList(); - while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname"))); - rs.close(); - item.setCommonNames(commonNames); - - } - - protected static void fillCommonNames(PluginSession session, int taxonNameId, ResultItem item) throws SQLException - { - PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId); - ResultSet rs = statement.executeQuery(); - List commonNames = new ArrayList(); - while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname"))); - rs.close(); - item.setCommonNames(commonNames); - } - - private static DataSet retrieveDatasetInformation(ResultSet rs) throws SQLException{ - int dataSetId = rs.getInt("datasetId"); - DataSet dataSet = new DataSet(String.valueOf(dataSetId)); - dataSet.setCitation(rs.getString("datasetCitation")); - dataSet.setName(rs.getString("datasetName")); - - DataProvider dataProvider = new DataProvider(String.valueOf(rs.getInt("providerId"))); - dataProvider.setName(rs.getString("providerName")); - dataSet.setDataProvider(dataProvider); - return dataSet; - } - - public static Set getCommonNames(PluginSession session, String scientificName) throws SQLException - { - PreparedStatement statement = session.getCommonNameFromScientificNamePreparedStatement(scientificName); - ResultSet rs = statement.executeQuery(); - - Set commonNames = new HashSet(); - - while(rs.next()) commonNames.add(rs.getString("cname")); - - rs.close(); - - return commonNames; - } - - public static void getScientificNames(PluginSession session, String commonName, Writer writer) throws SQLException - { - PreparedStatement statement = session.getScientificNameFromCommonNamePreparedStatement(commonName); - ResultSet rs = statement.executeQuery(); - while (rs.next() && writer.write(rs.getString("tname"))); - rs.close(); - } - - public static void getTaxonByScientificNames(PluginSession session, String scientificName, Writer writer) throws Exception - { - PreparedStatement statement = session.getScientificNamePreparedStatement(scientificName); - ResultSet rs = statement.executeQuery(); - generateTaxonomyItems(session, rs, writer); - } - - public static void getTaxonByCommonName(PluginSession session, String commonName, Writer writer) throws Exception - { - PreparedStatement statement = session.getCommonNamePreparedStatement(commonName); - ResultSet rs = statement.executeQuery(); - generateTaxonomyItems(session, rs, writer); - } - - protected static void generateTaxonomyItems(PluginSession session, ResultSet rs, Writer writer) throws SQLException, Exception - { - boolean continueWrite = true; - String credits = generateCredits(); - String citation = generateCitation(); - while(rs.next() && continueWrite) { - - Integer id = rs.getInt("id"); - - TaxonomyItem taxon = taxonomyItemCache.get(id); - if (taxon == null) { - taxon = new TaxonomyItem(String.valueOf(id)); - fillTaxonomyItem(session, id, taxon, credits, citation); - taxonomyItemCache.put(id, taxon); - } - - //TaxonomyItem taxon = new TaxonomyItem(String.valueOf(id)); - //fillTaxonomyItem(session, id, taxon); - continueWrite = writer.write(taxon); - } - - rs.close(); - } - - protected static List getChildrenTaxon(PluginSession session, int id) throws SQLException - { - PreparedStatement statement = session.getChildrenTaxonPreparedStatement(id); - - ResultSet rs = statement.executeQuery(); - - List children = new ArrayList(); - while (rs.next()) { - //taxon informations - int taxonId = rs.getInt("id"); - - //FIXME tmp workaround - if (taxonId == id) continue; - - TaxonomyItem child = new TaxonomyItem(String.valueOf(taxonId)); - child.setScientificName(rs.getString("tname")); - child.setCitation(rs.getString("tauthor")); - String rank = rs.getString("rank_name"); - child.setRank((rank!=null)?rank:""); - - child.setStatus(new TaxonomyStatus("",Status.ACCEPTED)); - - fillCommonNames(session, id, child); - - children.add(child); - } - rs.close(); - return children; - } - - protected static TaxonomyItem getTaxonById(PluginSession session, int id) throws Exception - { - TaxonomyItem item = new TaxonomyItem(String.valueOf(id)); - String credits = generateCredits(); - String citation = generateCitation(); - fillTaxonomyItem(session, id, item, credits, citation); - return item; - } - - protected static String generateCitation() - { - StringBuilder citation = new StringBuilder("Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org. (Consulted on "); - citation.append(sdf.format(Calendar.getInstance().getTime())); - citation.append(")"); - return citation.toString(); - } - - protected static String generateCredits() - { - //credits ="This information object has been generated via the Species Product Discovery service on 2012-11-26 by interfacing with the Interim Register of Marine and Nonmarine Genera (IRMNG) (http://www.obis.org.au/irmng/)"; - StringBuilder credits = new StringBuilder("This information object has been generated via the Species Product Discovery service on "); - credits.append(sdf.format(Calendar.getInstance().getTime())); - credits.append(" by interfacing with the Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org."); - return credits.toString(); - } - - static int counter = 0; - static int sum = 0; - - /** - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { - String url = "jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis"; - Properties props = new Properties(); - props.setProperty("user","postgres"); - props.setProperty("password","0b1s@d4sc13nc3"); - - - final Connection connection = DriverManager.getConnection(url, props); - - // System.out.println("Connected"); - - final PluginSession session = new PluginSession(connection); - session.preCacheStatements(); - - searchByScientificName(session, "gadus morhua", new SearchFilters(), new Writer() { - - @Override - public boolean write(ResultItem item) { - // System.out.println(item.getId()+" "+item.getScientificNameAuthorship()+" "+item.getScientificName()); - return true; - } - }); - - - - - /*getTaxonByScientificNames(session, "sarda sarda", new Writer() { - - @Override - public boolean write(TaxonomyItem item) { - System.out.println(item.getId()+" "+item.getAuthor()+" "+item.getScientificName()); - return true; - } - });*/ - - /*OccurrencePoint occurrencePoint = getOccurrenceById(session, "38069270"); - System.out.println(occurrencePoint);*/ - - - /*Taxon taxon = getTaxonByCommonName(session, "ruwe traliehoorn");//getTaxonByScientificNames(session, "Protozoa"); - System.out.println(taxon); - - List children = getChildrenTaxon(session, Integer.parseInt(taxon.getId())); - for (Taxon child:children) System.out.println(child);*/ - - /*searchByCommonName(session, "white shark", new SearchFilters(), new Writer() { - - @Override - public void write(ResultItem item) { - System.out.println("Item: "+item.getDataSet().getDataProvider().getName()+" <-> "+item.getDataSet().getName()); - } - });*/ - - /*final long start = System.currentTimeMillis(); - getTaxonByScientificNames(session, "Gadus macrocephalus", new Writer() { - long start = System.currentTimeMillis(); - - @Override - public void write(TaxonomyItem item) { - - - System.out.println(item); - - } - });*/ - - /*SearchFilters filters = new SearchFilters(); - ObisClient.searchByScientificName(session, "sarda sarda", filters, new Writer() { - - @Override - public void write(ResultItem item) { - System.out.println(item); - } - });*/ - - /*List taxa = getChildrenTaxon(session,769809); - for (TaxonomyItem taxon:taxa) System.out.println(taxon.getId());*/ - //navigate("", session, 741923); - - //System.out.println("result in "+(System.currentTimeMillis()-start)+" avg: "+(sum/counter)+" tot: "+counter); - - // System.out.println("done"); - session.expire(); - } - - protected static Set found = new HashSet(); - protected static Map foundTaxon = new HashMap(); - - protected static void navigate(String indentation, PluginSession session, int id) throws SQLException - { - //System.out.println("looking for children: "+id); - if (found.contains(id)) { - System.err.println("Already found "+id); - System.err.println(foundTaxon.get(id)); - System.exit(-1); - } - - List taxa = getChildrenTaxon(session,id); - - found.add(id); - - - for (TaxonomyItem taxon:taxa) { - System.out.println(indentation+taxon.getId()+" "+taxon.getRank()); - foundTaxon.put(taxon.getId(), taxon); - navigate(indentation+" ", session, Integer.valueOf(taxon.getId())); - - } - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java deleted file mode 100644 index 23f9f53..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin; - -import java.sql.SQLException; -import org.apache.log4j.Logger; -import org.gcube.data.spd.model.exceptions.StreamBlockingException; -import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; -import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability; -import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class ObisNameMapping implements MappingCapability { - - protected Logger logger = Logger.getLogger(ObisNameMapping.class); - protected PluginSessionPool sessionPool; - - /** - * @param sessionPool - */ - public ObisNameMapping(PluginSessionPool sessionPool) { - this.sessionPool = sessionPool; - } - - - /** - * {@inheritDoc} - */ - @Override - public void getRelatedScientificNames(final ObjectWriter writer, String commonName) { - logger.debug("retrieving mapping for "+commonName); - PluginSession session = sessionPool.checkOut(); - try { - ObisClient.getScientificNames(session, commonName, new Writer() { - - @Override - public boolean write(String item) { - writer.write(item); - return writer.isAlive(); - } - }); - } catch (SQLException e) { - logger.error("An error occurred retrieving the mapping for common name "+commonName, e); - writer.write(new StreamBlockingException("OBIS",commonName)); - } finally { - sessionPool.checkIn(session); - } - - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java deleted file mode 100644 index e0829cd..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import org.apache.log4j.Logger; -import org.gcube.data.spd.model.Conditions; -import org.gcube.data.spd.model.Condition; -import org.gcube.data.spd.model.products.OccurrencePoint; -import org.gcube.data.spd.model.products.Product; -import org.gcube.data.spd.model.products.Product.ProductType; -import org.gcube.data.spd.model.products.ResultItem; -import org.gcube.data.spd.obisplugin.data.SearchFilters; -import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; -import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability; -import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; -import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class ObisOccurrencesInterface extends OccurrencesCapability { - - protected static final Set SUPPORTED_PROPERTIES = new HashSet(Arrays.asList(Conditions.values())); - protected Logger logger = Logger.getLogger(ObisOccurrencesInterface.class); - - protected PluginSessionPool sessionPool; - - /** - * @param sessionPool - */ - public ObisOccurrencesInterface(PluginSessionPool sessionPool) { - this.sessionPool = sessionPool; - } - - /** - * {@inheritDoc} - */ - @Override - public Set getSupportedProperties() { - return SUPPORTED_PROPERTIES; - } - - @Override - public void searchByScientificName(String word, final ObjectWriter writer, Condition... properties) { - final PluginSession session = sessionPool.checkOut(); - try { - SearchFilters filters = new SearchFilters(properties); - logger.trace("filters: "+filters); - ObisClient.searchByScientificName(session, word, filters, new Writer() { - - @Override - public boolean write(ResultItem item) { - for (Product product:item.getProducts()){ - if (product.getType()==ProductType.Occurrence) { - String key = product.getKey(); - getOccurrencePoints(session, key, writer); - } - } - return writer.isAlive(); - } - }); - } catch (Exception e) { - logger.debug("searchByScientificName failed",e); - } finally { - sessionPool.checkIn(session); - } - - } - - - @Override - public void getOccurrencesByIds(ClosableWriter writer, Iterator reader) { - final PluginSession session = sessionPool.checkOut(); - try { - while(reader.hasNext() && writer.isAlive()){ - String id = reader.next(); - try { - OccurrencePoint occurrencePoint = ObisClient.getOccurrenceById(session, id); - if (occurrencePoint!=null) writer.write(occurrencePoint); - } catch (Exception e) { - logger.debug("searchByScientificName failed",e); - } - } - - writer.close(); - } finally { - sessionPool.checkIn(session); - } - - } - - - @Override - public void getOccurrencesByProductKeys(ClosableWriter writer, Iterator reader) { - PluginSession session = sessionPool.checkOut(); - - try { - while(reader.hasNext() && writer.isAlive()){ - String key = reader.next(); - getOccurrencePoints(session, key, writer); - } - writer.close(); - } finally { - sessionPool.checkIn(session); - } - } - - protected void getOccurrencePoints(PluginSession session, String key, final ObjectWriter writer) - { - try { - ObisClient.getOccurrences(session, key, new Writer() { - - @Override - public boolean write(OccurrencePoint item) { - writer.write(item); - return writer.isAlive(); - } - }); - } catch (Exception e) { - logger.error("Error getting occurrence points for key "+key, e); - } - } - - - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java index cf82e09..eb74696 100644 --- a/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java @@ -5,23 +5,21 @@ import java.util.Calendar; import java.util.HashSet; import java.util.Set; -import org.gcube.common.core.utils.logging.GCUBELog; import org.gcube.common.resources.gcore.ServiceEndpoint; -import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; -import org.gcube.common.encryption.StringEncrypter; -import org.gcube.data.spd.model.Conditions; import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.Conditions; import org.gcube.data.spd.model.RepositoryInfo; import org.gcube.data.spd.model.products.ResultItem; import org.gcube.data.spd.model.util.Capabilities; -import org.gcube.data.spd.obisplugin.data.SearchFilters; -import org.gcube.data.spd.obisplugin.pool.DatabaseCredential; -import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; +import org.gcube.data.spd.obisplugin.capabilities.OccurrencesCapabilityImpl; +import org.gcube.data.spd.obisplugin.search.ResultItemSearch; import org.gcube.data.spd.plugin.fwk.AbstractPlugin; import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability; import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability; import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability; import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author "Federico De Faveri defaveri@isti.cnr.it" @@ -38,33 +36,27 @@ public class ObisPlugin extends AbstractPlugin { protected static final RepositoryInfo REPOSITORY_INFO = new RepositoryInfo(LOGO_URL, HOME_URL, DESCRIPTION); protected static final String ENTRY_POINT_NAME = "jdbc"; - protected GCUBELog logger = new GCUBELog(ObisPlugin.class); - protected PluginSessionPool sessionPool; - protected ObisNameMapping nameMapping; - protected ObisOccurrencesInterface occurrencesInterface; - protected ObisClassification obisClassification; + protected static Logger logger = LoggerFactory.getLogger(ObisPlugin.class); + //protected ObisNameMapping nameMapping; + protected OccurrencesCapabilityImpl occurrencesInterface; + //protected ObisClassification obisClassification; protected static final SimpleDateFormat sdf = new SimpleDateFormat(); - /** - * @return the sessionPool - */ - public PluginSessionPool getSessionPool() { - return sessionPool; - } + private String baseUrl = "http://api.iobis.org/"; @SuppressWarnings("serial") @Override public Set getSupportedCapabilities() { return new HashSet(){{ - add(Capabilities.NamesMapping); + //add(Capabilities.NamesMapping); add(Capabilities.Occurrence); - add(Capabilities.Classification); + //add(Capabilities.Classification); }}; } @Override public String getRepositoryName() { - return "OBIS"; + return Constants.REPOSITORY_NAME; } @Override @@ -80,20 +72,9 @@ public class ObisPlugin extends AbstractPlugin { setUseCache(true); - DatabaseCredential databaseCredential = getDatabaseCredentials(resource); - sessionPool = new PluginSessionPool(databaseCredential); - nameMapping = new ObisNameMapping(sessionPool); - occurrencesInterface = new ObisOccurrencesInterface(sessionPool); - obisClassification = new ObisClassification(sessionPool); - } - - public void initialize(DatabaseCredential databaseCredential) throws Exception { - - setUseCache(true); - sessionPool = new PluginSessionPool(databaseCredential); - nameMapping = new ObisNameMapping(sessionPool); - occurrencesInterface = new ObisOccurrencesInterface(sessionPool); - obisClassification = new ObisClassification(sessionPool); + //nameMapping = new ObisNameMapping(); + occurrencesInterface = new OccurrencesCapabilityImpl(baseUrl); + //obisClassification = new ObisClassification(); } /** @@ -101,7 +82,6 @@ public class ObisPlugin extends AbstractPlugin { */ @Override public void shutdown() throws Exception { - sessionPool.shutdown(true); } /** @@ -121,29 +101,8 @@ public class ObisPlugin extends AbstractPlugin { */ @Override public void update(ServiceEndpoint resource) throws Exception { - DatabaseCredential databaseCredential = getDatabaseCredentials(resource); - sessionPool.setDatabaseCredential(databaseCredential); } - protected DatabaseCredential getDatabaseCredentials(ServiceEndpoint resource) throws Exception - { - AccessPoint jdbcAccessPoint = null; - for (AccessPoint accessPoint: resource.profile().accessPoints()) - { - if (ENTRY_POINT_NAME.equalsIgnoreCase(accessPoint.name())) { - jdbcAccessPoint = accessPoint; - break; - } - } - - if (jdbcAccessPoint==null) { - logger.error("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource"); - throw new Exception("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource"); - } - String password = StringEncrypter.getEncrypter().decrypt(jdbcAccessPoint.password()); - return new DatabaseCredential(jdbcAccessPoint.address(), jdbcAccessPoint.username(), password); - } - /** * {@inheritDoc} @@ -152,26 +111,11 @@ public class ObisPlugin extends AbstractPlugin { public void searchByScientificName(String searchTerm, final ObjectWriter writer, Condition... properties) { logger.debug("starting the search for obisPlugin word: "+searchTerm); - PluginSession session = sessionPool.checkOut(); try { - final String credits = getObisCredits(); - SearchFilters filters = new SearchFilters(properties); - logger.trace("filters: "+filters); - ObisClient.searchByScientificName(session, searchTerm, filters, new Writer() { - - @Override - public boolean write(ResultItem item) { - item.setCredits(credits); - writer.write(item); - return writer.isAlive(); - } - }); - + new ResultItemSearch(baseUrl, searchTerm, properties).search(writer,Constants.QUERY_LIMIT); } catch (Exception e) { logger.debug("searchByScientificName failed",e); - } finally { - sessionPool.checkIn(session); - } + } } /** @@ -179,7 +123,7 @@ public class ObisPlugin extends AbstractPlugin { */ @Override public MappingCapability getMappingInterface() { - return nameMapping; + return null; } /** @@ -203,7 +147,7 @@ public class ObisPlugin extends AbstractPlugin { */ @Override public ClassificationCapability getClassificationInterface() { - return obisClassification; + return null; } /** diff --git a/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java b/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java deleted file mode 100644 index 4703517..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java +++ /dev/null @@ -1,429 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin; - -import java.sql.Connection; -import java.sql.Date; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Calendar; -import java.util.List; - -import org.gcube.data.spd.model.Condition; -import org.gcube.data.spd.model.Coordinate; -import org.gcube.data.spd.obisplugin.data.SearchFilters; -import org.gcube.data.spd.obisplugin.pool.DatabaseCredential; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class PluginSession { - - protected static final String SCHEMA = "obis"; - - protected DatabaseCredential databaseCredential; - protected Connection connection; - protected PreparedStatement taxonPreparedStatement; - protected PreparedStatement taxonCommonNamePreparedStatement; - protected PreparedStatement datasetPreparedStatement; - protected PreparedStatement commonNameFromScientificNamePreparedStatement; - protected PreparedStatement scientificNameFromCommonNamePreparedStatement; - protected PreparedStatement searchCommonNamePreparedStatement; - protected PreparedStatement searchScientificNamePreparedStatement; - protected PreparedStatement scientificNamePreparedStatement; - protected PreparedStatement commonNamePreparedStatement; - protected PreparedStatement childrenTaxonPreparedStatement; - protected PreparedStatement occurrenceByIdPreparedStatement; - protected PreparedStatement datasetPerIdPreparedStatement; - - /** - * @param connection - */ - public PluginSession(Connection connection) { - this(null,connection); - } - /** - * @param credentialToken - * @param connection - */ - public PluginSession(DatabaseCredential databaseCredential, Connection connection) { - this.databaseCredential = databaseCredential; - this.connection = connection; - } - - /** - * @return the connection - */ - public Connection getConnection() { - return connection; - } - - /** - * The session is valid if the connection is OK and if the connection has been created using the same credentials. - * @param credentialToken - * @return - * @throws SQLException - */ - public boolean isValid(DatabaseCredential databaseCredential) throws SQLException - { - return (this.databaseCredential!=null?(this.databaseCredential.equals(databaseCredential)):true) && !connection.isClosed() && isValid(); - } - - protected boolean isValid() - { - try{ - ResultSet result = connection.createStatement().executeQuery("SELECT 1"); - result.close(); - }catch (Exception e) { - return false; - } - return true; - } - - public void expire() throws SQLException - { - connection.close(); - } - - public void preCacheStatements() throws SQLException - { - createTaxonPreparedStatement(); - createTaxonCommonNamePreparedStatemen(); - createDatasetPreparedStatement(); - createCommonNameFromScientificNamePreparedStatement(); - createScientificNameFromCommonNamePreparedStatement(); - createSearchCommonNamePreparedStatement(); - createSearchScientificNamePreparedStatement(); - createScientificNamePreparedStatement(); - createCommonNamePreparedStatement(); - createChildrenTaxonPreparedStatement(); - createOccurrenceByIdPreparedStatement(); - createDatasetPerDatasetIdPreparedStatement(); - } - - - public PreparedStatement getTaxonPreparedStatement(int id) throws SQLException - { - if (taxonPreparedStatement==null) createTaxonPreparedStatement(); - taxonPreparedStatement.clearParameters(); - taxonPreparedStatement.setInt(1, id); - return taxonPreparedStatement; - } - - protected void createTaxonPreparedStatement() throws SQLException - { - taxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id, t.parent_id, t.tauthor, t.worms_id, t.col_id, t.irmng_id, t.itis_id, r.rank_name " + - "FROM "+SCHEMA+".tnames t " + - "LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+ - "WHERE t.id = ?"); - } - - - public PreparedStatement getTaxonCommonNamePreparedStatement(int taxonNameId) throws SQLException - { - if (taxonCommonNamePreparedStatement==null) createTaxonCommonNamePreparedStatemen(); - taxonCommonNamePreparedStatement.clearParameters(); - taxonCommonNamePreparedStatement.setInt(1, taxonNameId); - return taxonCommonNamePreparedStatement; - } - - protected void createTaxonCommonNamePreparedStatemen() throws SQLException - { - taxonCommonNamePreparedStatement = connection.prepareStatement("select c.cname, l.lanname FROM "+SCHEMA+".cnames c, "+SCHEMA+".languages l WHERE c.tname_id = ? AND c.language_id = l.id"); - } - - public PreparedStatement getDatasetPreparedStatement(int id) throws SQLException - { - if (datasetPreparedStatement==null) createDatasetPreparedStatement(); - datasetPreparedStatement.clearParameters(); - datasetPreparedStatement.setInt(1, id); - return datasetPreparedStatement; - } - - public PreparedStatement getDatasetPerDatasetIdPreparedStatement(int dataSetId) throws SQLException - { - if (datasetPerIdPreparedStatement==null) createDatasetPerDatasetIdPreparedStatement(); - datasetPerIdPreparedStatement.clearParameters(); - datasetPerIdPreparedStatement.setInt(1, dataSetId); - return datasetPerIdPreparedStatement; - } - - public PreparedStatement getCommonNameFromScientificNamePreparedStatement(String scientificaName) throws SQLException - { - if (commonNameFromScientificNamePreparedStatement==null) createCommonNameFromScientificNamePreparedStatement(); - commonNameFromScientificNamePreparedStatement.clearParameters(); - commonNameFromScientificNamePreparedStatement.setString(1, scientificaName); - return commonNameFromScientificNamePreparedStatement; - } - - protected void createCommonNameFromScientificNamePreparedStatement() throws SQLException - { - commonNameFromScientificNamePreparedStatement = connection.prepareStatement("SELECT c.cname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE t.tname ILIKE ? AND c.tname_id = t.id"); - } - - public PreparedStatement getScientificNameFromCommonNamePreparedStatement(String commonName) throws SQLException - { - if (scientificNameFromCommonNamePreparedStatement==null) createScientificNameFromCommonNamePreparedStatement(); - scientificNameFromCommonNamePreparedStatement.clearParameters(); - scientificNameFromCommonNamePreparedStatement.setString(1, commonName); - return scientificNameFromCommonNamePreparedStatement; - } - - protected void createScientificNameFromCommonNamePreparedStatement() throws SQLException - { - scientificNameFromCommonNamePreparedStatement = connection.prepareStatement("SELECT DISTINCT t.tname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE c.cname ILIKE ? AND c.tname_id = t.id"); - } - - public PreparedStatement getSearchCommonNamePreparedStatement(String searchTerm) throws SQLException - { - if (searchCommonNamePreparedStatement == null) createSearchCommonNamePreparedStatement(); - searchCommonNamePreparedStatement.clearParameters(); - searchCommonNamePreparedStatement.setString(1, "%"+searchTerm+"%"); - return searchCommonNamePreparedStatement; - } - - protected void createSearchCommonNamePreparedStatement() throws SQLException - { - String query = "SELECT DISTINCT c.tname_id AS id FROM obis.cnames c WHERE c.cname ILIKE ?"; - searchCommonNamePreparedStatement = connection.prepareStatement(query); - } - - public PreparedStatement getSearchScientificNamePreparedStatement(String searchTerm) throws SQLException - { - if (searchScientificNamePreparedStatement == null) createSearchScientificNamePreparedStatement(); - searchScientificNamePreparedStatement.clearParameters(); - searchScientificNamePreparedStatement.setString(1, "%"+searchTerm+"%"); - return searchScientificNamePreparedStatement; - } - - protected void createSearchScientificNamePreparedStatement() throws SQLException - { - String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ? AND exists (SELECT 1 FROM obis.drs WHERE valid_id = t.id)"; - searchScientificNamePreparedStatement = connection.prepareStatement(query); - } - - public PreparedStatement getOccurrencesCountPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException{ - - //We don't cache it because in this case a PS is less performant - StringBuilder query = new StringBuilder("SELECT count(*) AS occurrences FROM "+SCHEMA+".drs WHERE valid_id = ? AND resource_id = ?"); - - buildConditions(query, "", filters.getConditions()); - - /*if (filters.getUpperBound()!=null) query.append(" AND latitude <= ? AND longitude <= ?"); - if (filters.getLowerBound()!=null) query.append(" AND latitude >= ? AND longitude >= ?"); - if (filters.getFromDate()!=null) query.append(" AND datecollected >= ?"); - if (filters.getToDate()!=null) query.append(" AND datecollected <= ?");*/ - - PreparedStatement occurrencesCountPreparedStatement = connection.prepareStatement(query.toString()); - - int parameterCounter = 1; - occurrencesCountPreparedStatement.setInt(parameterCounter++, taxonId); - occurrencesCountPreparedStatement.setInt(parameterCounter++, datasetId); - - addParameters(occurrencesCountPreparedStatement, parameterCounter, filters.getConditions()); - - /*for (Condition condition:filters.getConditions()) { - switch (condition.getProperty()) { - case COORDINATE: { - Coordinate coordinate = (Coordinate) condition.getValue(); - occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLatitude()); - occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLongitude()); - } break; - case EVENT_DATE: { - Calendar calendar = (Calendar) condition.getValue(); - occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis())); - } - } - }*/ - - /*if (filters.getUpperBound() != null) { - occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude()); - occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude()); - } - - if (filters.getLowerBound() != null) { - occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude()); - occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude()); - } - - if (filters.getFromDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis())); - if (filters.getToDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/ - - return occurrencesCountPreparedStatement; - } - - protected void buildConditions(StringBuilder query, String prefix, List conditions) - { - for (Condition condition:conditions) buildCondition(query, prefix, condition); - } - - protected void buildCondition(StringBuilder query, String prefix, Condition condition) - { - String op = ""; - switch (condition.getOp()) { - case EQ: op = "=="; break; - case GE: op = ">="; break; - case GT: op = ">"; break; - case LE: op = "<="; break; - case LT: op = "<"; break; - } - - switch (condition.getType()) { - case COORDINATE: { - query.append(" AND "); - query.append(prefix); - query.append("latitude "); - query.append(op); - query.append(" ? AND "); - query.append(prefix); - query.append("longitude "); - query.append(op); - query.append(" ?"); - } break; - case DATE: { - query.append(" AND "); - query.append(prefix); - query.append("datecollected "); - query.append(op); - query.append(" ?"); - } break; - } - } - - protected void addParameters(PreparedStatement preparedStatement, int parameterCounter, List conditions) throws SQLException { - for (Condition condition:conditions) { - switch (condition.getType()) { - case COORDINATE: { - Coordinate coordinate = (Coordinate) condition.getValue(); - preparedStatement.setDouble(parameterCounter++, coordinate.getLatitude()); - preparedStatement.setDouble(parameterCounter++, coordinate.getLongitude()); - } break; - case DATE: { - Calendar calendar = (Calendar) condition.getValue(); - preparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis())); - } - } - } - } - - protected PreparedStatement getOccurrencesPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException - { - //We don't cache it because in this case a PS is less performant - StringBuilder query = new StringBuilder("SELECT drs.id, drs.valid_id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.identifiedBy, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames tn WHERE drs.valid_id = ? AND drs.resource_id = ? AND drs.id = dxs.dr_id AND tn.id = drs.valid_id "); - - buildConditions(query, "drs.", filters.getConditions()); - - /*if (filters.getUpperBound() != null) query.append(" AND drs.latitude <= ? AND drs.longitude <= ?"); - if (filters.getLowerBound() != null) query.append(" AND drs.latitude >= ? AND drs.longitude >= ?"); - if (filters.getFromDate() != null) query.append(" AND drs.datecollected >= ?"); - if (filters.getToDate() != null) query.append(" AND drs.datecollected <= ?");*/ - - PreparedStatement occurrencesPreparedStatement = connection.prepareStatement(query.toString()); - - int parameterCounter = 1; - occurrencesPreparedStatement.setInt(parameterCounter++, taxonId); - occurrencesPreparedStatement.setInt(parameterCounter++, datasetId); - - addParameters(occurrencesPreparedStatement, parameterCounter, filters.getConditions()); - - /*if (filters.getUpperBound() != null) { - occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude()); - occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude()); - } - - if (filters.getLowerBound() != null) { - occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude()); - occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude()); - } - - if (filters.getFromDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis())); - if (filters.getToDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/ - - return occurrencesPreparedStatement; - } - - public PreparedStatement getScientificNamePreparedStatement(String scientificName) throws SQLException - { - if (scientificNamePreparedStatement == null) createScientificNamePreparedStatement(); - scientificNamePreparedStatement.clearParameters(); - scientificNamePreparedStatement.setString(1, "%"+scientificName+"%"); - return scientificNamePreparedStatement; - } - - protected void createScientificNamePreparedStatement() throws SQLException - { - String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ?"; - scientificNamePreparedStatement = connection.prepareStatement(query); - } - - public PreparedStatement getCommonNamePreparedStatement(String commonName) throws SQLException - { - if (commonNamePreparedStatement==null) createCommonNamePreparedStatement(); - commonNamePreparedStatement.clearParameters(); - commonNamePreparedStatement.setString(1, commonName); - return commonNamePreparedStatement; - } - - protected void createCommonNamePreparedStatement() throws SQLException - { - commonNamePreparedStatement = connection.prepareStatement("SELECT c.tname_id as id FROM "+SCHEMA+".cnames c WHERE c.cname LIKE ?"); - } - - public PreparedStatement getChildrenTaxonPreparedStatement(int id) throws SQLException - { - if (childrenTaxonPreparedStatement==null) createChildrenTaxonPreparedStatement(); - childrenTaxonPreparedStatement.clearParameters(); - childrenTaxonPreparedStatement.setInt(1, id); - return childrenTaxonPreparedStatement; - } - - protected void createChildrenTaxonPreparedStatement() throws SQLException - { - childrenTaxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id as id, t.parent_id, t.tauthor, r.rank_name " + - "FROM "+SCHEMA+".tnames t " + - "LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+ - "WHERE t.parent_id = ?"); - } - - protected PreparedStatement getOccurrenceByIdPreparedStatement(int occurrenceId) throws SQLException - { - if (occurrenceByIdPreparedStatement==null) createOccurrenceByIdPreparedStatement(); - occurrenceByIdPreparedStatement.clearParameters(); - occurrenceByIdPreparedStatement.setInt(1, occurrenceId); - return occurrenceByIdPreparedStatement; - } - - protected void createOccurrenceByIdPreparedStatement() throws SQLException - { - String query = "SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames as tn WHERE drs.id = ? AND drs.id = dxs.dr_id AND drs.valid_id = tn.id"; - occurrenceByIdPreparedStatement = connection.prepareStatement(query); - } - - protected void createDatasetPreparedStatement() throws SQLException - { - /*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + - "FROM obis.resources r, obis.providers p WHERE " + - "exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/ - - String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + - "FROM obis.resources r, obis.providers p WHERE " + - "r.id in (SELECT resource_id from portal.species_per_resource where valid_id = ?) AND r.provider_id = p.id"; - datasetPreparedStatement = connection.prepareStatement(query); - } - - protected void createDatasetPerDatasetIdPreparedStatement() throws SQLException - { - /*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + - "FROM obis.resources r, obis.providers p WHERE " + - "exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/ - - String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + - "FROM obis.resources r, obis.providers p WHERE " + - "r.id = ? AND r.provider_id = p.id"; - datasetPerIdPreparedStatement = connection.prepareStatement(query); - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/capabilities/OccurrencesCapabilityImpl.java b/src/main/java/org/gcube/data/spd/obisplugin/capabilities/OccurrencesCapabilityImpl.java new file mode 100644 index 0000000..1029dc0 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/capabilities/OccurrencesCapabilityImpl.java @@ -0,0 +1,110 @@ +package org.gcube.data.spd.obisplugin.capabilities; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.Conditions; +import org.gcube.data.spd.model.exceptions.StreamBlockingException; +import org.gcube.data.spd.model.exceptions.StreamNonBlockingException; +import org.gcube.data.spd.model.products.OccurrencePoint; +import org.gcube.data.spd.obisplugin.Constants; +import org.gcube.data.spd.obisplugin.search.OccurrenceSearch; +import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability; +import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OccurrencesCapabilityImpl extends OccurrencesCapability{ + + + private static Logger logger = LoggerFactory.getLogger(OccurrencesCapabilityImpl.class); + + private String baseUrl; + + + + public OccurrencesCapabilityImpl(String baseUrl) { + super(); + this.baseUrl = baseUrl; + } + + + + @SuppressWarnings("serial") + @Override + public Set getSupportedProperties() { + return new HashSet(){{ + add(Conditions.DATE); + add(Conditions.COORDINATE); + + }}; + } + + + + @Override + public void searchByScientificName(String word, + ObjectWriter writer, Condition... properties) { + try{ + new OccurrenceSearch(baseUrl).search(writer, word, Constants.QUERY_LIMIT, properties); + } catch (Exception e) { + logger.debug("search occurrences by ScientificName failed",e); + } + } + + @Override + public void getOccurrencesByProductKeys( + ClosableWriter writer, Iterator keys) { + OccurrenceSearch occSearch = null; + try{ + occSearch = new OccurrenceSearch(baseUrl); + }catch (Exception e) { + logger.error("error contacting obis server"); + writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); + return; + } + while (keys.hasNext()){ + String key = keys.next(); + try{ + occSearch.searchByKey(writer, key, Constants.QUERY_LIMIT); + }catch (Exception e) { + logger.warn("error retrieving key "+key, e); + writer.write(new StreamNonBlockingException(Constants.REPOSITORY_NAME,key)); + } + } + writer.close(); + } + + + + @Override + public void getOccurrencesByIds(ClosableWriter writer, + Iterator ids) { + OccurrenceSearch occSearch = null; + try{ + occSearch = new OccurrenceSearch(baseUrl); + }catch (Exception e) { + logger.error("error contacting gbif server"); + writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); + return; + } + while (ids.hasNext()){ + String id = ids.next(); + try{ + if (!writer.isAlive()){ + logger.trace("the writer is closed"); + return; + }else writer.write(occSearch.searchById(id)); + }catch (Exception e) { + logger.warn("error retrieving id "+id,e); + writer.write(new StreamNonBlockingException(Constants.REPOSITORY_NAME,id)); + } + } + writer.close(); + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java b/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java deleted file mode 100644 index 58fe2b1..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.data; - -import com.thoughtworks.xstream.XStream; - - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class ProductKey { - - protected static XStream stream; - - protected int taxonId; - protected int dataSetId; - protected SearchFilters filters; - - /** - * @param taxonId - * @param dataSetId - * @param filters - */ - public ProductKey(int taxonId, int dataSetId, SearchFilters filters) { - this.taxonId = taxonId; - this.dataSetId = dataSetId; - this.filters = filters; - } - - protected ProductKey() {} - - /** - * @return the taxonId - */ - public int getTaxonId() { - return taxonId; - } - - /** - * @return the dataSetId - */ - public int getDataSetId() { - return dataSetId; - } - - /** - * @return the filters - */ - public SearchFilters getFilters() { - return filters; - } - - protected static XStream getStream() - { - if (stream == null) stream = new XStream(); - return stream; - } - - public String serialize() - { - XStream stream = getStream(); - return stream.toXML(this); - } - - public static ProductKey deserialize(String key) - { - XStream stream = getStream(); - return (ProductKey) stream.fromXML(key); - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ProductKey [taxonId="); - builder.append(taxonId); - builder.append(", dataSetId="); - builder.append(dataSetId); - builder.append(", filters="); - builder.append(filters); - builder.append("]"); - return builder.toString(); - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java b/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java deleted file mode 100644 index 0586a78..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.data; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.gcube.data.spd.model.Condition; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class SearchFilters { - - protected List conditions; - - protected SearchFilters(){} - - public SearchFilters(Condition ... conditions){ - this.conditions = new ArrayList(Arrays.asList(conditions)); - } - - public void addCondition(Condition condition) - { - conditions.add(condition); - } - - /** - * @return the conditions - */ - public List getConditions() { - return conditions; - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("SearchFilters [conditions="); - builder.append(conditions); - builder.append("]"); - return builder.toString(); - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java deleted file mode 100644 index fb6a509..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.pool; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class DatabaseCredential { - - protected String url; - protected String user; - protected String password; - - /** - * @param url - * @param user - * @param password - */ - public DatabaseCredential(String url, String user, String password) { - this.url = url; - this.user = user; - this.password = password; - } - - /** - * @return the url - */ - public String getUrl() { - return url; - } - - /** - * @return the user - */ - public String getUser() { - return user; - } - - /** - * @return the password - */ - public String getPassword() { - return password; - } - - /** - * {@inheritDoc} - */ - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((password == null) ? 0 : password.hashCode()); - result = prime * result + ((url == null) ? 0 : url.hashCode()); - result = prime * result + ((user == null) ? 0 : user.hashCode()); - return result; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - DatabaseCredential other = (DatabaseCredential) obj; - if (password == null) { - if (other.password != null) - return false; - } else if (!password.equals(other.password)) - return false; - if (url == null) { - if (other.url != null) - return false; - } else if (!url.equals(other.url)) - return false; - if (user == null) { - if (other.user != null) - return false; - } else if (!user.equals(other.user)) - return false; - return true; - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java deleted file mode 100644 index 19c42dd..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.pool; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * Adapted from http://sourcemaking.com/design_patterns/object_pool/java - */ -public class JDBCConnectionPool extends ObjectPool { - - protected String url, username, password; - - public JDBCConnectionPool(String driver, String dsn, String usr, String pwd) { - super("JDBCConnectionPool", 10 * 60 * 1000); - try { - Class.forName(driver).newInstance(); - } catch (Exception e) { - e.printStackTrace(); - } - this.url = dsn; - this.username = usr; - this.password = pwd; - } - - @Override - protected Connection create() { - try { - Connection connection = DriverManager.getConnection(url, username, password); - connection.setAutoCommit(false); - connection.setReadOnly(true); - return connection; - } catch (SQLException e) { - e.printStackTrace(); - return null; - } - } - - @Override - protected void expire(Connection o) { - try { - o.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @Override - protected boolean validate(Connection o) { - try { - return !o.isClosed() && o.isValid(1000); - } catch (SQLException e) { - e.printStackTrace(); - return false; - } - } -} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java deleted file mode 100644 index 31f87e0..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.pool; - -import java.util.Enumeration; -import java.util.Hashtable; - -import org.gcube.common.core.utils.logging.GCUBELog; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * Adapted from http://sourcemaking.com/design_patterns/object_pool/java - */ -public abstract class ObjectPool { - - protected static GCUBELog logger = new GCUBELog(ObjectPool.class); - - protected long expirationTime; - - protected Hashtable locked, unlocked; - protected boolean closed = false; - protected String name; - - public ObjectPool(String name, long expirationTime) { - this.name = name; - this.expirationTime = expirationTime; - locked = new Hashtable(); - unlocked = new Hashtable(); - } - - protected abstract T create(); - - protected abstract boolean validate(T o); - - protected abstract void expire(T o); - - public synchronized T checkOut() { - long now = System.currentTimeMillis(); - T t; - if (unlocked.size() > 0) { - Enumeration e = unlocked.keys(); - while (e.hasMoreElements()) { - t = e.nextElement(); - if ((now - unlocked.get(t)) > expirationTime) { - // object has expired - unlocked.remove(t); - expire(t); - t = null; - } else { - if (validate(t)) { - unlocked.remove(t); - locked.put(t, now); - return t; - } else { - // object failed validation - unlocked.remove(t); - expire(t); - t = null; - } - } - } - } - - logger.trace("no objects available, create a new one, status: "+this); - // no objects available, create a new one - t = create(); - locked.put(t, now); - return (t); - } - - public synchronized void checkIn(T t) { - locked.remove(t); - if (!closed) unlocked.put(t, System.currentTimeMillis()); - else expire(t); - logger.trace("pool status: "+this); - } - - public synchronized void shutdown(boolean force) - { - closed = true; - expireAllUnlocked(); - if (force) expireAllLocked(); - } - - public synchronized void expireAllUnlocked() - { - Enumeration e = unlocked.keys(); - while (e.hasMoreElements()) { - T t = e.nextElement(); - unlocked.remove(t); - expire(t); - } - } - - public synchronized void expireAllLocked() - { - Enumeration e = locked.keys(); - while (e.hasMoreElements()) { - T t = e.nextElement(); - locked.remove(t); - expire(t); - } - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ObjectPool [name="); - builder.append(name); - builder.append(", expirationTime="); - builder.append(expirationTime); - builder.append(", locked="); - builder.append(locked.size()); - builder.append(", unlocked="); - builder.append(unlocked.size()); - builder.append(", closed="); - builder.append(closed); - builder.append("]"); - return builder.toString(); - } - -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java deleted file mode 100644 index f5599d7..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.pool; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Properties; - -import org.apache.log4j.Logger; -import org.gcube.data.spd.obisplugin.PluginSession; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class PluginSessionPool extends ObjectPool { - - protected Logger logger = Logger.getLogger(PluginSessionPool.class); - protected DatabaseCredential databaseCredential; - - public PluginSessionPool(DatabaseCredential databaseCredential) { - super("PluginSessionPool", 30*60*1000); - setDatabaseCredential(databaseCredential); - } - - /** - * @param databaseCredential the databaseCredential to set - */ - public void setDatabaseCredential(DatabaseCredential databaseCredential) { - this.databaseCredential = databaseCredential; - } - - /** - * {@inheritDoc} - */ - @Override - protected PluginSession create() { - - try { - Connection connection = createConnection(); - return new PluginSession(databaseCredential, connection); - } catch (SQLException e) { - logger.error("An error occurred creating the db connection", e); - return null; - } - } - - protected Connection createConnection() throws SQLException - { - Properties props = new Properties(); - props.setProperty("user", databaseCredential.getUser()); - props.setProperty("password", databaseCredential.getPassword()); - return DriverManager.getConnection(databaseCredential.getUrl(), props); - } - - /** - * {@inheritDoc} - */ - @Override - protected boolean validate(PluginSession session) { - try { - return session.isValid(databaseCredential); - } catch (Exception e) { - logger.warn("An error occurred validating the session", e); - return false; - } - } - - /** - * {@inheritDoc} - */ - @Override - protected void expire(PluginSession session) { - - try { - session.expire(); - } catch (SQLException e) { - logger.warn("An error occurred expiring the session", e); - } - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java new file mode 100644 index 0000000..4721264 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java @@ -0,0 +1,55 @@ +package org.gcube.data.spd.obisplugin.search; + +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; + +import java.util.List; +import java.util.Map; + +import org.gcube.data.spd.model.products.DataProvider; +import org.gcube.data.spd.model.products.DataSet; +import org.gcube.data.spd.obisplugin.Constants; +import org.gcube.data.spd.obisplugin.search.query.MappingUtils; +import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier; +import org.gcube.data.spd.obisplugin.search.query.QueryType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataSetRetreiver { + + private static Logger log = LoggerFactory.getLogger(DataSetRetreiver.class); + + @SuppressWarnings("unchecked") + public static DataSet get(String key, String baseURL) throws Exception{ + long start = System.currentTimeMillis(); + QueryByIdentifier datasetQuery = new QueryByIdentifier(baseURL, key, QueryType.Dataset); + Map mapping = MappingUtils.getObjectMapping(datasetQuery.build()); + DataSet dataset = new DataSet(key); + dataset.setName(getAsString(mapping,"name")); + + + + String citation = getAsString(mapping,"citation"); + if (citation ==null){ + List> institutionMapping = (List>)mapping.get("institutes"); + if (institutionMapping.size()>0){ + if (getAsString(institutionMapping.get(0),"parent")!=null) + citation += " - "+getAsString(institutionMapping.get(0),"parent"); + dataset.setCitation(citation); + } + } + + String providerKey = key; + Map providerMapping = (Map)mapping.get("provider"); + DataProvider provider = new DataProvider(providerKey); + if (providerMapping!=null) + provider.setName(getAsString(providerMapping,"name")); + else + provider.setName(Constants.REPOSITORY_NAME); + + dataset.setDataProvider(provider); + + log.trace("[Benchmark] time to retrieve dataset is "+(System.currentTimeMillis()-start)); + return dataset; + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java new file mode 100644 index 0000000..43b423f --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java @@ -0,0 +1,172 @@ +package org.gcube.data.spd.obisplugin.search; + +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsCalendar; +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsDouble; +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger; +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.List; +import java.util.Map; + +import org.gcube.data.spd.model.BasisOfRecord; +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.exceptions.StreamBlockingException; +import org.gcube.data.spd.model.products.DataProvider; +import org.gcube.data.spd.model.products.DataSet; +import org.gcube.data.spd.model.products.OccurrencePoint; +import org.gcube.data.spd.obisplugin.Constants; +import org.gcube.data.spd.obisplugin.search.query.MappingUtils; +import org.gcube.data.spd.obisplugin.search.query.PagedQueryIterator; +import org.gcube.data.spd.obisplugin.search.query.PagedQueryObject; +import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier; +import org.gcube.data.spd.obisplugin.search.query.QueryCondition; +import org.gcube.data.spd.obisplugin.search.query.QueryType; +import org.gcube.data.spd.obisplugin.search.query.ResultType; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OccurrenceSearch { + + private static Logger log = LoggerFactory.getLogger(OccurrenceSearch.class); + + private String baseURL; + + private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + + public OccurrenceSearch(String baseURL) { + this.baseURL = baseURL; + } + + public void search(ObjectWriter writer, String scientificName, int limit, Condition ...conditions) throws Exception{ + PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit); + List queryConditions = Utils.elaborateConditions(conditions); + occurrencesQuery.setConditions(QueryCondition.cond("scientificname",scientificName.replaceAll(" ", "%20"))); + occurrencesQuery.getConditions().addAll(queryConditions); + writeElements(writer, occurrencesQuery); + } + + public void searchByKey(ObjectWriter writer, String key, int limit) throws Exception{ + PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit); + ProductKey productKey = Utils.elaborateProductsKey(key); + occurrencesQuery.getConditions().addAll(productKey.getQueryCondition()); + writeElements(writer, occurrencesQuery); + } + + private void writeElements(ObjectWriter writer, PagedQueryObject occurrencesQuery){ + PagedQueryIterator pagedIterator = new PagedQueryIterator(occurrencesQuery) { + + @Override + protected OccurrencePoint getObject(Map mappedObject) + throws Exception { + OccurrencePoint op = retrieveElement(mappedObject); + Calendar now = Calendar.getInstance(); + String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, "+format.format(now.getTime())+")"; + op.setCredits(credits); + return op; + } + + }; + + try{ + while (pagedIterator.hasNext() && writer.isAlive()) + writer.write(pagedIterator.next()); + }catch(Exception e){ + log.error("error writing occurrences",e); + writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); + } + } + + public OccurrencePoint searchById(String id) throws Exception{ + QueryByIdentifier queryByIdentifier = new QueryByIdentifier(baseURL, id, QueryType.Occurrence); + return retrieveElement(MappingUtils.getObjectMapping(queryByIdentifier.build())); + } + + + /* + FOSSIL_SPECIMEN + An occurrence record describing a fossilized specimen. + HUMAN_OBSERVATION + An occurrence record describing an observation made by one or more people. + LITERATURE + An occurrence record based on literature alone. + LIVING_SPECIMEN + An occurrence record describing a living specimen, e.g. + MACHINE_OBSERVATION + An occurrence record describing an observation made by a machine. + MATERIAL_SAMPLE + An occurrence record based on samples taken from other specimens or the environment. + OBSERVATION + An occurrence record describing an observation. + PRESERVED_SPECIMEN + An occurrence record describing a preserved specimen. + UNKNOWN + */ + + + + + + private OccurrencePoint retrieveElement(Map mappedObj) throws Exception{ + /* + + {"id":772,"decimalLongitude":79.5,"decimalLatitude":-62.5,"eventDate":"1974-01-01 11:00:00", + "institutionCode":"AADC","collectionCode":"WC","catalogNumber":"1672","individualCount":193.0, + "datasetName":"Whale catches in the Southern Ocean","phylum":"Chordata","order":"Cetartiodactyla", + "family":"Balaenopteridae","genus":"Balaenoptera","scientificName":"Balaenoptera bonaerensis", + "originalScientificName":"Balaenoptera bonaerensis","scientificNameAuthorship":"Burmeister, 1867", + "obisID":409234,"resourceID":22,"yearcollected":1974,"species":"Balaenoptera bonaerensis","qc":1073217151,"aphiaID":231405 + ,"speciesID":409234,"scientificNameID":"urn:lsid:marinespecies.org:taxname:231405","class":"Mammalia"} + + + */ + + long start = System.currentTimeMillis(); + String occurrenceId = getAsInteger(mappedObj, "id").toString(); + OccurrencePoint occurrence = new OccurrencePoint(occurrenceId); + + occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude")); + occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude")); + + Calendar eventDate = getAsCalendar(mappedObj, "eventDate"); + occurrence.setEventDate(eventDate); + occurrence.setCollectionCode(getAsString(mappedObj, "collectionCode")); + occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode")); + occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber")); + + //occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy")); + //occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy")); + //occurrence.setCountry(getAsString(mappedObj, "country")); + //occurrence.setLocality(getAsString(mappedObj, "locality")); + + occurrence.setBasisOfRecord(BasisOfRecord.Unknown); + + /*occurrence.setMinDepth(getAsDouble(mappedObj, "elevation")); + occurrence.setMaxDepth(getAsDouble(mappedObj, "depth")); + */ + + occurrence.setKingdom("Animalia"); + occurrence.setFamily(getAsString(mappedObj, "family")); + + occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship")); + occurrence.setScientificName(getAsString(mappedObj, "scientificName")); + + String datasetName = getAsString(mappedObj, "datasetName"); + DataSet dataset = new DataSet(datasetName); + dataset.setCitation(datasetName); + dataset.setName(datasetName); + DataProvider dataProvider = new DataProvider("OBIS"); + dataProvider.setName("OBIS"); + dataset.setDataProvider(dataProvider); + + occurrence.setDataSet(dataset); + + //occurrence.setCitation(getAsString(mappedObj, "accordingTo")); + log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start)); + return occurrence; + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/ProductKey.java b/src/main/java/org/gcube/data/spd/obisplugin/search/ProductKey.java new file mode 100644 index 0000000..6f4333f --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/ProductKey.java @@ -0,0 +1,16 @@ +package org.gcube.data.spd.obisplugin.search; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import org.gcube.data.spd.obisplugin.search.query.QueryCondition; + +@AllArgsConstructor +@Getter +public class ProductKey { + + private List queryCondition; + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java new file mode 100644 index 0000000..3afe96b --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java @@ -0,0 +1,174 @@ +package org.gcube.data.spd.obisplugin.search; + +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString; +import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.exceptions.StreamBlockingException; +import org.gcube.data.spd.model.products.DataSet; +import org.gcube.data.spd.model.products.Product; +import org.gcube.data.spd.model.products.Product.ProductType; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.model.products.Taxon; +import org.gcube.data.spd.obisplugin.Constants; +import org.gcube.data.spd.obisplugin.search.query.MappingUtils; +import org.gcube.data.spd.obisplugin.search.query.PagedQueryIterator; +import org.gcube.data.spd.obisplugin.search.query.PagedQueryObject; +import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier; +import org.gcube.data.spd.obisplugin.search.query.QueryCondition; +import org.gcube.data.spd.obisplugin.search.query.QueryCount; +import org.gcube.data.spd.obisplugin.search.query.QueryType; +import org.gcube.data.spd.obisplugin.search.query.ResultType; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultItemSearch { + + private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + + private static Logger log = LoggerFactory.getLogger(ResultItemSearch.class); + + private List queryConditions = new ArrayList(); + + private String baseURL; + + String searchQuery; + + public ResultItemSearch(String baseURL, String searchQuery, Condition ... conditions){ + this.baseURL = baseURL; + this.searchQuery = searchQuery.replaceAll(" ", "%20").trim(); + this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase()+this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase(); + try{ + this.queryConditions = Utils.elaborateConditions(conditions); + }catch(Exception e){ + log.error("error elaborating conditions",e); + } + } + + public void search(ObjectWriter writer, int limit){ + PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence,limit); + queryObject.setConditions(QueryCondition.cond("scientificname",searchQuery)); + queryObject.getConditions().addAll(this.queryConditions); + try{ + PagedQueryIterator pagedIterator = new PagedQueryIterator(queryObject) { + + Set alreadyVisited =new HashSet(); + + @Override + protected ResultItem getObject(Map mappedObject) throws Exception { + log.debug("retrieved mapped object"); + return buildResult(mappedObject); + } + + @Override + protected boolean useIt(Map mappedObject) { + String datasetKey = ((Integer)mappedObject.get("resourceID")).toString(); + Integer taxonId = (Integer)mappedObject.get("obisID"); + String key = datasetKey+"|"+taxonId; + if (alreadyVisited.contains(key)) + return false; + alreadyVisited.add(key); + return true; + } + + }; + + while (pagedIterator.hasNext() && writer.isAlive()) + writer.write(pagedIterator.next()); + + }catch(Exception e){ + log.error("error writing resultItems",e); + writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME)); + } + + + } + + ResultItem buildResult(Map singleObject) throws Exception{ + try{ + long start = System.currentTimeMillis(); + Integer taxonId = getAsInteger(singleObject,"obisID"); + String scientificName = getAsString(singleObject,"scientificName"); + ResultItem resItem = new ResultItem(taxonId.toString(), scientificName ); + + String scientificNameAuthorship = getAsString(singleObject,"scientificNameAuthorship"); + + QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); + Map singleTaxon = MappingUtils.getObjectMapping(query.build()); + + + + resItem.setScientificNameAuthorship(scientificNameAuthorship); + + resItem.setRank(getAsString(singleTaxon, "rank_name")); + + //resItem.setCitation(getAsString(singleTaxon,"tauthor")); + + resItem.setParent(retrieveParentTaxon(getAsInteger(singleTaxon,"parent_id"))); + + DataSet dataset = DataSetRetreiver.get(getAsInteger(singleObject,"resourceID").toString(), baseURL); + resItem.setDataSet(dataset); + + List products = retrieveProducts(taxonId.toString(), dataset); + resItem.setProducts(products); + + String credits = "Biodiversity occurrence accessed through OBIS WebService, http://api.iobis.org/, "+format.format(Calendar.getInstance().getTime())+")"; + resItem.setCredits(credits); + log.trace("[Benchmark] time to retrieve ResultItem is "+(System.currentTimeMillis()-start)); + log.debug("found species {} with authorship {}",scientificName, scientificNameAuthorship); + return resItem; + }catch(Exception e){ + throw e; + } + } + + private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception { + if (parentTaxonId==0) return null; + long start = System.currentTimeMillis(); + + Integer taxonId = parentTaxonId; + Taxon previousTaxon = null; + Taxon taxonToReturn = null; + do{ + QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon); + Map singleTaxon = MappingUtils.getObjectMapping(query.build()); + + Taxon taxon = new Taxon(getAsInteger(singleTaxon, "id").toString(), getAsString(singleTaxon, "tname")); + taxon.setScientificNameAuthorship(getAsString(singleTaxon, "tauthor")); + //taxon.setCitation(getAsString(mappedObject, "accordingTo")); + taxon.setRank(getAsString(singleTaxon, "rank_name")); + if (previousTaxon!=null) + previousTaxon.setParent(taxon); + previousTaxon = taxon; + taxonId = getAsInteger(singleTaxon, "parent_id"); + if (taxonToReturn==null) + taxonToReturn = taxon; + } while (taxonId>0); + + log.trace("[Benchmark] time to retrieve taxon is "+(System.currentTimeMillis()-start)); + return taxonToReturn; + } + + private List retrieveProducts( String taxonId, DataSet dataset) throws Exception{ + long start = System.currentTimeMillis(); + QueryCount occurrencesQuery = new QueryCount(baseURL, ResultType.Occurrence); + occurrencesQuery.setConditions(QueryCondition.cond("obisid",taxonId), QueryCondition.cond("resourceid", dataset.getId())); + occurrencesQuery.getConditions().addAll(this.queryConditions); + String productId = Utils.createProductsKey(Utils.getDataSetAsString(dataset), taxonId, this.queryConditions); + Product product = new Product(ProductType.Occurrence, productId); + product.setCount(occurrencesQuery.getCount()); + log.trace("[Benchmark] time to retrieve product is "+(System.currentTimeMillis()-start)); + return Arrays.asList(product); + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java b/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java new file mode 100644 index 0000000..ab32bc5 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java @@ -0,0 +1,132 @@ +package org.gcube.data.spd.obisplugin.search; + +import static org.gcube.data.spd.obisplugin.search.query.QueryCondition.cond; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; + +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.Coordinate; +import org.gcube.data.spd.model.products.DataSet; +import org.gcube.data.spd.obisplugin.search.query.QueryCondition; + +public class Utils { + + protected static List elaborateConditions(Condition[] properties) throws Exception{ + List queryConditions = new ArrayList(); + List coordinateConditions = new ArrayList(); + for (Condition prop: properties){ + switch (prop.getType()) { + case COORDINATE: + coordinateConditions.add(prop); + break; + case DATE: + Calendar date = (Calendar) prop.getValue(); + queryConditions.addAll(getDateCondition(date, prop)); + break; + } + } + if (coordinateConditions.size()>0) + queryConditions.add(getCoordinateConditions(coordinateConditions)); + + return queryConditions; + } + + public static ProductKey elaborateProductsKey(String id) { + List queryConditions = new ArrayList(); + String[] splitString = id.split("\\|\\|"); + queryConditions.add(cond("resourceid", splitString[0])); + queryConditions.add(cond("obisid", splitString[1])); + if (splitString.length>2) + for (int i = 2; i queryConditions) { + StringBuilder conditionTransformer = new StringBuilder(); + for (QueryCondition cond : queryConditions) + conditionTransformer.append("||").append(cond.getKey()).append("=").append(cond.getValue()); + return dataResourceKey+"||"+taxonKey+conditionTransformer.toString(); + } + + public static QueryCondition getCoordinateConditions(List coordinateConditions){ + double lowerLong =-180, lowerLat= -90, upperLat= 90, upperLong=180; + double latitude, longitude; + for (Condition cond :coordinateConditions){ + switch (cond.getOp()) { + case EQ: + break; + case GT: + latitude = ((Coordinate)cond.getValue()).getLatitude()+0.01; + longitude = ((Coordinate)cond.getValue()).getLongitude()+0.01; + if (latitude>lowerLat) lowerLat = latitude; + if (longitude>lowerLong) lowerLong = longitude; + break; + case GE: + latitude = ((Coordinate)cond.getValue()).getLatitude(); + longitude = ((Coordinate)cond.getValue()).getLongitude(); + if (latitude>lowerLat) lowerLat = latitude; + if (longitude>lowerLong) lowerLong = longitude; + break; + case LT: + latitude = ((Coordinate)cond.getValue()).getLatitude()-0.01; + longitude = ((Coordinate)cond.getValue()).getLongitude()-0.01; + if (latitudeupperLong) upperLong = longitude; + break; + case LE: + latitude = ((Coordinate)cond.getValue()).getLatitude(); + longitude = ((Coordinate)cond.getValue()).getLongitude(); + if (latitudeupperLong) upperLong = longitude; + break; + default: + + break; + } + } + + return cond("geometry", String.format("POLYGON((%1$f %2$f,%3$f %4$f,%5$f %6$f,%1$f %2$f))" + ,lowerLat, lowerLong, upperLat, lowerLong, upperLat, upperLong, lowerLat, upperLong).replaceAll(" ", "%20")); + + } + + public static List getDateCondition(Calendar date, Condition prop){ + List conditions = new ArrayList(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + Calendar newDate = date; + switch (prop.getOp()) { + case EQ: + conditions.add(cond("eventDate",dateFormat.format(date.getTime()))); + break; + case GT: + newDate.add(Calendar.DAY_OF_MONTH, 1); + conditions.add(cond("startdate",dateFormat.format(newDate.getTime()))); + break; + case GE: + conditions.add(cond("startdate",dateFormat.format(date.getTime()))); + break; + case LT: + newDate.add(Calendar.DAY_OF_MONTH, -1); + conditions.add(cond("enddate",dateFormat.format(newDate.getTime()))); + break; + case LE: + conditions.add(cond("enddate",dateFormat.format(date.getTime()))); + break; + default: + break; + } + return conditions; + } + + protected static String getDataSetAsString(DataSet dataset){ + return dataset.getId(); + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java new file mode 100644 index 0000000..9d3963b --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java @@ -0,0 +1,109 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import java.io.StringReader; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import javax.ws.rs.core.MediaType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; + +public class MappingUtils { + + private static final int TIMEOUT = 40000; + private static final int retries = 5; + private static final int SLEEP_TIME = 10000; + + private static Logger log = LoggerFactory.getLogger(MappingUtils.class); + + @SuppressWarnings("unchecked") + public static Map getObjectMapping(String query) throws Exception{ + + String response = executeQuery(query); + + ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally + return mapper.readValue(new StringReader(response), Map.class); + + } + + public static List> getObjectList(String query) throws Exception{ + String response = executeQuery(query); + ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally + return mapper.readValue(new StringReader(response), new TypeReference>>() { + }); + + } + + public static String getAsString(Map map, String key){ + if (!map.containsKey(key)) return null; + return (String) map.get(key); + } + + public static Double getAsDouble(Map map, String key){ + if (!map.containsKey(key)) return 0d; + return (Double) map.get(key); + } + + public static Integer getAsInteger(Map map, String key){ + if (!map.containsKey(key)) return 0; + return (Integer) map.get(key); + } + + public static Calendar getAsCalendar(Map map, String key){ + if (!map.containsKey(key)) return null; + return parseCalendar((String) map.get(key)); + } + + + public static Calendar parseCalendar(String date){ + try{ + DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Calendar calendar= Calendar.getInstance(); + calendar.setTime(df.parse(date)); + return calendar; + }catch (ParseException e) { + log.warn("date discarded ("+date+")"); + return null; + } + } + + private static String executeQuery(String query){ + DefaultClientConfig clientConfig = new DefaultClientConfig(); + Client client = Client.create(clientConfig); + client.setConnectTimeout(TIMEOUT); + client.setReadTimeout(TIMEOUT); + WebResource target = client.resource(query); + //NameUsageWsClient nuws = new NameUsageWsClient(target); + int tries = 1; + String response = null; + do { + + log.debug("try number {} STARTED for query {} ", tries,query); + try{ + response = target.type(MediaType.APPLICATION_JSON).acceptLanguage(Locale.ENGLISH).get(String.class); + }catch (Exception e) { + log.debug("try number {} FAILED for query {} ", tries,query,e); + try { + if (tries implements Iterator{ + + private static Logger log = LoggerFactory.getLogger(PagedQueryIterator.class); + + private @NonNull PagedQueryObject pagedQuery; + + public PagedQueryIterator(@NonNull PagedQueryObject pagedQuery) { + this.pagedQuery = pagedQuery; + } + + protected abstract T getObject(Map mappedObject) throws Exception; + + Map mapping; + + Iterator> resultIterator; + + Map actualObject= null; + + Long start = null; + Long parsingStart = null; + + @SuppressWarnings("unchecked") + @Override + public boolean hasNext() { + try{ + if (resultIterator==null){ + String query = pagedQuery.buildNext(); + start = System.currentTimeMillis(); + mapping = MappingUtils.getObjectMapping(query); + if (mapping.get("results")== null) return false; + parsingStart = System.currentTimeMillis(); + log.trace("[Benchmark] got Elements with query "+query+" and took "+(parsingStart-start)); + resultIterator = ((Iterable>) mapping.get("results")).iterator(); + } + + if (!resultIterator.hasNext()){ + log.trace("[Benchmark] page retrieved and parsed in "+(System.currentTimeMillis()-start)); + if ((Boolean)mapping.get("lastpage")){ + log.trace("is end of records, no next element"); + return false; + } + resultIterator = null; + + } else{ + actualObject = resultIterator.next(); + if (useIt(actualObject)) + return true; + } + return this.hasNext(); + + }catch(Exception e){ + log.error("error computing hasNext",e); + throw new RuntimeException(e); + } + } + + @Override + public T next() { + try{ + return getObject(actualObject); + }catch(Exception e){ + log.error("error computing next",e); + throw new RuntimeException(e); + } + } + + protected boolean useIt(Map mappedObject){ + return true; + } + + @Override + public void remove() { + resultIterator = null; + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java new file mode 100644 index 0000000..63ca787 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java @@ -0,0 +1,50 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + + +@RequiredArgsConstructor +@Slf4j +public class PagedQueryObject { + + private @NonNull String baseUri; + + @Getter + List conditions = new ArrayList(); + + private @NonNull ResultType resultType; + + private @NonNull Integer resultPerQuery; + + private int offset = 0; + + public void setConditions(QueryCondition ... conditions){ + this.conditions.addAll(Arrays.asList(conditions)); + } + + + + public String buildNext(){ + StringBuilder query = new StringBuilder(baseUri); + if (!baseUri.endsWith("/")) query.append("/"); + query.append(this.resultType.getQueryEntry()).append("/"); + query.append("?limit=").append(resultPerQuery); + query.append("&offset=").append(offset); + + if (conditions.size()>0) + for (QueryCondition queryCond: conditions) + query.append("&").append(queryCond.getKey()).append("=").append(queryCond.getValue()); + offset = offset+resultPerQuery; + log.debug("executed query is "+query.toString()); + return query.toString(); + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java new file mode 100644 index 0000000..6dc280f --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java @@ -0,0 +1,38 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import java.util.ArrayList; +import java.util.List; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@RequiredArgsConstructor +@Slf4j +public class QueryByIdentifier { + + private @NonNull String baseUri; + + private @NonNull String key; + + private @NonNull QueryType type; + + private List paths = new ArrayList(); + + public void addPath(String path){ + paths.add(path); + } + + public String build(){ + StringBuilder query = new StringBuilder(baseUri); + if (!baseUri.endsWith("/")) query.append("/"); + query.append(this.type.getQueryEntry()).append("/"); + query.append(key); + for (String path : paths) + query.append("/").append(path); + log.trace("query by dentifier is "+query.toString()); + return query.toString(); + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCondition.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCondition.java new file mode 100644 index 0000000..1f70bea --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCondition.java @@ -0,0 +1,20 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + + +@RequiredArgsConstructor +public class QueryCondition{ + + public static QueryCondition cond(String key, String value){ + return new QueryCondition(key, value); + } + + @Getter + private @NonNull String key; + @Getter + private @NonNull String value; + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java new file mode 100644 index 0000000..ef6fea6 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java @@ -0,0 +1,63 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RequiredArgsConstructor +public class QueryCount { + + private static Logger log = LoggerFactory.getLogger(QueryCount.class); + + private @NonNull String baseUri; + + List conditions = new ArrayList(); + + private @NonNull ResultType resultType; + + public void setConditions(QueryCondition ... conditions){ + this.conditions.addAll(Arrays.asList(conditions)); + } + + public List getConditions() { + return conditions; + } + + + public int getCount(){ + Map mapping; + try { + mapping = MappingUtils.getObjectMapping(this.build()); + if (mapping.get("count")==null) return 0; + return (Integer)mapping.get("count"); + } catch (Exception e) { + log.error("error computing count, returning 0",e); + return 0; + } + + } + + + private String build(){ + StringBuilder query = new StringBuilder(baseUri); + if (!baseUri.endsWith("/")) query.append("/"); + query.append(this.resultType.getQueryEntry()).append("/"); + query.append("?limit=0"); + + if (conditions.size()>0) + for (QueryCondition queryCond: conditions) + query.append("&").append(queryCond.getKey().replaceAll(" ", "%20")).append("=").append(queryCond.getValue().replaceAll(" ", "%20")); + return query.toString(); + } + + + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java new file mode 100644 index 0000000..a7b1b53 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java @@ -0,0 +1,16 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; + + +public @AllArgsConstructor enum QueryType{ + Occurrence("occurrence"), + Taxon("taxon"), + Dataset("resource"); + + @Getter + private @NonNull String queryEntry; + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/search/query/ResultType.java b/src/main/java/org/gcube/data/spd/obisplugin/search/query/ResultType.java new file mode 100644 index 0000000..b3dffd7 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/search/query/ResultType.java @@ -0,0 +1,15 @@ +package org.gcube.data.spd.obisplugin.search.query; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ResultType{ + Occurrence("occurrence"), + Taxon("species"); + + @Getter + private @NonNull String queryEntry; + +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java b/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java deleted file mode 100644 index 3c6c0c8..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.util; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Map.Entry; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class Cache { - - protected int limit; - protected Map cache; - - protected boolean enableStatistics; - protected long hints; - protected long requests; - protected long removed; - - @SuppressWarnings("serial") - public Cache(final int limit) - { - this.limit = limit; - this.hints = 0; - - this.cache = new LinkedHashMap() { - - /** - * {@inheritDoc} - */ - @Override - protected boolean removeEldestEntry(Entry eldest) { - boolean remove = size() > limit; - if (remove) removed++; - return remove; - } - - }; - } - - /** - * @return the enableStatistics - */ - public boolean isEnableStatistics() { - return enableStatistics; - } - - /** - * @param enableStatistics the enableStatistics to set - */ - public void setEnableStatistics(boolean enableStatistics) { - this.enableStatistics = enableStatistics; - } - - /** - * @return the requests - */ - public long getRequests() { - return requests; - } - - /** - * @return the limit - */ - public int getLimit() { - return limit; - } - - /** - * @return the hints - */ - public long getHints() { - return hints; - } - - public V get(K key) - { - V value = cache.get(key); - if (enableStatistics) { - requests++; - if (value!=null) hints++; - } - return value; - } - - public void put(K key, V value) - { - cache.put(key, value); - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Cache [limit="); - builder.append(limit); - builder.append(", enableStatistics="); - builder.append(enableStatistics); - builder.append(", hints="); - builder.append(hints); - builder.append(", requests="); - builder.append(requests); - builder.append(", removed="); - builder.append(removed); - builder.append("]"); - return builder.toString(); - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java b/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java deleted file mode 100644 index 82c7a5d..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.util; - -import java.util.Date; - -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class DateUtil { - - //Unsupported: - //2010-05-03T14:44:04Z+01:00 - //2009-04-09TMountain Da:ylight Time - //2008-05-20T13:03:08OZ - //2008-11-01T14:45OZ - //0000-00-00 00:00:00 - - protected static final String[] DATE_FORMATS = new String[]{ - "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", //2005-10-24T13:33:11.000Z - "yyyy-MM-dd'T'HH:mm:ss:mm:ssZ", //2003-02-18T11:35:13:35:13Z - "yyyy-MM-dd' 'HH:mm:ss.SSSSSS", //2006-12-20 10:27:02.477563 - "yyyy-MM-dd'T'HH:mm:ssZZ", //2001-02-10T12:00:00+00:00 - "yyyy-MM-dd'T'HH:mm:sssZZ", //2011-09-25T11:00:000Z - "yyyy-MM-dd' 'HH:mm:ssZZ", //2007-05-11 14:01:15-04 - "yyyy-MM-dd'T'HH:mm:ssz", //2005-10-11T15:40:00Z - "yyyy-MM-dd' 'HH:mm:ss", //2007-07-18 06:13:06 - "yyyy-MM-dd'T'HH:mmZ", //2009-10-01T01:00Z - "MM/dd/yyyy' 'hh:mm:ss aa", //10/28/2010 10:12:43 AM - "MM/dd/yyyy' 'hh:mm", //12/9/2010 11:59 - "dd/MM/yyyyHH:mm:ssZ", //13/9/201012:00:00Z - "yyyy-MM-dd' 'HH:mm", //2005-12-20 17:12 - "yyyy-MM-dd'T'", //2010-06-09T - "yyyy-MM-dd-'T'", //2009-08-05-T - "yyyy-MM-dd", //2009-09-08 - "dd-MMM-yy", //28-MAR-01 08-AUG-96 - "dd/MM/yyyy", //11/2/2010 - "MM/dd/yyyy' 'HH:mm:ss", //8/23/2010 0:00:00 - "MM/dd/yyyy", //10/19/2010 - "yyyy/MM/dd' 'HH:mm:ss", //2010/10/27 22:29:04 - //FIXME Military Time Zone not supported - "yyyy-MM-dd'T'HH:mm:ss'B'", //2003-07-07T10:03:56B - }; - - - protected static DateUtil instance; - - public static DateUtil getInstance() - { - if (instance == null) { - instance = new DateUtil(); - instance.initialize(); - } - return instance; - } - - protected DateTimeFormatter[] FORMATS; - - protected DateUtil() { - FORMATS = new DateTimeFormatter[DATE_FORMATS.length]; - } - - protected void initialize() - { - int i = 0; - for (String dateFormat:DATE_FORMATS) { - FORMATS[i++] = DateTimeFormat.forPattern(dateFormat); - } - } - - public Date parse(String dateString) { - for (DateTimeFormatter formatter:FORMATS) { - try { - return formatter.parseDateTime(dateString).toDate(); - } catch (Exception e){} - } - return null; // Unknown format. - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java b/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java deleted file mode 100644 index 0288806..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.util; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class RegExpDateFormat { - - protected Pattern pattern; - protected DateFormat dateFormat; - - public RegExpDateFormat(String regExp, String datePattern) - { - pattern = Pattern.compile(regExp); - dateFormat = new SimpleDateFormat(datePattern); - } - - public boolean match(String input) - { - Matcher m = pattern.matcher(input); - return m.matches(); - } - - public Date parse(String input) throws ParseException - { - return dateFormat.parse(input); - } -} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java b/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java deleted file mode 100644 index 8981409..0000000 --- a/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * - */ -package org.gcube.data.spd.obisplugin.util; - -import java.util.ArrayList; -import java.util.List; - -import org.gcube.data.spd.model.CommonName; -import org.gcube.data.spd.model.products.ResultItem; -import org.gcube.data.spd.model.products.Taxon; - -/** - * @author "Federico De Faveri defaveri@isti.cnr.it" - * - */ -public class Util { - - - /** - * Partially clones the passed result item. - * @param item - * @return - */ - public static ResultItem cloneResultItem(ResultItem item) - { - if (item==null) return null; - - ResultItem clone = new ResultItem(item.getId(), item.getScientificName()); - copyTaxon(item, clone); - clone.setCommonNames(cloneCommonName(item.getCommonNames())); - - return clone; - } - - protected static Taxon cloneTaxon(Taxon taxon) - { - if (taxon==null) return null; - - Taxon clone = new Taxon(taxon.getId(), taxon.getScientificName()); - copyTaxon(taxon, clone); - return clone; - } - - protected static void copyTaxon(Taxon taxon, Taxon clone) - { - clone.setId(taxon.getId()); - clone.setScientificName(taxon.getScientificName()); - clone.setCitation(taxon.getCitation()); - clone.setCredits(taxon.getCredits()); - clone.setScientificNameAuthorship(taxon.getScientificNameAuthorship()); - clone.setRank(taxon.getRank()); - - clone.setParent(cloneTaxon(taxon.getParent())); - } - - protected static List cloneCommonName(List commonNames) - { - if (commonNames==null) return null; - List clones = new ArrayList(commonNames.size()); - for (CommonName commonName:commonNames) clones.add(cloneCommonName(commonName)); - return clones; - } - - protected static CommonName cloneCommonName(CommonName commonName) - { - if (commonName==null) return null; - return new CommonName(commonName.getLanguage(), commonName.getName()); - } - - public static String stripNotValidXMLCharacters(String input) { - - if (input == null) return null; - if (input.isEmpty()) return ""; - - StringBuffer out = new StringBuffer(); - - for (char current:input.toCharArray()){ - if ((current == 0x9) || - (current == 0xA) || - (current == 0xD) || - ((current >= 0x20) && (current <= 0xD7FF)) || - ((current >= 0xE000) && (current <= 0xFFFD)) || - ((current >= 0x10000) && (current <= 0x10FFFF))) - out.append(current); - } - return out.toString(); - } -} diff --git a/src/test/java/org/gcube/data/obisplugin/ObisTest.java b/src/test/java/org/gcube/data/obisplugin/ObisTest.java index a286c87..458cc45 100644 --- a/src/test/java/org/gcube/data/obisplugin/ObisTest.java +++ b/src/test/java/org/gcube/data/obisplugin/ObisTest.java @@ -1,9 +1,17 @@ package org.gcube.data.obisplugin; +import java.util.Arrays; + +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.Condition.Operator; +import org.gcube.data.spd.model.Conditions; +import org.gcube.data.spd.model.Coordinate; import org.gcube.data.spd.model.exceptions.StreamException; import org.gcube.data.spd.model.products.OccurrencePoint; -import org.gcube.data.spd.obisplugin.ObisPlugin; -import org.gcube.data.spd.obisplugin.pool.DatabaseCredential; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.obisplugin.capabilities.OccurrencesCapabilityImpl; +import org.gcube.data.spd.obisplugin.search.ResultItemSearch; +import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; import org.junit.Test; @@ -11,9 +19,10 @@ public class ObisTest { @Test public void search() throws Exception{ - ObisPlugin plugin= new ObisPlugin(); + /*ObisPlugin plugin= new ObisPlugin(); plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3")); - ObjectWriter writer = new ObjectWriter() { + plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/ + ClosableWriter writer = new ClosableWriter() { @Override public boolean isAlive() { @@ -22,7 +31,44 @@ public class ObisTest { @Override public boolean write(OccurrencePoint arg0) { - System.out.println(arg0.toString()); + return true; + } + + @Override + public boolean write(StreamException arg0) { + return false; + } + + @Override + public void close() { + // TODO Auto-generated method stub + + } + + }; + + + + OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("http://api.iobis.org/"); + //impl.searchByScientificName("Cetacea", writer); + impl.getOccurrencesByProductKeys(writer, Arrays.asList("3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))").iterator() ); + } + + @Test + public void searchRI() throws Exception{ + /*ObisPlugin plugin= new ObisPlugin(); + plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3")); + plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/ + ObjectWriter writer = new ObjectWriter() { + + @Override + public boolean isAlive() { + return true; + } + + @Override + public boolean write(ResultItem ri) { + System.out.println(ri); return true; } @@ -31,7 +77,11 @@ public class ObisTest { return false; } }; - plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer); + + //, new Condition(Conditions.COORDINATE, new Coordinate(20, 30) , Operator.GT) + ResultItemSearch search = new ResultItemSearch("http://api.iobis.org/", "Gamidae"); + + search.search(writer, 50); + } - } diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 0000000..3bf9377 --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,10 @@ +log4j.appender.ROOT=org.apache.log4j.ConsoleAppender +log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout +log4j.appender.ROOT.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %c{2} [%t,%M:%L] %m%n +log4j.rootLogger=INFO,ROOT + +log4j.appender.SPD-CL=org.apache.log4j.ConsoleAppender +log4j.appender.SPD-CL.layout=org.apache.log4j.PatternLayout +log4j.appender.SPD-CL.layout.ConversionPattern=[SPD-CL] %d{HH:mm:ss,SSS} %-5p %c{2} [%t,%M:%L] %m%n + +log4j.category.org.gcube.data.spd=DEBUG,SPD-CL \ No newline at end of file