gCubeSystem
/
obis-spd-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/branches/data-access/obis-spd-plugin/1.8@141980 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2017-02-01 15:23:25 +00:00
parent 4b4c80866f
commit cecdb72930
34 changed files with 1213 additions and 2366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
pom.xml
View File

@ -17,36 +17,56 @@
<properties>
<distroDirectory>${project.basedir}/distro</distroDirectory>
<jackson.version>2.4.4</jackson.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>gcf</artifactId>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.data.spd</groupId>
<artifactId>spd-plugin-framework</artifactId>
<version>[3.0.0-SNAPSHOT, 4.0.0-SNAPSHOT)</version>
</dependency>
<!-- lombok -->
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>8.4-702.jdbc4</version>
<scope>compile</scope>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.1</version>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-encryption</artifactId>
<version>[1.0.2-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19</version>
</dependency>
<dependency>
@ -56,6 +76,18 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.4</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>

21
src/main/java/org/gcube/data/spd/obisplugin/Constants.java Normal file
View File

@ -0,0 +1,21 @@
package org.gcube.data.spd.obisplugin;
public class Constants {
//public static final String BASE_URL = "http://api.gbif.org/v1";
public static final int QUERY_LIMIT = 50;
/*
public static final QName GBIFKEY_ATTR= new QName("gbifKey");
public static final QName ABOUT_ATTR= new QName("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "about");
public static final QName RESOURCE_ATTR= new QName("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "resource");
public static final QName TOTAL_MATCHED_ATTR = new QName("totalMatched");
public static final String CHILD_RELATIONSHIP_VALUE = "http://rs.tdwg.org/ontology/voc/TaxonConcept#IsChildTaxonOf";
*/
public static final String REPOSITORY_NAME="OBIS";
}

144
src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java
View File

@ -1,144 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin;
import java.sql.SQLException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.log4j.Logger;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.ExternalRepositoryException;
import org.gcube.data.spd.model.exceptions.IdNotValidException;
import org.gcube.data.spd.model.products.TaxonomyItem;
import org.gcube.data.spd.obisplugin.pool.PluginSessionPool;
import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability;
import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class ObisClassification extends ClassificationCapability {
protected Logger logger = Logger.getLogger(ObisClassification.class);
protected PluginSessionPool sessionPool;
/**
* @param sessionPool
*/
public ObisClassification(PluginSessionPool sessionPool) {
this.sessionPool = sessionPool;
}
/**
* {@inheritDoc}
*/
@Override
public Set<Conditions> getSupportedProperties() {
return Collections.emptySet();
}
/**
* {@inheritDoc}
*/
@Override
public void searchByScientificName(String word, final ObjectWriter<TaxonomyItem> writer, Condition... properties) {
PluginSession session = sessionPool.checkOut();
try {
ObisClient.getTaxonByScientificNames(session, word, new Writer<TaxonomyItem>() {
@Override
public boolean write(TaxonomyItem item) {
writer.write(item);
return writer.isAlive();
}
});
} catch (Exception e) {
logger.error("Error retrieving taxon with word \""+word+"\"", e);
} finally {
sessionPool.checkIn(session);
}
}
/**
* {@inheritDoc}
*/
@Override
public List<TaxonomyItem> retrieveTaxonChildrenByTaxonId(String taxonId) throws IdNotValidException, ExternalRepositoryException {
PluginSession session = sessionPool.checkOut();
try {
int id = converId(taxonId);
return ObisClient.getChildrenTaxon(session, id);
} catch (SQLException e) {
logger.error("Error retrieving TaxonChildsByTaxonId", e);
throw new ExternalRepositoryException(e);
} finally {
sessionPool.checkIn(session);
}
}
/**
* {@inheritDoc} writer.put(new StreamException());
*/
@Override
public void retrieveTaxonByIds(Iterator<String> reader, ClosableWriter<TaxonomyItem> writer) {
PluginSession session = sessionPool.checkOut();
try {
while(reader.hasNext() && writer.isAlive()) {
try {
String taxonId = reader.next();
int id = converId(taxonId);
TaxonomyItem item = ObisClient.getTaxonById(session, id);
writer.write(item);
} catch (Exception e) {
logger.error("Error retrieving TaxonById", e);
}
}
} finally {
sessionPool.checkIn(session);
}
}
@Override
public TaxonomyItem retrieveTaxonById(String taxonId) throws IdNotValidException {
PluginSession session = sessionPool.checkOut();
int id = converId(taxonId);
try {
TaxonomyItem item = ObisClient.getTaxonById(session, id);
return item;
} catch (IdNotValidException inve)
{
logger.error("Error retrieving TaxonById", inve);
throw inve;
} catch (Exception e) {
logger.error("Error retrieving TaxonById", e);
return null;
} finally {
sessionPool.checkIn(session);
}
}
protected int converId(String taxonId) throws IdNotValidException
{
try {
return Integer.parseInt(taxonId);
} catch(NumberFormatException nfe)
{
logger.error("Invalid id "+taxonId, nfe);
throw new IdNotValidException();
}
}
}

682
src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java
View File

@ -1,682 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.gcube.common.core.utils.logging.GCUBELog;
import org.gcube.data.spd.model.BasisOfRecord;
import org.gcube.data.spd.model.CommonName;
import org.gcube.data.spd.model.exceptions.IdNotValidException;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.model.products.Product;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.model.products.Taxon;
import org.gcube.data.spd.model.products.TaxonomyItem;
import org.gcube.data.spd.model.products.TaxonomyStatus;
import org.gcube.data.spd.model.products.Product.ProductType;
import org.gcube.data.spd.model.products.TaxonomyStatus.Status;
import org.gcube.data.spd.model.util.ElementProperty;
import org.gcube.data.spd.obisplugin.data.ProductKey;
import org.gcube.data.spd.obisplugin.data.SearchFilters;
import org.gcube.data.spd.obisplugin.util.Cache;
import org.gcube.data.spd.obisplugin.util.DateUtil;
import org.gcube.data.spd.obisplugin.util.Util;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class ObisClient {
protected static GCUBELog logger = new GCUBELog(ObisClient.class);
//"2009-12-11 11:30:00-07"
protected static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("y-M-d");
protected static final DateUtil DATE_UTIL = DateUtil.getInstance();
protected static final SimpleDateFormat sdf = new SimpleDateFormat();
public static final Cache<Integer, TaxonomyItem> taxonomyItemCache = new Cache<Integer, TaxonomyItem>(1000);
public static int getOccurrencesCount(PluginSession session, int taxonId, int datasetId, SearchFilters filters) throws SQLException
{
PreparedStatement statement = session.getOccurrencesCountPreparedStatement(taxonId, datasetId, filters);
ResultSet rs = statement.executeQuery();
if (!rs.next()) return 0;
int occurrences = rs.getInt("occurrences");
rs.close();
return occurrences;
}
public static void getOccurrences(PluginSession session, String key, Writer<OccurrencePoint> writer) throws Exception
{
ProductKey productKey = ProductKey.deserialize(key);
PreparedStatement statement = session.getOccurrencesPreparedStatement(productKey.getTaxonId(), productKey.getDataSetId(), productKey.getFilters());
ResultSet rs = statement.executeQuery();
String credits = generateCredits();
//String citation = generateCitation();
DataSet dataset = getDataSetById(session, productKey.getDataSetId());
boolean continueWrite = true;
while(rs.next() && continueWrite) {
OccurrencePoint occurrence = generateOccurrencePoint(rs, credits, dataset);
continueWrite = writer.write(occurrence);
};
rs.close();
}
private static DataSet getDataSetById(PluginSession session, int datasetId){
logger.debug("retrieving dataset for id "+datasetId);
DataSet dataset = null;
ResultSet datasetRs = null;
try{
PreparedStatement datasetStatement = session.getDatasetPerDatasetIdPreparedStatement(datasetId);
datasetRs = datasetStatement.executeQuery();
if (!datasetRs.next()) return null;
dataset = retrieveDatasetInformation(datasetRs);
}catch(Exception e){
logger.warn("cannot retrieve dataset for occurrence point with id "+datasetId,e);
}finally{
try{
if (datasetRs!=null)
datasetRs.close();
}catch(Exception e) {}
}
logger.debug("dataset returned "+dataset);
return dataset;
}
protected static OccurrencePoint generateOccurrencePoint(ResultSet rs, String credits, DataSet dataSet) throws SQLException
{
int id = rs.getInt("id");
OccurrencePoint occurrence = new OccurrencePoint(String.valueOf(id));
//drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord,
occurrence.setDecimalLatitude(rs.getDouble("latitude"));
occurrence.setDecimalLongitude(rs.getDouble("longitude"));
Timestamp dateValue = rs.getTimestamp("datecollected");
if (dateValue!=null) {
Calendar dateCollected = Calendar.getInstance();
dateCollected.setTimeInMillis(dateValue.getTime());
occurrence.setEventDate(dateCollected);
} else {
//dxs.yearcollected, dxs.monthcollected, dxs.daycollected
try {
Calendar dateCollected = Calendar.getInstance();
int year = Integer.parseInt(rs.getString("yearcollected"));
int month = Integer.parseInt(rs.getString("monthcollected"));
int date = Integer.parseInt(rs.getString("daycollected"));
dateCollected.set(year, month, date);
occurrence.setEventDate(dateCollected);
} catch(NumberFormatException nfe){}
}
String basisOfRecord = rs.getString("basisofrecord");
occurrence.setBasisOfRecord(getBasisOfRecord(basisOfRecord));
//dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector
occurrence.setCitation(rs.getString("citation"));
occurrence.setCredits(credits);
occurrence.setInstitutionCode(rs.getString("institutioncode"));
occurrence.setCollectionCode(rs.getString("collectioncode"));
occurrence.setCatalogueNumber(rs.getString("catalognumber"));
occurrence.setRecordedBy(rs.getString("collector"));
//, dxs.datelastmodified,
String datelastmodified = rs.getString("datelastmodified");
if (datelastmodified!=null) {
try {
java.util.Date date = DATE_UTIL.parse(datelastmodified);
Calendar lastmodified = Calendar.getInstance();
lastmodified.setTimeInMillis(date.getTime());
occurrence.setModified(lastmodified);
} catch (Exception e) {
//logger.warn("Unknow date format "+datelastmodified);
}
}
//dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated
occurrence.setCountry(rs.getString("country"));
occurrence.setLocality(rs.getString("locality"));
occurrence.setMinDepth(rs.getDouble("minimumdepth"));
occurrence.setMaxDepth(rs.getDouble("maximumdepth"));
occurrence.setCoordinateUncertaintyInMeters(rs.getString("coordinateprecision"));
//"Animalia|Chordata|Chondrichthyes|Lamniformes|Lamnidae|Carcharodon||carcharias||Carcharodon carcharias|Linnaeus"
/*
* Kingdom: Animalia
Phylum: Chordata
Class: Chondrichthyes
Subclass: Elasmobranchii
Order: Lamniformes
Family: Lamnidae
Genus: Carcharodon
A. Smith, 1838
Species: C. carcharias
*/
String concatenated = rs.getString("concatenated");
if (concatenated!=null) {
int authorStartIndex = concatenated.lastIndexOf('|');
if (authorStartIndex>0) {
String snPart = concatenated.substring(0, authorStartIndex);
int scientificNameStartIndex = snPart.lastIndexOf('|');
if (scientificNameStartIndex>0) {
String author = (authorStartIndex+1<concatenated.length())?concatenated.substring(authorStartIndex+1):"";
String sn = (scientificNameStartIndex+1<snPart.length())?snPart.substring(scientificNameStartIndex+1):"";
occurrence.setScientificName(sn/*+" ("+author+")"*/);
}
}
/*String[] taxon = concatenated.split("|");
if (taxon.length>0) occurrence.setKingdom(taxon[0]);
if (taxon.length>4) occurrence.setFamily(taxon[4]);
if (taxon.length>11) {
String scientific = taxon[9] +"("+taxon[10]+")";
occurrence.setScientificName(scientific);
}*/
}
occurrence.setScientificNameAuthorship(rs.getString("snAuthor"));
occurrence.setIdentifiedBy(rs.getString("identifiedBy"));
occurrence.setDataSet(dataSet);
//retrieving dataset
return occurrence;
}
public static OccurrencePoint getOccurrenceById(PluginSession session, String id) throws Exception
{
PreparedStatement statement = session.getOccurrenceByIdPreparedStatement(Integer.parseInt(id));
ResultSet rs = statement.executeQuery();
OccurrencePoint occurrence = null;
String credits = generateCredits();
if(rs.next()){
int objId = rs.getInt("valid_id");
occurrence = generateOccurrencePoint(rs, credits, getDataSetById(session, objId));
}
rs.close();
return occurrence;
}
public static BasisOfRecord getBasisOfRecord(String basis)
{
if (basis==null) return BasisOfRecord.Unknown;
if (basis.equalsIgnoreCase("HumanObservation") || basis.equalsIgnoreCase("O")) return BasisOfRecord.HumanObservation;
if (basis.equalsIgnoreCase("L")) return BasisOfRecord.LivingSpecimen;
if (basis.equalsIgnoreCase("P")) return BasisOfRecord.Literature;
if (basis.equalsIgnoreCase("PreservedSpecimen") || basis.equalsIgnoreCase("S")) return BasisOfRecord.PreservedSpecimen;
return BasisOfRecord.Unknown;
}
public static void searchByCommonName(PluginSession session, String searchTerm, SearchFilters filters, Writer<ResultItem> writer) throws Exception
{
PreparedStatement statement = session.getSearchCommonNamePreparedStatement(searchTerm);
ResultSet rs = statement.executeQuery();
generateResultItems(session, rs, filters, writer);
}
protected static void fillProducts(PluginSession session, int speciesId, int datasetId, String key, SearchFilters filters, ResultItem item) throws SQLException
{
List<Product> products = new LinkedList<Product>();
//OCCURRENCES
Product occurences = new Product(ProductType.Occurrence, key);
int occurencesCount = getOccurrencesCount(session, speciesId, datasetId, filters);
occurences.setCount(occurencesCount);
products.add(occurences);
logger.trace("product is "+occurences);
item.setProducts(products);
}
public static void searchByScientificName(PluginSession session, String searchTerm, SearchFilters filters, Writer<ResultItem> writer) throws Exception
{
PreparedStatement statement = session.getSearchScientificNamePreparedStatement(searchTerm);
ResultSet rs = statement.executeQuery();
generateResultItems(session, rs, filters, writer);
}
protected static void generateResultItems(PluginSession session, ResultSet rs, SearchFilters filters, Writer<ResultItem> writer) throws Exception
{
//System.out.println("generating records");
boolean continueWrite = true;
String credits = generateCredits();
String citation = generateCitation();
while(rs.next() && continueWrite) {
int id = rs.getInt("id");
//System.out.println("id "+id);
ResultItem baseItem = new ResultItem(String.valueOf(id), "");
fillTaxon(session, id, baseItem, credits, citation);
fillCommonNames(session, id, baseItem);
PreparedStatement datasetStatement = session.getDatasetPreparedStatement(id);
ResultSet dataSetrs = datasetStatement.executeQuery();
while(dataSetrs.next()) {
ResultItem item = Util.cloneResultItem(baseItem);
DataSet dataset = retrieveDatasetInformation(dataSetrs);
item.setDataSet(dataset);
int dataSetId = Integer.parseInt(dataset.getId());
ProductKey key = new ProductKey(id, dataSetId, filters);
logger.trace("datasetid is "+dataSetId+" and product key created is "+key);
fillProducts(session, id, dataSetId, key.serialize(), filters, item);
continueWrite = writer.write(item);
}
}
rs.close();
}
/**
* Fills the node with the taxon information. Also information about parent are retrieved.
* @param connection the db connection.
* @param id the taxon id.
* @param taxonNode the node to fill.
* @throws SQLException
*/
protected static void fillTaxon(PluginSession session, int id, Taxon taxon, String credits, String citation) throws SQLException
{
PreparedStatement statement = session.getTaxonPreparedStatement(id);
ResultSet rs = statement.executeQuery();
if (rs.next()) {
taxon.setCitation(citation);
taxon.setCredits(credits);
//taxon informations
taxon.setScientificName(rs.getString("tname"));
taxon.setScientificNameAuthorship(rs.getString("tauthor"));
String rank = rs.getString("rank_name");
taxon.setRank((rank!=null)?rank:"");
int parentId = rs.getInt("parent_id");
rs.close();
//check for parent
if (parentId!=id) {
//create and fill the parent
Taxon parent = new Taxon(String.valueOf(parentId));
fillTaxon(session, parentId, parent, credits, citation);
taxon.setParent(parent);
}
}
}
protected static void fillTaxonomyItem(PluginSession session, int id, TaxonomyItem item, String credits, String citation) throws Exception
{
PreparedStatement statement = session.getTaxonPreparedStatement(id);
ResultSet rs = statement.executeQuery();
if (rs.next()) {
//taxon informations
item.setScientificName(rs.getString("tname"));
String author = Util.stripNotValidXMLCharacters(rs.getString("tauthor"));
item.setScientificNameAuthorship(author);
//properties
item.addProperty(new ElementProperty("worms_id", rs.getString("worms_id")));
item.addProperty(new ElementProperty("col_id", rs.getString("col_id")));
item.addProperty(new ElementProperty("irmng_id", rs.getString("irmng_id")));
item.addProperty(new ElementProperty("itis_id", rs.getString("itis_id")));
item.setCredits(credits);
item.setCitation(citation);
String rank = rs.getString("rank_name");
item.setRank((rank!=null)?rank:"");
item.setStatus(new TaxonomyStatus("", Status.ACCEPTED));
boolean parentNull = rs.getObject("parent_id")==null;
int parentId = rs.getInt("parent_id");
rs.close();
//fill common names
fillCommonNames(session, id, item);
//check for parent
if (!parentNull && parentId!=id) {
//create and fill the parent
TaxonomyItem parent = taxonomyItemCache.get(parentId);
if (parent == null) {
parent = new TaxonomyItem(String.valueOf(parentId));
fillTaxonomyItem(session, parentId, parent, credits, citation);
}
item.setParent(parent);
}
} else throw new IdNotValidException("Taxon with id "+id+" not found");
}
protected static void fillCommonNames(PluginSession session, int taxonNameId, TaxonomyItem item) throws SQLException
{
PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId);
ResultSet rs = statement.executeQuery();
List<CommonName> commonNames = new ArrayList<CommonName>();
while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname")));
rs.close();
item.setCommonNames(commonNames);
}
protected static void fillCommonNames(PluginSession session, int taxonNameId, ResultItem item) throws SQLException
{
PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId);
ResultSet rs = statement.executeQuery();
List<CommonName> commonNames = new ArrayList<CommonName>();
while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname")));
rs.close();
item.setCommonNames(commonNames);
}
private static DataSet retrieveDatasetInformation(ResultSet rs) throws SQLException{
int dataSetId = rs.getInt("datasetId");
DataSet dataSet = new DataSet(String.valueOf(dataSetId));
dataSet.setCitation(rs.getString("datasetCitation"));
dataSet.setName(rs.getString("datasetName"));
DataProvider dataProvider = new DataProvider(String.valueOf(rs.getInt("providerId")));
dataProvider.setName(rs.getString("providerName"));
dataSet.setDataProvider(dataProvider);
return dataSet;
}
public static Set<String> getCommonNames(PluginSession session, String scientificName) throws SQLException
{
PreparedStatement statement = session.getCommonNameFromScientificNamePreparedStatement(scientificName);
ResultSet rs = statement.executeQuery();
Set<String> commonNames = new HashSet<String>();
while(rs.next()) commonNames.add(rs.getString("cname"));
rs.close();
return commonNames;
}
public static void getScientificNames(PluginSession session, String commonName, Writer<String> writer) throws SQLException
{
PreparedStatement statement = session.getScientificNameFromCommonNamePreparedStatement(commonName);
ResultSet rs = statement.executeQuery();
while (rs.next() && writer.write(rs.getString("tname")));
rs.close();
}
public static void getTaxonByScientificNames(PluginSession session, String scientificName, Writer<TaxonomyItem> writer) throws Exception
{
PreparedStatement statement = session.getScientificNamePreparedStatement(scientificName);
ResultSet rs = statement.executeQuery();
generateTaxonomyItems(session, rs, writer);
}
public static void getTaxonByCommonName(PluginSession session, String commonName, Writer<TaxonomyItem> writer) throws Exception
{
PreparedStatement statement = session.getCommonNamePreparedStatement(commonName);
ResultSet rs = statement.executeQuery();
generateTaxonomyItems(session, rs, writer);
}
protected static void generateTaxonomyItems(PluginSession session, ResultSet rs, Writer<TaxonomyItem> writer) throws SQLException, Exception
{
boolean continueWrite = true;
String credits = generateCredits();
String citation = generateCitation();
while(rs.next() && continueWrite) {
Integer id = rs.getInt("id");
TaxonomyItem taxon = taxonomyItemCache.get(id);
if (taxon == null) {
taxon = new TaxonomyItem(String.valueOf(id));
fillTaxonomyItem(session, id, taxon, credits, citation);
taxonomyItemCache.put(id, taxon);
}
//TaxonomyItem taxon = new TaxonomyItem(String.valueOf(id));
//fillTaxonomyItem(session, id, taxon);
continueWrite = writer.write(taxon);
}
rs.close();
}
protected static List<TaxonomyItem> getChildrenTaxon(PluginSession session, int id) throws SQLException
{
PreparedStatement statement = session.getChildrenTaxonPreparedStatement(id);
ResultSet rs = statement.executeQuery();
List<TaxonomyItem> children = new ArrayList<TaxonomyItem>();
while (rs.next()) {
//taxon informations
int taxonId = rs.getInt("id");
//FIXME tmp workaround
if (taxonId == id) continue;
TaxonomyItem child = new TaxonomyItem(String.valueOf(taxonId));
child.setScientificName(rs.getString("tname"));
child.setCitation(rs.getString("tauthor"));
String rank = rs.getString("rank_name");
child.setRank((rank!=null)?rank:"");
child.setStatus(new TaxonomyStatus("",Status.ACCEPTED));
fillCommonNames(session, id, child);
children.add(child);
}
rs.close();
return children;
}
protected static TaxonomyItem getTaxonById(PluginSession session, int id) throws Exception
{
TaxonomyItem item = new TaxonomyItem(String.valueOf(id));
String credits = generateCredits();
String citation = generateCitation();
fillTaxonomyItem(session, id, item, credits, citation);
return item;
}
protected static String generateCitation()
{
StringBuilder citation = new StringBuilder("Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org. (Consulted on ");
citation.append(sdf.format(Calendar.getInstance().getTime()));
citation.append(")");
return citation.toString();
}
protected static String generateCredits()
{
//credits ="This information object has been generated via the Species Product Discovery service on 2012-11-26 by interfacing with the Interim Register of Marine and Nonmarine Genera (IRMNG) (http://www.obis.org.au/irmng/)";
StringBuilder credits = new StringBuilder("This information object has been generated via the Species Product Discovery service on ");
credits.append(sdf.format(Calendar.getInstance().getTime()));
credits.append(" by interfacing with the Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org.");
return credits.toString();
}
static int counter = 0;
static int sum = 0;
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
String url = "jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis";
Properties props = new Properties();
props.setProperty("user","postgres");
props.setProperty("password","0b1s@d4sc13nc3");
final Connection connection = DriverManager.getConnection(url, props);
// System.out.println("Connected");
final PluginSession session = new PluginSession(connection);
session.preCacheStatements();
searchByScientificName(session, "gadus morhua", new SearchFilters(), new Writer<ResultItem>() {
@Override
public boolean write(ResultItem item) {
// System.out.println(item.getId()+" "+item.getScientificNameAuthorship()+" "+item.getScientificName());
return true;
}
});
/*getTaxonByScientificNames(session, "sarda sarda", new Writer<TaxonomyItem>() {
@Override
public boolean write(TaxonomyItem item) {
System.out.println(item.getId()+" "+item.getAuthor()+" "+item.getScientificName());
return true;
}
});*/
/*OccurrencePoint occurrencePoint = getOccurrenceById(session, "38069270");
System.out.println(occurrencePoint);*/
/*Taxon taxon = getTaxonByCommonName(session, "ruwe traliehoorn");//getTaxonByScientificNames(session, "Protozoa");
System.out.println(taxon);
List<Taxon> children = getChildrenTaxon(session, Integer.parseInt(taxon.getId()));
for (Taxon child:children) System.out.println(child);*/
/*searchByCommonName(session, "white shark", new SearchFilters(), new Writer<ResultItem>() {
@Override
public void write(ResultItem item) {
System.out.println("Item: "+item.getDataSet().getDataProvider().getName()+" <-> "+item.getDataSet().getName());
}
});*/
/*final long start = System.currentTimeMillis();
getTaxonByScientificNames(session, "Gadus macrocephalus", new Writer<TaxonomyItem>() {
long start = System.currentTimeMillis();
@Override
public void write(TaxonomyItem item) {
System.out.println(item);
}
});*/
/*SearchFilters filters = new SearchFilters();
ObisClient.searchByScientificName(session, "sarda sarda", filters, new Writer<ResultItem>() {
@Override
public void write(ResultItem item) {
System.out.println(item);
}
});*/
/*List<TaxonomyItem> taxa = getChildrenTaxon(session,769809);
for (TaxonomyItem taxon:taxa) System.out.println(taxon.getId());*/
//navigate("", session, 741923);
//System.out.println("result in "+(System.currentTimeMillis()-start)+" avg: "+(sum/counter)+" tot: "+counter);
// System.out.println("done");
session.expire();
}
protected static Set<Integer> found = new HashSet<Integer>();
protected static Map<String,TaxonomyItem> foundTaxon = new HashMap<String,TaxonomyItem>();
protected static void navigate(String indentation, PluginSession session, int id) throws SQLException
{
//System.out.println("looking for children: "+id);
if (found.contains(id)) {
System.err.println("Already found "+id);
System.err.println(foundTaxon.get(id));
System.exit(-1);
}
List<TaxonomyItem> taxa = getChildrenTaxon(session,id);
found.add(id);
for (TaxonomyItem taxon:taxa) {
System.out.println(indentation+taxon.getId()+" "+taxon.getRank());
foundTaxon.put(taxon.getId(), taxon);
navigate(indentation+" ", session, Integer.valueOf(taxon.getId()));
}
}
}

55
src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java
View File

@ -1,55 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin;
import java.sql.SQLException;
import org.apache.log4j.Logger;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.obisplugin.pool.PluginSessionPool;
import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class ObisNameMapping implements MappingCapability {
protected Logger logger = Logger.getLogger(ObisNameMapping.class);
protected PluginSessionPool sessionPool;
/**
* @param sessionPool
*/
public ObisNameMapping(PluginSessionPool sessionPool) {
this.sessionPool = sessionPool;
}
/**
* {@inheritDoc}
*/
@Override
public void getRelatedScientificNames(final ObjectWriter<String> writer, String commonName) {
logger.debug("retrieving mapping for "+commonName);
PluginSession session = sessionPool.checkOut();
try {
ObisClient.getScientificNames(session, commonName, new Writer<String>() {
@Override
public boolean write(String item) {
writer.write(item);
return writer.isAlive();
}
});
} catch (SQLException e) {
logger.error("An error occurred retrieving the mapping for common name "+commonName, e);
writer.write(new StreamBlockingException("OBIS",commonName));
} finally {
sessionPool.checkIn(session);
}
}
}

133
src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java
View File

@ -1,133 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.apache.log4j.Logger;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.model.products.Product;
import org.gcube.data.spd.model.products.Product.ProductType;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.obisplugin.data.SearchFilters;
import org.gcube.data.spd.obisplugin.pool.PluginSessionPool;
import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability;
import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class ObisOccurrencesInterface extends OccurrencesCapability {
protected static final Set<Conditions> SUPPORTED_PROPERTIES = new HashSet<Conditions>(Arrays.asList(Conditions.values()));
protected Logger logger = Logger.getLogger(ObisOccurrencesInterface.class);
protected PluginSessionPool sessionPool;
/**
* @param sessionPool
*/
public ObisOccurrencesInterface(PluginSessionPool sessionPool) {
this.sessionPool = sessionPool;
}
/**
* {@inheritDoc}
*/
@Override
public Set<Conditions> getSupportedProperties() {
return SUPPORTED_PROPERTIES;
}
@Override
public void searchByScientificName(String word, final ObjectWriter<OccurrencePoint> writer, Condition... properties) {
final PluginSession session = sessionPool.checkOut();
try {
SearchFilters filters = new SearchFilters(properties);
logger.trace("filters: "+filters);
ObisClient.searchByScientificName(session, word, filters, new Writer<ResultItem>() {
@Override
public boolean write(ResultItem item) {
for (Product product:item.getProducts()){
if (product.getType()==ProductType.Occurrence) {
String key = product.getKey();
getOccurrencePoints(session, key, writer);
}
}
return writer.isAlive();
}
});
} catch (Exception e) {
logger.debug("searchByScientificName failed",e);
} finally {
sessionPool.checkIn(session);
}
}
@Override
public void getOccurrencesByIds(ClosableWriter<OccurrencePoint> writer, Iterator<String> reader) {
final PluginSession session = sessionPool.checkOut();
try {
while(reader.hasNext() && writer.isAlive()){
String id = reader.next();
try {
OccurrencePoint occurrencePoint = ObisClient.getOccurrenceById(session, id);
if (occurrencePoint!=null) writer.write(occurrencePoint);
} catch (Exception e) {
logger.debug("searchByScientificName failed",e);
}
}
writer.close();
} finally {
sessionPool.checkIn(session);
}
}
@Override
public void getOccurrencesByProductKeys(ClosableWriter<OccurrencePoint> writer, Iterator<String> reader) {
PluginSession session = sessionPool.checkOut();
try {
while(reader.hasNext() && writer.isAlive()){
String key = reader.next();
getOccurrencePoints(session, key, writer);
}
writer.close();
} finally {
sessionPool.checkIn(session);
}
}
protected void getOccurrencePoints(PluginSession session, String key, final ObjectWriter<OccurrencePoint> writer)
{
try {
ObisClient.getOccurrences(session, key, new Writer<OccurrencePoint>() {
@Override
public boolean write(OccurrencePoint item) {
writer.write(item);
return writer.isAlive();
}
});
} catch (Exception e) {
logger.error("Error getting occurrence points for key "+key, e);
}
}
}

94
src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java
View File

@ -5,23 +5,21 @@ import java.util.Calendar;
import java.util.HashSet;
import java.util.Set;
import org.gcube.common.core.utils.logging.GCUBELog;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.encryption.StringEncrypter;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.RepositoryInfo;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.model.util.Capabilities;
import org.gcube.data.spd.obisplugin.data.SearchFilters;
import org.gcube.data.spd.obisplugin.pool.DatabaseCredential;
import org.gcube.data.spd.obisplugin.pool.PluginSessionPool;
import org.gcube.data.spd.obisplugin.capabilities.OccurrencesCapabilityImpl;
import org.gcube.data.spd.obisplugin.search.ResultItemSearch;
import org.gcube.data.spd.plugin.fwk.AbstractPlugin;
import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability;
import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability;
import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
@ -38,33 +36,27 @@ public class ObisPlugin extends AbstractPlugin {
protected static final RepositoryInfo REPOSITORY_INFO = new RepositoryInfo(LOGO_URL, HOME_URL, DESCRIPTION);
protected static final String ENTRY_POINT_NAME = "jdbc";
protected GCUBELog logger = new GCUBELog(ObisPlugin.class);
protected PluginSessionPool sessionPool;
protected ObisNameMapping nameMapping;
protected ObisOccurrencesInterface occurrencesInterface;
protected ObisClassification obisClassification;
protected static Logger logger = LoggerFactory.getLogger(ObisPlugin.class);
//protected ObisNameMapping nameMapping;
protected OccurrencesCapabilityImpl occurrencesInterface;
//protected ObisClassification obisClassification;
protected static final SimpleDateFormat sdf = new SimpleDateFormat();
/**
* @return the sessionPool
*/
public PluginSessionPool getSessionPool() {
return sessionPool;
}
private String baseUrl = "http://api.iobis.org/";
@SuppressWarnings("serial")
@Override
public Set<Capabilities> getSupportedCapabilities() {
return new HashSet<Capabilities>(){{
add(Capabilities.NamesMapping);
//add(Capabilities.NamesMapping);
add(Capabilities.Occurrence);
add(Capabilities.Classification);
//add(Capabilities.Classification);
}};
}
@Override
public String getRepositoryName() {
return "OBIS";
return Constants.REPOSITORY_NAME;
}
@Override
@ -80,20 +72,9 @@ public class ObisPlugin extends AbstractPlugin {
setUseCache(true);
DatabaseCredential databaseCredential = getDatabaseCredentials(resource);
sessionPool = new PluginSessionPool(databaseCredential);
nameMapping = new ObisNameMapping(sessionPool);
occurrencesInterface = new ObisOccurrencesInterface(sessionPool);
obisClassification = new ObisClassification(sessionPool);
}
public void initialize(DatabaseCredential databaseCredential) throws Exception {
setUseCache(true);
sessionPool = new PluginSessionPool(databaseCredential);
nameMapping = new ObisNameMapping(sessionPool);
occurrencesInterface = new ObisOccurrencesInterface(sessionPool);
obisClassification = new ObisClassification(sessionPool);
//nameMapping = new ObisNameMapping();
occurrencesInterface = new OccurrencesCapabilityImpl(baseUrl);
//obisClassification = new ObisClassification();
}
/**
@ -101,7 +82,6 @@ public class ObisPlugin extends AbstractPlugin {
*/
@Override
public void shutdown() throws Exception {
sessionPool.shutdown(true);
}
/**
@ -121,27 +101,6 @@ public class ObisPlugin extends AbstractPlugin {
*/
@Override
public void update(ServiceEndpoint resource) throws Exception {
DatabaseCredential databaseCredential = getDatabaseCredentials(resource);
sessionPool.setDatabaseCredential(databaseCredential);
}
protected DatabaseCredential getDatabaseCredentials(ServiceEndpoint resource) throws Exception
{
AccessPoint jdbcAccessPoint = null;
for (AccessPoint accessPoint: resource.profile().accessPoints())
{
if (ENTRY_POINT_NAME.equalsIgnoreCase(accessPoint.name())) {
jdbcAccessPoint = accessPoint;
break;
}
}
if (jdbcAccessPoint==null) {
logger.error("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource");
throw new Exception("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource");
}
String password = StringEncrypter.getEncrypter().decrypt(jdbcAccessPoint.password());
return new DatabaseCredential(jdbcAccessPoint.address(), jdbcAccessPoint.username(), password);
}
@ -152,25 +111,10 @@ public class ObisPlugin extends AbstractPlugin {
public void searchByScientificName(String searchTerm, final ObjectWriter<ResultItem> writer, Condition... properties) {
logger.debug("starting the search for obisPlugin word: "+searchTerm);
PluginSession session = sessionPool.checkOut();
try {
final String credits = getObisCredits();
SearchFilters filters = new SearchFilters(properties);
logger.trace("filters: "+filters);
ObisClient.searchByScientificName(session, searchTerm, filters, new Writer<ResultItem>() {
@Override
public boolean write(ResultItem item) {
item.setCredits(credits);
writer.write(item);
return writer.isAlive();
}
});
new ResultItemSearch(baseUrl, searchTerm, properties).search(writer,Constants.QUERY_LIMIT);
} catch (Exception e) {
logger.debug("searchByScientificName failed",e);
} finally {
sessionPool.checkIn(session);
}
}
@ -179,7 +123,7 @@ public class ObisPlugin extends AbstractPlugin {
*/
@Override
public MappingCapability getMappingInterface() {
return nameMapping;
return null;
}
/**
@ -203,7 +147,7 @@ public class ObisPlugin extends AbstractPlugin {
*/
@Override
public ClassificationCapability getClassificationInterface() {
return obisClassification;
return null;
}
/**

429
src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java
View File

@ -1,429 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Calendar;
import java.util.List;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Coordinate;
import org.gcube.data.spd.obisplugin.data.SearchFilters;
import org.gcube.data.spd.obisplugin.pool.DatabaseCredential;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class PluginSession {
protected static final String SCHEMA = "obis";
protected DatabaseCredential databaseCredential;
protected Connection connection;
protected PreparedStatement taxonPreparedStatement;
protected PreparedStatement taxonCommonNamePreparedStatement;
protected PreparedStatement datasetPreparedStatement;
protected PreparedStatement commonNameFromScientificNamePreparedStatement;
protected PreparedStatement scientificNameFromCommonNamePreparedStatement;
protected PreparedStatement searchCommonNamePreparedStatement;
protected PreparedStatement searchScientificNamePreparedStatement;
protected PreparedStatement scientificNamePreparedStatement;
protected PreparedStatement commonNamePreparedStatement;
protected PreparedStatement childrenTaxonPreparedStatement;
protected PreparedStatement occurrenceByIdPreparedStatement;
protected PreparedStatement datasetPerIdPreparedStatement;
/**
* @param connection
*/
public PluginSession(Connection connection) {
this(null,connection);
}
/**
* @param credentialToken
* @param connection
*/
public PluginSession(DatabaseCredential databaseCredential, Connection connection) {
this.databaseCredential = databaseCredential;
this.connection = connection;
}
/**
* @return the connection
*/
public Connection getConnection() {
return connection;
}
/**
* The session is valid if the connection is OK and if the connection has been created using the same credentials.
* @param credentialToken
* @return
* @throws SQLException
*/
public boolean isValid(DatabaseCredential databaseCredential) throws SQLException
{
return (this.databaseCredential!=null?(this.databaseCredential.equals(databaseCredential)):true) && !connection.isClosed() && isValid();
}
protected boolean isValid()
{
try{
ResultSet result = connection.createStatement().executeQuery("SELECT 1");
result.close();
}catch (Exception e) {
return false;
}
return true;
}
public void expire() throws SQLException
{
connection.close();
}
public void preCacheStatements() throws SQLException
{
createTaxonPreparedStatement();
createTaxonCommonNamePreparedStatemen();
createDatasetPreparedStatement();
createCommonNameFromScientificNamePreparedStatement();
createScientificNameFromCommonNamePreparedStatement();
createSearchCommonNamePreparedStatement();
createSearchScientificNamePreparedStatement();
createScientificNamePreparedStatement();
createCommonNamePreparedStatement();
createChildrenTaxonPreparedStatement();
createOccurrenceByIdPreparedStatement();
createDatasetPerDatasetIdPreparedStatement();
}
public PreparedStatement getTaxonPreparedStatement(int id) throws SQLException
{
if (taxonPreparedStatement==null) createTaxonPreparedStatement();
taxonPreparedStatement.clearParameters();
taxonPreparedStatement.setInt(1, id);
return taxonPreparedStatement;
}
protected void createTaxonPreparedStatement() throws SQLException
{
taxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id, t.parent_id, t.tauthor, t.worms_id, t.col_id, t.irmng_id, t.itis_id, r.rank_name " +
"FROM "+SCHEMA+".tnames t " +
"LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+
"WHERE t.id = ?");
}
public PreparedStatement getTaxonCommonNamePreparedStatement(int taxonNameId) throws SQLException
{
if (taxonCommonNamePreparedStatement==null) createTaxonCommonNamePreparedStatemen();
taxonCommonNamePreparedStatement.clearParameters();
taxonCommonNamePreparedStatement.setInt(1, taxonNameId);
return taxonCommonNamePreparedStatement;
}
protected void createTaxonCommonNamePreparedStatemen() throws SQLException
{
taxonCommonNamePreparedStatement = connection.prepareStatement("select c.cname, l.lanname FROM "+SCHEMA+".cnames c, "+SCHEMA+".languages l WHERE c.tname_id = ? AND c.language_id = l.id");
}
public PreparedStatement getDatasetPreparedStatement(int id) throws SQLException
{
if (datasetPreparedStatement==null) createDatasetPreparedStatement();
datasetPreparedStatement.clearParameters();
datasetPreparedStatement.setInt(1, id);
return datasetPreparedStatement;
}
public PreparedStatement getDatasetPerDatasetIdPreparedStatement(int dataSetId) throws SQLException
{
if (datasetPerIdPreparedStatement==null) createDatasetPerDatasetIdPreparedStatement();
datasetPerIdPreparedStatement.clearParameters();
datasetPerIdPreparedStatement.setInt(1, dataSetId);
return datasetPerIdPreparedStatement;
}
public PreparedStatement getCommonNameFromScientificNamePreparedStatement(String scientificaName) throws SQLException
{
if (commonNameFromScientificNamePreparedStatement==null) createCommonNameFromScientificNamePreparedStatement();
commonNameFromScientificNamePreparedStatement.clearParameters();
commonNameFromScientificNamePreparedStatement.setString(1, scientificaName);
return commonNameFromScientificNamePreparedStatement;
}
protected void createCommonNameFromScientificNamePreparedStatement() throws SQLException
{
commonNameFromScientificNamePreparedStatement = connection.prepareStatement("SELECT c.cname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE t.tname ILIKE ? AND c.tname_id = t.id");
}
public PreparedStatement getScientificNameFromCommonNamePreparedStatement(String commonName) throws SQLException
{
if (scientificNameFromCommonNamePreparedStatement==null) createScientificNameFromCommonNamePreparedStatement();
scientificNameFromCommonNamePreparedStatement.clearParameters();
scientificNameFromCommonNamePreparedStatement.setString(1, commonName);
return scientificNameFromCommonNamePreparedStatement;
}
protected void createScientificNameFromCommonNamePreparedStatement() throws SQLException
{
scientificNameFromCommonNamePreparedStatement = connection.prepareStatement("SELECT DISTINCT t.tname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE c.cname ILIKE ? AND c.tname_id = t.id");
}
public PreparedStatement getSearchCommonNamePreparedStatement(String searchTerm) throws SQLException
{
if (searchCommonNamePreparedStatement == null) createSearchCommonNamePreparedStatement();
searchCommonNamePreparedStatement.clearParameters();
searchCommonNamePreparedStatement.setString(1, "%"+searchTerm+"%");
return searchCommonNamePreparedStatement;
}
protected void createSearchCommonNamePreparedStatement() throws SQLException
{
String query = "SELECT DISTINCT c.tname_id AS id FROM obis.cnames c WHERE c.cname ILIKE ?";
searchCommonNamePreparedStatement = connection.prepareStatement(query);
}
public PreparedStatement getSearchScientificNamePreparedStatement(String searchTerm) throws SQLException
{
if (searchScientificNamePreparedStatement == null) createSearchScientificNamePreparedStatement();
searchScientificNamePreparedStatement.clearParameters();
searchScientificNamePreparedStatement.setString(1, "%"+searchTerm+"%");
return searchScientificNamePreparedStatement;
}
protected void createSearchScientificNamePreparedStatement() throws SQLException
{
String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ? AND exists (SELECT 1 FROM obis.drs WHERE valid_id = t.id)";
searchScientificNamePreparedStatement = connection.prepareStatement(query);
}
public PreparedStatement getOccurrencesCountPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException{
//We don't cache it because in this case a PS is less performant
StringBuilder query = new StringBuilder("SELECT count(*) AS occurrences FROM "+SCHEMA+".drs WHERE valid_id = ? AND resource_id = ?");
buildConditions(query, "", filters.getConditions());
/*if (filters.getUpperBound()!=null) query.append(" AND latitude <= ? AND longitude <= ?");
if (filters.getLowerBound()!=null) query.append(" AND latitude >= ? AND longitude >= ?");
if (filters.getFromDate()!=null) query.append(" AND datecollected >= ?");
if (filters.getToDate()!=null) query.append(" AND datecollected <= ?");*/
PreparedStatement occurrencesCountPreparedStatement = connection.prepareStatement(query.toString());
int parameterCounter = 1;
occurrencesCountPreparedStatement.setInt(parameterCounter++, taxonId);
occurrencesCountPreparedStatement.setInt(parameterCounter++, datasetId);
addParameters(occurrencesCountPreparedStatement, parameterCounter, filters.getConditions());
/*for (Condition condition:filters.getConditions()) {
switch (condition.getProperty()) {
case COORDINATE: {
Coordinate coordinate = (Coordinate) condition.getValue();
occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLatitude());
occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLongitude());
} break;
case EVENT_DATE: {
Calendar calendar = (Calendar) condition.getValue();
occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis()));
}
}
}*/
/*if (filters.getUpperBound() != null) {
occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude());
occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude());
}
if (filters.getLowerBound() != null) {
occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude());
occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude());
}
if (filters.getFromDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis()));
if (filters.getToDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/
return occurrencesCountPreparedStatement;
}
protected void buildConditions(StringBuilder query, String prefix, List<Condition> conditions)
{
for (Condition condition:conditions) buildCondition(query, prefix, condition);
}
protected void buildCondition(StringBuilder query, String prefix, Condition condition)
{
String op = "";
switch (condition.getOp()) {
case EQ: op = "=="; break;
case GE: op = ">="; break;
case GT: op = ">"; break;
case LE: op = "<="; break;
case LT: op = "<"; break;
}
switch (condition.getType()) {
case COORDINATE: {
query.append(" AND ");
query.append(prefix);
query.append("latitude ");
query.append(op);
query.append(" ? AND ");
query.append(prefix);
query.append("longitude ");
query.append(op);
query.append(" ?");
} break;
case DATE: {
query.append(" AND ");
query.append(prefix);
query.append("datecollected ");
query.append(op);
query.append(" ?");
} break;
}
}
protected void addParameters(PreparedStatement preparedStatement, int parameterCounter, List<Condition> conditions) throws SQLException {
for (Condition condition:conditions) {
switch (condition.getType()) {
case COORDINATE: {
Coordinate coordinate = (Coordinate) condition.getValue();
preparedStatement.setDouble(parameterCounter++, coordinate.getLatitude());
preparedStatement.setDouble(parameterCounter++, coordinate.getLongitude());
} break;
case DATE: {
Calendar calendar = (Calendar) condition.getValue();
preparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis()));
}
}
}
}
protected PreparedStatement getOccurrencesPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException
{
//We don't cache it because in this case a PS is less performant
StringBuilder query = new StringBuilder("SELECT drs.id, drs.valid_id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.identifiedBy, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames tn WHERE drs.valid_id = ? AND drs.resource_id = ? AND drs.id = dxs.dr_id AND tn.id = drs.valid_id ");
buildConditions(query, "drs.", filters.getConditions());
/*if (filters.getUpperBound() != null) query.append(" AND drs.latitude <= ? AND drs.longitude <= ?");
if (filters.getLowerBound() != null) query.append(" AND drs.latitude >= ? AND drs.longitude >= ?");
if (filters.getFromDate() != null) query.append(" AND drs.datecollected >= ?");
if (filters.getToDate() != null) query.append(" AND drs.datecollected <= ?");*/
PreparedStatement occurrencesPreparedStatement = connection.prepareStatement(query.toString());
int parameterCounter = 1;
occurrencesPreparedStatement.setInt(parameterCounter++, taxonId);
occurrencesPreparedStatement.setInt(parameterCounter++, datasetId);
addParameters(occurrencesPreparedStatement, parameterCounter, filters.getConditions());
/*if (filters.getUpperBound() != null) {
occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude());
occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude());
}
if (filters.getLowerBound() != null) {
occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude());
occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude());
}
if (filters.getFromDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis()));
if (filters.getToDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/
return occurrencesPreparedStatement;
}
public PreparedStatement getScientificNamePreparedStatement(String scientificName) throws SQLException
{
if (scientificNamePreparedStatement == null) createScientificNamePreparedStatement();
scientificNamePreparedStatement.clearParameters();
scientificNamePreparedStatement.setString(1, "%"+scientificName+"%");
return scientificNamePreparedStatement;
}
protected void createScientificNamePreparedStatement() throws SQLException
{
String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ?";
scientificNamePreparedStatement = connection.prepareStatement(query);
}
public PreparedStatement getCommonNamePreparedStatement(String commonName) throws SQLException
{
if (commonNamePreparedStatement==null) createCommonNamePreparedStatement();
commonNamePreparedStatement.clearParameters();
commonNamePreparedStatement.setString(1, commonName);
return commonNamePreparedStatement;
}
protected void createCommonNamePreparedStatement() throws SQLException
{
commonNamePreparedStatement = connection.prepareStatement("SELECT c.tname_id as id FROM "+SCHEMA+".cnames c WHERE c.cname LIKE ?");
}
public PreparedStatement getChildrenTaxonPreparedStatement(int id) throws SQLException
{
if (childrenTaxonPreparedStatement==null) createChildrenTaxonPreparedStatement();
childrenTaxonPreparedStatement.clearParameters();
childrenTaxonPreparedStatement.setInt(1, id);
return childrenTaxonPreparedStatement;
}
protected void createChildrenTaxonPreparedStatement() throws SQLException
{
childrenTaxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id as id, t.parent_id, t.tauthor, r.rank_name " +
"FROM "+SCHEMA+".tnames t " +
"LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+
"WHERE t.parent_id = ?");
}
protected PreparedStatement getOccurrenceByIdPreparedStatement(int occurrenceId) throws SQLException
{
if (occurrenceByIdPreparedStatement==null) createOccurrenceByIdPreparedStatement();
occurrenceByIdPreparedStatement.clearParameters();
occurrenceByIdPreparedStatement.setInt(1, occurrenceId);
return occurrenceByIdPreparedStatement;
}
protected void createOccurrenceByIdPreparedStatement() throws SQLException
{
String query = "SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames as tn WHERE drs.id = ? AND drs.id = dxs.dr_id AND drs.valid_id = tn.id";
occurrenceByIdPreparedStatement = connection.prepareStatement(query);
}
protected void createDatasetPreparedStatement() throws SQLException
{
/*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/
String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"r.id in (SELECT resource_id from portal.species_per_resource where valid_id = ?) AND r.provider_id = p.id";
datasetPreparedStatement = connection.prepareStatement(query);
}
protected void createDatasetPerDatasetIdPreparedStatement() throws SQLException
{
/*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/
String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"r.id = ? AND r.provider_id = p.id";
datasetPerIdPreparedStatement = connection.prepareStatement(query);
}
}

110
src/main/java/org/gcube/data/spd/obisplugin/capabilities/OccurrencesCapabilityImpl.java Normal file
View File

@ -0,0 +1,110 @@
package org.gcube.data.spd.obisplugin.capabilities;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.exceptions.StreamNonBlockingException;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.obisplugin.Constants;
import org.gcube.data.spd.obisplugin.search.OccurrenceSearch;
import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability;
import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class OccurrencesCapabilityImpl extends OccurrencesCapability{
private static Logger logger = LoggerFactory.getLogger(OccurrencesCapabilityImpl.class);
private String baseUrl;
public OccurrencesCapabilityImpl(String baseUrl) {
super();
this.baseUrl = baseUrl;
}
@SuppressWarnings("serial")
@Override
public Set<Conditions> getSupportedProperties() {
return new HashSet<Conditions>(){{
add(Conditions.DATE);
add(Conditions.COORDINATE);
}};
}
@Override
public void searchByScientificName(String word,
ObjectWriter<OccurrencePoint> writer, Condition... properties) {
try{
new OccurrenceSearch(baseUrl).search(writer, word, Constants.QUERY_LIMIT, properties);
} catch (Exception e) {
logger.debug("search occurrences by ScientificName failed",e);
}
}
@Override
public void getOccurrencesByProductKeys(
ClosableWriter<OccurrencePoint> writer, Iterator<String> keys) {
OccurrenceSearch occSearch = null;
try{
occSearch = new OccurrenceSearch(baseUrl);
}catch (Exception e) {
logger.error("error contacting obis server");
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
return;
}
while (keys.hasNext()){
String key = keys.next();
try{
occSearch.searchByKey(writer, key, Constants.QUERY_LIMIT);
}catch (Exception e) {
logger.warn("error retrieving key "+key, e);
writer.write(new StreamNonBlockingException(Constants.REPOSITORY_NAME,key));
}
}
writer.close();
}
@Override
public void getOccurrencesByIds(ClosableWriter<OccurrencePoint> writer,
Iterator<String> ids) {
OccurrenceSearch occSearch = null;
try{
occSearch = new OccurrenceSearch(baseUrl);
}catch (Exception e) {
logger.error("error contacting gbif server");
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
return;
}
while (ids.hasNext()){
String id = ids.next();
try{
if (!writer.isAlive()){
logger.trace("the writer is closed");
return;
}else writer.write(occSearch.searchById(id));
}catch (Exception e) {
logger.warn("error retrieving id "+id,e);
writer.write(new StreamNonBlockingException(Constants.REPOSITORY_NAME,id));
}
}
writer.close();
}
}

88
src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java
View File

@ -1,88 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.data;
import com.thoughtworks.xstream.XStream;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class ProductKey {
protected static XStream stream;
protected int taxonId;
protected int dataSetId;
protected SearchFilters filters;
/**
* @param taxonId
* @param dataSetId
* @param filters
*/
public ProductKey(int taxonId, int dataSetId, SearchFilters filters) {
this.taxonId = taxonId;
this.dataSetId = dataSetId;
this.filters = filters;
}
protected ProductKey() {}
/**
* @return the taxonId
*/
public int getTaxonId() {
return taxonId;
}
/**
* @return the dataSetId
*/
public int getDataSetId() {
return dataSetId;
}
/**
* @return the filters
*/
public SearchFilters getFilters() {
return filters;
}
protected static XStream getStream()
{
if (stream == null) stream = new XStream();
return stream;
}
public String serialize()
{
XStream stream = getStream();
return stream.toXML(this);
}
public static ProductKey deserialize(String key)
{
XStream stream = getStream();
return (ProductKey) stream.fromXML(key);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ProductKey [taxonId=");
builder.append(taxonId);
builder.append(", dataSetId=");
builder.append(dataSetId);
builder.append(", filters=");
builder.append(filters);
builder.append("]");
return builder.toString();
}
}

50
src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java
View File

@ -1,50 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.data;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.gcube.data.spd.model.Condition;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class SearchFilters {
protected List<Condition> conditions;
protected SearchFilters(){}
public SearchFilters(Condition ... conditions){
this.conditions = new ArrayList<Condition>(Arrays.asList(conditions));
}
public void addCondition(Condition condition)
{
conditions.add(condition);
}
/**
* @return the conditions
*/
public List<Condition> getConditions() {
return conditions;
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("SearchFilters [conditions=");
builder.append(conditions);
builder.append("]");
return builder.toString();
}
}

90
src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java
View File

@ -1,90 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.pool;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class DatabaseCredential {
protected String url;
protected String user;
protected String password;
/**
* @param url
* @param user
* @param password
*/
public DatabaseCredential(String url, String user, String password) {
this.url = url;
this.user = user;
this.password = password;
}
/**
* @return the url
*/
public String getUrl() {
return url;
}
/**
* @return the user
*/
public String getUser() {
return user;
}
/**
* @return the password
*/
public String getPassword() {
return password;
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((password == null) ? 0 : password.hashCode());
result = prime * result + ((url == null) ? 0 : url.hashCode());
result = prime * result + ((user == null) ? 0 : user.hashCode());
return result;
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
DatabaseCredential other = (DatabaseCredential) obj;
if (password == null) {
if (other.password != null)
return false;
} else if (!password.equals(other.password))
return false;
if (url == null) {
if (other.url != null)
return false;
} else if (!url.equals(other.url))
return false;
if (user == null) {
if (other.user != null)
return false;
} else if (!user.equals(other.user))
return false;
return true;
}
}

61
src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java
View File

@ -1,61 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.pool;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
* Adapted from http://sourcemaking.com/design_patterns/object_pool/java
*/
public class JDBCConnectionPool extends ObjectPool<Connection> {
protected String url, username, password;
public JDBCConnectionPool(String driver, String dsn, String usr, String pwd) {
super("JDBCConnectionPool", 10 * 60 * 1000);
try {
Class.forName(driver).newInstance();
} catch (Exception e) {
e.printStackTrace();
}
this.url = dsn;
this.username = usr;
this.password = pwd;
}
@Override
protected Connection create() {
try {
Connection connection = DriverManager.getConnection(url, username, password);
connection.setAutoCommit(false);
connection.setReadOnly(true);
return connection;
} catch (SQLException e) {
e.printStackTrace();
return null;
}
}
@Override
protected void expire(Connection o) {
try {
o.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
@Override
protected boolean validate(Connection o) {
try {
return !o.isClosed() && o.isValid(1000);
} catch (SQLException e) {
e.printStackTrace();
return false;
}
}
}

126
src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java
View File

@ -1,126 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.pool;
import java.util.Enumeration;
import java.util.Hashtable;
import org.gcube.common.core.utils.logging.GCUBELog;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
* Adapted from http://sourcemaking.com/design_patterns/object_pool/java
*/
public abstract class ObjectPool<T> {
protected static GCUBELog logger = new GCUBELog(ObjectPool.class);
protected long expirationTime;
protected Hashtable<T, Long> locked, unlocked;
protected boolean closed = false;
protected String name;
public ObjectPool(String name, long expirationTime) {
this.name = name;
this.expirationTime = expirationTime;
locked = new Hashtable<T, Long>();
unlocked = new Hashtable<T, Long>();
}
protected abstract T create();
protected abstract boolean validate(T o);
protected abstract void expire(T o);
public synchronized T checkOut() {
long now = System.currentTimeMillis();
T t;
if (unlocked.size() > 0) {
Enumeration<T> e = unlocked.keys();
while (e.hasMoreElements()) {
t = e.nextElement();
if ((now - unlocked.get(t)) > expirationTime) {
// object has expired
unlocked.remove(t);
expire(t);
t = null;
} else {
if (validate(t)) {
unlocked.remove(t);
locked.put(t, now);
return t;
} else {
// object failed validation
unlocked.remove(t);
expire(t);
t = null;
}
}
}
}
logger.trace("no objects available, create a new one, status: "+this);
// no objects available, create a new one
t = create();
locked.put(t, now);
return (t);
}
public synchronized void checkIn(T t) {
locked.remove(t);
if (!closed) unlocked.put(t, System.currentTimeMillis());
else expire(t);
logger.trace("pool status: "+this);
}
public synchronized void shutdown(boolean force)
{
closed = true;
expireAllUnlocked();
if (force) expireAllLocked();
}
public synchronized void expireAllUnlocked()
{
Enumeration<T> e = unlocked.keys();
while (e.hasMoreElements()) {
T t = e.nextElement();
unlocked.remove(t);
expire(t);
}
}
public synchronized void expireAllLocked()
{
Enumeration<T> e = locked.keys();
while (e.hasMoreElements()) {
T t = e.nextElement();
locked.remove(t);
expire(t);
}
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ObjectPool [name=");
builder.append(name);
builder.append(", expirationTime=");
builder.append(expirationTime);
builder.append(", locked=");
builder.append(locked.size());
builder.append(", unlocked=");
builder.append(unlocked.size());
builder.append(", closed=");
builder.append(closed);
builder.append("]");
return builder.toString();
}
}

83
src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java
View File

@ -1,83 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.pool;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.log4j.Logger;
import org.gcube.data.spd.obisplugin.PluginSession;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class PluginSessionPool extends ObjectPool<PluginSession> {
protected Logger logger = Logger.getLogger(PluginSessionPool.class);
protected DatabaseCredential databaseCredential;
public PluginSessionPool(DatabaseCredential databaseCredential) {
super("PluginSessionPool", 30*60*1000);
setDatabaseCredential(databaseCredential);
}
/**
* @param databaseCredential the databaseCredential to set
*/
public void setDatabaseCredential(DatabaseCredential databaseCredential) {
this.databaseCredential = databaseCredential;
}
/**
* {@inheritDoc}
*/
@Override
protected PluginSession create() {
try {
Connection connection = createConnection();
return new PluginSession(databaseCredential, connection);
} catch (SQLException e) {
logger.error("An error occurred creating the db connection", e);
return null;
}
}
protected Connection createConnection() throws SQLException
{
Properties props = new Properties();
props.setProperty("user", databaseCredential.getUser());
props.setProperty("password", databaseCredential.getPassword());
return DriverManager.getConnection(databaseCredential.getUrl(), props);
}
/**
* {@inheritDoc}
*/
@Override
protected boolean validate(PluginSession session) {
try {
return session.isValid(databaseCredential);
} catch (Exception e) {
logger.warn("An error occurred validating the session", e);
return false;
}
}
/**
* {@inheritDoc}
*/
@Override
protected void expire(PluginSession session) {
try {
session.expire();
} catch (SQLException e) {
logger.warn("An error occurred expiring the session", e);
}
}
}

55
src/main/java/org/gcube/data/spd/obisplugin/search/DataSetRetreiver.java Normal file
View File

@ -0,0 +1,55 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import java.util.List;
import java.util.Map;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.obisplugin.Constants;
import org.gcube.data.spd.obisplugin.search.query.MappingUtils;
import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier;
import org.gcube.data.spd.obisplugin.search.query.QueryType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataSetRetreiver {
private static Logger log = LoggerFactory.getLogger(DataSetRetreiver.class);
@SuppressWarnings("unchecked")
public static DataSet get(String key, String baseURL) throws Exception{
long start = System.currentTimeMillis();
QueryByIdentifier datasetQuery = new QueryByIdentifier(baseURL, key, QueryType.Dataset);
Map<String, Object> mapping = MappingUtils.getObjectMapping(datasetQuery.build());
DataSet dataset = new DataSet(key);
dataset.setName(getAsString(mapping,"name"));
String citation = getAsString(mapping,"citation");
if (citation ==null){
List<Map<String, Object>> institutionMapping = (List<Map<String, Object>>)mapping.get("institutes");
if (institutionMapping.size()>0){
if (getAsString(institutionMapping.get(0),"parent")!=null)
citation += " - "+getAsString(institutionMapping.get(0),"parent");
dataset.setCitation(citation);
}
}
String providerKey = key;
Map<String, Object> providerMapping = (Map<String, Object>)mapping.get("provider");
DataProvider provider = new DataProvider(providerKey);
if (providerMapping!=null)
provider.setName(getAsString(providerMapping,"name"));
else
provider.setName(Constants.REPOSITORY_NAME);
dataset.setDataProvider(provider);
log.trace("[Benchmark] time to retrieve dataset is "+(System.currentTimeMillis()-start));
return dataset;
}
}

172
src/main/java/org/gcube/data/spd/obisplugin/search/OccurrenceSearch.java Normal file
View File

@ -0,0 +1,172 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsCalendar;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsDouble;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import org.gcube.data.spd.model.BasisOfRecord;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.products.DataProvider;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.obisplugin.Constants;
import org.gcube.data.spd.obisplugin.search.query.MappingUtils;
import org.gcube.data.spd.obisplugin.search.query.PagedQueryIterator;
import org.gcube.data.spd.obisplugin.search.query.PagedQueryObject;
import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier;
import org.gcube.data.spd.obisplugin.search.query.QueryCondition;
import org.gcube.data.spd.obisplugin.search.query.QueryType;
import org.gcube.data.spd.obisplugin.search.query.ResultType;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class OccurrenceSearch {
private static Logger log = LoggerFactory.getLogger(OccurrenceSearch.class);
private String baseURL;
private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
public OccurrenceSearch(String baseURL) {
this.baseURL = baseURL;
}
public void search(ObjectWriter<OccurrencePoint> writer, String scientificName, int limit, Condition ...conditions) throws Exception{
PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit);
List<QueryCondition> queryConditions = Utils.elaborateConditions(conditions);
occurrencesQuery.setConditions(QueryCondition.cond("scientificname",scientificName.replaceAll(" ", "%20")));
occurrencesQuery.getConditions().addAll(queryConditions);
writeElements(writer, occurrencesQuery);
}
public void searchByKey(ObjectWriter<OccurrencePoint> writer, String key, int limit) throws Exception{
PagedQueryObject occurrencesQuery = new PagedQueryObject(baseURL, ResultType.Occurrence, limit);
ProductKey productKey = Utils.elaborateProductsKey(key);
occurrencesQuery.getConditions().addAll(productKey.getQueryCondition());
writeElements(writer, occurrencesQuery);
}
private void writeElements(ObjectWriter<OccurrencePoint> writer, PagedQueryObject occurrencesQuery){
PagedQueryIterator<OccurrencePoint> pagedIterator = new PagedQueryIterator<OccurrencePoint>(occurrencesQuery) {
@Override
protected OccurrencePoint getObject(Map<String, Object> mappedObject)
throws Exception {
OccurrencePoint op = retrieveElement(mappedObject);
Calendar now = Calendar.getInstance();
String credits = "Biodiversity occurrence data accessed through OBIS WebService, http://api.iobis.org/, "+format.format(now.getTime())+")";
op.setCredits(credits);
return op;
}
};
try{
while (pagedIterator.hasNext() && writer.isAlive())
writer.write(pagedIterator.next());
}catch(Exception e){
log.error("error writing occurrences",e);
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
}
}
public OccurrencePoint searchById(String id) throws Exception{
QueryByIdentifier queryByIdentifier = new QueryByIdentifier(baseURL, id, QueryType.Occurrence);
return retrieveElement(MappingUtils.getObjectMapping(queryByIdentifier.build()));
}
/*
FOSSIL_SPECIMEN
An occurrence record describing a fossilized specimen.
HUMAN_OBSERVATION
An occurrence record describing an observation made by one or more people.
LITERATURE
An occurrence record based on literature alone.
LIVING_SPECIMEN
An occurrence record describing a living specimen, e.g.
MACHINE_OBSERVATION
An occurrence record describing an observation made by a machine.
MATERIAL_SAMPLE
An occurrence record based on samples taken from other specimens or the environment.
OBSERVATION
An occurrence record describing an observation.
PRESERVED_SPECIMEN
An occurrence record describing a preserved specimen.
UNKNOWN
*/
private OccurrencePoint retrieveElement(Map<String, Object> mappedObj) throws Exception{
/*
{"id":772,"decimalLongitude":79.5,"decimalLatitude":-62.5,"eventDate":"1974-01-01 11:00:00",
"institutionCode":"AADC","collectionCode":"WC","catalogNumber":"1672","individualCount":193.0,
"datasetName":"Whale catches in the Southern Ocean","phylum":"Chordata","order":"Cetartiodactyla",
"family":"Balaenopteridae","genus":"Balaenoptera","scientificName":"Balaenoptera bonaerensis",
"originalScientificName":"Balaenoptera bonaerensis","scientificNameAuthorship":"Burmeister, 1867",
"obisID":409234,"resourceID":22,"yearcollected":1974,"species":"Balaenoptera bonaerensis","qc":1073217151,"aphiaID":231405
,"speciesID":409234,"scientificNameID":"urn:lsid:marinespecies.org:taxname:231405","class":"Mammalia"}
*/
long start = System.currentTimeMillis();
String occurrenceId = getAsInteger(mappedObj, "id").toString();
OccurrencePoint occurrence = new OccurrencePoint(occurrenceId);
occurrence.setDecimalLatitude(getAsDouble(mappedObj, "decimalLatitude"));
occurrence.setDecimalLongitude(getAsDouble(mappedObj, "decimalLongitude"));
Calendar eventDate = getAsCalendar(mappedObj, "eventDate");
occurrence.setEventDate(eventDate);
occurrence.setCollectionCode(getAsString(mappedObj, "collectionCode"));
occurrence.setInstitutionCode(getAsString(mappedObj, "institutionCode"));
occurrence.setCatalogueNumber(getAsString(mappedObj, "catalogNumber"));
//occurrence.setRecordedBy(getAsString(mappedObj, "recordedBy"));
//occurrence.setIdentifiedBy(getAsString(mappedObj, "identifiedBy"));
//occurrence.setCountry(getAsString(mappedObj, "country"));
//occurrence.setLocality(getAsString(mappedObj, "locality"));
occurrence.setBasisOfRecord(BasisOfRecord.Unknown);
/*occurrence.setMinDepth(getAsDouble(mappedObj, "elevation"));
occurrence.setMaxDepth(getAsDouble(mappedObj, "depth"));
*/
occurrence.setKingdom("Animalia");
occurrence.setFamily(getAsString(mappedObj, "family"));
occurrence.setScientificNameAuthorship(getAsString(mappedObj, "scientificNameAuthorship"));
occurrence.setScientificName(getAsString(mappedObj, "scientificName"));
String datasetName = getAsString(mappedObj, "datasetName");
DataSet dataset = new DataSet(datasetName);
dataset.setCitation(datasetName);
dataset.setName(datasetName);
DataProvider dataProvider = new DataProvider("OBIS");
dataProvider.setName("OBIS");
dataset.setDataProvider(dataProvider);
occurrence.setDataSet(dataset);
//occurrence.setCitation(getAsString(mappedObj, "accordingTo"));
log.trace("[Benchmark] time to retrieve occurrence is "+(System.currentTimeMillis()-start));
return occurrence;
}
}

16
src/main/java/org/gcube/data/spd/obisplugin/search/ProductKey.java Normal file
View File

@ -0,0 +1,16 @@
package org.gcube.data.spd.obisplugin.search;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.gcube.data.spd.obisplugin.search.query.QueryCondition;
@AllArgsConstructor
@Getter
public class ProductKey {
private List<QueryCondition> queryCondition;
}

174
src/main/java/org/gcube/data/spd/obisplugin/search/ResultItemSearch.java Normal file
View File

@ -0,0 +1,174 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsString;
import static org.gcube.data.spd.obisplugin.search.query.MappingUtils.getAsInteger;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.exceptions.StreamBlockingException;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.model.products.Product;
import org.gcube.data.spd.model.products.Product.ProductType;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.model.products.Taxon;
import org.gcube.data.spd.obisplugin.Constants;
import org.gcube.data.spd.obisplugin.search.query.MappingUtils;
import org.gcube.data.spd.obisplugin.search.query.PagedQueryIterator;
import org.gcube.data.spd.obisplugin.search.query.PagedQueryObject;
import org.gcube.data.spd.obisplugin.search.query.QueryByIdentifier;
import org.gcube.data.spd.obisplugin.search.query.QueryCondition;
import org.gcube.data.spd.obisplugin.search.query.QueryCount;
import org.gcube.data.spd.obisplugin.search.query.QueryType;
import org.gcube.data.spd.obisplugin.search.query.ResultType;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ResultItemSearch {
private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
private static Logger log = LoggerFactory.getLogger(ResultItemSearch.class);
private List<QueryCondition> queryConditions = new ArrayList<QueryCondition>();
private String baseURL;
String searchQuery;
public ResultItemSearch(String baseURL, String searchQuery, Condition ... conditions){
this.baseURL = baseURL;
this.searchQuery = searchQuery.replaceAll(" ", "%20").trim();
this.searchQuery = this.searchQuery.substring(0, 1).toUpperCase()+this.searchQuery.substring(1, this.searchQuery.length()).toLowerCase();
try{
this.queryConditions = Utils.elaborateConditions(conditions);
}catch(Exception e){
log.error("error elaborating conditions",e);
}
}
public void search(ObjectWriter<ResultItem> writer, int limit){
PagedQueryObject queryObject = new PagedQueryObject(baseURL, ResultType.Occurrence,limit);
queryObject.setConditions(QueryCondition.cond("scientificname",searchQuery));
queryObject.getConditions().addAll(this.queryConditions);
try{
PagedQueryIterator<ResultItem> pagedIterator = new PagedQueryIterator<ResultItem>(queryObject) {
Set<String> alreadyVisited =new HashSet<String>();
@Override
protected ResultItem getObject(Map<String, Object> mappedObject) throws Exception {
log.debug("retrieved mapped object");
return buildResult(mappedObject);
}
@Override
protected boolean useIt(Map<String, Object> mappedObject) {
String datasetKey = ((Integer)mappedObject.get("resourceID")).toString();
Integer taxonId = (Integer)mappedObject.get("obisID");
String key = datasetKey+"|"+taxonId;
if (alreadyVisited.contains(key))
return false;
alreadyVisited.add(key);
return true;
}
};
while (pagedIterator.hasNext() && writer.isAlive())
writer.write(pagedIterator.next());
}catch(Exception e){
log.error("error writing resultItems",e);
writer.write(new StreamBlockingException(Constants.REPOSITORY_NAME));
}
}
ResultItem buildResult(Map<String,Object> singleObject) throws Exception{
try{
long start = System.currentTimeMillis();
Integer taxonId = getAsInteger(singleObject,"obisID");
String scientificName = getAsString(singleObject,"scientificName");
ResultItem resItem = new ResultItem(taxonId.toString(), scientificName );
String scientificNameAuthorship = getAsString(singleObject,"scientificNameAuthorship");
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> singleTaxon = MappingUtils.getObjectMapping(query.build());
resItem.setScientificNameAuthorship(scientificNameAuthorship);
resItem.setRank(getAsString(singleTaxon, "rank_name"));
//resItem.setCitation(getAsString(singleTaxon,"tauthor"));
resItem.setParent(retrieveParentTaxon(getAsInteger(singleTaxon,"parent_id")));
DataSet dataset = DataSetRetreiver.get(getAsInteger(singleObject,"resourceID").toString(), baseURL);
resItem.setDataSet(dataset);
List<Product> products = retrieveProducts(taxonId.toString(), dataset);
resItem.setProducts(products);
String credits = "Biodiversity occurrence accessed through OBIS WebService, http://api.iobis.org/, "+format.format(Calendar.getInstance().getTime())+")";
resItem.setCredits(credits);
log.trace("[Benchmark] time to retrieve ResultItem is "+(System.currentTimeMillis()-start));
log.debug("found species {} with authorship {}",scientificName, scientificNameAuthorship);
return resItem;
}catch(Exception e){
throw e;
}
}
private Taxon retrieveParentTaxon(Integer parentTaxonId) throws Exception {
if (parentTaxonId==0) return null;
long start = System.currentTimeMillis();
Integer taxonId = parentTaxonId;
Taxon previousTaxon = null;
Taxon taxonToReturn = null;
do{
QueryByIdentifier query = new QueryByIdentifier(baseURL, taxonId.toString(), QueryType.Taxon);
Map<String, Object> singleTaxon = MappingUtils.getObjectMapping(query.build());
Taxon taxon = new Taxon(getAsInteger(singleTaxon, "id").toString(), getAsString(singleTaxon, "tname"));
taxon.setScientificNameAuthorship(getAsString(singleTaxon, "tauthor"));
//taxon.setCitation(getAsString(mappedObject, "accordingTo"));
taxon.setRank(getAsString(singleTaxon, "rank_name"));
if (previousTaxon!=null)
previousTaxon.setParent(taxon);
previousTaxon = taxon;
taxonId = getAsInteger(singleTaxon, "parent_id");
if (taxonToReturn==null)
taxonToReturn = taxon;
} while (taxonId>0);
log.trace("[Benchmark] time to retrieve taxon is "+(System.currentTimeMillis()-start));
return taxonToReturn;
}
private List<Product> retrieveProducts( String taxonId, DataSet dataset) throws Exception{
long start = System.currentTimeMillis();
QueryCount occurrencesQuery = new QueryCount(baseURL, ResultType.Occurrence);
occurrencesQuery.setConditions(QueryCondition.cond("obisid",taxonId), QueryCondition.cond("resourceid", dataset.getId()));
occurrencesQuery.getConditions().addAll(this.queryConditions);
String productId = Utils.createProductsKey(Utils.getDataSetAsString(dataset), taxonId, this.queryConditions);
Product product = new Product(ProductType.Occurrence, productId);
product.setCount(occurrencesQuery.getCount());
log.trace("[Benchmark] time to retrieve product is "+(System.currentTimeMillis()-start));
return Arrays.asList(product);
}
}

132
src/main/java/org/gcube/data/spd/obisplugin/search/Utils.java Normal file
View File

@ -0,0 +1,132 @@
package org.gcube.data.spd.obisplugin.search;
import static org.gcube.data.spd.obisplugin.search.query.QueryCondition.cond;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Coordinate;
import org.gcube.data.spd.model.products.DataSet;
import org.gcube.data.spd.obisplugin.search.query.QueryCondition;
public class Utils {
protected static List<QueryCondition> elaborateConditions(Condition[] properties) throws Exception{
List<QueryCondition> queryConditions = new ArrayList<QueryCondition>();
List<Condition> coordinateConditions = new ArrayList<Condition>();
for (Condition prop: properties){
switch (prop.getType()) {
case COORDINATE:
coordinateConditions.add(prop);
break;
case DATE:
Calendar date = (Calendar) prop.getValue();
queryConditions.addAll(getDateCondition(date, prop));
break;
}
}
if (coordinateConditions.size()>0)
queryConditions.add(getCoordinateConditions(coordinateConditions));
return queryConditions;
}
public static ProductKey elaborateProductsKey(String id) {
List<QueryCondition> queryConditions = new ArrayList<QueryCondition>();
String[] splitString = id.split("\\|\\|");
queryConditions.add(cond("resourceid", splitString[0]));
queryConditions.add(cond("obisid", splitString[1]));
if (splitString.length>2)
for (int i = 2; i<splitString.length; i++){
String[] equalSplit = splitString[i].split("=");
queryConditions.add(cond(equalSplit[0], equalSplit[1].replaceAll(" ", "%20")));
}
return new ProductKey(queryConditions);
}
protected static String createProductsKey(String dataResourceKey, String taxonKey, List<QueryCondition> queryConditions) {
StringBuilder conditionTransformer = new StringBuilder();
for (QueryCondition cond : queryConditions)
conditionTransformer.append("||").append(cond.getKey()).append("=").append(cond.getValue());
return dataResourceKey+"||"+taxonKey+conditionTransformer.toString();
}
public static QueryCondition getCoordinateConditions(List<Condition> coordinateConditions){
double lowerLong =-180, lowerLat= -90, upperLat= 90, upperLong=180;
double latitude, longitude;
for (Condition cond :coordinateConditions){
switch (cond.getOp()) {
case EQ:
break;
case GT:
latitude = ((Coordinate)cond.getValue()).getLatitude()+0.01;
longitude = ((Coordinate)cond.getValue()).getLongitude()+0.01;
if (latitude>lowerLat) lowerLat = latitude;
if (longitude>lowerLong) lowerLong = longitude;
break;
case GE:
latitude = ((Coordinate)cond.getValue()).getLatitude();
longitude = ((Coordinate)cond.getValue()).getLongitude();
if (latitude>lowerLat) lowerLat = latitude;
if (longitude>lowerLong) lowerLong = longitude;
break;
case LT:
latitude = ((Coordinate)cond.getValue()).getLatitude()-0.01;
longitude = ((Coordinate)cond.getValue()).getLongitude()-0.01;
if (latitude<upperLat) upperLat = latitude;
if (longitude>upperLong) upperLong = longitude;
break;
case LE:
latitude = ((Coordinate)cond.getValue()).getLatitude();
longitude = ((Coordinate)cond.getValue()).getLongitude();
if (latitude<upperLat) upperLat = latitude;
if (longitude>upperLong) upperLong = longitude;
break;
default:
break;
}
}
return cond("geometry", String.format("POLYGON((%1$f %2$f,%3$f %4$f,%5$f %6$f,%1$f %2$f))"
,lowerLat, lowerLong, upperLat, lowerLong, upperLat, upperLong, lowerLat, upperLong).replaceAll(" ", "%20"));
}
public static List<QueryCondition> getDateCondition(Calendar date, Condition prop){
List<QueryCondition> conditions = new ArrayList<QueryCondition>();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Calendar newDate = date;
switch (prop.getOp()) {
case EQ:
conditions.add(cond("eventDate",dateFormat.format(date.getTime())));
break;
case GT:
newDate.add(Calendar.DAY_OF_MONTH, 1);
conditions.add(cond("startdate",dateFormat.format(newDate.getTime())));
break;
case GE:
conditions.add(cond("startdate",dateFormat.format(date.getTime())));
break;
case LT:
newDate.add(Calendar.DAY_OF_MONTH, -1);
conditions.add(cond("enddate",dateFormat.format(newDate.getTime())));
break;
case LE:
conditions.add(cond("enddate",dateFormat.format(date.getTime())));
break;
default:
break;
}
return conditions;
}
protected static String getDataSetAsString(DataSet dataset){
return dataset.getId();
}
}

109
src/main/java/org/gcube/data/spd/obisplugin/search/query/MappingUtils.java Normal file
View File

@ -0,0 +1,109 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.io.StringReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.ws.rs.core.MediaType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
public class MappingUtils {
private static final int TIMEOUT = 40000;
private static final int retries = 5;
private static final int SLEEP_TIME = 10000;
private static Logger log = LoggerFactory.getLogger(MappingUtils.class);
@SuppressWarnings("unchecked")
public static Map<String, Object> getObjectMapping(String query) throws Exception{
String response = executeQuery(query);
ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally
return mapper.readValue(new StringReader(response), Map.class);
}
public static List<Map<String, Object>> getObjectList(String query) throws Exception{
String response = executeQuery(query);
ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally
return mapper.readValue(new StringReader(response), new TypeReference<LinkedList<HashMap<String, Object>>>() {
});
}
public static String getAsString(Map<String, Object> map, String key){
if (!map.containsKey(key)) return null;
return (String) map.get(key);
}
public static Double getAsDouble(Map<String, Object> map, String key){
if (!map.containsKey(key)) return 0d;
return (Double) map.get(key);
}
public static Integer getAsInteger(Map<String, Object> map, String key){
if (!map.containsKey(key)) return 0;
return (Integer) map.get(key);
}
public static Calendar getAsCalendar(Map<String, Object> map, String key){
if (!map.containsKey(key)) return null;
return parseCalendar((String) map.get(key));
}
public static Calendar parseCalendar(String date){
try{
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar calendar= Calendar.getInstance();
calendar.setTime(df.parse(date));
return calendar;
}catch (ParseException e) {
log.warn("date discarded ("+date+")");
return null;
}
}
private static String executeQuery(String query){
DefaultClientConfig clientConfig = new DefaultClientConfig();
Client client = Client.create(clientConfig);
client.setConnectTimeout(TIMEOUT);
client.setReadTimeout(TIMEOUT);
WebResource target = client.resource(query);
//NameUsageWsClient nuws = new NameUsageWsClient(target);
int tries = 1;
String response = null;
do {
log.debug("try number {} STARTED for query {} ", tries,query);
try{
response = target.type(MediaType.APPLICATION_JSON).acceptLanguage(Locale.ENGLISH).get(String.class);
}catch (Exception e) {
log.debug("try number {} FAILED for query {} ", tries,query,e);
try {
if (tries<retries)Thread.sleep(SLEEP_TIME);
} catch (InterruptedException e1) {}
}
tries++;
}while (response==null && tries<=retries);
return response;
}
}

86
src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryIterator.java Normal file
View File

@ -0,0 +1,86 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.util.Iterator;
import java.util.Map;
import lombok.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class PagedQueryIterator<T> implements Iterator<T>{
private static Logger log = LoggerFactory.getLogger(PagedQueryIterator.class);
private @NonNull PagedQueryObject pagedQuery;
public PagedQueryIterator(@NonNull PagedQueryObject pagedQuery) {
this.pagedQuery = pagedQuery;
}
protected abstract T getObject(Map<String,Object> mappedObject) throws Exception;
Map<String, Object> mapping;
Iterator<Map<String,Object>> resultIterator;
Map<String,Object> actualObject= null;
Long start = null;
Long parsingStart = null;
@SuppressWarnings("unchecked")
@Override
public boolean hasNext() {
try{
if (resultIterator==null){
String query = pagedQuery.buildNext();
start = System.currentTimeMillis();
mapping = MappingUtils.getObjectMapping(query);
if (mapping.get("results")== null) return false;
parsingStart = System.currentTimeMillis();
log.trace("[Benchmark] got Elements with query "+query+" and took "+(parsingStart-start));
resultIterator = ((Iterable<Map<String,Object>>) mapping.get("results")).iterator();
}
if (!resultIterator.hasNext()){
log.trace("[Benchmark] page retrieved and parsed in "+(System.currentTimeMillis()-start));
if ((Boolean)mapping.get("lastpage")){
log.trace("is end of records, no next element");
return false;
}
resultIterator = null;
} else{
actualObject = resultIterator.next();
if (useIt(actualObject))
return true;
}
return this.hasNext();
}catch(Exception e){
log.error("error computing hasNext",e);
throw new RuntimeException(e);
}
}
@Override
public T next() {
try{
return getObject(actualObject);
}catch(Exception e){
log.error("error computing next",e);
throw new RuntimeException(e);
}
}
protected boolean useIt(Map<String,Object> mappedObject){
return true;
}
@Override
public void remove() {
resultIterator = null;
}
}

50
src/main/java/org/gcube/data/spd/obisplugin/search/query/PagedQueryObject.java Normal file
View File

@ -0,0 +1,50 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class PagedQueryObject {
private @NonNull String baseUri;
@Getter
List<QueryCondition> conditions = new ArrayList<QueryCondition>();
private @NonNull ResultType resultType;
private @NonNull Integer resultPerQuery;
private int offset = 0;
public void setConditions(QueryCondition ... conditions){
this.conditions.addAll(Arrays.asList(conditions));
}
public String buildNext(){
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/")) query.append("/");
query.append(this.resultType.getQueryEntry()).append("/");
query.append("?limit=").append(resultPerQuery);
query.append("&offset=").append(offset);
if (conditions.size()>0)
for (QueryCondition queryCond: conditions)
query.append("&").append(queryCond.getKey()).append("=").append(queryCond.getValue());
offset = offset+resultPerQuery;
log.debug("executed query is "+query.toString());
return query.toString();
}
}

38
src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryByIdentifier.java Normal file
View File

@ -0,0 +1,38 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.util.ArrayList;
import java.util.List;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class QueryByIdentifier {
private @NonNull String baseUri;
private @NonNull String key;
private @NonNull QueryType type;
private List<String> paths = new ArrayList<String>();
public void addPath(String path){
paths.add(path);
}
public String build(){
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/")) query.append("/");
query.append(this.type.getQueryEntry()).append("/");
query.append(key);
for (String path : paths)
query.append("/").append(path);
log.trace("query by dentifier is "+query.toString());
return query.toString();
}
}

20
src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCondition.java Normal file
View File

@ -0,0 +1,20 @@
package org.gcube.data.spd.obisplugin.search.query;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class QueryCondition{
public static QueryCondition cond(String key, String value){
return new QueryCondition(key, value);
}
@Getter
private @NonNull String key;
@Getter
private @NonNull String value;
}

63
src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryCount.java Normal file
View File

@ -0,0 +1,63 @@
package org.gcube.data.spd.obisplugin.search.query;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RequiredArgsConstructor
public class QueryCount {
private static Logger log = LoggerFactory.getLogger(QueryCount.class);
private @NonNull String baseUri;
List<QueryCondition> conditions = new ArrayList<QueryCondition>();
private @NonNull ResultType resultType;
public void setConditions(QueryCondition ... conditions){
this.conditions.addAll(Arrays.asList(conditions));
}
public List<QueryCondition> getConditions() {
return conditions;
}
public int getCount(){
Map<String, Object> mapping;
try {
mapping = MappingUtils.getObjectMapping(this.build());
if (mapping.get("count")==null) return 0;
return (Integer)mapping.get("count");
} catch (Exception e) {
log.error("error computing count, returning 0",e);
return 0;
}
}
private String build(){
StringBuilder query = new StringBuilder(baseUri);
if (!baseUri.endsWith("/")) query.append("/");
query.append(this.resultType.getQueryEntry()).append("/");
query.append("?limit=0");
if (conditions.size()>0)
for (QueryCondition queryCond: conditions)
query.append("&").append(queryCond.getKey().replaceAll(" ", "%20")).append("=").append(queryCond.getValue().replaceAll(" ", "%20"));
return query.toString();
}
}

16
src/main/java/org/gcube/data/spd/obisplugin/search/query/QueryType.java Normal file
View File

@ -0,0 +1,16 @@
package org.gcube.data.spd.obisplugin.search.query;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NonNull;
public @AllArgsConstructor enum QueryType{
Occurrence("occurrence"),
Taxon("taxon"),
Dataset("resource");
@Getter
private @NonNull String queryEntry;
}

15
src/main/java/org/gcube/data/spd/obisplugin/search/query/ResultType.java Normal file
View File

@ -0,0 +1,15 @@
package org.gcube.data.spd.obisplugin.search.query;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public enum ResultType{
Occurrence("occurrence"),
Taxon("species");
@Getter
private @NonNull String queryEntry;
}

114
src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java
View File

@ -1,114 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.util;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class Cache<K,V> {
protected int limit;
protected Map<K, V> cache;
protected boolean enableStatistics;
protected long hints;
protected long requests;
protected long removed;
@SuppressWarnings("serial")
public Cache(final int limit)
{
this.limit = limit;
this.hints = 0;
this.cache = new LinkedHashMap<K, V>() {
/**
* {@inheritDoc}
*/
@Override
protected boolean removeEldestEntry(Entry<K, V> eldest) {
boolean remove = size() > limit;
if (remove) removed++;
return remove;
}
};
}
/**
* @return the enableStatistics
*/
public boolean isEnableStatistics() {
return enableStatistics;
}
/**
* @param enableStatistics the enableStatistics to set
*/
public void setEnableStatistics(boolean enableStatistics) {
this.enableStatistics = enableStatistics;
}
/**
* @return the requests
*/
public long getRequests() {
return requests;
}
/**
* @return the limit
*/
public int getLimit() {
return limit;
}
/**
* @return the hints
*/
public long getHints() {
return hints;
}
public V get(K key)
{
V value = cache.get(key);
if (enableStatistics) {
requests++;
if (value!=null) hints++;
}
return value;
}
public void put(K key, V value)
{
cache.put(key, value);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("Cache [limit=");
builder.append(limit);
builder.append(", enableStatistics=");
builder.append(enableStatistics);
builder.append(", hints=");
builder.append(hints);
builder.append(", requests=");
builder.append(requests);
builder.append(", removed=");
builder.append(removed);
builder.append("]");
return builder.toString();
}
}

84
src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java
View File

@ -1,84 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.util;
import java.util.Date;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class DateUtil {
//Unsupported:
//2010-05-03T14:44:04Z+01:00
//2009-04-09TMountain Da:ylight Time
//2008-05-20T13:03:08OZ
//2008-11-01T14:45OZ
//0000-00-00 00:00:00
protected static final String[] DATE_FORMATS = new String[]{
"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", //2005-10-24T13:33:11.000Z
"yyyy-MM-dd'T'HH:mm:ss:mm:ssZ", //2003-02-18T11:35:13:35:13Z
"yyyy-MM-dd' 'HH:mm:ss.SSSSSS", //2006-12-20 10:27:02.477563
"yyyy-MM-dd'T'HH:mm:ssZZ", //2001-02-10T12:00:00+00:00
"yyyy-MM-dd'T'HH:mm:sssZZ", //2011-09-25T11:00:000Z
"yyyy-MM-dd' 'HH:mm:ssZZ", //2007-05-11 14:01:15-04
"yyyy-MM-dd'T'HH:mm:ssz", //2005-10-11T15:40:00Z
"yyyy-MM-dd' 'HH:mm:ss", //2007-07-18 06:13:06
"yyyy-MM-dd'T'HH:mmZ", //2009-10-01T01:00Z
"MM/dd/yyyy' 'hh:mm:ss aa", //10/28/2010 10:12:43 AM
"MM/dd/yyyy' 'hh:mm", //12/9/2010 11:59
"dd/MM/yyyyHH:mm:ssZ", //13/9/201012:00:00Z
"yyyy-MM-dd' 'HH:mm", //2005-12-20 17:12
"yyyy-MM-dd'T'", //2010-06-09T
"yyyy-MM-dd-'T'", //2009-08-05-T
"yyyy-MM-dd", //2009-09-08
"dd-MMM-yy", //28-MAR-01 08-AUG-96
"dd/MM/yyyy", //11/2/2010
"MM/dd/yyyy' 'HH:mm:ss", //8/23/2010 0:00:00
"MM/dd/yyyy", //10/19/2010
"yyyy/MM/dd' 'HH:mm:ss", //2010/10/27 22:29:04
//FIXME Military Time Zone not supported
"yyyy-MM-dd'T'HH:mm:ss'B'", //2003-07-07T10:03:56B
};
protected static DateUtil instance;
public static DateUtil getInstance()
{
if (instance == null) {
instance = new DateUtil();
instance.initialize();
}
return instance;
}
protected DateTimeFormatter[] FORMATS;
protected DateUtil() {
FORMATS = new DateTimeFormatter[DATE_FORMATS.length];
}
protected void initialize()
{
int i = 0;
for (String dateFormat:DATE_FORMATS) {
FORMATS[i++] = DateTimeFormat.forPattern(dateFormat);
}
}
public Date parse(String dateString) {
for (DateTimeFormatter formatter:FORMATS) {
try {
return formatter.parseDateTime(dateString).toDate();
} catch (Exception e){}
}
return null; // Unknown format.
}
}

38
src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java
View File

@ -1,38 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.util;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class RegExpDateFormat {
protected Pattern pattern;
protected DateFormat dateFormat;
public RegExpDateFormat(String regExp, String datePattern)
{
pattern = Pattern.compile(regExp);
dateFormat = new SimpleDateFormat(datePattern);
}
public boolean match(String input)
{
Matcher m = pattern.matcher(input);
return m.matches();
}
public Date parse(String input) throws ParseException
{
return dateFormat.parse(input);
}
}

89
src/main/java/org/gcube/data/spd/obisplugin/util/Util.java
View File

@ -1,89 +0,0 @@
/**
*
*/
package org.gcube.data.spd.obisplugin.util;
import java.util.ArrayList;
import java.util.List;
import org.gcube.data.spd.model.CommonName;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.model.products.Taxon;
/**
* @author "Federico De Faveri defaveri@isti.cnr.it"
*
*/
public class Util {
/**
* Partially clones the passed result item.
* @param item
* @return
*/
public static ResultItem cloneResultItem(ResultItem item)
{
if (item==null) return null;
ResultItem clone = new ResultItem(item.getId(), item.getScientificName());
copyTaxon(item, clone);
clone.setCommonNames(cloneCommonName(item.getCommonNames()));
return clone;
}
protected static Taxon cloneTaxon(Taxon taxon)
{
if (taxon==null) return null;
Taxon clone = new Taxon(taxon.getId(), taxon.getScientificName());
copyTaxon(taxon, clone);
return clone;
}
protected static void copyTaxon(Taxon taxon, Taxon clone)
{
clone.setId(taxon.getId());
clone.setScientificName(taxon.getScientificName());
clone.setCitation(taxon.getCitation());
clone.setCredits(taxon.getCredits());
clone.setScientificNameAuthorship(taxon.getScientificNameAuthorship());
clone.setRank(taxon.getRank());
clone.setParent(cloneTaxon(taxon.getParent()));
}
protected static List<CommonName> cloneCommonName(List<CommonName> commonNames)
{
if (commonNames==null) return null;
List<CommonName> clones = new ArrayList<CommonName>(commonNames.size());
for (CommonName commonName:commonNames) clones.add(cloneCommonName(commonName));
return clones;
}
protected static CommonName cloneCommonName(CommonName commonName)
{
if (commonName==null) return null;
return new CommonName(commonName.getLanguage(), commonName.getName());
}
public static String stripNotValidXMLCharacters(String input) {
if (input == null) return null;
if (input.isEmpty()) return "";
StringBuffer out = new StringBuffer();
for (char current:input.toCharArray()){
if ((current == 0x9) ||
(current == 0xA) ||
(current == 0xD) ||
((current >= 0x20) && (current <= 0xD7FF)) ||
((current >= 0xE000) && (current <= 0xFFFD)) ||
((current >= 0x10000) && (current <= 0x10FFFF)))
out.append(current);
}
return out.toString();
}
}

64
src/test/java/org/gcube/data/obisplugin/ObisTest.java
View File

@ -1,9 +1,17 @@
package org.gcube.data.obisplugin;
import java.util.Arrays;
import org.gcube.data.spd.model.Condition;
import org.gcube.data.spd.model.Condition.Operator;
import org.gcube.data.spd.model.Conditions;
import org.gcube.data.spd.model.Coordinate;
import org.gcube.data.spd.model.exceptions.StreamException;
import org.gcube.data.spd.model.products.OccurrencePoint;
import org.gcube.data.spd.obisplugin.ObisPlugin;
import org.gcube.data.spd.obisplugin.pool.DatabaseCredential;
import org.gcube.data.spd.model.products.ResultItem;
import org.gcube.data.spd.obisplugin.capabilities.OccurrencesCapabilityImpl;
import org.gcube.data.spd.obisplugin.search.ResultItemSearch;
import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter;
import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter;
import org.junit.Test;
@ -11,9 +19,10 @@ public class ObisTest {
@Test
public void search() throws Exception{
ObisPlugin plugin= new ObisPlugin();
/*ObisPlugin plugin= new ObisPlugin();
plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3"));
ObjectWriter<OccurrencePoint> writer = new ObjectWriter<OccurrencePoint>() {
plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/
ClosableWriter<OccurrencePoint> writer = new ClosableWriter<OccurrencePoint>() {
@Override
public boolean isAlive() {
@ -22,7 +31,44 @@ public class ObisTest {
@Override
public boolean write(OccurrencePoint arg0) {
System.out.println(arg0.toString());
return true;
}
@Override
public boolean write(StreamException arg0) {
return false;
}
@Override
public void close() {
// TODO Auto-generated method stub
}
};
OccurrencesCapabilityImpl impl = new OccurrencesCapabilityImpl("http://api.iobis.org/");
//impl.searchByScientificName("Cetacea", writer);
impl.getOccurrencesByProductKeys(writer, Arrays.asList("3422||513384||geometry=POLYGON((30.000000%2020.000000,90.000000%2020.000000,90.000000%20180.000000,30.000000%2020.000000))").iterator() );
}
@Test
public void searchRI() throws Exception{
/*ObisPlugin plugin= new ObisPlugin();
plugin.initialize(new DatabaseCredential("jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis", "postgres", "0b1s@d4sc13nc3"));
plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);*/
ObjectWriter<ResultItem> writer = new ObjectWriter<ResultItem>() {
@Override
public boolean isAlive() {
return true;
}
@Override
public boolean write(ResultItem ri) {
System.out.println(ri);
return true;
}
@ -31,7 +77,11 @@ public class ObisTest {
return false;
}
};
plugin.getOccurrencesInterface().searchByScientificName("Architeuthis dux", writer);
}
//, new Condition(Conditions.COORDINATE, new Coordinate(20, 30) , Operator.GT)
ResultItemSearch search = new ResultItemSearch("http://api.iobis.org/", "Gamidae");
search.search(writer, 50);
}
}

10
src/test/resources/log4j.properties Normal file
View File

@ -0,0 +1,10 @@
log4j.appender.ROOT=org.apache.log4j.ConsoleAppender
log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
log4j.appender.ROOT.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %c{2} [%t,%M:%L] %m%n
log4j.rootLogger=INFO,ROOT
log4j.appender.SPD-CL=org.apache.log4j.ConsoleAppender
log4j.appender.SPD-CL.layout=org.apache.log4j.PatternLayout
log4j.appender.SPD-CL.layout.ConversionPattern=[SPD-CL] %d{HH:mm:ss,SSS} %-5p %c{2} [%t,%M:%L] %m%n
log4j.category.org.gcube.data.spd=DEBUG,SPD-CL