commit dcc51657cb4de1d0a6fb8a08172cb20d3d838e24 Author: Lucio Lelii Date: Tue Apr 29 14:42:09 2014 +0000 branch for release 3.1 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/branches/data-access/obis-spd-plugin/1.8@95221 82a268e6-3cf1-43bd-a215-b396298e98cf diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..0f53f3e --- /dev/null +++ b/.classpath @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..65753d3 --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + ObisPlugin + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..ad6cf5d --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,7 @@ +#Tue Feb 14 11:06:25 CET 2012 +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..5d06b25 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,6 @@ +#Tue Feb 14 11:06:25 CET 2012 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..7a7b954 --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,5 @@ +#Mon Feb 13 12:19:08 CET 2012 +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/distro/INSTALL b/distro/INSTALL new file mode 100644 index 0000000..7e9ee1c --- /dev/null +++ b/distro/INSTALL @@ -0,0 +1 @@ +Used as a Plugin in the gCube Framework \ No newline at end of file diff --git a/distro/LICENSE b/distro/LICENSE new file mode 100644 index 0000000..630ba97 --- /dev/null +++ b/distro/LICENSE @@ -0,0 +1,6 @@ +gCube System - License +------------------------------------------------------------ + +The gCube/gCore software is licensed as Free Open Source software conveying to the EUPL (http://ec.europa.eu/idabc/eupl). +The software and documentation is provided by its authors/distributors "as is" and no expressed or +implied warranty is given for its use, quality or fitness for a particular case. diff --git a/distro/MAINTAINERS b/distro/MAINTAINERS new file mode 100644 index 0000000..237e600 --- /dev/null +++ b/distro/MAINTAINERS @@ -0,0 +1 @@ +* Valentina Marioli (valentina.marioli@isti.cnr.it), CNR Pisa, Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo". \ No newline at end of file diff --git a/distro/README b/distro/README new file mode 100644 index 0000000..2d27b0d --- /dev/null +++ b/distro/README @@ -0,0 +1,44 @@ +The gCube System - ${name} +------------------------------------------------------------ + +This work has been partially supported by the following European projects: DILIGENT (FP6-2003-IST-2), +D4Science (FP7-INFRA-2007-1.2.2), D4Science-II (FP7-INFRA-2008-1.2.2), iMarine (FP7-INFRASTRUCTURES-2011-2), +and EUBrazilOpenBio (FP7-ICT-2011-EU-Brazil). + + +Authors +------- + +* Federico De Faveri (federico.defaveri@isti.cnr.it), CNR Pisa, Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" + +Version and Release Date +------------------------ +${version} + +Description +----------- +${description} + +Description +----------- +A plugin for the Ocean Bibliographic Information System. + + +Source code is available from SVN: +${scm.url} + +Binaries can be downloaded from: + + +Documentation +------------- +https://gcube.wiki.gcube-system.org/gcube/index.php + + +Licensing +--------- + +This software is licensed under the terms you may find in the file named "LICENSE" in this directory. + + + diff --git a/distro/changelog.xml b/distro/changelog.xml new file mode 100644 index 0000000..654df1d --- /dev/null +++ b/distro/changelog.xml @@ -0,0 +1,31 @@ + + + Sync with spd-model-library changes + + + #1533: Obis + Sync with service changes + + + #547: Obis plugin occurrence point retrieving failure + + + #1090: Obis plugin have to be synchronized with SPD service changes + Added RR decription + + + Enabled plugin cache provided by the plugin framework + Fixed wrong PS initialization in Plugin Session + Synchronized changes with spd service + fixed #496: Too many connections from Obis plugin to Postgres + + + Synchronized changes with spd service + + + Synchronized changes with spd service + + + obis plugin for species product discovery release + + \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml new file mode 100644 index 0000000..21d8c88 --- /dev/null +++ b/distro/descriptor.xml @@ -0,0 +1,42 @@ + + servicearchive + + tar.gz + + / + + + ${distroDirectory} + / + true + + README + LICENSE + INSTALL + MAINTAINERS + changelog.xml + + 755 + true + + + + + ${distroDirectory}/profile.xml + / + true + + + target/${build.finalName}.jar + /${artifactId} + + + ${distroDirectory}/svnpath.txt + /${artifactId} + true + + + \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml new file mode 100644 index 0000000..62b13c8 --- /dev/null +++ b/distro/profile.xml @@ -0,0 +1,28 @@ + + + + Service + + ${description} + DataAccess + ${artifactId} + 1.0.0 + + + ${description} + ${artifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + + library + + ${build.finalName}.jar + + + + + + diff --git a/distro/svnpath.txt b/distro/svnpath.txt new file mode 100644 index 0000000..e697b00 --- /dev/null +++ b/distro/svnpath.txt @@ -0,0 +1 @@ +https://svn.d4science-ii.research-infrastructures.eu/gcube/trunk/data-access/obis-spd-plugin \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..3e9fe4f --- /dev/null +++ b/pom.xml @@ -0,0 +1,112 @@ + + 4.0.0 + + + maven-parent + org.gcube.tools + 1.0.0 + + + + org.gcube.data.spd + obis-spd-plugin + 1.8.0-SNAPSHOT + ObisPlugin + obis plugin for species manager service + + + ${project.basedir}/distro + + + + + org.gcube.core + gcf + [1.6.0-SNAPSHOT,2.0.0) + + + + org.gcube.data.spd + spd-plugin-framework + [3.0.0-SNAPSHOT, 4.0.0-SNAPSHOT) + + + + postgresql + postgresql + 8.4-702.jdbc4 + compile + + + joda-time + joda-time + 2.1 + + + org.gcube.core + common-utils-encryption + [1.0.2-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + org.apache.maven.plugins + maven-resources-plugin + 2.5 + + + copy-profile + install + + copy-resources + + + target + + + ${distroDirectory} + true + + profile.xml + + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.2 + + + ${distroDirectory}/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java new file mode 100644 index 0000000..9c85678 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisClassification.java @@ -0,0 +1,144 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.gcube.data.spd.model.Conditions; +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.exceptions.ExternalRepositoryException; +import org.gcube.data.spd.model.exceptions.IdNotValidException; +import org.gcube.data.spd.model.products.TaxonomyItem; +import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; +import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability; +import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ObisClassification extends ClassificationCapability { + + protected Logger logger = Logger.getLogger(ObisClassification.class); + protected PluginSessionPool sessionPool; + + /** + * @param sessionPool + */ + public ObisClassification(PluginSessionPool sessionPool) { + this.sessionPool = sessionPool; + } + + /** + * {@inheritDoc} + */ + @Override + public Set getSupportedProperties() { + return Collections.emptySet(); + } + + + /** + * {@inheritDoc} + */ + @Override + public void searchByScientificName(String word, final ObjectWriter writer, Condition... properties) { + PluginSession session = sessionPool.checkOut(); + + try { + ObisClient.getTaxonByScientificNames(session, word, new Writer() { + + @Override + public boolean write(TaxonomyItem item) { + writer.write(item); + return writer.isAlive(); + } + }); + } catch (Exception e) { + logger.error("Error retrieving taxon with word \""+word+"\"", e); + } finally { + sessionPool.checkIn(session); + } + + } + + /** + * {@inheritDoc} + */ + @Override + public List retrieveTaxonChildrenByTaxonId(String taxonId) throws IdNotValidException, ExternalRepositoryException { + PluginSession session = sessionPool.checkOut(); + + try { + int id = converId(taxonId); + return ObisClient.getChildrenTaxon(session, id); + } catch (SQLException e) { + logger.error("Error retrieving TaxonChildsByTaxonId", e); + throw new ExternalRepositoryException(e); + } finally { + sessionPool.checkIn(session); + } + } + + /** + * {@inheritDoc} writer.put(new StreamException()); + */ + @Override + public void retrieveTaxonByIds(Iterator reader, ClosableWriter writer) { + PluginSession session = sessionPool.checkOut(); + + try { + while(reader.hasNext() && writer.isAlive()) { + try { + String taxonId = reader.next(); + int id = converId(taxonId); + TaxonomyItem item = ObisClient.getTaxonById(session, id); + writer.write(item); + } catch (Exception e) { + logger.error("Error retrieving TaxonById", e); + } + } + } finally { + sessionPool.checkIn(session); + } + + } + + @Override + public TaxonomyItem retrieveTaxonById(String taxonId) throws IdNotValidException { + PluginSession session = sessionPool.checkOut(); + int id = converId(taxonId); + try { + TaxonomyItem item = ObisClient.getTaxonById(session, id); + return item; + } catch (IdNotValidException inve) + { + logger.error("Error retrieving TaxonById", inve); + throw inve; + } catch (Exception e) { + logger.error("Error retrieving TaxonById", e); + return null; + } finally { + sessionPool.checkIn(session); + } + } + + protected int converId(String taxonId) throws IdNotValidException + { + try { + return Integer.parseInt(taxonId); + } catch(NumberFormatException nfe) + { + logger.error("Invalid id "+taxonId, nfe); + throw new IdNotValidException(); + } + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java new file mode 100644 index 0000000..549014d --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisClient.java @@ -0,0 +1,652 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.gcube.common.core.utils.logging.GCUBELog; +import org.gcube.data.spd.model.BasisOfRecord; +import org.gcube.data.spd.model.CommonName; +import org.gcube.data.spd.model.exceptions.IdNotValidException; +import org.gcube.data.spd.model.products.DataProvider; +import org.gcube.data.spd.model.products.DataSet; +import org.gcube.data.spd.model.products.OccurrencePoint; +import org.gcube.data.spd.model.products.Product; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.model.products.Taxon; +import org.gcube.data.spd.model.products.TaxonomyItem; +import org.gcube.data.spd.model.products.TaxonomyStatus; +import org.gcube.data.spd.model.products.Product.ProductType; +import org.gcube.data.spd.model.products.TaxonomyStatus.Status; +import org.gcube.data.spd.model.util.ElementProperty; +import org.gcube.data.spd.obisplugin.data.ProductKey; +import org.gcube.data.spd.obisplugin.data.SearchFilters; +import org.gcube.data.spd.obisplugin.util.Cache; +import org.gcube.data.spd.obisplugin.util.DateUtil; +import org.gcube.data.spd.obisplugin.util.Util; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ObisClient { + + protected static GCUBELog logger = new GCUBELog(ObisClient.class); + //"2009-12-11 11:30:00-07" + protected static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("y-M-d"); + protected static final DateUtil DATE_UTIL = DateUtil.getInstance(); + protected static final SimpleDateFormat sdf = new SimpleDateFormat(); + + public static final Cache taxonomyItemCache = new Cache(1000); + + public static int getOccurrencesCount(PluginSession session, int taxonId, int datasetId, SearchFilters filters) throws SQLException + { + PreparedStatement statement = session.getOccurrencesCountPreparedStatement(taxonId, datasetId, filters); + + ResultSet rs = statement.executeQuery(); + + if (!rs.next()) return 0; + int occurrences = rs.getInt("occurrences"); + rs.close(); + + return occurrences; + } + + public static void getOccurrences(PluginSession session, String key, Writer writer) throws Exception + { + + ProductKey productKey = ProductKey.deserialize(key); + + PreparedStatement statement = session.getOccurrencesPreparedStatement(productKey.getTaxonId(), productKey.getDataSetId(), productKey.getFilters()); + ResultSet rs = statement.executeQuery(); + + String credits = generateCredits(); + //String citation = generateCitation(); + + boolean continueWrite = true; + while(rs.next() && continueWrite) { + + OccurrencePoint occurrence = generateOccurrencePoint(rs, credits); + continueWrite = writer.write(occurrence); + }; + + rs.close(); + } + + protected static OccurrencePoint generateOccurrencePoint(ResultSet rs, String credits) throws SQLException + { + int id = rs.getInt("id"); + OccurrencePoint occurrence = new OccurrencePoint(String.valueOf(id)); + + //drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, + occurrence.setDecimalLatitude(rs.getDouble("latitude")); + occurrence.setDecimalLongitude(rs.getDouble("longitude")); + + Timestamp dateValue = rs.getTimestamp("datecollected"); + if (dateValue!=null) { + Calendar dateCollected = Calendar.getInstance(); + dateCollected.setTimeInMillis(dateValue.getTime()); + occurrence.setEventDate(dateCollected); + } else { + //dxs.yearcollected, dxs.monthcollected, dxs.daycollected + try { + Calendar dateCollected = Calendar.getInstance(); + int year = Integer.parseInt(rs.getString("yearcollected")); + int month = Integer.parseInt(rs.getString("monthcollected")); + int date = Integer.parseInt(rs.getString("daycollected")); + dateCollected.set(year, month, date); + occurrence.setEventDate(dateCollected); + } catch(NumberFormatException nfe){} + } + + String basisOfRecord = rs.getString("basisofrecord"); + occurrence.setBasisOfRecord(getBasisOfRecord(basisOfRecord)); + + //dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector + occurrence.setCitation(rs.getString("citation")); + occurrence.setCredits(credits); + occurrence.setInstitutionCode(rs.getString("institutioncode")); + occurrence.setCollectionCode(rs.getString("collectioncode")); + occurrence.setCatalogueNumber(rs.getString("catalognumber")); + occurrence.setRecordedBy(rs.getString("collector")); + + //, dxs.datelastmodified, + + String datelastmodified = rs.getString("datelastmodified"); + if (datelastmodified!=null) { + + try { + java.util.Date date = DATE_UTIL.parse(datelastmodified); + Calendar lastmodified = Calendar.getInstance(); + lastmodified.setTimeInMillis(date.getTime()); + occurrence.setModified(lastmodified); + } catch (Exception e) { + //logger.warn("Unknow date format "+datelastmodified); + } + + } + + //dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated + occurrence.setCountry(rs.getString("country")); + occurrence.setLocality(rs.getString("locality")); + occurrence.setMinDepth(rs.getDouble("minimumdepth")); + occurrence.setMaxDepth(rs.getDouble("maximumdepth")); + occurrence.setCoordinateUncertaintyInMeters(rs.getString("coordinateprecision")); + //"Animalia|Chordata|Chondrichthyes|Lamniformes|Lamnidae|Carcharodon||carcharias||Carcharodon carcharias|Linnaeus" + + /* + * Kingdom: Animalia +Phylum: Chordata +Class: Chondrichthyes +Subclass: Elasmobranchii +Order: Lamniformes +Family: Lamnidae +Genus: Carcharodon +A. Smith, 1838 +Species: C. carcharias + */ + + String concatenated = rs.getString("concatenated"); + if (concatenated!=null) { + int authorStartIndex = concatenated.lastIndexOf('|'); + if (authorStartIndex>0) { + String snPart = concatenated.substring(0, authorStartIndex); + int scientificNameStartIndex = snPart.lastIndexOf('|'); + if (scientificNameStartIndex>0) { + String author = (authorStartIndex+10) occurrence.setKingdom(taxon[0]); + if (taxon.length>4) occurrence.setFamily(taxon[4]); + if (taxon.length>11) { + String scientific = taxon[9] +"("+taxon[10]+")"; + occurrence.setScientificName(scientific); + }*/ + } + + occurrence.setScientificNameAuthorship(rs.getString("identifiedBy")); + + return occurrence; + } + + public static OccurrencePoint getOccurrenceById(PluginSession session, String id) throws Exception + { + PreparedStatement statement = session.getOccurrenceByIdPreparedStatement(Integer.parseInt(id)); + ResultSet rs = statement.executeQuery(); + + OccurrencePoint occurrence = null; + String credits = generateCredits(); + if(rs.next()) occurrence = generateOccurrencePoint(rs, credits); + rs.close(); + + return occurrence; + } + + + public static BasisOfRecord getBasisOfRecord(String basis) + { + + if (basis==null) return BasisOfRecord.HumanObservation; + + //"HumanObservation" + if (basis.equals("HumanObservation")) return BasisOfRecord.HumanObservation; + + //"preservedspecimen" + //"Preservedspecimen" + //"PreservedSpecimen" + if (basis.equalsIgnoreCase("PreservedSpecimen")) return BasisOfRecord.PreservedSpecimen; + + //others + return BasisOfRecord.HumanObservation; + + } + + public static void searchByCommonName(PluginSession session, String searchTerm, SearchFilters filters, Writer writer) throws Exception + { + + PreparedStatement statement = session.getSearchCommonNamePreparedStatement(searchTerm); + + ResultSet rs = statement.executeQuery(); + + generateResultItems(session, rs, filters, writer); + } + + protected static void fillProducts(PluginSession session, int speciesId, int datasetId, String key, SearchFilters filters, ResultItem item) throws SQLException + { + + List products = new LinkedList(); + + //OCCURRENCES + Product occurences = new Product(ProductType.Occurrence, key); + int occurencesCount = getOccurrencesCount(session, speciesId, datasetId, filters); + occurences.setCount(occurencesCount); + products.add(occurences); + + item.setProducts(products); + } + + public static void searchByScientificName(PluginSession session, String searchTerm, SearchFilters filters, Writer writer) throws Exception + { + + PreparedStatement statement = session.getSearchScientificNamePreparedStatement(searchTerm); + + ResultSet rs = statement.executeQuery(); + + generateResultItems(session, rs, filters, writer); + } + + protected static void generateResultItems(PluginSession session, ResultSet rs, SearchFilters filters, Writer writer) throws Exception + { + //System.out.println("generating records"); + boolean continueWrite = true; + String credits = generateCredits(); + String citation = generateCitation(); + while(rs.next() && continueWrite) { + + int id = rs.getInt("id"); + //System.out.println("id "+id); + + ResultItem baseItem = new ResultItem(String.valueOf(id), ""); + + fillTaxon(session, id, baseItem, credits, citation); + + fillCommonNames(session, id, baseItem); + + PreparedStatement datasetStatement = session.getDatasetPreparedStatement(id); + ResultSet dataSetrs = datasetStatement.executeQuery(); + + while(dataSetrs.next()) { + + ResultItem item = Util.cloneResultItem(baseItem); + + int dataSetId = fillDatasetInformation(dataSetrs, item); + + ProductKey key = new ProductKey(id, dataSetId, filters); + + fillProducts(session, id, dataSetId, key.serialize(), filters, item); + + continueWrite = writer.write(item); + } + } + rs.close(); + } + + /** + * Fills the node with the taxon information. Also information about parent are retrieved. + * @param connection the db connection. + * @param id the taxon id. + * @param taxonNode the node to fill. + * @throws SQLException + */ + protected static void fillTaxon(PluginSession session, int id, Taxon taxon, String credits, String citation) throws SQLException + { + + PreparedStatement statement = session.getTaxonPreparedStatement(id); + + ResultSet rs = statement.executeQuery(); + + if (rs.next()) { + taxon.setCitation(citation); + taxon.setCredits(credits); + + //taxon informations + taxon.setScientificName(rs.getString("tname")); + taxon.setScientificNameAuthorship(rs.getString("tauthor")); + + String rank = rs.getString("rank_name"); + taxon.setRank((rank!=null)?rank:""); + + int parentId = rs.getInt("parent_id"); + rs.close(); + + //check for parent + if (parentId!=id) { + + //create and fill the parent + Taxon parent = new Taxon(String.valueOf(parentId)); + fillTaxon(session, parentId, parent, credits, citation); + taxon.setParent(parent); + } + } + } + + protected static void fillTaxonomyItem(PluginSession session, int id, TaxonomyItem item, String credits, String citation) throws Exception + { + PreparedStatement statement = session.getTaxonPreparedStatement(id); + + ResultSet rs = statement.executeQuery(); + + if (rs.next()) { + + //taxon informations + item.setScientificName(rs.getString("tname")); + + String author = Util.stripNotValidXMLCharacters(rs.getString("tauthor")); + item.setScientificNameAuthorship(author); + + //properties + item.addProperty(new ElementProperty("worms_id", rs.getString("worms_id"))); + item.addProperty(new ElementProperty("col_id", rs.getString("col_id"))); + item.addProperty(new ElementProperty("irmng_id", rs.getString("irmng_id"))); + item.addProperty(new ElementProperty("itis_id", rs.getString("itis_id"))); + + item.setCredits(credits); + item.setCitation(citation); + + String rank = rs.getString("rank_name"); + item.setRank((rank!=null)?rank:""); + + item.setStatus(new TaxonomyStatus("", Status.ACCEPTED)); + + boolean parentNull = rs.getObject("parent_id")==null; + int parentId = rs.getInt("parent_id"); + rs.close(); + + //fill common names + fillCommonNames(session, id, item); + + //check for parent + if (!parentNull && parentId!=id) { + + //create and fill the parent + TaxonomyItem parent = taxonomyItemCache.get(parentId); + if (parent == null) { + parent = new TaxonomyItem(String.valueOf(parentId)); + fillTaxonomyItem(session, parentId, parent, credits, citation); + } + item.setParent(parent); + } + } else throw new IdNotValidException("Taxon with id "+id+" not found"); + } + + protected static void fillCommonNames(PluginSession session, int taxonNameId, TaxonomyItem item) throws SQLException + { + PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId); + ResultSet rs = statement.executeQuery(); + + List commonNames = new ArrayList(); + while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname"))); + rs.close(); + item.setCommonNames(commonNames); + + } + + protected static void fillCommonNames(PluginSession session, int taxonNameId, ResultItem item) throws SQLException + { + PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId); + ResultSet rs = statement.executeQuery(); + List commonNames = new ArrayList(); + while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname"))); + rs.close(); + item.setCommonNames(commonNames); + } + + protected static int fillDatasetInformation(ResultSet rs, ResultItem item) throws SQLException + { + int dataSetId = rs.getInt("datasetId"); + DataSet dataSet = new DataSet(String.valueOf(dataSetId)); + dataSet.setCitation(rs.getString("datasetCitation")); + dataSet.setName(rs.getString("datasetName")); + + DataProvider dataProvider = new DataProvider(String.valueOf(rs.getInt("providerId"))); + dataProvider.setName(rs.getString("providerName")); + dataSet.setDataProvider(dataProvider); + + item.setDataSet(dataSet); + + return dataSetId; + } + + public static Set getCommonNames(PluginSession session, String scientificName) throws SQLException + { + PreparedStatement statement = session.getCommonNameFromScientificNamePreparedStatement(scientificName); + ResultSet rs = statement.executeQuery(); + + Set commonNames = new HashSet(); + + while(rs.next()) commonNames.add(rs.getString("cname")); + + rs.close(); + + return commonNames; + } + + public static void getScientificNames(PluginSession session, String commonName, Writer writer) throws SQLException + { + PreparedStatement statement = session.getScientificNameFromCommonNamePreparedStatement(commonName); + ResultSet rs = statement.executeQuery(); + while (rs.next() && writer.write(rs.getString("tname"))); + rs.close(); + } + + public static void getTaxonByScientificNames(PluginSession session, String scientificName, Writer writer) throws Exception + { + PreparedStatement statement = session.getScientificNamePreparedStatement(scientificName); + ResultSet rs = statement.executeQuery(); + generateTaxonomyItems(session, rs, writer); + } + + public static void getTaxonByCommonName(PluginSession session, String commonName, Writer writer) throws Exception + { + PreparedStatement statement = session.getCommonNamePreparedStatement(commonName); + ResultSet rs = statement.executeQuery(); + generateTaxonomyItems(session, rs, writer); + } + + protected static void generateTaxonomyItems(PluginSession session, ResultSet rs, Writer writer) throws SQLException, Exception + { + boolean continueWrite = true; + String credits = generateCredits(); + String citation = generateCitation(); + while(rs.next() && continueWrite) { + + Integer id = rs.getInt("id"); + + TaxonomyItem taxon = taxonomyItemCache.get(id); + if (taxon == null) { + taxon = new TaxonomyItem(String.valueOf(id)); + fillTaxonomyItem(session, id, taxon, credits, citation); + taxonomyItemCache.put(id, taxon); + } + + //TaxonomyItem taxon = new TaxonomyItem(String.valueOf(id)); + //fillTaxonomyItem(session, id, taxon); + continueWrite = writer.write(taxon); + } + + rs.close(); + } + + protected static List getChildrenTaxon(PluginSession session, int id) throws SQLException + { + PreparedStatement statement = session.getChildrenTaxonPreparedStatement(id); + + ResultSet rs = statement.executeQuery(); + + List children = new ArrayList(); + while (rs.next()) { + //taxon informations + int taxonId = rs.getInt("id"); + + //FIXME tmp workaround + if (taxonId == id) continue; + + TaxonomyItem child = new TaxonomyItem(String.valueOf(taxonId)); + child.setScientificName(rs.getString("tname")); + child.setCitation(rs.getString("tauthor")); + String rank = rs.getString("rank_name"); + child.setRank((rank!=null)?rank:""); + + child.setStatus(new TaxonomyStatus("",Status.ACCEPTED)); + + fillCommonNames(session, id, child); + + children.add(child); + } + rs.close(); + return children; + } + + protected static TaxonomyItem getTaxonById(PluginSession session, int id) throws Exception + { + TaxonomyItem item = new TaxonomyItem(String.valueOf(id)); + String credits = generateCredits(); + String citation = generateCitation(); + fillTaxonomyItem(session, id, item, credits, citation); + return item; + } + + protected static String generateCitation() + { + StringBuilder citation = new StringBuilder("Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org. (Consulted on "); + citation.append(sdf.format(Calendar.getInstance().getTime())); + citation.append(")"); + return citation.toString(); + } + + protected static String generateCredits() + { + //credits ="This information object has been generated via the Species Product Discovery service on 2012-11-26 by interfacing with the Interim Register of Marine and Nonmarine Genera (IRMNG) (http://www.obis.org.au/irmng/)"; + StringBuilder credits = new StringBuilder("This information object has been generated via the Species Product Discovery service on "); + credits.append(sdf.format(Calendar.getInstance().getTime())); + credits.append(" by interfacing with the Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org."); + return credits.toString(); + } + + static int counter = 0; + static int sum = 0; + + /** + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + String url = "jdbc:postgresql://geoserver2.i-marine.research-infrastructures.eu/obis"; + Properties props = new Properties(); + props.setProperty("user","postgres"); + props.setProperty("password","0b1s@d4sc13nc3"); + + + final Connection connection = DriverManager.getConnection(url, props); + +// System.out.println("Connected"); + + final PluginSession session = new PluginSession(connection); + session.preCacheStatements(); + + searchByScientificName(session, "gadus morhua", new SearchFilters(), new Writer() { + + @Override + public boolean write(ResultItem item) { +// System.out.println(item.getId()+" "+item.getScientificNameAuthorship()+" "+item.getScientificName()); + return true; + } + }); + + + + + /*getTaxonByScientificNames(session, "sarda sarda", new Writer() { + + @Override + public boolean write(TaxonomyItem item) { + System.out.println(item.getId()+" "+item.getAuthor()+" "+item.getScientificName()); + return true; + } + });*/ + + /*OccurrencePoint occurrencePoint = getOccurrenceById(session, "38069270"); + System.out.println(occurrencePoint);*/ + + + /*Taxon taxon = getTaxonByCommonName(session, "ruwe traliehoorn");//getTaxonByScientificNames(session, "Protozoa"); + System.out.println(taxon); + + List children = getChildrenTaxon(session, Integer.parseInt(taxon.getId())); + for (Taxon child:children) System.out.println(child);*/ + + /*searchByCommonName(session, "white shark", new SearchFilters(), new Writer() { + + @Override + public void write(ResultItem item) { + System.out.println("Item: "+item.getDataSet().getDataProvider().getName()+" <-> "+item.getDataSet().getName()); + } + });*/ + + /*final long start = System.currentTimeMillis(); + getTaxonByScientificNames(session, "Gadus macrocephalus", new Writer() { + long start = System.currentTimeMillis(); + + @Override + public void write(TaxonomyItem item) { + + + System.out.println(item); + + } + });*/ + + /*SearchFilters filters = new SearchFilters(); + ObisClient.searchByScientificName(session, "sarda sarda", filters, new Writer() { + + @Override + public void write(ResultItem item) { + System.out.println(item); + } + });*/ + + /*List taxa = getChildrenTaxon(session,769809); + for (TaxonomyItem taxon:taxa) System.out.println(taxon.getId());*/ + //navigate("", session, 741923); + + //System.out.println("result in "+(System.currentTimeMillis()-start)+" avg: "+(sum/counter)+" tot: "+counter); + +// System.out.println("done"); + session.expire(); + } + + protected static Set found = new HashSet(); + protected static Map foundTaxon = new HashMap(); + + protected static void navigate(String indentation, PluginSession session, int id) throws SQLException + { + //System.out.println("looking for children: "+id); + if (found.contains(id)) { + System.err.println("Already found "+id); + System.err.println(foundTaxon.get(id)); + System.exit(-1); + } + + List taxa = getChildrenTaxon(session,id); + + found.add(id); + + + for (TaxonomyItem taxon:taxa) { + System.out.println(indentation+taxon.getId()+" "+taxon.getRank()); + foundTaxon.put(taxon.getId(), taxon); + navigate(indentation+" ", session, Integer.valueOf(taxon.getId())); + + } + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java new file mode 100644 index 0000000..23f9f53 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisNameMapping.java @@ -0,0 +1,55 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +import java.sql.SQLException; +import org.apache.log4j.Logger; +import org.gcube.data.spd.model.exceptions.StreamBlockingException; +import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; +import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ObisNameMapping implements MappingCapability { + + protected Logger logger = Logger.getLogger(ObisNameMapping.class); + protected PluginSessionPool sessionPool; + + /** + * @param sessionPool + */ + public ObisNameMapping(PluginSessionPool sessionPool) { + this.sessionPool = sessionPool; + } + + + /** + * {@inheritDoc} + */ + @Override + public void getRelatedScientificNames(final ObjectWriter writer, String commonName) { + logger.debug("retrieving mapping for "+commonName); + PluginSession session = sessionPool.checkOut(); + try { + ObisClient.getScientificNames(session, commonName, new Writer() { + + @Override + public boolean write(String item) { + writer.write(item); + return writer.isAlive(); + } + }); + } catch (SQLException e) { + logger.error("An error occurred retrieving the mapping for common name "+commonName, e); + writer.write(new StreamBlockingException("OBIS",commonName)); + } finally { + sessionPool.checkIn(session); + } + + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java new file mode 100644 index 0000000..e0829cd --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisOccurrencesInterface.java @@ -0,0 +1,133 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.gcube.data.spd.model.Conditions; +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.products.OccurrencePoint; +import org.gcube.data.spd.model.products.Product; +import org.gcube.data.spd.model.products.Product.ProductType; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.obisplugin.data.SearchFilters; +import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; +import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability; +import org.gcube.data.spd.plugin.fwk.writers.ClosableWriter; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ObisOccurrencesInterface extends OccurrencesCapability { + + protected static final Set SUPPORTED_PROPERTIES = new HashSet(Arrays.asList(Conditions.values())); + protected Logger logger = Logger.getLogger(ObisOccurrencesInterface.class); + + protected PluginSessionPool sessionPool; + + /** + * @param sessionPool + */ + public ObisOccurrencesInterface(PluginSessionPool sessionPool) { + this.sessionPool = sessionPool; + } + + /** + * {@inheritDoc} + */ + @Override + public Set getSupportedProperties() { + return SUPPORTED_PROPERTIES; + } + + @Override + public void searchByScientificName(String word, final ObjectWriter writer, Condition... properties) { + final PluginSession session = sessionPool.checkOut(); + try { + SearchFilters filters = new SearchFilters(properties); + logger.trace("filters: "+filters); + ObisClient.searchByScientificName(session, word, filters, new Writer() { + + @Override + public boolean write(ResultItem item) { + for (Product product:item.getProducts()){ + if (product.getType()==ProductType.Occurrence) { + String key = product.getKey(); + getOccurrencePoints(session, key, writer); + } + } + return writer.isAlive(); + } + }); + } catch (Exception e) { + logger.debug("searchByScientificName failed",e); + } finally { + sessionPool.checkIn(session); + } + + } + + + @Override + public void getOccurrencesByIds(ClosableWriter writer, Iterator reader) { + final PluginSession session = sessionPool.checkOut(); + try { + while(reader.hasNext() && writer.isAlive()){ + String id = reader.next(); + try { + OccurrencePoint occurrencePoint = ObisClient.getOccurrenceById(session, id); + if (occurrencePoint!=null) writer.write(occurrencePoint); + } catch (Exception e) { + logger.debug("searchByScientificName failed",e); + } + } + + writer.close(); + } finally { + sessionPool.checkIn(session); + } + + } + + + @Override + public void getOccurrencesByProductKeys(ClosableWriter writer, Iterator reader) { + PluginSession session = sessionPool.checkOut(); + + try { + while(reader.hasNext() && writer.isAlive()){ + String key = reader.next(); + getOccurrencePoints(session, key, writer); + } + writer.close(); + } finally { + sessionPool.checkIn(session); + } + } + + protected void getOccurrencePoints(PluginSession session, String key, final ObjectWriter writer) + { + try { + ObisClient.getOccurrences(session, key, new Writer() { + + @Override + public boolean write(OccurrencePoint item) { + writer.write(item); + return writer.isAlive(); + } + }); + } catch (Exception e) { + logger.error("Error getting occurrence points for key "+key, e); + } + } + + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java new file mode 100644 index 0000000..70a2bbc --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/ObisPlugin.java @@ -0,0 +1,208 @@ +package org.gcube.data.spd.obisplugin; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.HashSet; +import java.util.Set; + +import org.gcube.common.core.utils.logging.GCUBELog; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.utils.encryption.StringEncrypter; +import org.gcube.data.spd.model.Conditions; +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.RepositoryInfo; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.model.util.Capabilities; +import org.gcube.data.spd.obisplugin.data.SearchFilters; +import org.gcube.data.spd.obisplugin.pool.DatabaseCredential; +import org.gcube.data.spd.obisplugin.pool.PluginSessionPool; +import org.gcube.data.spd.plugin.fwk.AbstractPlugin; +import org.gcube.data.spd.plugin.fwk.capabilities.ClassificationCapability; +import org.gcube.data.spd.plugin.fwk.capabilities.MappingCapability; +import org.gcube.data.spd.plugin.fwk.capabilities.OccurrencesCapability; +import org.gcube.data.spd.plugin.fwk.writers.ObjectWriter; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ObisPlugin extends AbstractPlugin { + + protected static final String LOGO_URL = "http://iobis.org/sites/all/themes/corolla/logo.png"; + protected static final String HOME_URL = "http://iobis.org"; + protected static final String DESCRIPTION = "The Ocean Biogeographic information System (OBIS) seeks to absorb, integrate, and assess isolated datasets into a larger, more comprehensive pictures of life in our oceans. " + + "The system hopes to stimulate research about our oceans to generate new hypotheses concerning evolutionary processes, species distributions, and roles of organisms in marine systems on a global scale. " + + "Created by the Census of Marine Life, OBIS is now part of the Intergovernmental Oceanographic Commission (IOC) of UNESCO, under its International Oceanographic Data and Information Exchange (IODE) programme."; + + + protected static final RepositoryInfo REPOSITORY_INFO = new RepositoryInfo(LOGO_URL, HOME_URL, DESCRIPTION); + protected static final String ENTRY_POINT_NAME = "jdbc"; + protected GCUBELog logger = new GCUBELog(ObisPlugin.class); + protected PluginSessionPool sessionPool; + protected ObisNameMapping nameMapping; + protected ObisOccurrencesInterface occurrencesInterface; + protected ObisClassification obisClassification; + protected static final SimpleDateFormat sdf = new SimpleDateFormat(); + + /** + * @return the sessionPool + */ + public PluginSessionPool getSessionPool() { + return sessionPool; + } + + @SuppressWarnings("serial") + @Override + public Set getSupportedCapabilities() { + return new HashSet(){{ + add(Capabilities.NamesMapping); + add(Capabilities.Occurrence); + add(Capabilities.Classification); + }}; + } + + @Override + public String getRepositoryName() { + return "OBIS"; + } + + @Override + public String getDescription() { + return "A plugin for OBIS interaction"; + } + + /** + * {@inheritDoc} + */ + @Override + public void initialize(ServiceEndpoint resource) throws Exception { + + setUseCache(true); + + DatabaseCredential databaseCredential = getDatabaseCredentials(resource); + sessionPool = new PluginSessionPool(databaseCredential); + nameMapping = new ObisNameMapping(sessionPool); + occurrencesInterface = new ObisOccurrencesInterface(sessionPool); + obisClassification = new ObisClassification(sessionPool); + } + + /** + * {@inheritDoc} + */ + @Override + public void shutdown() throws Exception { + sessionPool.shutdown(true); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("serial") + @Override + public Set getSupportedProperties() { + return new HashSet(){{ + add(Conditions.DATE); + add(Conditions.COORDINATE); + }}; + } + + /** + * {@inheritDoc} + */ + @Override + public void update(ServiceEndpoint resource) throws Exception { + DatabaseCredential databaseCredential = getDatabaseCredentials(resource); + sessionPool.setDatabaseCredential(databaseCredential); + } + + protected DatabaseCredential getDatabaseCredentials(ServiceEndpoint resource) throws Exception + { + AccessPoint jdbcAccessPoint = null; + for (AccessPoint accessPoint: resource.profile().accessPoints()) + { + if (ENTRY_POINT_NAME.equalsIgnoreCase(accessPoint.name())) { + jdbcAccessPoint = accessPoint; + break; + } + } + + if (jdbcAccessPoint==null) { + logger.error("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource"); + throw new Exception("AccessPoint with entry name "+ENTRY_POINT_NAME+" not found in the plugin RuntimeResource"); + } + String password = StringEncrypter.getEncrypter().decrypt(jdbcAccessPoint.password()); + return new DatabaseCredential(jdbcAccessPoint.address(), jdbcAccessPoint.username(), password); + } + + + /** + * {@inheritDoc} + */ + @Override + public void searchByScientificName(String searchTerm, final ObjectWriter writer, Condition... properties) { + logger.debug("starting the search for obisPlugin word: "+searchTerm); + + PluginSession session = sessionPool.checkOut(); + try { + final String credits = getObisCredits(); + SearchFilters filters = new SearchFilters(properties); + logger.trace("filters: "+filters); + ObisClient.searchByScientificName(session, searchTerm, filters, new Writer() { + + @Override + public boolean write(ResultItem item) { + item.setCredits(credits); + writer.write(item); + return writer.isAlive(); + } + }); + + } catch (Exception e) { + logger.debug("searchByScientificName failed",e); + } finally { + sessionPool.checkIn(session); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MappingCapability getMappingInterface() { + return nameMapping; + } + + /** + * {@inheritDoc} + */ + @Override + public OccurrencesCapability getOccurrencesInterface() { + return occurrencesInterface; + } + + protected String getObisCredits() + { + StringBuilder credits = new StringBuilder("Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org. (Consulted on "); + credits.append(sdf.format(Calendar.getInstance().getTime())); + credits.append(")"); + return credits.toString(); + } + + /** + * {@inheritDoc} + */ + @Override + public ClassificationCapability getClassificationInterface() { + return obisClassification; + } + + /** + * {@inheritDoc} + */ + @Override + public RepositoryInfo getRepositoryInfo() { + return REPOSITORY_INFO; + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java b/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java new file mode 100644 index 0000000..15e09f6 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/PluginSession.java @@ -0,0 +1,408 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Calendar; +import java.util.List; + +import org.gcube.data.spd.model.Condition; +import org.gcube.data.spd.model.Coordinate; +import org.gcube.data.spd.obisplugin.data.SearchFilters; +import org.gcube.data.spd.obisplugin.pool.DatabaseCredential; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class PluginSession { + + protected static final String SCHEMA = "obis"; + + protected DatabaseCredential databaseCredential; + protected Connection connection; + protected PreparedStatement taxonPreparedStatement; + protected PreparedStatement taxonCommonNamePreparedStatement; + protected PreparedStatement datasetPreparedStatement; + protected PreparedStatement commonNameFromScientificNamePreparedStatement; + protected PreparedStatement scientificNameFromCommonNamePreparedStatement; + protected PreparedStatement searchCommonNamePreparedStatement; + protected PreparedStatement searchScientificNamePreparedStatement; + protected PreparedStatement scientificNamePreparedStatement; + protected PreparedStatement commonNamePreparedStatement; + protected PreparedStatement childrenTaxonPreparedStatement; + protected PreparedStatement occurrenceByIdPreparedStatement; + + /** + * @param connection + */ + public PluginSession(Connection connection) { + this(null,connection); + } + /** + * @param credentialToken + * @param connection + */ + public PluginSession(DatabaseCredential databaseCredential, Connection connection) { + this.databaseCredential = databaseCredential; + this.connection = connection; + } + + /** + * @return the connection + */ + public Connection getConnection() { + return connection; + } + + /** + * The session is valid if the connection is OK and if the connection has been created using the same credentials. + * @param credentialToken + * @return + * @throws SQLException + */ + public boolean isValid(DatabaseCredential databaseCredential) throws SQLException + { + return (this.databaseCredential!=null?(this.databaseCredential.equals(databaseCredential)):true) && !connection.isClosed() && isValid(); + } + + protected boolean isValid() + { + try{ + ResultSet result = connection.createStatement().executeQuery("SELECT 1"); + result.close(); + }catch (Exception e) { + return false; + } + return true; + } + + public void expire() throws SQLException + { + connection.close(); + } + + public void preCacheStatements() throws SQLException + { + createTaxonPreparedStatement(); + createTaxonCommonNamePreparedStatemen(); + createDatasetPreparedStatement(); + createCommonNameFromScientificNamePreparedStatement(); + createScientificNameFromCommonNamePreparedStatement(); + createSearchCommonNamePreparedStatement(); + createSearchScientificNamePreparedStatement(); + createScientificNamePreparedStatement(); + createCommonNamePreparedStatement(); + createChildrenTaxonPreparedStatement(); + createOccurrenceByIdPreparedStatement(); + } + + + public PreparedStatement getTaxonPreparedStatement(int id) throws SQLException + { + if (taxonPreparedStatement==null) createTaxonPreparedStatement(); + taxonPreparedStatement.clearParameters(); + taxonPreparedStatement.setInt(1, id); + return taxonPreparedStatement; + } + + protected void createTaxonPreparedStatement() throws SQLException + { + taxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id, t.parent_id, t.tauthor, t.worms_id, t.col_id, t.irmng_id, t.itis_id, r.rank_name " + + "FROM "+SCHEMA+".tnames t " + + "LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+ + "WHERE t.id = ?"); + } + + + public PreparedStatement getTaxonCommonNamePreparedStatement(int taxonNameId) throws SQLException + { + if (taxonCommonNamePreparedStatement==null) createTaxonCommonNamePreparedStatemen(); + taxonCommonNamePreparedStatement.clearParameters(); + taxonCommonNamePreparedStatement.setInt(1, taxonNameId); + return taxonCommonNamePreparedStatement; + } + + protected void createTaxonCommonNamePreparedStatemen() throws SQLException + { + taxonCommonNamePreparedStatement = connection.prepareStatement("select c.cname, l.lanname FROM "+SCHEMA+".cnames c, "+SCHEMA+".languages l WHERE c.tname_id = ? AND c.language_id = l.id"); + } + + public PreparedStatement getDatasetPreparedStatement(int id) throws SQLException + { + if (datasetPreparedStatement==null) createDatasetPreparedStatement(); + datasetPreparedStatement.clearParameters(); + datasetPreparedStatement.setInt(1, id); + return datasetPreparedStatement; + } + + protected void createDatasetPreparedStatement() throws SQLException + { + /*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + + "FROM obis.resources r, obis.providers p WHERE " + + "exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/ + + String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " + + "FROM obis.resources r, obis.providers p WHERE " + + "r.id in (SELECT resource_id from portal.species_per_resource where valid_id = ?) AND r.provider_id = p.id"; + datasetPreparedStatement = connection.prepareStatement(query); + } + + public PreparedStatement getCommonNameFromScientificNamePreparedStatement(String scientificaName) throws SQLException + { + if (commonNameFromScientificNamePreparedStatement==null) createCommonNameFromScientificNamePreparedStatement(); + commonNameFromScientificNamePreparedStatement.clearParameters(); + commonNameFromScientificNamePreparedStatement.setString(1, scientificaName); + return commonNameFromScientificNamePreparedStatement; + } + + protected void createCommonNameFromScientificNamePreparedStatement() throws SQLException + { + commonNameFromScientificNamePreparedStatement = connection.prepareStatement("SELECT c.cname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE t.tname ILIKE ? AND c.tname_id = t.id"); + } + + public PreparedStatement getScientificNameFromCommonNamePreparedStatement(String commonName) throws SQLException + { + if (scientificNameFromCommonNamePreparedStatement==null) createScientificNameFromCommonNamePreparedStatement(); + scientificNameFromCommonNamePreparedStatement.clearParameters(); + scientificNameFromCommonNamePreparedStatement.setString(1, commonName); + return scientificNameFromCommonNamePreparedStatement; + } + + protected void createScientificNameFromCommonNamePreparedStatement() throws SQLException + { + scientificNameFromCommonNamePreparedStatement = connection.prepareStatement("SELECT DISTINCT t.tname FROM "+SCHEMA+".cnames c, "+SCHEMA+".tnames t WHERE c.cname ILIKE ? AND c.tname_id = t.id"); + } + + public PreparedStatement getSearchCommonNamePreparedStatement(String searchTerm) throws SQLException + { + if (searchCommonNamePreparedStatement == null) createSearchCommonNamePreparedStatement(); + searchCommonNamePreparedStatement.clearParameters(); + searchCommonNamePreparedStatement.setString(1, "%"+searchTerm+"%"); + return searchCommonNamePreparedStatement; + } + + protected void createSearchCommonNamePreparedStatement() throws SQLException + { + String query = "SELECT DISTINCT c.tname_id AS id FROM obis.cnames c WHERE c.cname ILIKE ?"; + searchCommonNamePreparedStatement = connection.prepareStatement(query); + } + + public PreparedStatement getSearchScientificNamePreparedStatement(String searchTerm) throws SQLException + { + if (searchScientificNamePreparedStatement == null) createSearchScientificNamePreparedStatement(); + searchScientificNamePreparedStatement.clearParameters(); + searchScientificNamePreparedStatement.setString(1, "%"+searchTerm+"%"); + return searchScientificNamePreparedStatement; + } + + protected void createSearchScientificNamePreparedStatement() throws SQLException + { + String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ? AND exists (SELECT 1 FROM obis.drs WHERE valid_id = t.id)"; + searchScientificNamePreparedStatement = connection.prepareStatement(query); + } + + public PreparedStatement getOccurrencesCountPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException{ + + //We don't cache it because in this case a PS is less performant + StringBuilder query = new StringBuilder("SELECT count(*) AS occurrences FROM "+SCHEMA+".drs WHERE valid_id = ? AND resource_id = ?"); + + buildConditions(query, "", filters.getConditions()); + + /*if (filters.getUpperBound()!=null) query.append(" AND latitude <= ? AND longitude <= ?"); + if (filters.getLowerBound()!=null) query.append(" AND latitude >= ? AND longitude >= ?"); + if (filters.getFromDate()!=null) query.append(" AND datecollected >= ?"); + if (filters.getToDate()!=null) query.append(" AND datecollected <= ?");*/ + + PreparedStatement occurrencesCountPreparedStatement = connection.prepareStatement(query.toString()); + + int parameterCounter = 1; + occurrencesCountPreparedStatement.setInt(parameterCounter++, taxonId); + occurrencesCountPreparedStatement.setInt(parameterCounter++, datasetId); + + addParameters(occurrencesCountPreparedStatement, parameterCounter, filters.getConditions()); + + /*for (Condition condition:filters.getConditions()) { + switch (condition.getProperty()) { + case COORDINATE: { + Coordinate coordinate = (Coordinate) condition.getValue(); + occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLatitude()); + occurrencesCountPreparedStatement.setDouble(parameterCounter++, coordinate.getLongitude()); + } break; + case EVENT_DATE: { + Calendar calendar = (Calendar) condition.getValue(); + occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis())); + } + } + }*/ + + /*if (filters.getUpperBound() != null) { + occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude()); + occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude()); + } + + if (filters.getLowerBound() != null) { + occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude()); + occurrencesCountPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude()); + } + + if (filters.getFromDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis())); + if (filters.getToDate() != null) occurrencesCountPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/ + + return occurrencesCountPreparedStatement; + } + + protected void buildConditions(StringBuilder query, String prefix, List conditions) + { + for (Condition condition:conditions) buildCondition(query, prefix, condition); + } + + protected void buildCondition(StringBuilder query, String prefix, Condition condition) + { + String op = ""; + switch (condition.getOp()) { + case EQ: op = "=="; break; + case GE: op = ">="; break; + case GT: op = ">"; break; + case LE: op = "<="; break; + case LT: op = "<"; break; + } + + switch (condition.getType()) { + case COORDINATE: { + query.append(" AND "); + query.append(prefix); + query.append("latitude "); + query.append(op); + query.append(" ? AND "); + query.append(prefix); + query.append("longitude "); + query.append(op); + query.append(" ?"); + } break; + case DATE: { + query.append(" AND "); + query.append(prefix); + query.append("datecollected "); + query.append(op); + query.append(" ?"); + } break; + } + } + + protected void addParameters(PreparedStatement preparedStatement, int parameterCounter, List conditions) throws SQLException { + for (Condition condition:conditions) { + switch (condition.getType()) { + case COORDINATE: { + Coordinate coordinate = (Coordinate) condition.getValue(); + preparedStatement.setDouble(parameterCounter++, coordinate.getLatitude()); + preparedStatement.setDouble(parameterCounter++, coordinate.getLongitude()); + } break; + case DATE: { + Calendar calendar = (Calendar) condition.getValue(); + preparedStatement.setDate(parameterCounter++, new Date(calendar.getTimeInMillis())); + } + } + } + } + + protected PreparedStatement getOccurrencesPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException + { + //We don't cache it because in this case a PS is less performant + StringBuilder query = new StringBuilder("SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.identifiedBy, dxs.yearcollected, dxs.monthcollected, dxs.daycollected FROM obis.drs drs, obis.dxs dxs WHERE drs.valid_id = ? AND drs.resource_id = ? AND drs.id = dxs.dr_id"); + + buildConditions(query, "drs.", filters.getConditions()); + + /*if (filters.getUpperBound() != null) query.append(" AND drs.latitude <= ? AND drs.longitude <= ?"); + if (filters.getLowerBound() != null) query.append(" AND drs.latitude >= ? AND drs.longitude >= ?"); + if (filters.getFromDate() != null) query.append(" AND drs.datecollected >= ?"); + if (filters.getToDate() != null) query.append(" AND drs.datecollected <= ?");*/ + + PreparedStatement occurrencesPreparedStatement = connection.prepareStatement(query.toString()); + + int parameterCounter = 1; + occurrencesPreparedStatement.setInt(parameterCounter++, taxonId); + occurrencesPreparedStatement.setInt(parameterCounter++, datasetId); + + addParameters(occurrencesPreparedStatement, parameterCounter, filters.getConditions()); + + /*if (filters.getUpperBound() != null) { + occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLatitude()); + occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getUpperBound().getLongitude()); + } + + if (filters.getLowerBound() != null) { + occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLatitude()); + occurrencesPreparedStatement.setDouble(parameterCounter++, filters.getLowerBound().getLongitude()); + } + + if (filters.getFromDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getFromDate().getTimeInMillis())); + if (filters.getToDate() != null) occurrencesPreparedStatement.setDate(parameterCounter++, new Date(filters.getToDate().getTimeInMillis()));*/ + + return occurrencesPreparedStatement; + } + + public PreparedStatement getScientificNamePreparedStatement(String scientificName) throws SQLException + { + if (scientificNamePreparedStatement == null) createScientificNamePreparedStatement(); + scientificNamePreparedStatement.clearParameters(); + scientificNamePreparedStatement.setString(1, "%"+scientificName+"%"); + return scientificNamePreparedStatement; + } + + protected void createScientificNamePreparedStatement() throws SQLException + { + String query = "SELECT t.id as id FROM obis.tnames t WHERE t.tname ILIKE ?"; + scientificNamePreparedStatement = connection.prepareStatement(query); + } + + public PreparedStatement getCommonNamePreparedStatement(String commonName) throws SQLException + { + if (commonNamePreparedStatement==null) createCommonNamePreparedStatement(); + commonNamePreparedStatement.clearParameters(); + commonNamePreparedStatement.setString(1, commonName); + return commonNamePreparedStatement; + } + + protected void createCommonNamePreparedStatement() throws SQLException + { + commonNamePreparedStatement = connection.prepareStatement("SELECT c.tname_id as id FROM "+SCHEMA+".cnames c WHERE c.cname LIKE ?"); + } + + public PreparedStatement getChildrenTaxonPreparedStatement(int id) throws SQLException + { + if (childrenTaxonPreparedStatement==null) createChildrenTaxonPreparedStatement(); + childrenTaxonPreparedStatement.clearParameters(); + childrenTaxonPreparedStatement.setInt(1, id); + return childrenTaxonPreparedStatement; + } + + protected void createChildrenTaxonPreparedStatement() throws SQLException + { + childrenTaxonPreparedStatement = connection.prepareStatement("SELECT t.tname, t.id as id, t.parent_id, t.tauthor, r.rank_name " + + "FROM "+SCHEMA+".tnames t " + + "LEFT JOIN "+SCHEMA+".ranks r ON t.rank_id = r.rank_id and r.kingdom_id = CASE WHEN t.rank_id = 10 THEN 738303 ELSE (string_to_array(storedpath, 'x')::text[])[3]::int END "+ + "WHERE t.parent_id = ?"); + } + + protected PreparedStatement getOccurrenceByIdPreparedStatement(int occurrenceId) throws SQLException + { + if (occurrenceByIdPreparedStatement==null) createOccurrenceByIdPreparedStatement(); + occurrenceByIdPreparedStatement.clearParameters(); + occurrenceByIdPreparedStatement.setInt(1, occurrenceId); + return occurrenceByIdPreparedStatement; + } + + protected void createOccurrenceByIdPreparedStatement() throws SQLException + { + String query = "SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.yearcollected, dxs.monthcollected, dxs.daycollected FROM obis.drs drs, obis.dxs dxs WHERE drs.id = ? AND drs.id = dxs.dr_id"; + occurrenceByIdPreparedStatement = connection.prepareStatement(query); + } + + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/Writer.java b/src/main/java/org/gcube/data/spd/obisplugin/Writer.java new file mode 100644 index 0000000..82890c5 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/Writer.java @@ -0,0 +1,19 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public interface Writer { + + /** + * Writes the specified element. + * @param item the item to write. + * @return false if the writer has been closed and no more elements are accepted. + */ + public boolean write(T item); + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java b/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java new file mode 100644 index 0000000..2284a79 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/data/ProductKey.java @@ -0,0 +1,86 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.data; + +import com.thoughtworks.xstream.XStream; + + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class ProductKey { + + protected static XStream stream; + + protected int taxonId; + protected int dataSetId; + protected SearchFilters filters; + + /** + * @param taxonId + * @param dataSetId + * @param filters + */ + public ProductKey(int taxonId, int dataSetId, SearchFilters filters) { + this.taxonId = taxonId; + this.dataSetId = dataSetId; + this.filters = filters; + } + + /** + * @return the taxonId + */ + public int getTaxonId() { + return taxonId; + } + + /** + * @return the dataSetId + */ + public int getDataSetId() { + return dataSetId; + } + + /** + * @return the filters + */ + public SearchFilters getFilters() { + return filters; + } + + protected static XStream getStream() + { + if (stream == null) stream = new XStream(); + return stream; + } + + public String serialize() + { + XStream stream = getStream(); + return stream.toXML(this); + } + + public static ProductKey deserialize(String key) + { + XStream stream = getStream(); + return (ProductKey) stream.fromXML(key); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ProductKey [taxonId="); + builder.append(taxonId); + builder.append(", dataSetId="); + builder.append(dataSetId); + builder.append(", filters="); + builder.append(filters); + builder.append("]"); + return builder.toString(); + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java b/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java new file mode 100644 index 0000000..51fe142 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/data/SearchFilters.java @@ -0,0 +1,48 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.data; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.gcube.data.spd.model.Condition; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class SearchFilters { + + protected List conditions; + + public SearchFilters(Condition ... conditions){ + this.conditions = new ArrayList(Arrays.asList(conditions)); + } + + public void addCondition(Condition condition) + { + conditions.add(condition); + } + + /** + * @return the conditions + */ + public List getConditions() { + return conditions; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("SearchFilters [conditions="); + builder.append(conditions); + builder.append("]"); + return builder.toString(); + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java new file mode 100644 index 0000000..fb6a509 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/pool/DatabaseCredential.java @@ -0,0 +1,90 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.pool; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class DatabaseCredential { + + protected String url; + protected String user; + protected String password; + + /** + * @param url + * @param user + * @param password + */ + public DatabaseCredential(String url, String user, String password) { + this.url = url; + this.user = user; + this.password = password; + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @return the user + */ + public String getUser() { + return user; + } + + /** + * @return the password + */ + public String getPassword() { + return password; + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((password == null) ? 0 : password.hashCode()); + result = prime * result + ((url == null) ? 0 : url.hashCode()); + result = prime * result + ((user == null) ? 0 : user.hashCode()); + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + DatabaseCredential other = (DatabaseCredential) obj; + if (password == null) { + if (other.password != null) + return false; + } else if (!password.equals(other.password)) + return false; + if (url == null) { + if (other.url != null) + return false; + } else if (!url.equals(other.url)) + return false; + if (user == null) { + if (other.user != null) + return false; + } else if (!user.equals(other.user)) + return false; + return true; + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java new file mode 100644 index 0000000..19c42dd --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/pool/JDBCConnectionPool.java @@ -0,0 +1,61 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.pool; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * Adapted from http://sourcemaking.com/design_patterns/object_pool/java + */ +public class JDBCConnectionPool extends ObjectPool { + + protected String url, username, password; + + public JDBCConnectionPool(String driver, String dsn, String usr, String pwd) { + super("JDBCConnectionPool", 10 * 60 * 1000); + try { + Class.forName(driver).newInstance(); + } catch (Exception e) { + e.printStackTrace(); + } + this.url = dsn; + this.username = usr; + this.password = pwd; + } + + @Override + protected Connection create() { + try { + Connection connection = DriverManager.getConnection(url, username, password); + connection.setAutoCommit(false); + connection.setReadOnly(true); + return connection; + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + } + + @Override + protected void expire(Connection o) { + try { + o.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Override + protected boolean validate(Connection o) { + try { + return !o.isClosed() && o.isValid(1000); + } catch (SQLException e) { + e.printStackTrace(); + return false; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java new file mode 100644 index 0000000..31f87e0 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/pool/ObjectPool.java @@ -0,0 +1,126 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.pool; + +import java.util.Enumeration; +import java.util.Hashtable; + +import org.gcube.common.core.utils.logging.GCUBELog; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * Adapted from http://sourcemaking.com/design_patterns/object_pool/java + */ +public abstract class ObjectPool { + + protected static GCUBELog logger = new GCUBELog(ObjectPool.class); + + protected long expirationTime; + + protected Hashtable locked, unlocked; + protected boolean closed = false; + protected String name; + + public ObjectPool(String name, long expirationTime) { + this.name = name; + this.expirationTime = expirationTime; + locked = new Hashtable(); + unlocked = new Hashtable(); + } + + protected abstract T create(); + + protected abstract boolean validate(T o); + + protected abstract void expire(T o); + + public synchronized T checkOut() { + long now = System.currentTimeMillis(); + T t; + if (unlocked.size() > 0) { + Enumeration e = unlocked.keys(); + while (e.hasMoreElements()) { + t = e.nextElement(); + if ((now - unlocked.get(t)) > expirationTime) { + // object has expired + unlocked.remove(t); + expire(t); + t = null; + } else { + if (validate(t)) { + unlocked.remove(t); + locked.put(t, now); + return t; + } else { + // object failed validation + unlocked.remove(t); + expire(t); + t = null; + } + } + } + } + + logger.trace("no objects available, create a new one, status: "+this); + // no objects available, create a new one + t = create(); + locked.put(t, now); + return (t); + } + + public synchronized void checkIn(T t) { + locked.remove(t); + if (!closed) unlocked.put(t, System.currentTimeMillis()); + else expire(t); + logger.trace("pool status: "+this); + } + + public synchronized void shutdown(boolean force) + { + closed = true; + expireAllUnlocked(); + if (force) expireAllLocked(); + } + + public synchronized void expireAllUnlocked() + { + Enumeration e = unlocked.keys(); + while (e.hasMoreElements()) { + T t = e.nextElement(); + unlocked.remove(t); + expire(t); + } + } + + public synchronized void expireAllLocked() + { + Enumeration e = locked.keys(); + while (e.hasMoreElements()) { + T t = e.nextElement(); + locked.remove(t); + expire(t); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ObjectPool [name="); + builder.append(name); + builder.append(", expirationTime="); + builder.append(expirationTime); + builder.append(", locked="); + builder.append(locked.size()); + builder.append(", unlocked="); + builder.append(unlocked.size()); + builder.append(", closed="); + builder.append(closed); + builder.append("]"); + return builder.toString(); + } + +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java b/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java new file mode 100644 index 0000000..f5599d7 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/pool/PluginSessionPool.java @@ -0,0 +1,83 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.pool; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.log4j.Logger; +import org.gcube.data.spd.obisplugin.PluginSession; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class PluginSessionPool extends ObjectPool { + + protected Logger logger = Logger.getLogger(PluginSessionPool.class); + protected DatabaseCredential databaseCredential; + + public PluginSessionPool(DatabaseCredential databaseCredential) { + super("PluginSessionPool", 30*60*1000); + setDatabaseCredential(databaseCredential); + } + + /** + * @param databaseCredential the databaseCredential to set + */ + public void setDatabaseCredential(DatabaseCredential databaseCredential) { + this.databaseCredential = databaseCredential; + } + + /** + * {@inheritDoc} + */ + @Override + protected PluginSession create() { + + try { + Connection connection = createConnection(); + return new PluginSession(databaseCredential, connection); + } catch (SQLException e) { + logger.error("An error occurred creating the db connection", e); + return null; + } + } + + protected Connection createConnection() throws SQLException + { + Properties props = new Properties(); + props.setProperty("user", databaseCredential.getUser()); + props.setProperty("password", databaseCredential.getPassword()); + return DriverManager.getConnection(databaseCredential.getUrl(), props); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean validate(PluginSession session) { + try { + return session.isValid(databaseCredential); + } catch (Exception e) { + logger.warn("An error occurred validating the session", e); + return false; + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void expire(PluginSession session) { + + try { + session.expire(); + } catch (SQLException e) { + logger.warn("An error occurred expiring the session", e); + } + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java b/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java new file mode 100644 index 0000000..3c6c0c8 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/util/Cache.java @@ -0,0 +1,114 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.util; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class Cache { + + protected int limit; + protected Map cache; + + protected boolean enableStatistics; + protected long hints; + protected long requests; + protected long removed; + + @SuppressWarnings("serial") + public Cache(final int limit) + { + this.limit = limit; + this.hints = 0; + + this.cache = new LinkedHashMap() { + + /** + * {@inheritDoc} + */ + @Override + protected boolean removeEldestEntry(Entry eldest) { + boolean remove = size() > limit; + if (remove) removed++; + return remove; + } + + }; + } + + /** + * @return the enableStatistics + */ + public boolean isEnableStatistics() { + return enableStatistics; + } + + /** + * @param enableStatistics the enableStatistics to set + */ + public void setEnableStatistics(boolean enableStatistics) { + this.enableStatistics = enableStatistics; + } + + /** + * @return the requests + */ + public long getRequests() { + return requests; + } + + /** + * @return the limit + */ + public int getLimit() { + return limit; + } + + /** + * @return the hints + */ + public long getHints() { + return hints; + } + + public V get(K key) + { + V value = cache.get(key); + if (enableStatistics) { + requests++; + if (value!=null) hints++; + } + return value; + } + + public void put(K key, V value) + { + cache.put(key, value); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Cache [limit="); + builder.append(limit); + builder.append(", enableStatistics="); + builder.append(enableStatistics); + builder.append(", hints="); + builder.append(hints); + builder.append(", requests="); + builder.append(requests); + builder.append(", removed="); + builder.append(removed); + builder.append("]"); + return builder.toString(); + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java b/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java new file mode 100644 index 0000000..82c7a5d --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/util/DateUtil.java @@ -0,0 +1,84 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.util; + +import java.util.Date; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class DateUtil { + + //Unsupported: + //2010-05-03T14:44:04Z+01:00 + //2009-04-09TMountain Da:ylight Time + //2008-05-20T13:03:08OZ + //2008-11-01T14:45OZ + //0000-00-00 00:00:00 + + protected static final String[] DATE_FORMATS = new String[]{ + "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", //2005-10-24T13:33:11.000Z + "yyyy-MM-dd'T'HH:mm:ss:mm:ssZ", //2003-02-18T11:35:13:35:13Z + "yyyy-MM-dd' 'HH:mm:ss.SSSSSS", //2006-12-20 10:27:02.477563 + "yyyy-MM-dd'T'HH:mm:ssZZ", //2001-02-10T12:00:00+00:00 + "yyyy-MM-dd'T'HH:mm:sssZZ", //2011-09-25T11:00:000Z + "yyyy-MM-dd' 'HH:mm:ssZZ", //2007-05-11 14:01:15-04 + "yyyy-MM-dd'T'HH:mm:ssz", //2005-10-11T15:40:00Z + "yyyy-MM-dd' 'HH:mm:ss", //2007-07-18 06:13:06 + "yyyy-MM-dd'T'HH:mmZ", //2009-10-01T01:00Z + "MM/dd/yyyy' 'hh:mm:ss aa", //10/28/2010 10:12:43 AM + "MM/dd/yyyy' 'hh:mm", //12/9/2010 11:59 + "dd/MM/yyyyHH:mm:ssZ", //13/9/201012:00:00Z + "yyyy-MM-dd' 'HH:mm", //2005-12-20 17:12 + "yyyy-MM-dd'T'", //2010-06-09T + "yyyy-MM-dd-'T'", //2009-08-05-T + "yyyy-MM-dd", //2009-09-08 + "dd-MMM-yy", //28-MAR-01 08-AUG-96 + "dd/MM/yyyy", //11/2/2010 + "MM/dd/yyyy' 'HH:mm:ss", //8/23/2010 0:00:00 + "MM/dd/yyyy", //10/19/2010 + "yyyy/MM/dd' 'HH:mm:ss", //2010/10/27 22:29:04 + //FIXME Military Time Zone not supported + "yyyy-MM-dd'T'HH:mm:ss'B'", //2003-07-07T10:03:56B + }; + + + protected static DateUtil instance; + + public static DateUtil getInstance() + { + if (instance == null) { + instance = new DateUtil(); + instance.initialize(); + } + return instance; + } + + protected DateTimeFormatter[] FORMATS; + + protected DateUtil() { + FORMATS = new DateTimeFormatter[DATE_FORMATS.length]; + } + + protected void initialize() + { + int i = 0; + for (String dateFormat:DATE_FORMATS) { + FORMATS[i++] = DateTimeFormat.forPattern(dateFormat); + } + } + + public Date parse(String dateString) { + for (DateTimeFormatter formatter:FORMATS) { + try { + return formatter.parseDateTime(dateString).toDate(); + } catch (Exception e){} + } + return null; // Unknown format. + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java b/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java new file mode 100644 index 0000000..0288806 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/util/RegExpDateFormat.java @@ -0,0 +1,38 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.util; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class RegExpDateFormat { + + protected Pattern pattern; + protected DateFormat dateFormat; + + public RegExpDateFormat(String regExp, String datePattern) + { + pattern = Pattern.compile(regExp); + dateFormat = new SimpleDateFormat(datePattern); + } + + public boolean match(String input) + { + Matcher m = pattern.matcher(input); + return m.matches(); + } + + public Date parse(String input) throws ParseException + { + return dateFormat.parse(input); + } +} diff --git a/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java b/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java new file mode 100644 index 0000000..8981409 --- /dev/null +++ b/src/main/java/org/gcube/data/spd/obisplugin/util/Util.java @@ -0,0 +1,89 @@ +/** + * + */ +package org.gcube.data.spd.obisplugin.util; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.data.spd.model.CommonName; +import org.gcube.data.spd.model.products.ResultItem; +import org.gcube.data.spd.model.products.Taxon; + +/** + * @author "Federico De Faveri defaveri@isti.cnr.it" + * + */ +public class Util { + + + /** + * Partially clones the passed result item. + * @param item + * @return + */ + public static ResultItem cloneResultItem(ResultItem item) + { + if (item==null) return null; + + ResultItem clone = new ResultItem(item.getId(), item.getScientificName()); + copyTaxon(item, clone); + clone.setCommonNames(cloneCommonName(item.getCommonNames())); + + return clone; + } + + protected static Taxon cloneTaxon(Taxon taxon) + { + if (taxon==null) return null; + + Taxon clone = new Taxon(taxon.getId(), taxon.getScientificName()); + copyTaxon(taxon, clone); + return clone; + } + + protected static void copyTaxon(Taxon taxon, Taxon clone) + { + clone.setId(taxon.getId()); + clone.setScientificName(taxon.getScientificName()); + clone.setCitation(taxon.getCitation()); + clone.setCredits(taxon.getCredits()); + clone.setScientificNameAuthorship(taxon.getScientificNameAuthorship()); + clone.setRank(taxon.getRank()); + + clone.setParent(cloneTaxon(taxon.getParent())); + } + + protected static List cloneCommonName(List commonNames) + { + if (commonNames==null) return null; + List clones = new ArrayList(commonNames.size()); + for (CommonName commonName:commonNames) clones.add(cloneCommonName(commonName)); + return clones; + } + + protected static CommonName cloneCommonName(CommonName commonName) + { + if (commonName==null) return null; + return new CommonName(commonName.getLanguage(), commonName.getName()); + } + + public static String stripNotValidXMLCharacters(String input) { + + if (input == null) return null; + if (input.isEmpty()) return ""; + + StringBuffer out = new StringBuffer(); + + for (char current:input.toCharArray()){ + if ((current == 0x9) || + (current == 0xA) || + (current == 0xD) || + ((current >= 0x20) && (current <= 0xD7FF)) || + ((current >= 0xE000) && (current <= 0xFFFD)) || + ((current >= 0x10000) && (current <= 0x10FFFF))) + out.append(current); + } + return out.toString(); + } +} diff --git a/src/main/resources/META-INF/services/org.gcube.data.spd.plugin.fwk.AbstractPlugin b/src/main/resources/META-INF/services/org.gcube.data.spd.plugin.fwk.AbstractPlugin new file mode 100644 index 0000000..c4dc4a2 --- /dev/null +++ b/src/main/resources/META-INF/services/org.gcube.data.spd.plugin.fwk.AbstractPlugin @@ -0,0 +1 @@ +org.gcube.data.spd.obisplugin.ObisPlugin \ No newline at end of file