This commit is contained in:
Lucio Lelii 2014-12-04 11:44:44 +00:00
parent dcc51657cb
commit 77c461eeaa
6 changed files with 159 additions and 94 deletions

View File

@ -1,4 +1,7 @@
<ReleaseNotes>
<Changeset component="org.gcube.data-access.obis-spd-plugin.1-8-1" date="2013-11-28">
<Change> Added field SNAutorship and Dataset to occurrence points</Change>
</Changeset>
<Changeset component="org.gcube.data-access.obis-spd-plugin.1-7-0" date="2013-07-18">
<Change> Sync with spd-model-library changes</Change>
</Changeset>

View File

@ -11,7 +11,7 @@
<groupId>org.gcube.data.spd</groupId>
<artifactId>obis-spd-plugin</artifactId>
<version>1.8.0-SNAPSHOT</version>
<version>1.8.1-SNAPSHOT</version>
<name>ObisPlugin</name>
<description>obis plugin for species manager service</description>

View File

@ -52,7 +52,7 @@ public class ObisClient {
protected static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("y-M-d");
protected static final DateUtil DATE_UTIL = DateUtil.getInstance();
protected static final SimpleDateFormat sdf = new SimpleDateFormat();
public static final Cache<Integer, TaxonomyItem> taxonomyItemCache = new Cache<Integer, TaxonomyItem>(1000);
public static int getOccurrencesCount(PluginSession session, int taxonId, int datasetId, SearchFilters filters) throws SQLException
@ -75,21 +75,44 @@ public class ObisClient {
PreparedStatement statement = session.getOccurrencesPreparedStatement(productKey.getTaxonId(), productKey.getDataSetId(), productKey.getFilters());
ResultSet rs = statement.executeQuery();
String credits = generateCredits();
//String citation = generateCitation();
DataSet dataset = getDataSetById(session, productKey.getDataSetId());
boolean continueWrite = true;
while(rs.next() && continueWrite) {
OccurrencePoint occurrence = generateOccurrencePoint(rs, credits);
while(rs.next() && continueWrite) {
OccurrencePoint occurrence = generateOccurrencePoint(rs, credits, dataset);
continueWrite = writer.write(occurrence);
};
rs.close();
}
private static DataSet getDataSetById(PluginSession session, int datasetId){
logger.debug("retrieving dataset for id "+datasetId);
DataSet dataset = null;
ResultSet datasetRs = null;
try{
PreparedStatement datasetStatement = session.getDatasetPerDatasetIdPreparedStatement(datasetId);
datasetRs = datasetStatement.executeQuery();
if (!datasetRs.next()) return null;
dataset = retrieveDatasetInformation(datasetRs);
}catch(Exception e){
logger.warn("cannot retrieve dataset for occurrence point with id "+datasetId,e);
}finally{
try{
if (datasetRs!=null)
datasetRs.close();
}catch(Exception e) {}
}
logger.debug("dataset returned "+dataset);
return dataset;
}
protected static OccurrencePoint generateOccurrencePoint(ResultSet rs, String credits) throws SQLException
protected static OccurrencePoint generateOccurrencePoint(ResultSet rs, String credits, DataSet dataSet) throws SQLException
{
int id = rs.getInt("id");
OccurrencePoint occurrence = new OccurrencePoint(String.valueOf(id));
@ -106,12 +129,12 @@ public class ObisClient {
} else {
//dxs.yearcollected, dxs.monthcollected, dxs.daycollected
try {
Calendar dateCollected = Calendar.getInstance();
int year = Integer.parseInt(rs.getString("yearcollected"));
int month = Integer.parseInt(rs.getString("monthcollected"));
int date = Integer.parseInt(rs.getString("daycollected"));
dateCollected.set(year, month, date);
occurrence.setEventDate(dateCollected);
Calendar dateCollected = Calendar.getInstance();
int year = Integer.parseInt(rs.getString("yearcollected"));
int month = Integer.parseInt(rs.getString("monthcollected"));
int date = Integer.parseInt(rs.getString("daycollected"));
dateCollected.set(year, month, date);
occurrence.setEventDate(dateCollected);
} catch(NumberFormatException nfe){}
}
@ -152,14 +175,14 @@ public class ObisClient {
/*
* Kingdom: Animalia
Phylum: Chordata
Class: Chondrichthyes
Subclass: Elasmobranchii
Order: Lamniformes
Family: Lamnidae
Genus: Carcharodon
A. Smith, 1838
Species: C. carcharias
Phylum: Chordata
Class: Chondrichthyes
Subclass: Elasmobranchii
Order: Lamniformes
Family: Lamnidae
Genus: Carcharodon
A. Smith, 1838
Species: C. carcharias
*/
String concatenated = rs.getString("concatenated");
@ -171,7 +194,7 @@ Species: C. carcharias
if (scientificNameStartIndex>0) {
String author = (authorStartIndex+1<concatenated.length())?concatenated.substring(authorStartIndex+1):"";
String sn = (scientificNameStartIndex+1<snPart.length())?snPart.substring(scientificNameStartIndex+1):"";
occurrence.setScientificName(sn+" ("+author+")");
occurrence.setScientificName(sn/*+" ("+author+")"*/);
}
}
/*String[] taxon = concatenated.split("|");
@ -182,12 +205,17 @@ Species: C. carcharias
occurrence.setScientificName(scientific);
}*/
}
occurrence.setScientificNameAuthorship(rs.getString("snAuthor"));
occurrence.setScientificNameAuthorship(rs.getString("identifiedBy"));
occurrence.setIdentifiedBy(rs.getString("identifiedBy"));
occurrence.setDataSet(dataSet);
//retrieving dataset
return occurrence;
}
public static OccurrencePoint getOccurrenceById(PluginSession session, String id) throws Exception
{
PreparedStatement statement = session.getOccurrenceByIdPreparedStatement(Integer.parseInt(id));
@ -195,9 +223,12 @@ Species: C. carcharias
OccurrencePoint occurrence = null;
String credits = generateCredits();
if(rs.next()) occurrence = generateOccurrencePoint(rs, credits);
if(rs.next()){
int objId = rs.getInt("valid_id");
occurrence = generateOccurrencePoint(rs, credits, getDataSetById(session, objId));
}
rs.close();
return occurrence;
}
@ -237,10 +268,13 @@ Species: C. carcharias
//OCCURRENCES
Product occurences = new Product(ProductType.Occurrence, key);
int occurencesCount = getOccurrencesCount(session, speciesId, datasetId, filters);
occurences.setCount(occurencesCount);
products.add(occurences);
logger.trace("product is "+occurences);
item.setProducts(products);
}
@ -266,22 +300,29 @@ Species: C. carcharias
//System.out.println("id "+id);
ResultItem baseItem = new ResultItem(String.valueOf(id), "");
fillTaxon(session, id, baseItem, credits, citation);
fillCommonNames(session, id, baseItem);
PreparedStatement datasetStatement = session.getDatasetPreparedStatement(id);
ResultSet dataSetrs = datasetStatement.executeQuery();
while(dataSetrs.next()) {
ResultItem item = Util.cloneResultItem(baseItem);
int dataSetId = fillDatasetInformation(dataSetrs, item);
DataSet dataset = retrieveDatasetInformation(dataSetrs);
item.setDataSet(dataset);
int dataSetId = Integer.parseInt(dataset.getId());
ProductKey key = new ProductKey(id, dataSetId, filters);
logger.trace("datasetid is "+dataSetId+" and product key created is "+key);
fillProducts(session, id, dataSetId, key.serialize(), filters, item);
continueWrite = writer.write(item);
@ -299,7 +340,7 @@ Species: C. carcharias
*/
protected static void fillTaxon(PluginSession session, int id, Taxon taxon, String credits, String citation) throws SQLException
{
PreparedStatement statement = session.getTaxonPreparedStatement(id);
ResultSet rs = statement.executeQuery();
@ -314,7 +355,7 @@ Species: C. carcharias
String rank = rs.getString("rank_name");
taxon.setRank((rank!=null)?rank:"");
int parentId = rs.getInt("parent_id");
rs.close();
@ -339,28 +380,28 @@ Species: C. carcharias
//taxon informations
item.setScientificName(rs.getString("tname"));
String author = Util.stripNotValidXMLCharacters(rs.getString("tauthor"));
item.setScientificNameAuthorship(author);
//properties
item.addProperty(new ElementProperty("worms_id", rs.getString("worms_id")));
item.addProperty(new ElementProperty("col_id", rs.getString("col_id")));
item.addProperty(new ElementProperty("irmng_id", rs.getString("irmng_id")));
item.addProperty(new ElementProperty("itis_id", rs.getString("itis_id")));
item.setCredits(credits);
item.setCitation(citation);
String rank = rs.getString("rank_name");
item.setRank((rank!=null)?rank:"");
item.setStatus(new TaxonomyStatus("", Status.ACCEPTED));
boolean parentNull = rs.getObject("parent_id")==null;
int parentId = rs.getInt("parent_id");
rs.close();
//fill common names
fillCommonNames(session, id, item);
@ -377,7 +418,7 @@ Species: C. carcharias
}
} else throw new IdNotValidException("Taxon with id "+id+" not found");
}
protected static void fillCommonNames(PluginSession session, int taxonNameId, TaxonomyItem item) throws SQLException
{
PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId);
@ -387,9 +428,9 @@ Species: C. carcharias
while(rs.next()) commonNames.add(new CommonName(rs.getString("lanname"), rs.getString("cname")));
rs.close();
item.setCommonNames(commonNames);
}
protected static void fillCommonNames(PluginSession session, int taxonNameId, ResultItem item) throws SQLException
{
PreparedStatement statement = session.getTaxonCommonNamePreparedStatement(taxonNameId);
@ -400,8 +441,7 @@ Species: C. carcharias
item.setCommonNames(commonNames);
}
protected static int fillDatasetInformation(ResultSet rs, ResultItem item) throws SQLException
{
private static DataSet retrieveDatasetInformation(ResultSet rs) throws SQLException{
int dataSetId = rs.getInt("datasetId");
DataSet dataSet = new DataSet(String.valueOf(dataSetId));
dataSet.setCitation(rs.getString("datasetCitation"));
@ -410,10 +450,7 @@ Species: C. carcharias
DataProvider dataProvider = new DataProvider(String.valueOf(rs.getInt("providerId")));
dataProvider.setName(rs.getString("providerName"));
dataSet.setDataProvider(dataProvider);
item.setDataSet(dataSet);
return dataSetId;
return dataSet;
}
public static Set<String> getCommonNames(PluginSession session, String scientificName) throws SQLException
@ -451,7 +488,7 @@ Species: C. carcharias
ResultSet rs = statement.executeQuery();
generateTaxonomyItems(session, rs, writer);
}
protected static void generateTaxonomyItems(PluginSession session, ResultSet rs, Writer<TaxonomyItem> writer) throws SQLException, Exception
{
boolean continueWrite = true;
@ -460,7 +497,7 @@ Species: C. carcharias
while(rs.next() && continueWrite) {
Integer id = rs.getInt("id");
TaxonomyItem taxon = taxonomyItemCache.get(id);
if (taxon == null) {
taxon = new TaxonomyItem(String.valueOf(id));
@ -472,7 +509,7 @@ Species: C. carcharias
//fillTaxonomyItem(session, id, taxon);
continueWrite = writer.write(taxon);
}
rs.close();
}
@ -486,18 +523,18 @@ Species: C. carcharias
while (rs.next()) {
//taxon informations
int taxonId = rs.getInt("id");
//FIXME tmp workaround
if (taxonId == id) continue;
TaxonomyItem child = new TaxonomyItem(String.valueOf(taxonId));
child.setScientificName(rs.getString("tname"));
child.setCitation(rs.getString("tauthor"));
String rank = rs.getString("rank_name");
child.setRank((rank!=null)?rank:"");
child.setStatus(new TaxonomyStatus("",Status.ACCEPTED));
fillCommonNames(session, id, child);
children.add(child);
@ -505,7 +542,7 @@ Species: C. carcharias
rs.close();
return children;
}
protected static TaxonomyItem getTaxonById(PluginSession session, int id) throws Exception
{
TaxonomyItem item = new TaxonomyItem(String.valueOf(id));
@ -514,7 +551,7 @@ Species: C. carcharias
fillTaxonomyItem(session, id, item, credits, citation);
return item;
}
protected static String generateCitation()
{
StringBuilder citation = new StringBuilder("Intergovernmental Oceanographic Commission (IOC) of UNESCO. The Ocean Biogeographic Information System. Web. http://www.iobis.org. (Consulted on ");
@ -522,7 +559,7 @@ Species: C. carcharias
citation.append(")");
return citation.toString();
}
protected static String generateCredits()
{
//credits ="This information object has been generated via the Species Product Discovery service on 2012-11-26 by interfacing with the Interim Register of Marine and Nonmarine Genera (IRMNG) (http://www.obis.org.au/irmng/)";
@ -548,32 +585,32 @@ Species: C. carcharias
final Connection connection = DriverManager.getConnection(url, props);
// System.out.println("Connected");
// System.out.println("Connected");
final PluginSession session = new PluginSession(connection);
session.preCacheStatements();
searchByScientificName(session, "gadus morhua", new SearchFilters(), new Writer<ResultItem>() {
@Override
public boolean write(ResultItem item) {
// System.out.println(item.getId()+" "+item.getScientificNameAuthorship()+" "+item.getScientificName());
// System.out.println(item.getId()+" "+item.getScientificNameAuthorship()+" "+item.getScientificName());
return true;
}
});
/*getTaxonByScientificNames(session, "sarda sarda", new Writer<TaxonomyItem>() {
@Override
public boolean write(TaxonomyItem item) {
System.out.println(item.getId()+" "+item.getAuthor()+" "+item.getScientificName());
return true;
}
});*/
/*OccurrencePoint occurrencePoint = getOccurrenceById(session, "38069270");
System.out.println(occurrencePoint);*/
@ -583,9 +620,9 @@ Species: C. carcharias
List<Taxon> children = getChildrenTaxon(session, Integer.parseInt(taxon.getId()));
for (Taxon child:children) System.out.println(child);*/
/*searchByCommonName(session, "white shark", new SearchFilters(), new Writer<ResultItem>() {
@Override
public void write(ResultItem item) {
System.out.println("Item: "+item.getDataSet().getDataProvider().getName()+" <-> "+item.getDataSet().getName());
@ -599,12 +636,12 @@ Species: C. carcharias
@Override
public void write(TaxonomyItem item) {
System.out.println(item);
}
});*/
/*SearchFilters filters = new SearchFilters();
ObisClient.searchByScientificName(session, "sarda sarda", filters, new Writer<ResultItem>() {
@ -613,20 +650,20 @@ Species: C. carcharias
System.out.println(item);
}
});*/
/*List<TaxonomyItem> taxa = getChildrenTaxon(session,769809);
for (TaxonomyItem taxon:taxa) System.out.println(taxon.getId());*/
//navigate("", session, 741923);
//System.out.println("result in "+(System.currentTimeMillis()-start)+" avg: "+(sum/counter)+" tot: "+counter);
// System.out.println("done");
// System.out.println("done");
session.expire();
}
protected static Set<Integer> found = new HashSet<Integer>();
protected static Map<String,TaxonomyItem> foundTaxon = new HashMap<String,TaxonomyItem>();
protected static void navigate(String indentation, PluginSession session, int id) throws SQLException
{
//System.out.println("looking for children: "+id);
@ -635,17 +672,17 @@ Species: C. carcharias
System.err.println(foundTaxon.get(id));
System.exit(-1);
}
List<TaxonomyItem> taxa = getChildrenTaxon(session,id);
found.add(id);
for (TaxonomyItem taxon:taxa) {
System.out.println(indentation+taxon.getId()+" "+taxon.getRank());
foundTaxon.put(taxon.getId(), taxon);
navigate(indentation+" ", session, Integer.valueOf(taxon.getId()));
}
}

View File

@ -37,6 +37,7 @@ public class PluginSession {
protected PreparedStatement commonNamePreparedStatement;
protected PreparedStatement childrenTaxonPreparedStatement;
protected PreparedStatement occurrenceByIdPreparedStatement;
protected PreparedStatement datasetPerIdPreparedStatement;
/**
* @param connection
@ -100,6 +101,7 @@ public class PluginSession {
createCommonNamePreparedStatement();
createChildrenTaxonPreparedStatement();
createOccurrenceByIdPreparedStatement();
createDatasetPerDatasetIdPreparedStatement();
}
@ -140,19 +142,15 @@ public class PluginSession {
datasetPreparedStatement.setInt(1, id);
return datasetPreparedStatement;
}
protected void createDatasetPreparedStatement() throws SQLException
public PreparedStatement getDatasetPerDatasetIdPreparedStatement(int dataSetId) throws SQLException
{
/*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/
String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"r.id in (SELECT resource_id from portal.species_per_resource where valid_id = ?) AND r.provider_id = p.id";
datasetPreparedStatement = connection.prepareStatement(query);
if (datasetPerIdPreparedStatement==null) createDatasetPerDatasetIdPreparedStatement();
datasetPerIdPreparedStatement.clearParameters();
datasetPerIdPreparedStatement.setInt(1, dataSetId);
return datasetPerIdPreparedStatement;
}
public PreparedStatement getCommonNameFromScientificNamePreparedStatement(String scientificaName) throws SQLException
{
if (commonNameFromScientificNamePreparedStatement==null) createCommonNameFromScientificNamePreparedStatement();
@ -314,7 +312,7 @@ public class PluginSession {
protected PreparedStatement getOccurrencesPreparedStatement(int taxonId, int datasetId, SearchFilters filters) throws SQLException
{
//We don't cache it because in this case a PS is less performant
StringBuilder query = new StringBuilder("SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.identifiedBy, dxs.yearcollected, dxs.monthcollected, dxs.daycollected FROM obis.drs drs, obis.dxs dxs WHERE drs.valid_id = ? AND drs.resource_id = ? AND drs.id = dxs.dr_id");
StringBuilder query = new StringBuilder("SELECT drs.id, drs.valid_id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.identifiedBy, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames tn WHERE drs.valid_id = ? AND drs.resource_id = ? AND drs.id = dxs.dr_id AND tn.id = drs.valid_id ");
buildConditions(query, "drs.", filters.getConditions());
@ -400,9 +398,32 @@ public class PluginSession {
protected void createOccurrenceByIdPreparedStatement() throws SQLException
{
String query = "SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.yearcollected, dxs.monthcollected, dxs.daycollected FROM obis.drs drs, obis.dxs dxs WHERE drs.id = ? AND drs.id = dxs.dr_id";
String query = "SELECT drs.id, drs.latitude, drs.longitude, drs.datecollected, drs.basisofrecord, dxs.citation, dxs.institutioncode, dxs.collectioncode, dxs.catalognumber, dxs.collector, dxs.datelastmodified, dxs.country, dxs.locality, dxs.minimumdepth, dxs.maximumdepth, dxs.coordinateprecision, dxs.concatenated, dxs.yearcollected, dxs.monthcollected, dxs.daycollected, tn.tauthor as snAuthor FROM obis.drs drs, obis.dxs dxs, obis.tnames as tn WHERE drs.id = ? AND drs.id = dxs.dr_id AND drs.valid_id = tn.id";
occurrenceByIdPreparedStatement = connection.prepareStatement(query);
}
protected void createDatasetPreparedStatement() throws SQLException
{
/*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/
String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"r.id in (SELECT resource_id from portal.species_per_resource where valid_id = ?) AND r.provider_id = p.id";
datasetPreparedStatement = connection.prepareStatement(query);
}
protected void createDatasetPerDatasetIdPreparedStatement() throws SQLException
{
/*String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"exists (SELECT r.id FROM obis.drs d WHERE d.valid_id = ? AND d.resource_id = r.id) AND r.provider_id = p.id";*/
String query = "SELECT r.id as datasetId, r.resname as datasetName, r.citation as datasetCitation, p.id as providerId, p.providername as providerName " +
"FROM obis.resources r, obis.providers p WHERE " +
"r.id = ? AND r.provider_id = p.id";
datasetPerIdPreparedStatement = connection.prepareStatement(query);
}
}

View File

@ -28,7 +28,9 @@ public class ProductKey {
this.dataSetId = dataSetId;
this.filters = filters;
}
protected ProductKey() {}
/**
* @return the taxonId
*/

View File

@ -17,6 +17,8 @@ public class SearchFilters {
protected List<Condition> conditions;
protected SearchFilters(){}
public SearchFilters(Condition ... conditions){
this.conditions = new ArrayList<Condition>(Arrays.asList(conditions));
}