gCubeSystem
/
ecological-engine
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@61658 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2012-11-30 09:41:06 +00:00
parent 0b9aac8a87
commit e732849b50
6 changed files with 219 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java
View File

@ -12,7 +12,7 @@ public class AnalysisLogger {
public static Logger getLogger(){
if (logger == null){
setLogger("./ALog.properties");
// setLogger("./ALog.properties");
logger = Logger.getLogger("AnalysisLogger");
}

7
src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/ActorNode.java
View File

@ -11,7 +11,7 @@ public abstract class ActorNode implements GenericAlgorithm{
public abstract float getInternalStatus();
// execute a single node
public abstract int executeNode(int cellStarIndex, int numberOfRightElementsToProcess, int leftStartIndex, int numberOfLeftElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce);
public abstract int executeNode(int rightStartIndex, int numberOfRightElementsToProcess, int leftStartIndex, int numberOfLeftElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce);
// An initialization phase in which the inputs are initialized
public abstract void setup(AlgorithmConfiguration config) throws Exception;
@ -22,9 +22,6 @@ public abstract class ActorNode implements GenericAlgorithm{
// get overall number of geographical information to process
public abstract int getNumberOfLeftElements();
// get overall number of processed species
public abstract int getNumberOfProcessedElements();
// stop the sexecution of the node
public abstract void stop();
@ -65,7 +62,7 @@ public abstract class ActorNode implements GenericAlgorithm{
System.out.println("Generic Node: executing class");
SpatialProbabilityDistributionNode node = (SpatialProbabilityDistributionNode) Class.forName(algorithmClass).newInstance();
ActorNode node = (ActorNode) Class.forName(algorithmClass).newInstance();
node.executeNode(order, chunksize, speciesOrder, speciesChunksize, duplicate, path, nodeConfiguration, logfile);
} catch (Exception e) {

2
src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionNode.java
View File

@ -28,7 +28,7 @@ public abstract class SpatialProbabilityDistributionNode implements GenericAlgor
//get overall number of processed species
public abstract int getNumberOfProcessedSpecies();
//stop the sexecution of the node
//stop the execution of the node
public abstract void stop();
//prostprocess after the whole calculation : reduce operation

2
src/main/java/org/gcube/dataanalysis/ecoengine/models/testing/FeedForwardNNFile.java
View File

@ -43,7 +43,7 @@ public class FeedForwardNNFile extends ModelAquamapsNN{
public StatisticalType getOutput() {
HashMap<String,StatisticalType> map = new HashMap<String, StatisticalType>();
PrimitiveType p = new PrimitiveType(File.class.getName(), new File(fileName), PrimitiveTypes.FILE, "NeuralNetwork","Trained Neural Network");
PrimitiveType score = new PrimitiveType(String.class.getName(), nn.en, PrimitiveTypes.STRING, "LearningScore","Learning Score");
PrimitiveType score = new PrimitiveType(String.class.getName(), ""+nn.en, PrimitiveTypes.STRING, "LearningScore","Learning Score");
List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.GENERIC);
OutputTable outTable = new OutputTable(template, trainingDataSet, trainingDataSet, "Output table");

291
src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java
View File

@ -42,8 +42,8 @@ public class OccurrencePointsMerger implements Transducerer {
static String finalTableNameF = "finalTableName";
static String spatialTolerance = "spatialTolerance";
static String confidence = "confidence";
static String sqlDateFormat="MM/DD/YYYY HH24:MI:SS";
static String javaDateFormat="MM/dd/yyyy HH:mm:ss";
static String sqlDateFormat = "MM/DD/YYYY HH24:MI:SS";
static String javaDateFormat = "MM/dd/yyyy HH:mm:ss";
protected List<OccurrenceRecord> records_left;
protected List<OccurrenceRecord> records_right;
@ -68,11 +68,11 @@ public class OccurrencePointsMerger implements Transducerer {
protected SessionFactory dbconnection;
protected float status;
protected boolean firstbest;
public OccurrencePointsMerger(){
firstbest=true;
public OccurrencePointsMerger() {
firstbest = true;
}
protected class OccurrenceRecord {
public String scientificName;
@ -167,7 +167,7 @@ public class OccurrencePointsMerger implements Transducerer {
} else if (name$.equalsIgnoreCase(eventDatFld)) {
if (record.eventdate != null) {
String dat = convert2conventionalFormat(record.eventdate);
if ((dat!=null)&&(dat.length()>0))
if ((dat != null) && (dat.length() > 0))
value$ = "'" + convert2conventionalFormat(record.eventdate) + "'";
else
value$ = "NULL";
@ -175,10 +175,10 @@ public class OccurrencePointsMerger implements Transducerer {
}
} else if (name$.equalsIgnoreCase(modifDatFld)) {
if (record.modifdate != null) {
String dat = convert2conventionalFormat(record.modifdate) ;
if ((dat!=null)&&(dat.length()>0))
String dat = convert2conventionalFormat(record.modifdate);
if ((dat != null) && (dat.length() > 0))
value$ = "'" + convert2conventionalFormat(record.modifdate) + "'";
else
else
value$ = "NULL";
// value$="'"+record.modifdate.getTimeInMillis()+"'";
}
@ -190,7 +190,7 @@ public class OccurrencePointsMerger implements Transducerer {
}
if (value$.equals("'null'"))
value$ = "NULL";
buffer.append(value$);
if (index < nNames - 1) {
buffer.append(",");
@ -202,14 +202,13 @@ public class OccurrencePointsMerger implements Transducerer {
return buffer.toString();
}
@Override
public List<StatisticalType> getInputParameters() {
List<TableTemplates> templatesOccurrence = new ArrayList<TableTemplates>();
templatesOccurrence.add(TableTemplates.OCCURRENCE_SPECIES);
// occurrence points tables
PrimitiveType p0 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableNameL,"the name of the produced table", "Occ_");
PrimitiveType p0 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableNameL, "the name of the produced table", "Occ_");
InputTable p1 = new InputTable(templatesOccurrence, leftTableNameF, "the First table containing the occurrence points", "");
InputTable p2 = new InputTable(templatesOccurrence, rightTableNameF, "the Second table containing the occurrence points", "");
@ -237,7 +236,7 @@ public class OccurrencePointsMerger implements Transducerer {
inputs.add(p9);
inputs.add(p10);
inputs.add(p11);
DatabaseType.addDefaultDBPars(inputs);
return inputs;
}
@ -282,7 +281,7 @@ public class OccurrencePointsMerger implements Transducerer {
@Override
public void init() throws Exception {
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
// AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
lonFld = config.getParam(longitudeColumn);
latFld = config.getParam(latitudeColumn);
recordedByFld = config.getParam(recordedByColumn);
@ -306,11 +305,6 @@ public class OccurrencePointsMerger implements Transducerer {
this.config = config;
}
@Override
public void shutdown() {
}
@Override
public String getDescription() {
return "An algorithm for merging two sets of occurrence points of species coming from the Species Discovery Facility of D4Science";
@ -351,14 +345,9 @@ public class OccurrencePointsMerger implements Transducerer {
// insert the most recent:
// if it is the left then leave it as is
// otherwise put the left in the deletion list and the right in the insertion list
if (
((leftOcc.modifdate!=null)&&(rightOcc.modifdate!=null)&&leftOcc.modifdate.before(rightOcc.modifdate))
||
(leftOcc.modifdate==null)&&(rightOcc.modifdate!=null)
)
{
if (((leftOcc.modifdate != null) && (rightOcc.modifdate != null) && leftOcc.modifdate.before(rightOcc.modifdate)) || (leftOcc.modifdate == null) && (rightOcc.modifdate != null)) {
objectstodelete.add(leftOcc);
objectstoinsert.add(rightOcc);
}
@ -369,7 +358,7 @@ public class OccurrencePointsMerger implements Transducerer {
objectstoinsert.add(rightOcc);
}
protected void persist() throws Exception{
protected void persist() throws Exception {
// DELETE ELEMENTS IN THE DELETION LIST
int todel = objectstodelete.size();
@ -378,26 +367,26 @@ public class OccurrencePointsMerger implements Transducerer {
AnalysisLogger.getLogger().debug("Deleting " + todel + " objects");
if (todel > 0) {
for (OccurrenceRecord record : objectstodelete) {
String rec = recordedByFld + "='" + record.recordedby.replace("'","")+"'";
String sci = scientificNameFld + "='" + record.scientificName.replace("'","")+"'";
String x = lonFld + "='" + record.x$+"'";
String y = latFld + "='" + record.y$+"'";
String rec = recordedByFld + "='" + record.recordedby.replace("'", "") + "'";
String sci = scientificNameFld + "='" + record.scientificName.replace("'", "") + "'";
String x = lonFld + "='" + record.x$ + "'";
String y = latFld + "='" + record.y$ + "'";
String event = null;
String modified = null;
if (record.eventdate!=null)
//to_timestamp('09/30/56 11:00:00 PM', 'MM/DD/YY HH12:MI:SS a')
event = eventDatFld + "=to_timestamp('" + convert2conventionalFormat(record.eventdate)+"','"+sqlDateFormat+"')";
if (record.modifdate!=null)
modified = modifDatFld + "=to_timestamp('" + convert2conventionalFormat(record.modifdate)+"','"+sqlDateFormat+"')";
if (record.eventdate != null)
// to_timestamp('09/30/56 11:00:00 PM', 'MM/DD/YY HH12:MI:SS a')
event = eventDatFld + "=to_timestamp('" + convert2conventionalFormat(record.eventdate) + "','" + sqlDateFormat + "')";
if (record.modifdate != null)
modified = modifDatFld + "=to_timestamp('" + convert2conventionalFormat(record.modifdate) + "','" + sqlDateFormat + "')";
buffer.append("(");
buffer.append(rec + " AND " + sci + " AND " + x + " AND " + y);
if (event!=null)
buffer.append(" AND "+event);
if (modified!=null)
buffer.append(" AND "+modified);
if (event != null)
buffer.append(" AND " + event);
if (modified != null)
buffer.append(" AND " + modified);
buffer.append(")");
if (counter < todel - 1)
buffer.append(" OR ");
@ -406,7 +395,7 @@ public class OccurrencePointsMerger implements Transducerer {
}
String updateQ = DatabaseUtils.deleteFromBuffer(finalTableName, buffer);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
DatabaseFactory.executeSQLUpdate(updateQ, dbconnection);
AnalysisLogger.getLogger().debug("Objects deleted");
}
@ -427,34 +416,165 @@ public class OccurrencePointsMerger implements Transducerer {
}
String updateQ = DatabaseUtils.insertFromBuffer(finalTableName, columns.toString(), buffer);
// System.out.println("Update:\n"+updateQ);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
// System.out.println("Update:\n"+updateQ);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
DatabaseFactory.executeSQLUpdate(updateQ, dbconnection);
AnalysisLogger.getLogger().debug("Objects inserted");
}
objectstoinsert = null;
objectstodelete= null;
objectstodelete = null;
objectstoinsert = new ArrayList<OccurrencePointsMerger.OccurrenceRecord>();
objectstodelete = new ArrayList<OccurrencePointsMerger.OccurrenceRecord>();
System.gc();
}
protected void prepareFinalTable() throws Exception{
protected void prepareFinalTable() throws Exception {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.duplicateTableStatement(leftTableName, finalTableName), dbconnection);
}
protected void extractColumnNames() throws Exception{
// take the description of the table
columnsNames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(rightTableName), dbconnection);
int nCols = columnsNames.size();
columns = new StringBuffer();
for (int i = 0; i < nCols; i++) {
columns.append("\"" + columnsNames.get(i) + "\"");
if (i < nCols - 1)
columns.append(",");
}
protected void extractColumnNames() throws Exception {
// take the description of the table
columnsNames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(rightTableName), dbconnection);
int nCols = columnsNames.size();
columns = new StringBuffer();
for (int i = 0; i < nCols; i++) {
columns.append("\"" + columnsNames.get(i) + "\"");
if (i < nCols - 1)
columns.append(",");
}
}
public void initDB(boolean buildTable) throws Exception {
// init DB connection
AnalysisLogger.getLogger().trace("Initializing DB Connection");
dbconnection = DatabaseUtils.initDBSession(config);
if (buildTable) {
AnalysisLogger.getLogger().trace("Taking Table Description");
extractColumnNames();
AnalysisLogger.getLogger().trace("Taken Table Description: " + columns);
AnalysisLogger.getLogger().trace("Creating final table: " + finalTableName);
// create new merged table
try {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
} catch (Exception e1) {
}
prepareFinalTable();
}
}
@Override
public void shutdown() {
if (dbconnection != null)
try {
dbconnection.close();
} catch (Exception e) {
}
}
public List<Object> leftRows;
public List<Object> rightRows;
public int getNumLeftObjects(){
if (leftRows!=null)
return leftRows.size();
else return 0;
}
public int getNumRightObjects(){
if (rightRows!=null)
return rightRows.size();
else return 0;
}
public void takeFullRanges() {
// take the elements from sx table
AnalysisLogger.getLogger().trace("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), ""), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().trace("Taking elements from right table: " + rightTableName);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), ""), dbconnection);
}
public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) {
// take the elements from sx table
AnalysisLogger.getLogger().trace("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), "offset " + offsetLeft + " limit " + numLeft), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().trace("Taking elements from right table: " + rightTableName);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), "offset " + offsetRight + " limit " + numRight), dbconnection);
}
public void computeRange() throws Exception {
try {
AnalysisLogger.getLogger().trace("Processing " + leftTableName + " vs " + rightTableName);
status = 10;
int rightCounter = 0;
int similaritiesCounter = 0;
int allrightrows = rightRows.size();
for (Object rRow : rightRows) {
// transform into an occurrence object
OccurrenceRecord rightOcc = row2OccurrenceRecord((Object[]) rRow);
// for each element in sx
int k = 0;
boolean found = false;
float p = 0;
OccurrenceRecord bestleftOcc = null;
for (Object lRow : leftRows) {
OccurrenceRecord leftOcc = null;
leftOcc = row2OccurrenceRecord((Object[]) lRow);
p = extProb(leftOcc, rightOcc);
if (p >= confidenceValue) {
bestleftOcc = leftOcc;
found = true;
similaritiesCounter++;
AnalysisLogger.getLogger().trace("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
// break;
if (!firstbest)
manageHighProbability(p, bestleftOcc, rightOcc);
else
break;
} else if (!firstbest)
manageLowProbability(p, bestleftOcc, rightOcc);
k++;
}
rightCounter++;
if (firstbest) {
if (found)
manageHighProbability(p, bestleftOcc, rightOcc);
else
manageLowProbability(p, bestleftOcc, rightOcc);
}
status = Math.min(90, 10f + (80 * ((float) rightCounter) / ((float) allrightrows)));
if (rightCounter % 500 == 0) {
AnalysisLogger.getLogger().trace("Persisting ... " + rightCounter + " over " + allrightrows);
persist();
}
}
AnalysisLogger.getLogger().trace("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements");
status = 90;
// transform the complete list into a table
persist();
// close DB connection
} catch (Exception e) {
throw e;
} finally {
shutdown();
status = 100;
AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed");
}
}
@Override
public void compute() throws Exception {
@ -468,10 +588,10 @@ public class OccurrencePointsMerger implements Transducerer {
AnalysisLogger.getLogger().trace("Taken Table Description: " + columns);
AnalysisLogger.getLogger().trace("Creating final table: " + finalTableName);
// create new merged table
try{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
}catch(Exception e1){
try {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
} catch (Exception e1) {
}
prepareFinalTable();
// take the elements from sx table
@ -500,14 +620,7 @@ public class OccurrencePointsMerger implements Transducerer {
OccurrenceRecord leftOcc = null;
// only for the first iteration on the left occurrences perform the transformation
/*
if (leftrecordsSize <= k) {
// transform into an occurrence object
leftOcc = row2OccurrenceRecord((Object[]) lRow);
leftRecords.add(leftOcc);
leftrecordsSize++;
// System.out.println("ADDED "+k+"-th elements size: "+leftRecords.size());
} else
leftOcc = leftRecords.get(k);
* if (leftrecordsSize <= k) { // transform into an occurrence object leftOcc = row2OccurrenceRecord((Object[]) lRow); leftRecords.add(leftOcc); leftrecordsSize++; // System.out.println("ADDED "+k+"-th elements size: "+leftRecords.size()); } else leftOcc = leftRecords.get(k);
*/
leftOcc = row2OccurrenceRecord((Object[]) lRow);
// evaluate P(dx,sx)
@ -518,29 +631,28 @@ public class OccurrencePointsMerger implements Transducerer {
found = true;
similaritiesCounter++;
AnalysisLogger.getLogger().trace("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
// break;
// break;
if (!firstbest)
manageHighProbability(p, bestleftOcc, rightOcc);
else
else
break;
}
else if (!firstbest)
} else if (!firstbest)
manageLowProbability(p, bestleftOcc, rightOcc);
k++;
}
rightCounter++;
if (firstbest){
if (firstbest) {
if (found)
manageHighProbability(p, bestleftOcc, rightOcc);
else
manageLowProbability(p, bestleftOcc, rightOcc);
}
status = Math.min(90, 10f + (80 * ((float) rightCounter) / ((float) allrightrows)));
if (rightCounter%500==0){
AnalysisLogger.getLogger().trace("Persisting ... "+rightCounter+" over "+allrightrows);
if (rightCounter % 500 == 0) {
AnalysisLogger.getLogger().trace("Persisting ... " + rightCounter + " over " + allrightrows);
persist();
}
}
@ -554,14 +666,15 @@ public class OccurrencePointsMerger implements Transducerer {
throw e;
} finally {
if (dbconnection != null)
try{
dbconnection.close();
}catch(Exception e){}
try {
dbconnection.close();
} catch (Exception e) {
}
status = 100;
AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed");
}
}
public static void main(String[] args) throws Exception {
AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1);

13
src/main/java/org/gcube/dataanalysis/ecoengine/utils/Transformations.java
View File

@ -1,11 +1,12 @@
package org.gcube.dataanalysis.ecoengine.utils;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import com.rapidminer.example.ExampleSet;
import com.rapidminer.example.table.DataRow;
@ -147,6 +148,16 @@ public class Transformations {
*/
}
public static void dumpConfig(String pathToFile, AlgorithmConfiguration config) throws Exception {
Transformations.dumpObjectToFile(pathToFile, config);
}
public static AlgorithmConfiguration restoreConfig(String configFile) throws Exception{
FileInputStream fis = new FileInputStream(new File(configFile));
AlgorithmConfiguration config = (AlgorithmConfiguration) new XStream().fromXML(fis);
fis.close();
return config;
}
public static double indexString(String string) {
// string = Sha1.SHA1(string);