gCubeSystem
/
ecological-engine
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

modifications to run transducers in parallel 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@61733 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2012-11-30 14:57:41 +00:00
parent e732849b50
commit ad3ce33d46
3 changed files with 50 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java
View File

@ -1,5 +1,6 @@
package org.gcube.contentmanagement.lexicalmatcher.utils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
@ -13,7 +14,10 @@ public class AnalysisLogger {
if (logger == null){
// setLogger("./ALog.properties");
logger = Logger.getLogger("AnalysisLogger");
org.apache.log4j.BasicConfigurator.configure();
// logger = Logger.getLogger("ConsoleAppender");
logger = Logger.getRootLogger();
logger.setLevel(Level.INFO);
}
return logger;

2
src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java
View File

@ -233,7 +233,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
@Override
public ALG_PROPS[] getProperties() {
// ALG_PROPS [] p = {ALG_PROPS.SPECIES_VS_CSQUARE_FROM_DATABASE, ALG_PROPS.SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE};
ALG_PROPS [] p = {ALG_PROPS.PARALLEL_SPECIES_VS_CSQUARE_FROM_DATABASE};
ALG_PROPS [] p = {ALG_PROPS.PARALLEL_SPECIES_VS_CSQUARE_FROM_DATABASE,ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON};
return p;
}

89
src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java
View File

@ -31,19 +31,19 @@ import org.hibernate.SessionFactory;
public class OccurrencePointsMerger implements Transducerer {
static protected String finalTableNameL = "final_Table_Name";
static String longitudeColumn = "longitudeColumn";
static String latitudeColumn = "latitudeColumn";
static String recordedByColumn = "recordedByColumn";
static String scientificNameColumn = "scientificNameColumn";
static String eventDateColumn = "eventDateColumn";
static String lastModificationColumn = "lastModificationColumn";
static String rightTableNameF = "rightTableName";
static String leftTableNameF = "leftTableName";
static String finalTableNameF = "finalTableName";
static String spatialTolerance = "spatialTolerance";
static String confidence = "confidence";
static String sqlDateFormat = "MM/DD/YYYY HH24:MI:SS";
static String javaDateFormat = "MM/dd/yyyy HH:mm:ss";
static protected String longitudeColumn = "longitudeColumn";
static protected String latitudeColumn = "latitudeColumn";
static protected String recordedByColumn = "recordedByColumn";
static protected String scientificNameColumn = "scientificNameColumn";
static protected String eventDateColumn = "eventDateColumn";
static protected String lastModificationColumn = "lastModificationColumn";
static protected String rightTableNameF = "rightTableName";
static protected String leftTableNameF = "leftTableName";
static protected String finalTableNameF = "finalTableName";
static protected String spatialTolerance = "spatialTolerance";
static protected String confidence = "confidence";
static protected String sqlDateFormat = "MM/DD/YYYY HH24:MI:SS";
static protected String javaDateFormat = "MM/dd/yyyy HH:mm:ss";
protected List<OccurrenceRecord> records_left;
protected List<OccurrenceRecord> records_right;
@ -129,7 +129,7 @@ public class OccurrencePointsMerger implements Transducerer {
*/
record.eventdate = DateGuesser.convertDate(value$);
if (displaydateconvert) {
AnalysisLogger.getLogger().trace("From " + value$ + "->" + convert2conventionalFormat(record.eventdate) + " pattern " + DateGuesser.getPattern(value$));
AnalysisLogger.getLogger().info("From " + value$ + "->" + convert2conventionalFormat(record.eventdate) + " pattern " + DateGuesser.getPattern(value$));
displaydateconvert = false;
}
@ -364,7 +364,7 @@ public class OccurrencePointsMerger implements Transducerer {
int todel = objectstodelete.size();
int counter = 0;
StringBuffer buffer = new StringBuffer();
AnalysisLogger.getLogger().debug("Deleting " + todel + " objects");
AnalysisLogger.getLogger().info("Deleting " + todel + " objects");
if (todel > 0) {
for (OccurrenceRecord record : objectstodelete) {
String rec = recordedByFld + "='" + record.recordedby.replace("'", "") + "'";
@ -397,12 +397,12 @@ public class OccurrencePointsMerger implements Transducerer {
String updateQ = DatabaseUtils.deleteFromBuffer(finalTableName, buffer);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
DatabaseFactory.executeSQLUpdate(updateQ, dbconnection);
AnalysisLogger.getLogger().debug("Objects deleted");
AnalysisLogger.getLogger().info("Objects deleted");
}
buffer = new StringBuffer();
int toins = objectstoinsert.size();
AnalysisLogger.getLogger().debug("Inserting " + toins + " objects");
AnalysisLogger.getLogger().info("Inserting " + toins + " objects");
counter = 0;
if (toins > 0) {
for (OccurrenceRecord record : objectstoinsert) {
@ -419,7 +419,7 @@ public class OccurrencePointsMerger implements Transducerer {
// System.out.println("Update:\n"+updateQ);
// AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
DatabaseFactory.executeSQLUpdate(updateQ, dbconnection);
AnalysisLogger.getLogger().debug("Objects inserted");
AnalysisLogger.getLogger().info("Objects inserted");
}
objectstoinsert = null;
@ -433,7 +433,7 @@ public class OccurrencePointsMerger implements Transducerer {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.duplicateTableStatement(leftTableName, finalTableName), dbconnection);
}
protected void extractColumnNames() throws Exception {
public void extractColumnNames() throws Exception {
// take the description of the table
columnsNames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(rightTableName), dbconnection);
@ -448,15 +448,14 @@ public class OccurrencePointsMerger implements Transducerer {
public void initDB(boolean buildTable) throws Exception {
// init DB connection
AnalysisLogger.getLogger().trace("Initializing DB Connection");
AnalysisLogger.getLogger().info("Initializing DB Connection");
dbconnection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().info("Taking Table Description");
extractColumnNames();
if (buildTable) {
AnalysisLogger.getLogger().trace("Taking Table Description");
extractColumnNames();
AnalysisLogger.getLogger().trace("Taken Table Description: " + columns);
AnalysisLogger.getLogger().trace("Creating final table: " + finalTableName);
AnalysisLogger.getLogger().info("Taken Table Description: " + columns);
AnalysisLogger.getLogger().info("Creating final table: " + finalTableName);
// create new merged table
try {
@ -494,25 +493,25 @@ public class OccurrencePointsMerger implements Transducerer {
public void takeFullRanges() {
// take the elements from sx table
AnalysisLogger.getLogger().trace("Taking elements from left table: " + leftTableName);
AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), ""), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().trace("Taking elements from right table: " + rightTableName);
AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), ""), dbconnection);
}
public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) {
// take the elements from sx table
AnalysisLogger.getLogger().trace("Taking elements from left table: " + leftTableName);
AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), "offset " + offsetLeft + " limit " + numLeft), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().trace("Taking elements from right table: " + rightTableName);
AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), "offset " + offsetRight + " limit " + numRight), dbconnection);
}
public void computeRange() throws Exception {
try {
AnalysisLogger.getLogger().trace("Processing " + leftTableName + " vs " + rightTableName);
AnalysisLogger.getLogger().info("Processing " + leftTableName + " vs " + rightTableName);
status = 10;
int rightCounter = 0;
int similaritiesCounter = 0;
@ -534,7 +533,7 @@ public class OccurrencePointsMerger implements Transducerer {
bestleftOcc = leftOcc;
found = true;
similaritiesCounter++;
AnalysisLogger.getLogger().trace("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
AnalysisLogger.getLogger().info("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
// break;
if (!firstbest)
manageHighProbability(p, bestleftOcc, rightOcc);
@ -556,12 +555,12 @@ public class OccurrencePointsMerger implements Transducerer {
status = Math.min(90, 10f + (80 * ((float) rightCounter) / ((float) allrightrows)));
if (rightCounter % 500 == 0) {
AnalysisLogger.getLogger().trace("Persisting ... " + rightCounter + " over " + allrightrows);
AnalysisLogger.getLogger().info("Persisting ... " + rightCounter + " over " + allrightrows);
persist();
}
}
AnalysisLogger.getLogger().trace("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements");
AnalysisLogger.getLogger().info("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements");
status = 90;
// transform the complete list into a table
persist();
@ -571,7 +570,7 @@ public class OccurrencePointsMerger implements Transducerer {
} finally {
shutdown();
status = 100;
AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed");
AnalysisLogger.getLogger().info("Occ Points Processing Finished and db closed");
}
}
@ -580,13 +579,13 @@ public class OccurrencePointsMerger implements Transducerer {
try {
// init DB connection
AnalysisLogger.getLogger().trace("Initializing DB Connection");
AnalysisLogger.getLogger().info("Initializing DB Connection");
dbconnection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().trace("Taking Table Description");
AnalysisLogger.getLogger().info("Taking Table Description");
extractColumnNames();
AnalysisLogger.getLogger().trace("Taken Table Description: " + columns);
AnalysisLogger.getLogger().trace("Creating final table: " + finalTableName);
AnalysisLogger.getLogger().info("Taken Table Description: " + columns);
AnalysisLogger.getLogger().info("Creating final table: " + finalTableName);
// create new merged table
try {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
@ -595,14 +594,14 @@ public class OccurrencePointsMerger implements Transducerer {
}
prepareFinalTable();
// take the elements from sx table
AnalysisLogger.getLogger().trace("Taking elements from left table: " + leftTableName);
AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName);
List<Object> leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), ""), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().trace("Taking elements from right table: " + rightTableName);
AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName);
List<Object> rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), ""), dbconnection);
// for each element in dx
List<OccurrenceRecord> leftRecords = new ArrayList<OccurrencePointsMerger.OccurrenceRecord>();
AnalysisLogger.getLogger().trace("Processing " + leftTableName + " vs " + rightTableName);
AnalysisLogger.getLogger().info("Processing " + leftTableName + " vs " + rightTableName);
status = 10;
int rightCounter = 0;
int similaritiesCounter = 0;
@ -630,7 +629,7 @@ public class OccurrencePointsMerger implements Transducerer {
bestleftOcc = leftOcc;
found = true;
similaritiesCounter++;
AnalysisLogger.getLogger().trace("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
AnalysisLogger.getLogger().info("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")");
// break;
if (!firstbest)
manageHighProbability(p, bestleftOcc, rightOcc);
@ -652,12 +651,12 @@ public class OccurrencePointsMerger implements Transducerer {
status = Math.min(90, 10f + (80 * ((float) rightCounter) / ((float) allrightrows)));
if (rightCounter % 500 == 0) {
AnalysisLogger.getLogger().trace("Persisting ... " + rightCounter + " over " + allrightrows);
AnalysisLogger.getLogger().info("Persisting ... " + rightCounter + " over " + allrightrows);
persist();
}
}
AnalysisLogger.getLogger().trace("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements");
AnalysisLogger.getLogger().info("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements");
status = 90;
// transform the complete list into a table
persist();
@ -671,7 +670,7 @@ public class OccurrencePointsMerger implements Transducerer {
} catch (Exception e) {
}
status = 100;
AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed");
AnalysisLogger.getLogger().info("Occ Points Processing Finished and db closed");
}
}