gCubeSystem
/
ecological-engine
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@85692 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2013-11-19 14:16:48 +00:00
parent 1708eaca43
commit ef043ffcc4
5 changed files with 156 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/StandardLocalExternalAlgorithm.java
View File

@ -63,7 +63,7 @@ public abstract class StandardLocalExternalAlgorithm implements Transducerer {
}
protected void log(Object message){
AnalysisLogger.getLogger().trace(message);
AnalysisLogger.getLogger().debug(message);
}
@Override

140
src/main/java/org/gcube/dataanalysis/ecoengine/test/checks/CheckOccurrences.java Normal file
View File

@ -0,0 +1,140 @@
package org.gcube.dataanalysis.ecoengine.test.checks;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.evaluation.bioclimate.InterpolateTables.INTERPOLATIONFUNCTIONS;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class CheckOccurrences {
public static void main(String[] args) throws Exception {
System.out.println("TEST 1");
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(testConfigLocal5());
trans.get(0).init();
Regressor.process(trans.get(0));
trans = null;
}
private static AlgorithmConfiguration testConfigLocal() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEC");
config.setParam("HSPEC_TABLE_LIST", "hspec_validation"+AlgorithmConfiguration.getListSeparator()+"hspec_validation2");
config.setParam("HSPEC_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test");
config.setParam("Threshold", "0.5");
return config;
}
private static AlgorithmConfiguration testConfigLocal2() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("BIOCLIMATE_HCAF");
config.setParam("HCAF_TABLE_LIST","hcaf_d"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01332632269756"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01336062995861"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2050");
config.setParam("HCAF_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
return config;
}
private static AlgorithmConfiguration testConfigLocal3() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEN");
config.setParam("HSPEN_TABLE_LIST","hspen"+AlgorithmConfiguration.getListSeparator()+"hspen_2016"+AlgorithmConfiguration.getListSeparator()+"hspen_2020"+AlgorithmConfiguration.getListSeparator()+"hspen_2050");
config.setParam("HSPEN_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
return config;
}
private static AlgorithmConfiguration testConfigLocal4() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("HCAF_INTERPOLATION");
config.setParam("FirstHCAF","hcaf_d");
config.setParam("SecondHCAF","hcaf_d_2050");
config.setParam("YearStart","2012");
config.setParam("YearEnd","2050");
config.setParam("NumberOfInterpolations","2");
config.setParam("InterpolationFunction",INTERPOLATIONFUNCTIONS.LINEAR.name());
return config;
}
private static AlgorithmConfiguration testConfigLocal5() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("OCCURRENCES_MERGER");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("longitudeColumn", "decimallongitude");
config.setParam("latitudeColumn", "decimallatitude");
config.setParam("recordedByColumn", "recordedby");
config.setParam("scientificNameColumn", "scientificname");
config.setParam("eventDateColumn", "eventdate");
config.setParam("lastModificationColumn", "modified");
config.setParam("rightTableName", "occurrence_species_id72d89d3d_103d_4c78_a922_a0ed620b393d");
config.setParam("leftTableName", "occurrence_species_id72d89d3d_103d_4c78_a922_a0ed620b393d");
config.setParam("finalTableName", "occurrence_species_merged");
config.setParam("spatialTolerance", "0.1");
config.setParam("confidence", "80");
return config;
}
private static AlgorithmConfiguration testConfigLocal6() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setAgent("OCCURRENCES_INSEAS_ONEARTH");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("longitudeColumn", "decimallongitude");
config.setParam("latitudeColumn", "decimallatitude");
config.setParam("OccurrencePointsTableName", "occurrence_species_id72d89d3d_103d_4c78_a922_a0ed620b393d");
config.setParam("finalTableName", "occurrence_species_seas");
config.setParam("FilterType", "IN_THE_WATER");
// config.setParam("FilterType", "ON_EARTH");
return config;
}
private static AlgorithmConfiguration testConfigLocal7() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setAgent("OCCURRENCES_DUPLICATES_DELETER");
config.setParam("longitudeColumn", "decimallongitude");
config.setParam("latitudeColumn", "decimallatitude");
config.setParam("recordedByColumn", "recordedby");
config.setParam("scientificNameColumn", "scientificname");
config.setParam("eventDateColumn", "eventdate");
config.setParam("lastModificationColumn", "modified");
config.setParam("OccurrencePointsTableName", "occurrence_species_id72d89d3d_103d_4c78_a922_a0ed620b393d");
config.setParam("finalTableName", "occurrence_species_deleted");
config.setParam("spatialTolerance", "0.5");
config.setParam("confidence", "80");
return config;
}
}

6
src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/RegressionTestTransducers.java
View File

@ -138,6 +138,12 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal7() {
AlgorithmConfiguration config = Regressor.getConfig();
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setAgent("OCCURRENCES_DUPLICATES_DELETER");
config.setParam("longitudeColumn", "decimallongitude");

2
src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsDuplicatesDeleter.java
View File

@ -106,7 +106,7 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger {
public void takeFullRanges() {
// take the elements from sx table
AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(tableName, columns.toString(), "limit 100000"), dbconnection);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(tableName, columns.toString(),"")+" limit 100000", dbconnection);
}
public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) {

11
src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java
View File

@ -496,7 +496,12 @@ public class OccurrencePointsMerger implements Transducerer {
}
protected void insertBuffer(StringBuffer buffer) throws Exception{
String subBuffer = buffer.substring(0, buffer.length()-1);
String subBuffer = "";
if (buffer.charAt(buffer.length()-1)!=')')
subBuffer = buffer.substring(0, buffer.length()-1);
else
subBuffer = buffer.toString();
String updateQ = "SET datestyle = \"ISO, MDY\"; "+DatabaseUtils.insertFromString(finalTableName, columns.toString(), subBuffer);
// System.out.println("Update:\n"+updateQ);
AnalysisLogger.getLogger().debug("Update:\n"+updateQ);
@ -580,10 +585,10 @@ public class OccurrencePointsMerger implements Transducerer {
public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) {
// take the elements from sx table
AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), "offset " + offsetLeft + " limit " + numLeft), dbconnection);
leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), "order by "+columns.toString()+" offset " + offsetLeft + " limit " + numLeft), dbconnection);
// take the elements from dx table
AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), "offset " + offsetRight + " limit " + numRight), dbconnection);
rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), "order by "+columns.toString()+" offset " + offsetRight + " limit " + numRight), dbconnection);
}
public void computeRange() throws Exception {