This commit is contained in:
Gianpaolo Coro 2012-09-28 14:45:31 +00:00
parent 07eb4a545f
commit 6a8f08a1ca
13 changed files with 95 additions and 54 deletions

View File

@ -142,7 +142,7 @@ public class DatabaseFactory {
}
public static void executeHQLUpdate(String query, SessionFactory DBSessionFactory, boolean useSQL) {
public static void executeHQLUpdate(String query, SessionFactory DBSessionFactory, boolean useSQL) throws Exception{
// System.out.println("executing query: " + query);
Session ss = null;
@ -164,10 +164,11 @@ public class DatabaseFactory {
} catch (Exception e) {
rollback(ss);
AnalysisLogger.getLogger().debug("ERROR IN UPDATE: "+e.getMessage());
throw e;
}
}
public static void executeSQLUpdate(String query, SessionFactory DBSessionFactory) {
public static void executeSQLUpdate(String query, SessionFactory DBSessionFactory) throws Exception{
executeHQLUpdate(query, DBSessionFactory, true);
}

View File

@ -34,7 +34,7 @@ public class DistributionQualityAnalysis extends DataAnalysis {
static String getProbabilititesQuery = "select count(*) as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s and b.%5$s %6$s %7$s";
static String getNumberOfElementsQuery = "select count(*) from %1$s";
static String getValuesQuery = "select %5$s as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s";
static String getValuesQuery = "select %5$s as distribprob (select distinct * from %1$s as a join %2$s as b on a.%3$s=b.%4$s) as b";
float threshold = 0.1f;
String configPath = "./cfg/";
@ -160,7 +160,7 @@ public class DistributionQualityAnalysis extends DataAnalysis {
String acceptanceThreshold = config.getParam("PositiveThreshold");
String rejectionThreshold = config.getParam("NegativeThreshold");
int numberOfPositiveCases = calculateNumberOfPoints(config.getParam("PositiveCasesTable"));
int numberOfPositiveCases = calculateNumberOfPoints(positiveCasesTable);
int truePositives = calculateCaughtPoints(positiveCasesTable, distributionTable, positiveCasesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, ">", acceptanceThreshold);

View File

@ -22,7 +22,7 @@ public class ExperimentsForLatimeria {
static String absenceRandomTable = "absence_data_latimeria_random";
static String absenceStaticTable = "absence_data_latimeria";
static String presenceTable = "presence_data_latimeria";
static String presenceTable = "presence_data_latimeria_2";
static String presenceTableNoEarth = "presence_data_latimeria_sea";
static String envelopeTable = "hspen_latimeria";
@ -277,14 +277,14 @@ public class ExperimentsForLatimeria {
// generateAquamapsNativeSuitable();
//generate presence and absence hcafs
// generatePresenceTable();
generatePresenceTable();
// generateAbsenceTable();
// generateHCAFFilter();
//train the neural networks on these tables
trainSuitableNeuralNetworks();
trainNativeNeuralNetworks();
// trainSuitableNeuralNetworks();
// trainNativeNeuralNetworks();
//project the neural networks
generateAquamapsNativeSuitableNeuralNetwokrs();
// generateAquamapsNativeSuitableNeuralNetwokrs();
//Analysis
/*
@ -293,13 +293,16 @@ public class ExperimentsForLatimeria {
calcdiscrepancy(aquamapsSuitableTable, nnsuitableTable);
calcdiscrepancy(aquamapsNativeTable, nnnativeTable);
*/
/*
calcquality(aquamapsSuitableTable, presenceTableNoEarth, absenceStaticTable);
calcquality(nnsuitableTable, presenceTableNoEarth, absenceStaticTable);
*/
calcquality(aquamapsSuitableTable, presenceTable, absenceStaticTable);
calcquality(nnsuitableTable, presenceTable, absenceStaticTable);
calcquality(aquamapsNativeTable, presenceTable, absenceStaticTable);
calcquality(nnnativeTable, presenceTable, absenceStaticTable);
// calcquality(aquamapsNativeTable, presenceTableNoEarth, absenceStaticTable);
// calcquality(nnnativeTable, presenceTableNoEarth, absenceStaticTable);
/*
calcHRS(filteredhcaf,absenceStaticTable,presenceTableNoEarth);
calcHRS(filteredhcaf,null,presenceTableNoEarth);
*/

View File

@ -103,7 +103,7 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger{
}
@Override
protected void prepareFinalTable(){
protected void prepareFinalTable() throws Exception{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(tableName, finalTableName), dbconnection);
}

View File

@ -70,7 +70,7 @@ public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{
}
@Override
protected void prepareFinalTable(){
protected void prepareFinalTable() throws Exception{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(tableName, finalTableName), dbconnection);
}

View File

@ -17,7 +17,7 @@ public class OccurrencePointsIntersector extends OccurrencePointsMerger{
}
@Override
protected void prepareFinalTable(){
protected void prepareFinalTable() throws Exception{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(leftTableName, finalTableName), dbconnection);
}

View File

@ -358,7 +358,7 @@ public class OccurrencePointsMerger implements Transducerer {
objectstoinsert.add(rightOcc);
}
protected void persist() {
protected void persist() throws Exception{
// DELETE ELEMENTS IN THE DELETION LIST
int todel = objectstodelete.size();
@ -422,11 +422,11 @@ public class OccurrencePointsMerger implements Transducerer {
}
}
protected void prepareFinalTable(){
protected void prepareFinalTable() throws Exception{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.duplicateTableStatement(leftTableName, finalTableName), dbconnection);
}
protected void extractColumnNames(){
protected void extractColumnNames() throws Exception{
// take the description of the table
columnsNames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(rightTableName), dbconnection);

View File

@ -19,8 +19,11 @@ public class QueryExecutor implements Transducerer {
protected String query = "";
protected String finalTableName = "";
protected String finalTableLabel = "";
protected float status = 0;
protected AlgorithmConfiguration config;
protected static String finalTable = "Table_Name";
protected static String finalTableLabel$ = "Table_Label";
@Override
public List<StatisticalType> getInputParameters() {
@ -81,15 +84,22 @@ public class QueryExecutor implements Transducerer {
public void compute() throws Exception {
SessionFactory dbconnection = null;
try{
AnalysisLogger.getLogger().trace("Initializing DB Connection");
dbconnection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().trace("Deleting Previous Table");
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
AnalysisLogger.getLogger().trace("Deleting Previous Table "+DatabaseUtils.dropTableStatement(finalTableName));
try{
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection);
}catch(Exception ee){
}
status = 10;
AnalysisLogger.getLogger().trace("Deleted");
AnalysisLogger.getLogger().trace("Executing query: "+query);
DatabaseFactory.executeSQLUpdate(query, dbconnection);
AnalysisLogger.getLogger().trace("Executed!");
} catch (Exception e) {
AnalysisLogger.getLogger().trace("ERROR:",e);
throw e;
} finally {
if (dbconnection != null)

View File

@ -3,21 +3,22 @@ package org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.transducers.QueryExecutor;
public class HcafFilter extends QueryExecutor {
static String finalTable = "FINAL_TABLE_NAME";
static String bbx1 = "BOUNDING_BOX_LEFT_LOWER_LONG";
static String bbx2 = "BOUNDING_BOX_RIGHT_UPPER_LONG";
static String bby1 = "BOUNDING_BOX_LEFT_LOWER_LAT";
static String bby2 = "BOUNDING_BOX_RIGHT_UPPER_LAT";
static String bbx1 = "B_Box_Left_Lower_Long";
static String bbx2 = "B_Box_Right_Upper_Long";
static String bby1 = "B_Box_Left_Lower_Lat";
static String bby2 = "B_Box_Right_Upper_Lat";
String bbx1$;
String bbx2$;
@ -30,6 +31,7 @@ public class HcafFilter extends QueryExecutor {
public void init() throws Exception {
finalTableName = config.getParam(finalTable);
finalTableLabel = config.getParam(finalTableLabel$);
bbx1$ = config.getParam(bbx1);
bbx2$ = config.getParam(bbx2);
bby1$ = config.getParam(bby1);
@ -41,19 +43,23 @@ public class HcafFilter extends QueryExecutor {
@Override
public List<StatisticalType> getInputParameters() {
PrimitiveType p1 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable, "the name of the Filtered Hcaf", "hspen_filtered");
PrimitiveType p0 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableLabel$,"the name of the Filtered Hcaf", "hcaf_filtered");
ServiceType p1 = new ServiceType(ServiceParameters.RANDOMSTRING, finalTable, "the name of the Filtered Hcaf", "hspen_filtered");
PrimitiveType p2 = new PrimitiveType(Float.class.getName(), null, PrimitiveTypes.NUMBER, bbx1, "the left lower longitude of the bounding box (range [-180,+180])", "0");
PrimitiveType p3 = new PrimitiveType(Float.class.getName(), null, PrimitiveTypes.NUMBER, bbx2, "the right upper longitude of the bounding box (range [-180,+180])", "0");
PrimitiveType p4 = new PrimitiveType(Float.class.getName(), null, PrimitiveTypes.NUMBER, bby1, "the left lower latitude of the bounding box (range [-90,+90])", "10");
PrimitiveType p5 = new PrimitiveType(Float.class.getName(), null, PrimitiveTypes.NUMBER, bby2, "the right upper latitude of the bounding box (range [-90,+90])", "10");
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
parameters.add(p0);
parameters.add(p1);
parameters.add(p2);
parameters.add(p3);
parameters.add(p4);
parameters.add(p5);
DatabaseType.addDefaultDBPars(parameters);
return parameters;
}
@ -61,7 +67,7 @@ public class HcafFilter extends QueryExecutor {
public StatisticalType getOutput() {
List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.HCAF);
return new OutputTable(template, finalTableName, finalTableName, "a HCAF table focusing on the selected Bounding Box");
return new OutputTable(template, finalTableLabel, finalTableName, "a HCAF table focusing on the selected Bounding Box");
}
@Override

View File

@ -4,24 +4,27 @@ import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.transducers.QueryExecutor;
public class HspenFilter extends QueryExecutor {
static String finalTable = "FINAL_TABLE_NAME";
static String speciesCodes = "SPECIES_CODES";
static String speciesCodes = "Species_Codes";
String species;
@Override
public void init() throws Exception {
finalTableName = config.getParam(finalTable);
finalTableLabel = config.getParam(finalTableLabel$);
species = config.getParam(speciesCodes).replace(AlgorithmConfiguration.getListSeparator(), ",");
query = "select * into "+finalTableName+" from hspen where speciesid in ('"+species+"')";
@ -30,21 +33,25 @@ public class HspenFilter extends QueryExecutor {
@Override
public List<StatisticalType> getInputParameters() {
PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"the name of the Filtered Hspen","hspen_filtered");
// PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"the name of the Filtered Hspen","hspen_filtered");
PrimitiveType p1 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableLabel$,"the name of the Filtered Hspen","hspen_filtered");
ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, finalTable,"the name of the Filtered Hspen","hspen_filtered");
PrimitiveTypesList pl = new PrimitiveTypesList(String.class.getName(),PrimitiveTypes.STRING, speciesCodes, "A list of species codes (Fish Base Format) to take. E.g. Fis-30189", false);
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
parameters.add(p1);
parameters.add(p3);
parameters.add(pl);
DatabaseType.addDefaultDBPars(parameters);
return parameters;
}
@Override
public StatisticalType getOutput() {
List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.HCAF);
return new OutputTable(template, finalTableName, finalTableName, "a HSPEN table containing only selected species");
template.add(TableTemplates.HSPEN);
return new OutputTable(template, finalTableLabel, finalTableName, "a HSPEN table containing only selected species");
}
@Override

View File

@ -3,21 +3,23 @@ package org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.transducers.QueryExecutor;
public class MarineAbsencePointsFromAquamapsDistribution extends QueryExecutor {
static String doRandom = "RANDOM_TAKE";
static String AquamapsHSpecTable = "AQUAMAPS_HSPEC";
static String finalTable = "FINAL_TABLE_NAME";
static String numberOfPoints = "NUMBER_OF_POINTS";
static String speciesCode = "SPECIES_CODE";
static String doRandom = "Take_Randomly";
static String AquamapsHSpecTable = "Aquamaps_HSPEC";
static String numberOfPoints = "Number_of_Points";
static String speciesCode = "Species_Code";
String dorandom;
String aquamapsTable;
@ -28,6 +30,7 @@ public class MarineAbsencePointsFromAquamapsDistribution extends QueryExecutor {
public void init() throws Exception {
dorandom = config.getParam(doRandom);
finalTableName = config.getParam(finalTable);
finalTableLabel = config.getParam(finalTableLabel$);
aquamapsTable= config.getParam(AquamapsHSpecTable);
nPoints= config.getParam(numberOfPoints);
species = config.getParam(speciesCode);
@ -44,18 +47,23 @@ public class MarineAbsencePointsFromAquamapsDistribution extends QueryExecutor {
List<TableTemplates> templateHspec = new ArrayList<TableTemplates>();
templateHspec.add(TableTemplates.HSPEC);
PrimitiveType p0 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableLabel$,"the name of the Filtered Hcaf", "AbsenceCells_");
InputTable p1 = new InputTable(templateHspec,AquamapsHSpecTable,"an Aquamaps table from which to produce the absence points","hspec");
PrimitiveType p2 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, doRandom, "a flag for taking points randomly (true) or close together (false)","true");
PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"Name of the HCAF table to produce containing Absence Cells","absence_hcaf");
// PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"Name of the HCAF table to produce containing Absence Cells","absence_hcaf");
ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, finalTable,"Name of the HCAF table to produce containing Absence Cells","absence_hcaf");
PrimitiveType p4 = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, numberOfPoints,"Number of points to take","20");
PrimitiveType p5 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, speciesCode,"the species code according to the Fish-Base conventions","Fis-30189");
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
parameters.add(p0);
parameters.add(p1);
parameters.add(p2);
parameters.add(p3);
parameters.add(p4);
parameters.add(p5);
DatabaseType.addDefaultDBPars(parameters);
return parameters;
}
@ -63,7 +71,7 @@ public class MarineAbsencePointsFromAquamapsDistribution extends QueryExecutor {
public StatisticalType getOutput() {
List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.HCAF);
return new OutputTable(template, finalTableName, finalTableName, "a HCAF table containing Absence Points cells");
return new OutputTable(template, finalTableLabel, finalTableName, "a HCAF table containing Absence Points cells");
}
@Override

View File

@ -3,18 +3,20 @@ package org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.transducers.QueryExecutor;
public class MarinePresencePoints extends QueryExecutor {
static String finalTable = "FINAL_TABLE_NAME";
static String numberOfPoints = "NUMBER_OF_POINTS";
static String speciesCode = "SPECIES_CODE";
static String numberOfPoints = "Number_of_Points";
static String speciesCode = "Species_Code";
String nPoints;
String species;
@ -22,6 +24,7 @@ public class MarinePresencePoints extends QueryExecutor {
public void init() throws Exception {
finalTableName = config.getParam(finalTable);
finalTableLabel = config.getParam(finalTableLabel$);
nPoints= config.getParam(numberOfPoints);
species = config.getParam(speciesCode);
@ -30,21 +33,24 @@ public class MarinePresencePoints extends QueryExecutor {
points = "limit "+nPoints;
}
query = "select * into "+finalTableName+" from hcaf_d where csquarecode in (select csquarecode from occurrencecells where speciesid = '"+species+"' "+points+")";
query = "select * into "+finalTableName+" from hcaf_d where csquarecode in (select csquarecode from occurrencecells where speciesid = '"+species+"' limit 100000) and oceanarea>0 "+points;
}
@Override
public List<StatisticalType> getInputParameters() {
PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"Name of the HCAF table to produce containing Presence Cells","presence_hcaf");
PrimitiveType p0 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, finalTableLabel$,"the name of the Filtered Hcaf", "PresenceCells_");
// PrimitiveType p3 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.RANDOM, finalTable,"Name of the HCAF table to produce containing Presence Cells","presence_hcaf");
ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, finalTable,"Name of the HCAF table to produce containing Presence Cells","presence_hcaf");
PrimitiveType p4 = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, numberOfPoints,"Maximum number of points to take (-1 to take all)","-1");
PrimitiveType p5 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, speciesCode,"the species code according to the Fish-Base conventions","Fis-30189");
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
parameters.add(p0);
parameters.add(p3);
parameters.add(p4);
parameters.add(p5);
DatabaseType.addDefaultDBPars(parameters);
return parameters;
}
@ -52,7 +58,7 @@ public class MarinePresencePoints extends QueryExecutor {
public StatisticalType getOutput() {
List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.HCAF);
return new OutputTable(template, finalTableName, finalTableName, "a HCAF table containing Presence Points cells");
return new OutputTable(template, finalTableLabel, finalTableName, "a HCAF table containing Presence Points cells");
}
@Override

View File

@ -169,10 +169,10 @@ public class PresetConfigGenerator {
AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1);
config.setAgent("PRESENCE_CELLS_GENERATION");
config.setParam("FINAL_TABLE_NAME", presenceTable);
config.setParam("SPECIES_CODE", speciesCode);
config.setParam("NUMBER_OF_POINTS", ""+numberOfPoints);
config.setParam("Table_Label", presenceTable);
config.setParam("Table_Name", presenceTable);
config.setParam("Species_Code", speciesCode);
config.setParam("Number_of_Points", ""+numberOfPoints);
return config;
}