git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@58301 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
254f87cb73
commit
d6686c91dc
|
|
@ -0,0 +1,243 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.aquamaps;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class MaxMinGenerator {
|
||||
|
||||
String selectionQuery = "SELECT DISTINCT Max(hcaf_s.CenterLat) AS maxCLat, Min(hcaf_s.CenterLat) AS minCLat,speciesid FROM #occurrencecells# INNER JOIN hcaf_s ON #occurrencecells#.CsquareCode = hcaf_s.CsquareCode WHERE (((hcaf_s.oceanarea > 0))) AND #occurrencecells#.SpeciesID in (%1$s) AND #occurrencecells#.GoodCell <> 0 group by speciesid; ";
|
||||
|
||||
String insertionQuery = "insert into maxminlat_%4$s values ('%1$s','%2$s','%3$s'); ";
|
||||
String creationQuery = "CREATE TABLE %1$s( speciesid character varying, maxclat real, minclat real) WITH ( OIDS=FALSE)";
|
||||
SessionFactory vreConnection;
|
||||
List<Object> selectedSpecies;
|
||||
// static String SpeciesListQuery = "select distinct speciesid from hspen;";
|
||||
String SpeciesListQuery = "select distinct s.speciesid from %1$s s;";
|
||||
String CheckListQuery = "select speciesid from maxminlat where speciesid = '%1$s';";
|
||||
String vreDatabaseConfig = "DestinationDBHibernate.cfg.xml";
|
||||
|
||||
private boolean checkElement(String speciesid) {
|
||||
|
||||
List<Object> maxMin = DatabaseFactory.executeSQLQuery(String.format(CheckListQuery, speciesid), vreConnection);
|
||||
|
||||
if ((maxMin != null) && (maxMin.size() > 0))
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
public void checkAllElements() {
|
||||
int i = 1;
|
||||
for (Object species : selectedSpecies) {
|
||||
// System.out.println("CHECKING: "+i+" SPECIES: "+species);
|
||||
boolean check = checkElement((String) species);
|
||||
if (!check) {
|
||||
System.err.println("MISSING: " + species);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
int counter = 0;
|
||||
StringBuffer buffer = new StringBuffer();
|
||||
private double status = 0;
|
||||
|
||||
public double getstatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
|
||||
private void insertQuery(List<String> species, String hspenTable) {
|
||||
|
||||
StringBuffer sb = new StringBuffer();
|
||||
int size = species.size();
|
||||
int i = 0;
|
||||
for (String sp: species){
|
||||
sb.append("'"+sp+"'");
|
||||
i++;
|
||||
if (i<size)
|
||||
sb.append(",");
|
||||
}
|
||||
|
||||
String query = String.format(selectionQuery, sb.toString());
|
||||
|
||||
List<Object> maxMin = DatabaseFactory.executeSQLQuery(query, vreConnection);
|
||||
int sizemm = maxMin.size();
|
||||
StringBuffer buffer = new StringBuffer();
|
||||
|
||||
for (int j=0;j<sizemm;j++){
|
||||
Object[] maxMinRow = (Object[]) maxMin.get(0);
|
||||
String insert = String.format(insertionQuery, maxMinRow[2], maxMinRow[0], maxMinRow[1], hspenTable);
|
||||
|
||||
buffer.append(insert);
|
||||
|
||||
}
|
||||
try {
|
||||
AnalysisLogger.getLogger().debug("inserting...");
|
||||
// DatabaseFactory.executeSQLUpdate(insert, vreConnection);
|
||||
DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void executeQuery(String species, String hspenTable) {
|
||||
|
||||
String query = String.format(selectionQuery, species);
|
||||
List<Object> maxMin = DatabaseFactory.executeSQLQuery(query, vreConnection);
|
||||
Object[] maxMinRow = (Object[]) maxMin.get(0);
|
||||
String insert = String.format(insertionQuery, species, maxMinRow[0], maxMinRow[1], hspenTable);
|
||||
|
||||
buffer.append(insert);
|
||||
if (counter % 100 == 0) {
|
||||
|
||||
try {
|
||||
AnalysisLogger.getLogger().debug("inserting...");
|
||||
// DatabaseFactory.executeSQLUpdate(insert, vreConnection);
|
||||
DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
buffer = new StringBuffer();
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
|
||||
public void getAllSpecies(String hspentable) {
|
||||
// populates the selectedSpecies variable by getting species from db
|
||||
String query = String.format(SpeciesListQuery, hspentable);
|
||||
AnalysisLogger.getLogger().warn("Distribution Generator ->getting all species list from DB");
|
||||
AnalysisLogger.getLogger().warn("Distribution Generator ->" + query);
|
||||
List<Object> allspecies = DatabaseFactory.executeSQLQuery(query, vreConnection);
|
||||
selectedSpecies = allspecies;
|
||||
// AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + selectedSpecies.size());
|
||||
}
|
||||
|
||||
protected void deleteDestinationTable(String table) {
|
||||
String deleteQuery = "drop table " + table + ";";
|
||||
// clean the corresponding table on destination db
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(deleteQuery, vreConnection);
|
||||
AnalysisLogger.getLogger().debug("destination table dropped");
|
||||
} catch (Exception e) {
|
||||
AnalysisLogger.getLogger().debug("destination table NOT dropped");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected void buildDestinationTable(String destinationTable) {
|
||||
|
||||
String createQuery = String.format(creationQuery, destinationTable);
|
||||
AnalysisLogger.getLogger().debug("Creating new table or destination schema: " + destinationTable);
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(String.format(createQuery, destinationTable), vreConnection);
|
||||
AnalysisLogger.getLogger().debug("Table created");
|
||||
} catch (Exception e) {
|
||||
AnalysisLogger.getLogger().debug("Table NOT created");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void populatemaxminlat2(String hspenTable) {
|
||||
|
||||
int i = 1;
|
||||
try {
|
||||
long t0 = System.currentTimeMillis();
|
||||
getAllSpecies(hspenTable);
|
||||
deleteDestinationTable("maxminlat_" + hspenTable);
|
||||
buildDestinationTable("maxminlat_" + hspenTable);
|
||||
int size = selectedSpecies.size();
|
||||
AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size);
|
||||
for (Object species : selectedSpecies) {
|
||||
String speciesid = (String) species;
|
||||
executeQuery(speciesid, hspenTable);
|
||||
i++;
|
||||
|
||||
status = MathFunctions.roundDecimal(((double) i *100/ (double) size), 2);
|
||||
// status = (double) i / (double) size;
|
||||
if (i%10==0)
|
||||
AnalysisLogger.getLogger().debug("status " + status);
|
||||
}
|
||||
long t1 = System.currentTimeMillis();
|
||||
AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0));
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
||||
} finally {
|
||||
vreConnection.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public String populatemaxminlat(String destinationTable, String hspenTable, String occurrenceCellsTable) {
|
||||
|
||||
int i = 1;
|
||||
|
||||
try {
|
||||
selectionQuery = selectionQuery.replace("#occurrencecells#",occurrenceCellsTable);
|
||||
long t0 = System.currentTimeMillis();
|
||||
getAllSpecies(hspenTable);
|
||||
deleteDestinationTable(destinationTable);
|
||||
buildDestinationTable(destinationTable);
|
||||
int size = selectedSpecies.size();
|
||||
AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size);
|
||||
List<String> specieslist = new ArrayList<String>();
|
||||
for (Object species : selectedSpecies) {
|
||||
String speciesid = (String) species;
|
||||
// executeQuery(speciesid, hspenTable);
|
||||
i++;
|
||||
specieslist.add(speciesid);
|
||||
|
||||
status = MathFunctions.roundDecimal(((double) i *100/ (double) size), 2);
|
||||
if (status >= 100)
|
||||
status = 99.00;
|
||||
|
||||
// status = (double) i / (double) size;
|
||||
if (i%100==0){
|
||||
AnalysisLogger.getLogger().debug("status " + status+" species processed "+i);
|
||||
insertQuery(specieslist, hspenTable);
|
||||
specieslist = null;
|
||||
specieslist = new ArrayList<String>();
|
||||
|
||||
//long t1 = System.currentTimeMillis();
|
||||
//AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (specieslist.size()>0){
|
||||
AnalysisLogger.getLogger().debug("final status " + status+" species processed "+i);
|
||||
insertQuery(specieslist, hspenTable);
|
||||
}
|
||||
|
||||
long t1 = System.currentTimeMillis();
|
||||
AnalysisLogger.getLogger().debug("overall elapsed Time: " + (t1 - t0));
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
||||
} finally {
|
||||
status = 100;
|
||||
}
|
||||
|
||||
return destinationTable;
|
||||
}
|
||||
|
||||
|
||||
public MaxMinGenerator(SessionFactory connection) {
|
||||
vreConnection = connection;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
|||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionTable;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.MaxMinGenerator;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
String destinationTableLabel;
|
||||
String metainfo ="boundboxyn, faoareayn, faoaream, eezall, lme";
|
||||
String selectAllSpeciesObservationQuery = "SELECT speciesid,maxclat,minclat from %1$s;";
|
||||
String hspenMinMaxLat = "maxminlat_hspen";
|
||||
String hspenMinMaxLat;
|
||||
AquamapsAlgorithmCore core;
|
||||
protected String currentFAOAreas;
|
||||
protected HashMap<String,String> currentSpeciesBoundingBoxInfo;
|
||||
|
|
@ -39,6 +40,14 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
//to overwrite in case of 2050
|
||||
protected String type = null;
|
||||
|
||||
protected String generateMaxMinHspec(String minmaxTableName, String hspenTable, String occurrencePointsTable, SessionFactory dbHibConnection){
|
||||
MaxMinGenerator maxmin = new MaxMinGenerator(dbHibConnection);
|
||||
if (occurrencePointsTable==null)
|
||||
occurrencePointsTable = "occurrencecells";
|
||||
|
||||
return maxmin.populatemaxminlat(minmaxTableName, hspenTable,occurrencePointsTable);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection) {
|
||||
selectAllSpeciesQuery = String.format(selectAllSpeciesQuery, config.getParam("EnvelopeTable"));
|
||||
|
|
@ -52,7 +61,28 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
if ((config.getParam("PreprocessedTable")!=null)&&(config.getParam("PreprocessedTable").length()>0))
|
||||
hspenMinMaxLat = config.getParam("PreprocessedTable");
|
||||
|
||||
//TO-DO: if not preprocessed then generate a preprocessed table
|
||||
//if not preprocessed then generate a preprocessed table
|
||||
if (hspenMinMaxLat==null){
|
||||
//take the name of the hspen table
|
||||
String hspenTable = config.getParam("EnvelopeTable");
|
||||
//check if the table exists
|
||||
String supposedminmaxlattable = "maxminlat_"+hspenTable;
|
||||
List<Object> select = null;
|
||||
try{
|
||||
select = DatabaseFactory.executeSQLQuery("select * from "+supposedminmaxlattable+" limit 1",dbHibConnection);
|
||||
}catch(Exception ee){}
|
||||
//if it exists then set the table name
|
||||
if (select!=null){
|
||||
hspenMinMaxLat = supposedminmaxlattable;
|
||||
AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table yet exists "+hspenMinMaxLat);
|
||||
}
|
||||
else{
|
||||
//otherwise create it by calling the creator
|
||||
AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table does not exist! - generating");
|
||||
hspenMinMaxLat = generateMaxMinHspec(supposedminmaxlattable, hspenTable,config.getParam("OccurrencePointsTable"), dbHibConnection);
|
||||
AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->min max latitudes table created in "+hspenMinMaxLat);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
AnalysisLogger.getLogger().trace("Aquamaps Algorithm Init ->getting min max latitudes from "+hspenMinMaxLat);
|
||||
|
|
@ -220,8 +250,8 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
@Override
|
||||
public List<StatisticalType> getInputParameters() {
|
||||
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
|
||||
List<TableTemplates> templatesMinmax = new ArrayList<TableTemplates>();
|
||||
templatesMinmax.add(TableTemplates.MINMAXLAT);
|
||||
List<TableTemplates> templatesOccurrence = new ArrayList<TableTemplates>();
|
||||
templatesOccurrence.add(TableTemplates.OCCURRENCE);
|
||||
List<TableTemplates> templateHspen = new ArrayList<TableTemplates>();
|
||||
templateHspen.add(TableTemplates.HSPEN);
|
||||
List<TableTemplates> templateHcaf = new ArrayList<TableTemplates>();
|
||||
|
|
@ -231,7 +261,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
InputTable p2 = new InputTable(templateHcaf,"CsquarecodesTable","HCaf Table","hcaf_d");
|
||||
ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, "DistributionTable","Table name of the distribution","hspec_");
|
||||
PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, "DistributionTableLabel","Name of the HSPEC probability distribution","hspec");
|
||||
InputTable p5 = new InputTable(templatesMinmax,"PreprocessedTable","Minimum maximum latitudes table for species","maxminlat_hspen");
|
||||
InputTable p5 = new InputTable(templatesOccurrence,"OccurrencePointsTable","The Occurrence points table for calculating the bounding box","occurrencecells");
|
||||
PrimitiveType p6 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.CONSTANT, "CreateTable","Create New Table for each computation","true");
|
||||
|
||||
DatabaseType p7 = new DatabaseType(DatabaseParameters.DATABASEUSERNAME, "DatabaseUserName", "db user name");
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ public static void main(String[] args) throws Exception {
|
|||
|
||||
config.setParam("DistributionTable","hspec_suitable_test_gp");
|
||||
config.setParam("CsquarecodesTable","hcaf_d");
|
||||
config.setParam("EnvelopeTable","hspen_micro");
|
||||
config.setParam("PreprocessedTable", "maxminlat_hspen");
|
||||
config.setParam("EnvelopeTable","hspen_micro_1");
|
||||
config.setParam("OccurrencePointsTable", "occurrencecells");
|
||||
config.setParam("CreateTable","true");
|
||||
|
||||
return config;
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ public class DatabaseFactory{
|
|||
obj = result;
|
||||
}
|
||||
|
||||
rollback(ss);
|
||||
// rollback(ss);
|
||||
|
||||
return obj;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue