git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@58301 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/MaxMinGenerator.java

@@ -0,0 +1,243 @@
+package org.gcube.dataanalysis.ecoengine.models.cores.aquamaps;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
+import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
+import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory;
+import org.hibernate.SessionFactory;
+
+public class MaxMinGenerator {
+
+	String selectionQuery = "SELECT DISTINCT Max(hcaf_s.CenterLat) AS maxCLat, Min(hcaf_s.CenterLat) AS minCLat,speciesid FROM #occurrencecells# INNER JOIN hcaf_s ON #occurrencecells#.CsquareCode = hcaf_s.CsquareCode WHERE (((hcaf_s.oceanarea > 0))) AND #occurrencecells#.SpeciesID in (%1$s) AND #occurrencecells#.GoodCell <> 0 group by speciesid; ";
+	
+	String insertionQuery = "insert into maxminlat_%4$s values ('%1$s','%2$s','%3$s'); ";
+	String creationQuery = "CREATE TABLE %1$s(  speciesid character varying,  maxclat real,  minclat real) WITH (  OIDS=FALSE)";
+	SessionFactory vreConnection;
+	List<Object> selectedSpecies;
+	// static String SpeciesListQuery = "select distinct speciesid from hspen;";
+	String SpeciesListQuery = "select distinct s.speciesid from %1$s s;";
+	String CheckListQuery = "select speciesid from maxminlat where speciesid = '%1$s';";
+	String vreDatabaseConfig = "DestinationDBHibernate.cfg.xml";
+
+	private boolean checkElement(String speciesid) {
+
+		List<Object> maxMin = DatabaseFactory.executeSQLQuery(String.format(CheckListQuery, speciesid), vreConnection);
+
+		if ((maxMin != null) && (maxMin.size() > 0))
+			return true;
+		else
+			return false;
+	}
+
+	public void checkAllElements() {
+		int i = 1;
+		for (Object species : selectedSpecies) {
+			// System.out.println("CHECKING: "+i+" SPECIES: "+species);
+			boolean check = checkElement((String) species);
+			if (!check) {
+				System.err.println("MISSING: " + species);
+			}
+			i++;
+		}
+	}
+
+	int counter = 0;
+	StringBuffer buffer = new StringBuffer();
+	private double status = 0;
+
+	public double getstatus() {
+		return status;
+	}
+
+	
+	private void insertQuery(List<String> species, String hspenTable) {
+
+		StringBuffer sb = new StringBuffer();
+		int size = species.size();
+		int i = 0;
+		for (String sp: species){
+			sb.append("'"+sp+"'");
+			i++;
+			if (i<size)
+				sb.append(",");
+		}
+		
+		String query = String.format(selectionQuery, sb.toString());
+		
+		List<Object> maxMin = DatabaseFactory.executeSQLQuery(query, vreConnection);
+		int sizemm = maxMin.size();
+		StringBuffer buffer = new StringBuffer();
+		
+		for (int j=0;j<sizemm;j++){
+		Object[] maxMinRow = (Object[]) maxMin.get(0);
+		String insert = String.format(insertionQuery,  maxMinRow[2], maxMinRow[0], maxMinRow[1], hspenTable);
+
+		buffer.append(insert);
+		
+		}
+		try {
+				AnalysisLogger.getLogger().debug("inserting...");
+				// DatabaseFactory.executeSQLUpdate(insert, vreConnection);
+				DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection);
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+		
+	}
+
+
+	
+	private void executeQuery(String species, String hspenTable) {
+
+		String query = String.format(selectionQuery, species);
+		List<Object> maxMin = DatabaseFactory.executeSQLQuery(query, vreConnection);
+		Object[] maxMinRow = (Object[]) maxMin.get(0);
+		String insert = String.format(insertionQuery, species, maxMinRow[0], maxMinRow[1], hspenTable);
+
+		buffer.append(insert);
+		if (counter % 100 == 0) {
+
+			try {
+				AnalysisLogger.getLogger().debug("inserting...");
+				// DatabaseFactory.executeSQLUpdate(insert, vreConnection);
+				DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection);
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+			buffer = new StringBuffer();
+		}
+		counter++;
+	}
+
+	public void getAllSpecies(String hspentable) {
+		// populates the selectedSpecies variable by getting species from db
+		String query = String.format(SpeciesListQuery, hspentable);
+		AnalysisLogger.getLogger().warn("Distribution Generator ->getting all species list from DB");
+		AnalysisLogger.getLogger().warn("Distribution Generator ->" + query);
+		List<Object> allspecies = DatabaseFactory.executeSQLQuery(query, vreConnection);
+		selectedSpecies = allspecies;
+		// AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + selectedSpecies.size());
+	}
+
+	protected void deleteDestinationTable(String table) {
+		String deleteQuery = "drop table " + table + ";";
+		// clean the corresponding table on destination db
+		try {
+			DatabaseFactory.executeSQLUpdate(deleteQuery, vreConnection);
+			AnalysisLogger.getLogger().debug("destination table dropped");
+		} catch (Exception e) {
+			AnalysisLogger.getLogger().debug("destination table NOT dropped");
+		}
+
+	}
+
+	protected void buildDestinationTable(String destinationTable) {
+
+		String createQuery = String.format(creationQuery, destinationTable);
+		AnalysisLogger.getLogger().debug("Creating new table or destination schema: " + destinationTable);
+		try {
+			DatabaseFactory.executeSQLUpdate(String.format(createQuery, destinationTable), vreConnection);
+			AnalysisLogger.getLogger().debug("Table created");
+		} catch (Exception e) {
+			AnalysisLogger.getLogger().debug("Table NOT created");
+		}
+	}
+
+	
+	public void populatemaxminlat2(String hspenTable) {
+
+		int i = 1;
+		try {
+			long t0 = System.currentTimeMillis();
+			getAllSpecies(hspenTable);
+			deleteDestinationTable("maxminlat_" + hspenTable);
+			buildDestinationTable("maxminlat_" + hspenTable);
+			int size = selectedSpecies.size();
+			AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size);
+			for (Object species : selectedSpecies) {
+				String speciesid = (String) species;
+				executeQuery(speciesid, hspenTable);
+				i++;
+				
+				status = MathFunctions.roundDecimal(((double) i *100/ (double) size), 2);
+//				status = (double) i / (double) size;
+				if (i%10==0)
+					AnalysisLogger.getLogger().debug("status " + status);
+			}
+			long t1 = System.currentTimeMillis();
+			AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0));
+
+		} catch (Exception e) {
+			e.printStackTrace();
+
+		} finally {
+			vreConnection.close();
+		}
+	}
+	
+	
+	public String populatemaxminlat(String destinationTable, String hspenTable, String occurrenceCellsTable) {
+
+		int i = 1;
+		
+		try {
+			selectionQuery = selectionQuery.replace("#occurrencecells#",occurrenceCellsTable);
+			long t0 = System.currentTimeMillis();
+			getAllSpecies(hspenTable);
+			deleteDestinationTable(destinationTable);
+			buildDestinationTable(destinationTable);
+			int size = selectedSpecies.size();
+			AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size);
+			List<String> specieslist = new ArrayList<String>();
+			for (Object species : selectedSpecies) {
+				String speciesid = (String) species;
+//				executeQuery(speciesid, hspenTable);
+				i++;
+				specieslist.add(speciesid);
+				
+				status = MathFunctions.roundDecimal(((double) i *100/ (double) size), 2);
+				if (status >= 100)
+					status = 99.00;
+				
+//				status = (double) i / (double) size;
+				if (i%100==0){
+					AnalysisLogger.getLogger().debug("status " + status+" species processed "+i);
+					insertQuery(specieslist, hspenTable);
+					specieslist = null;
+					specieslist = new ArrayList<String>();
+					
+					//long t1 = System.currentTimeMillis();
+					//AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0));
+				}
+				
+			}
+			
+			if (specieslist.size()>0){
+				AnalysisLogger.getLogger().debug("final status " + status+" species processed "+i);
+				insertQuery(specieslist, hspenTable);
+			}
+			
+			long t1 = System.currentTimeMillis();
+			AnalysisLogger.getLogger().debug("overall elapsed Time: " + (t1 - t0));
+			
+		} catch (Exception e) {
+			e.printStackTrace();
+
+		} finally {
+			status = 100;
+		}
+		
+		return destinationTable;
+	}
+
+	
+	public MaxMinGenerator(SessionFactory connection) {
+		vreConnection = connection;
+	}
+	
+
+	
+
+}

src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java

@@ -19,6 +19,7 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
 import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
 import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
 import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionTable;
+import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.MaxMinGenerator;
 import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
 import org.hibernate.SessionFactory;
 
@@ -31,7 +32,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
 	String destinationTableLabel;
 	String metainfo ="boundboxyn, faoareayn, faoaream, eezall, lme";
 	String selectAllSpeciesObservationQuery = "SELECT speciesid,maxclat,minclat from %1$s;";
-	String hspenMinMaxLat = "maxminlat_hspen";
+	String hspenMinMaxLat;
 	AquamapsAlgorithmCore core;
 	protected String currentFAOAreas;
 	protected HashMap<String,String> currentSpeciesBoundingBoxInfo;
@@ -39,6 +40,14 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
 	//to overwrite in case of 2050
 	protected String type = null;
 	
+	protected String generateMaxMinHspec(String minmaxTableName, String hspenTable, String occurrencePointsTable, SessionFactory dbHibConnection){
+		MaxMinGenerator maxmin = new MaxMinGenerator(dbHibConnection);
+		if (occurrencePointsTable==null) 
+			occurrencePointsTable = "occurrencecells";
+		
+		return maxmin.populatemaxminlat(minmaxTableName, hspenTable,occurrencePointsTable);
+	}
+	
 	@Override
 	public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection) {
 		selectAllSpeciesQuery = String.format(selectAllSpeciesQuery, config.getParam("EnvelopeTable"));
@@ -52,8 +61,29 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
 		if ((config.getParam("PreprocessedTable")!=null)&&(config.getParam("PreprocessedTable").length()>0))
 			hspenMinMaxLat = config.getParam("PreprocessedTable");
 		
-		//TO-DO: if not preprocessed then generate a preprocessed table
+		//if not preprocessed then generate a preprocessed table
-		
+		if (hspenMinMaxLat==null){
+			//take the name of the hspen table
+			String hspenTable = config.getParam("EnvelopeTable");
+			//check if the table exists
+			String supposedminmaxlattable = "maxminlat_"+hspenTable;
+			List<Object> select = null;
+			try{
+				select = DatabaseFactory.executeSQLQuery("select * from "+supposedminmaxlattable+" limit 1",dbHibConnection);
+			}catch(Exception ee){}
+			//if it exists then set the table name
+			if (select!=null){
+				hspenMinMaxLat = supposedminmaxlattable;
+				AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table yet exists "+hspenMinMaxLat);
+			}
+			else{
+				//otherwise create it by calling the creator
+				AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table does not exist! - generating");
+				hspenMinMaxLat = generateMaxMinHspec(supposedminmaxlattable, hspenTable,config.getParam("OccurrencePointsTable"), dbHibConnection);
+				AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->min max latitudes table created in "+hspenMinMaxLat);
+			}
+		}
+			
 		
 		AnalysisLogger.getLogger().trace("Aquamaps Algorithm Init ->getting min max latitudes from "+hspenMinMaxLat);
 		
@@ -220,8 +250,8 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
 	@Override
 	public List<StatisticalType> getInputParameters() {
 		List<StatisticalType> parameters = new ArrayList<StatisticalType>();
-		List<TableTemplates> templatesMinmax = new ArrayList<TableTemplates>();
+		List<TableTemplates> templatesOccurrence = new ArrayList<TableTemplates>();
-		templatesMinmax.add(TableTemplates.MINMAXLAT);
+		templatesOccurrence.add(TableTemplates.OCCURRENCE);
 		List<TableTemplates> templateHspen = new ArrayList<TableTemplates>();
 		templateHspen.add(TableTemplates.HSPEN);
 		List<TableTemplates> templateHcaf = new ArrayList<TableTemplates>();
@@ -231,7 +261,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
 		InputTable p2 = new InputTable(templateHcaf,"CsquarecodesTable","HCaf Table","hcaf_d");
 		ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, "DistributionTable","Table name of the distribution","hspec_");
 		PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, "DistributionTableLabel","Name of the HSPEC probability distribution","hspec");
-		InputTable p5 = new InputTable(templatesMinmax,"PreprocessedTable","Minimum maximum latitudes table for species","maxminlat_hspen");
+		InputTable p5 = new InputTable(templatesOccurrence,"OccurrencePointsTable","The Occurrence points table for calculating the bounding box","occurrencecells");
 		PrimitiveType p6  = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.CONSTANT, "CreateTable","Create New Table for each computation","true");		
 
 		DatabaseType p7 = new DatabaseType(DatabaseParameters.DATABASEUSERNAME, "DatabaseUserName", "db user name");

src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/RegressionTestGenerators.java

@@ -32,8 +32,8 @@ public static void main(String[] args) throws Exception {
 		
 		config.setParam("DistributionTable","hspec_suitable_test_gp");
 		config.setParam("CsquarecodesTable","hcaf_d");
-		config.setParam("EnvelopeTable","hspen_micro");
+		config.setParam("EnvelopeTable","hspen_micro_1");
-		config.setParam("PreprocessedTable", "maxminlat_hspen");
+		config.setParam("OccurrencePointsTable", "occurrencecells");
 		config.setParam("CreateTable","true");
 
 		return config;

src/main/java/org/gcube/dataanalysis/ecoengine/utils/DatabaseFactory.java

@@ -175,7 +175,7 @@ public class DatabaseFactory{
 				obj = result;
 			}
 
-			rollback(ss);
+//			rollback(ss);
 
 		return obj;