diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/CSquare2Coordinates.java b/src/main/java/org/gcube/dataanalysis/geo/vti/CSquare2Coordinates.java new file mode 100644 index 0000000..9c58db1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/CSquare2Coordinates.java @@ -0,0 +1,60 @@ +package org.gcube.dataanalysis.geo.vti; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter; + +public class CSquare2Coordinates extends GridCWP2Coordinates{ + + + @Override + protected void setInputParameters() { + + List templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm will add information"); + inputs.add(tinput); + + ColumnType Dimension = new ColumnType(inputTableParameter, CodeColumn, "The column containing c-square codes", "GRID", false); + + inputs.add(Dimension); + + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "csq_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public String getDescription() { + return "An algorithm that adds longitude, latitude and resolution columns analysing a column containing c-square codes."; + } + + public void rowToCoords (Object[] rowArray) { + // take x and y + Object grid = null; + try{ + grid=rowArray[rowArray.length - 1]; + String gridValue = ""+ grid; + // generate csquarecodes + CSquareCodesConverter converter = new CSquareCodesConverter(); + converter.parse(gridValue); + + currentLat= converter.getCurrentLat(); + currentLong=converter.getCurrentLong(); + currentRes=converter.getCurrentResolution(); + }catch(Exception e){ + AnalysisLogger.getLogger().debug("Error converting grid: "+grid+" - "+e.getLocalizedMessage()); + currentLat= 0; + currentLong=0; + currentRes=0; + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/EstimateFishingActivity.java b/src/main/java/org/gcube/dataanalysis/geo/vti/EstimateFishingActivity.java new file mode 100644 index 0000000..7c088d9 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/EstimateFishingActivity.java @@ -0,0 +1,242 @@ +package org.gcube.dataanalysis.geo.vti; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.gcube.contentmanagement.graphtools.utils.DateGuesser; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import org.gcube.dataanalysis.geo.vti.vesselsprocessing.Bathymetry; +import org.gcube.dataanalysis.geo.vti.vesselsprocessing.FishingHoursCalculator; +import org.gcube.dataanalysis.geo.vti.vesselsprocessing.VTIClassificator; + +public class EstimateFishingActivity extends GridCWP2Coordinates{ + + static String VesselsIDColumn = "VesselsIDColumn"; + static String VesselsSpeedsColumn = "VesselsSpeedsColumn"; + static String VesselsTimestampsColumn = "VesselsTimestampsColumn"; + static String VesselsLatitudesColumn = "VesselsLatitudesColumn"; + static String VesselsLongitudesColumn = "VesselsLongitudesColumn"; + + @Override + protected void setInputParameters() { + + List templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm will add information"); + inputs.add(tinput); + + ColumnType Dimension1 = new ColumnType(inputTableParameter, VesselsIDColumn, "A column containing (anonymised) unique vessels identifiers", "vessel_id", false); + ColumnType Dimension2 = new ColumnType(inputTableParameter, VesselsSpeedsColumn, "The column containing vessels speeds", "speed", false); + ColumnType Dimension3 = new ColumnType(inputTableParameter, VesselsTimestampsColumn, "The column containing the time stamp of the vessels transmitted information", "datetime", false); + ColumnType Dimension4 = new ColumnType(inputTableParameter, VesselsLatitudesColumn, "The column containing vessels latitudes", "y", false); + ColumnType Dimension5 = new ColumnType(inputTableParameter, VesselsLongitudesColumn, "The column containing vessels longitudes", "x", false); + + inputs.add(Dimension1); + inputs.add(Dimension2); + inputs.add(Dimension3); + inputs.add(Dimension4); + inputs.add(Dimension5); + + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "fish_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public String getDescription() { + return "An algorithm that estimates activity hours (fishing or other) from vessels trajectories, " + + "adds bathymetry information to the table and classifies (point-by-point) fishing activity of the involved vessels according to two algorithms: " + + "one based on speed (activity_class_speed output column) and the other based on speed and bathymetry (activity_class_speed_bath output column). " + + "The algorithm produces new columns containing this information. " + + "This algorithm is based on the paper 'Deriving Fishing Monthly Effort and Caught Species' (Coro et al. 2013, in proc. of OCEANS - Bergen, 2013 MTS/IEEE). " + + "Example of input table (NAFO anonymised data): http://goo.gl/3auJkM"; + } + + @Override + protected void process() throws Exception { + status = 0; + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + long t0 = System.currentTimeMillis(); + String table = IOHelper.getInputParameter(config, inputTableParameter); + outTable = ("code_" + UUID.randomUUID()).replace("-", ""); + outTableLabel = IOHelper.getInputParameter(config, outputTableParameter); + + AnalysisLogger.getLogger().debug("EstimateFishingActivity: received parameters: " + config.getGeneralProperties()); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: input table: "+ table +" outputTable: " + outTable + " outLabel: " + outTableLabel); + + status = 10; + try{ + addInformationColumsToTable(table); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: finished in "+(System.currentTimeMillis()-t0)+" ms"); + } catch (Throwable e) { + throw new Exception(e.getMessage()); + } finally { + status = 100; + } + } + + public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){ + + String d = IOHelper.getInputParameter(config, CodeColumn); + + String select = "select *," + d + " as loforcs01 from " + table + " limit " + limit + " offset " + offset; + return select; + } + + public void addInformationColumsToTable(String table) throws Exception{ + AnalysisLogger.getLogger().debug("EstimateFishingActivity: initializing connection"); + long t0 = System.currentTimeMillis(); + try { + connection = DatabaseUtils.initDBSession(config); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: database: " + config.getDatabaseURL()); + // create a new output table + AnalysisLogger.getLogger().debug("EstimateFishingActivity: dropping table " + outTable + " if exists"); + try { + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); + } catch (Exception e) { + AnalysisLogger.getLogger().debug("EstimateFishingActivity: table " + outTable + " does not exist yet"); + } + AnalysisLogger.getLogger().debug("EstimateFishingActivity: creating the new table " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: adding new columns to " + outTable); + + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "activity_hours", "real"), connection); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "bathymetry", "real"), connection); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "activity_class_speed", "character varying"), connection); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "activity_class_speed_bath", "character varying"), connection); + + AnalysisLogger.getLogger().debug("EstimateFishingActivity: getting columns from " + outTable); + // get columns names + List names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection); + StringBuffer colnames = new StringBuffer(); + int nnames = names.size(); + for (int i = 0; i < nnames; i++) { + colnames.append(names.get(i)); + if (i < nnames - 1) + colnames.append(","); + } + AnalysisLogger.getLogger().debug("EstimateFishingActivity: columns are: " + colnames.toString()); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: taking chunks ... "); + + List vessels = DatabaseFactory.executeSQLQuery("select distinct "+config.getParam(VesselsIDColumn)+" from "+table, connection); + int nvessels = vessels.size(); + int k=0; + + //error check + if (config.getParam(VesselsIDColumn)==null || config.getParam(VesselsSpeedsColumn)==null || config.getParam(VesselsTimestampsColumn)==null || config.getParam(VesselsLongitudesColumn)==null || config.getParam(VesselsLatitudesColumn)==null) + throw new Exception ("Error with input parameters, please check that all the required inputs have been provided."); + for (Object vesselrow:vessels){ + String vesselID = ""+vesselrow; + AnalysisLogger.getLogger().debug("EstimateFishingActivity: Analysing vessel "+vesselID+" "+(k+1)+" of "+nvessels); + //extract single vessel trajectory information + String selectTrajectory = "select *, "+ + "\""+config.getParam(VesselsIDColumn)+"\""+" as fhv01,"+ + "\""+config.getParam(VesselsSpeedsColumn)+"\""+" as fhv02,"+ + "\""+config.getParam(VesselsTimestampsColumn)+"\""+" as fhv03,"+ + "\""+config.getParam(VesselsLongitudesColumn)+"\""+" as fhv04,"+ + "\""+config.getParam(VesselsLatitudesColumn)+"\""+" as fhv05 "+ + " from "+table+ + " where "+"\""+config.getParam(VesselsIDColumn)+"\""+" ="+vesselID+" order by CAST("+"\""+config.getParam(VesselsTimestampsColumn)+"\""+" as timestamp)"; + + AnalysisLogger.getLogger().debug("EstimateFishingActivity: EstimateFishingActivity: Select trajectory: "+selectTrajectory); + + List vesselTrajectory = DatabaseFactory.executeSQLQuery(selectTrajectory, connection); + + int nvesselpoints = vesselTrajectory.size(); + String[] vesselIDs = new String[nvesselpoints]; + Date[] timeStamps = new Date[nvesselpoints]; + java.awt.geom.Point2D.Double [] coordinates = new java.awt.geom.Point2D.Double[nvesselpoints]; + Tuple[] bathspeedpairs = new Tuple [nvesselpoints]; + String[] speeds = new String[nvesselpoints]; + AnalysisLogger.getLogger().debug("EstimateFishingActivity: EstimateFishingActivity: building information: "+selectTrajectory); + int i=0; + for (Object trajectorrow:vesselTrajectory){ + Object[] trajectvector = (Object[]) trajectorrow; + int lenvector = trajectvector.length; + vesselIDs[i] = ""+trajectvector[lenvector-5]; + speeds[i] = ""+trajectvector[lenvector-4]; + timeStamps[i] = DateGuesser.convertDate(""+trajectvector[lenvector-3]).getTime(); + if (timeStamps[i]==null) + throw new Exception ("Cannot parse time "+trajectvector[lenvector-3]+" for vessel "+vesselIDs[i]+". please try specifying time as MM/dd/yy KK:mm:ss a"); + if (i==0){ + String pattern = DateGuesser.getPattern(""+trajectvector[lenvector-3]); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: sample time conversion: original "+trajectvector[lenvector-3]+" guessed: "+timeStamps[i]+" pattern: "+pattern); + } + try{ + coordinates [i] = new java.awt.geom.Point2D.Double(Double.parseDouble(""+trajectvector[lenvector-2]),Double.parseDouble(""+trajectvector[lenvector-1])); + }catch(Exception e){ + AnalysisLogger.getLogger().debug("EstimateFishingActivity: Warning - wrong coordinates: "+trajectvector[lenvector-2]+","+trajectvector[lenvector-1]); + coordinates [i] = new java.awt.geom.Point2D.Double(0,0); + } + i++; + } + AnalysisLogger.getLogger().debug("EstimateFishingActivity: estimating fishing hours"); + double [] hours = FishingHoursCalculator.calculateFishingHours(vesselIDs, timeStamps); + AnalysisLogger.getLogger().debug("EstimateFishingActivity: estimating bathymetry"); + + short [] bathymetry = null; + try{ + Bathymetry bathymetryprocessor = new Bathymetry(config.getConfigPath()+"gebco_08.nc"); + bathymetry = bathymetryprocessor.compute(coordinates); + }catch(Exception e){ + AnalysisLogger.getLogger().debug("EstimateFishingActivity: Error - Bathymetry resource not available for the service "+e.getLocalizedMessage()); + throw new Exception("Error - Bathymetry resource not available for the service"); + } + for (int g=0;g(""+speeds[g],""+bathymetry[g]); + } + AnalysisLogger.getLogger().debug("EstimateFishingActivity: classifying routes"); + Tuple[] classifications = VTIClassificator.classify(bathspeedpairs); + AnalysisLogger.getLogger().debug("EstimateFishingActivity:building rows for the final table"); + List stringrows = new ArrayList(); + i=0; + for (Object trajectorrow:vesselTrajectory){ + Object[] trajectvector = (Object[]) trajectorrow; + String[] extendedrow = new String[nnames]; + for(int j=0;j templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm will add information"); + inputs.add(tinput); + + ColumnType Dimension = new ColumnType(inputTableParameter, CodeColumn, "The column containing FAO Ocean Area codes in CWP format", "GRID", false); + + inputs.add(Dimension); + + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "cwp_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public StatisticalType getOutput() { + List template = new ArrayList(); + template.add(TableTemplates.GENERIC); + OutputTable p = new OutputTable(template, outTableLabel, outTable, "Output table"); + return p; + } + + @Override + public String getDescription() { + return "An algorithm that adds longitude, latitude and resolution columns analysing a column containing FAO Ocean Area codes (CWP format)."; + } + + public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){ + + String d = IOHelper.getInputParameter(config, CodeColumn); + + String select = "select *," + d + " as loforcs01 from " + table + " limit " + limit + " offset " + offset; + return select; + } + + + public void rowToCoords (Object[] rowArray) { + // take x and y + Object grid = null; + try{ + grid=rowArray[rowArray.length - 1]; + String gridValue = ""+(int)Double.parseDouble("" + grid); + // generate csquarecodes + GridCWPConverter gridder = null; + gridder = new GridCWPConverter(); + gridder.gridCodeToPair(gridValue); + + currentLat= gridder.outlat; + currentLong=gridder.outlon; + currentRes=gridder.gridresolution; + }catch(Exception e){ + AnalysisLogger.getLogger().debug("Error converting grid: "+grid+" - "+e.getLocalizedMessage()); + currentLat= 0; + currentLong=0; + currentRes=0; + } + } + + @Override + protected void process() throws Exception { + status = 0; + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + long t0 = System.currentTimeMillis(); + String gridField = IOHelper.getInputParameter(config, CodeColumn); + String table = IOHelper.getInputParameter(config, inputTableParameter); + outTable = ("code_" + UUID.randomUUID()).replace("-", ""); + outTableLabel = IOHelper.getInputParameter(config, outputTableParameter); + + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: received parameters: code column " + gridField +", table " + table + ", outputTable " + outTable + " outLabel " + outTableLabel); + + status = 10; + if (gridField == null || gridField.trim().length() == 0 ) + throw new Exception("Error please provide information for the code column"); + try{ + addInformationColumsToTable(table); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: finished in "+(System.currentTimeMillis()-t0)+" ms"); + + } catch (Throwable e) { + throw new Exception(e.getMessage()); + } finally { + status = 100; + } + } + + public void addInformationColumsToTable(String table) throws Exception{ + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: initializing connection"); + long t0 = System.currentTimeMillis(); + try { + connection = DatabaseUtils.initDBSession(config); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: database: " + config.getDatabaseURL()); + // create a new output table + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: dropping table " + outTable + " if exists"); + try { + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); + } catch (Exception e) { + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: table " + outTable + " does not exist yet"); + } + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: creating the new table " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: adding new columns to " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "long_estim", "real"), connection); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "lat_estim", "real"), connection); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, "res_estim", "real"), connection); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: getting columns from " + outTable); + // get columns names + List names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection); + StringBuffer colnames = new StringBuffer(); + int nnames = names.size(); + for (int i = 0; i < nnames; i++) { + colnames.append(names.get(i)); + if (i < nnames - 1) + colnames.append(","); + } + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: columns are: " + colnames.toString()); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: taking chunks ... "); + // take chunks of the table + int initIdx = 0; + int limit = 5000; + + long maxRows = DatabaseUtils.estimateNumberofRows(table, connection); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: estimated number of rows " + maxRows); + status = 20; + while (true) { + + String select = selectInformationForTransformation(config, table, limit, initIdx); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: executing query: "+select); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: from " + initIdx + " to " + (initIdx + limit) + " limit "+limit); + List rows = DatabaseFactory.executeSQLQuery(select, connection); + + if (rows == null || rows.size() == 0) { + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: no more rows"); + break; + } + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: transforming "); + // take x and y + List stringrows = new ArrayList(); + for (Object row : rows) { + Object[] rowArray = (Object[]) row; +// AnalysisLogger.getLogger().debug("ROW: "+Arrays.toString(rowArray)); + rowToCoords(rowArray); + String[] stringArray = new String[nnames]; + // convert all the objects into Strings + for (int k = 0; k < rowArray.length-1; k++) { + stringArray[k] = "" + rowArray[k]; + } + stringArray[nnames-3] = ""+currentLong; + stringArray[nnames-2] = ""+currentLat; + stringArray[nnames-1] = ""+currentRes; + + stringrows.add(stringArray); + } + + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: inserting chunks into the table"); + // write the vector into the table + DatabaseUtils.insertChunksIntoTable(outTable, colnames.toString(), stringrows, limit, connection, true); + initIdx = initIdx+limit; + status = Math.min(90, 20 + (70 * initIdx / maxRows)); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: status " + status); + } + + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: finished"); + } catch (Throwable e) { + e.printStackTrace(); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates : ERROR!: " + e.getLocalizedMessage()); + try { + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: dropping " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); + } catch (Exception e1) { + AnalysisLogger.getLogger().debug("GridCWP2Coordinates: could not drop " + outTable); + } + throw new Exception(e.getMessage()); + } finally { + shutdown(); + AnalysisLogger.getLogger().debug("GridCWP2Coordinates finished in " + (System.currentTimeMillis() - t0) + " ms"); + } + } + + @Override + public void init() throws Exception { + + } + + @Override + public void shutdown() { + AnalysisLogger.getLogger().debug("GridCWP2Coordinates shutdown"); + DatabaseUtils.closeDBConnection(connection); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/Bathymetry.java b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/Bathymetry.java new file mode 100644 index 0000000..4a0b2f1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/Bathymetry.java @@ -0,0 +1,94 @@ +package org.gcube.dataanalysis.geo.vti.vesselsprocessing; + + +public class Bathymetry { + + private static VTIBathymetry bathymetryObj; + + public static void init() throws Exception { + bathymetryObj.open(); + } + + public static void close() throws Exception { + bathymetryObj.close(); + } + + public static void initInstance(String bathfile ) throws Exception{ + if (bathymetryObj==null){ + bathymetryObj = new VTIBathymetry(bathfile); + try { + init(); + } catch (Exception e) { + e.printStackTrace(); + close(); + } + } + } + public Bathymetry(String bathfile) throws Exception { + + initInstance(bathfile); + } + + public synchronized short[] compute(java.awt.geom.Point2D.Double [] points) throws Exception{ + + return bathymetryObj.getZ(points); + + } + + public static void main(String[] args) { + + VTIBathymetry vti = new VTIBathymetry("./cfg/gebco_08.nc"); + try { + long t0 = System.currentTimeMillis(); + vti.open(); + System.out.println("file open"); + + long t00 = System.currentTimeMillis(); + + // x = long + // y = lat + + double z = vti.getZ(1.75, 0.25); + System.out.println("Z:" + z); + long t01 = System.currentTimeMillis(); + System.out.println("computation elapsed " + (t01 - t00) + " ms"); + + t00 = System.currentTimeMillis(); + java.awt.geom.Point2D.Double[] pp = new java.awt.geom.Point2D.Double[9]; + + java.awt.geom.Point2D.Double point = new java.awt.geom.Point2D.Double(0.25f, 0.75f); + pp[0] = point; + point = new java.awt.geom.Point2D.Double(0.25f, 0.75f); + pp[1] = point; + point = new java.awt.geom.Point2D.Double(50f, 75f); + pp[2] = point; + point = new java.awt.geom.Point2D.Double(45f, 0.75f); + pp[3] = point; + point = new java.awt.geom.Point2D.Double(30f, 0.75f); + pp[4] = point; + point = new java.awt.geom.Point2D.Double(25f, 0.75f); + pp[5] = point; + point = new java.awt.geom.Point2D.Double(90f, 0.75f); + pp[6] = point; + point = new java.awt.geom.Point2D.Double(180f, 180f); + pp[7] = point; + point = new java.awt.geom.Point2D.Double(0.25f, 0.75f); + pp[8] = point; + + short[] zz = vti.getZ(pp); + System.out.println("ZZ:" + zz); + t01 = System.currentTimeMillis(); + System.out.println("computation elapsed " + (t01 - t00) + " ms"); + for (short z1 : zz) + System.out.println("Zz:" + z1); + + vti.close(); + System.out.println("file closed"); + long t1 = System.currentTimeMillis(); + System.out.println("elapsed " + (t1 - t0) + " ms"); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/FishingHoursCalculator.java b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/FishingHoursCalculator.java new file mode 100644 index 0000000..5261d21 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/FishingHoursCalculator.java @@ -0,0 +1,48 @@ +package org.gcube.dataanalysis.geo.vti.vesselsprocessing; + +import java.util.Date; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; + +public class FishingHoursCalculator { + + + + + public static double[] calculateFishingHours(String [] vesselIDsField,Date[] timeStamps){ + int vNumber = vesselIDsField.length; + double [] hours = new double[vNumber]; + Date previousDate = null; + String previousVessel = null; + for (int i=0;i=0) && (timediff<=4*60*60*1000)){ + if ((timediff>=0) && (timediff<=14400000)){ + hours [i] = MathFunctions.roundDecimal((double)timediff/(double)(60*60*1000),2); + } + else + hours [i] = 0.0f; + } + +// System.out.println(" hours:"+hours[i]); + + previousDate = timeStamps[i]; + previousVessel = vesselIDsField[i]; + } + +// System.exit(0); + return hours; + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/MonthlyFishingEffortCalculator.java b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/MonthlyFishingEffortCalculator.java new file mode 100644 index 0000000..e1a10f6 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/MonthlyFishingEffortCalculator.java @@ -0,0 +1,97 @@ +package org.gcube.dataanalysis.geo.vti.vesselsprocessing; + +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter; + +public class MonthlyFishingEffortCalculator { + + private static int monthsDifference(Date minDate, Date maxDate) { + Calendar min = Calendar.getInstance(); + min.setTime(minDate); + Calendar max = Calendar.getInstance(); + max.setTime(maxDate); + + int monthsDiff = (max.get(Calendar.YEAR) - min.get(Calendar.YEAR))*12 + (max.get(Calendar.MONTH) - min.get(Calendar.MONTH)) + ((max.get(Calendar.DAY_OF_MONTH) - min.get(Calendar.DAY_OF_MONTH)) / 27); + + return monthsDiff; + } + + // positional Object: x,y,date,hours + private double minEffort; + private double maxEffort; + + public Map calculateMonthlyFishingEffort(List rows, Date minDate, Date maxDate) { + + HashMap csquare2month = new HashMap(); + HashMap csquaremonths = new HashMap(); + int numOfPoints = rows.size(); + int numOfMonths = monthsDifference(minDate, maxDate); + System.out.println("min:"+minDate+" max "+maxDate+" months diffs "+numOfMonths); + + for (int i = 0; i < numOfPoints; i++) { + Object[] singleRow = (Object[]) rows.get(i); + double x = Float.parseFloat(""+singleRow[0]); + double y = Float.parseFloat(""+singleRow[1]); + Date d = (Date) singleRow[2]; + double hours = (Float) singleRow[3]; + double speed = Double.parseDouble("" + singleRow[4]); + + if ((speed >= 2) && (speed <= 6)) { + String csquare = CSquareCodesConverter.convertHalfDegree(x, y); + double[] csquaremontheffort = csquaremonths.get(csquare); + + if (csquaremontheffort == null) + csquaremontheffort = new double[numOfMonths + 1]; + + int index = monthsDifference(minDate, d); + csquaremontheffort[index] = csquaremontheffort[index] + hours; + // update the map + csquaremonths.put(csquare, csquaremontheffort); + } + } + + // calculate mean monthly effort + for (String keySquare : csquaremonths.keySet()) { + + double[] monthsEffort = csquaremonths.get(keySquare); + + double sum = 0; + for (int i = 0; i < numOfMonths + 1; i++) { + sum += monthsEffort[i]; + } + sum = sum / (double) (numOfMonths + 1); + sum = MathFunctions.roundDecimal(sum, 2); + if (sum > maxEffort) + maxEffort = sum; + if (sum < minEffort) + minEffort = sum; + + csquare2month.put(keySquare, sum); + } + + return csquare2month; + } + + public double getMinEffort() { + return minEffort; + } + + public void setMinEffort(double minEffort) { + this.minEffort = minEffort; + } + + public double getMaxEffort() { + return maxEffort; + } + + public void setMaxEffort(double maxEffort) { + this.maxEffort = maxEffort; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIBathymetry.java b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIBathymetry.java new file mode 100644 index 0000000..d6f39e1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIBathymetry.java @@ -0,0 +1,290 @@ +package org.gcube.dataanalysis.geo.vti.vesselsprocessing; + +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; + +/** + * Class for obtaining altimetry and bathymetry values from a netCDF file. + * Requires the netCDF libraries (see here: http://www.unidata.ucar.edu/software/netcdf-java/documentation.htm) + * + * You should use the class in the following way: + * 
+ *   VTIBathymetry bath = new VTIBathymetry("/path/to/bath.nc");   // e.g. gebco_08.nc
+ *   bath.open();                                                  // file must be explicitly opened, this takes a short time, avoid opening it several times
+ *   short value1 = bath.getZ(142.2, 11.35);                       // e.g. Mariana Trench
+ *   short value2 = bath.getZ(0, 0);
+ *   // ...
+ *   bath.close();                                                 // file should be closed when not needed any longer
+ *
+ * + * @author Frank Loeschau, Terradue Srl. + */ +public class VTIBathymetry { + + private String filename; + private String mode; + private java.awt.geom.Point2D.Double[] points; + private ucar.nc2.NetcdfFile ncFile; + private ucar.nc2.Variable var; + //private ucar.ma2.Array arr; + private boolean debug = false; + public final int GRID_COLS = 360 * 120; + public final int GRID_ROWS = 180 * 120; + + /** + * The class can also be used as command-line tool. + * The syntax is:

java org.d4science2.vtivre.VTIBathymetry get x1,y1 [x2,y2 [...]] filename

+ * If only one pair of coordinates is specified, debug information about the calculation is written. + * @param args the command-line arguments + */ + public static void main(String[] args) { + int status = 0; + VTIBathymetry bm = new VTIBathymetry(); + + try { + if (!bm.parseArguments(args)) System.exit(1); + + bm.open(); + + short[][] res = new short[361][181]; + + if (bm.mode.compareTo("draw") == 0) { + bm.draw(); + } else if (bm.mode.compareTo("get") == 0) { + bm.debug = false; + + if (bm.points == null || bm.points.length == 0) { + System.err.println("No point specified"); + System.exit(1); + } else if (bm.points.length == 1) { + bm.debug = true; + short z = bm.getZ(bm.points[0]); + System.out.println(z); + } else { + short[] zs = bm.getZ(bm.points); + for (int i = 0; i < zs.length; i++) System.out.println(zs[i]); + } + } + + } catch (Exception e) { + e.printStackTrace(); + System.out.println("ERROR: " + e.getMessage()); + status = 1; + } finally { + try { + bm.close(); + } catch (Exception e2) {} + } + System.exit(status); + } + + private static void printUsage() { + System.out.println("Usage: org.d4science2.vtivre.VTIBathymetry (get x,y [x2,y2 [x3,y3 ...]]) file"); + System.out.println(); + System.exit(1); + } + + private VTIBathymetry() {} + + private boolean parseArguments(String[] args) throws Exception { + try { + if (args.length >= 1) mode = args[0]; + if (args.length >= 2) filename = args[args.length - 1]; + + points = new java.awt.geom.Point2D.Double[args.length > 2 ? args.length - 2 : 0]; + for (byte i = 0; i < args.length - 2; i++) { + String[] s = args[i + 1].split(","); + if (s.length != 2) { + System.err.println("Invalid coordinate format: must be x,y (e.g. -12.34,56.78)"); + System.exit(1); + } + points[i] = new java.awt.geom.Point2D.Double(); + points[i].setLocation(Double.parseDouble(s[0]), Double.parseDouble(s[1])); + } + } catch (Exception e) { + printUsage(); + return false; + } + + if (mode == null) { + System.err.println("No mode specified"); + printUsage(); + return false; + } else if (filename == null) { + System.err.println("No filename specified"); + return false; + } + + return true; + } + + /** + * Creates an instance of VTIBathymetry with the specified file. + * @param filename the netCDF filename containing the altimetry/bathymetry data + */ + public VTIBathymetry(String filename) { + this.filename = filename; + } + + /** + * Opens the netCDF file and loads the data array. + * This operation is somewhat slow and requires the creation of a big object in memory. Ideally, this method is called only once, before the altrimetry/bathymetry values are needed. + * @throws java.io.IOException + */ + public void open() throws java.io.IOException { + ncFile = ucar.nc2.dataset.NetcdfDataset.openFile(filename, null); + var = ncFile.findVariable("z"); + //arr = var.read(); + + } + + /** + * Closes the netCDF file. + * @throws java.io.IOException + */ + public void close() throws java.io.IOException { + if (ncFile != null) ncFile.close(); + } + + /** + * Returns the altimetric/bathymetric value of the given geographical coordinates. + * The value derives from the corresponding cell in the coordinate grid (120 x 120 cells per degree). + * If a coordinate is close to the cell border (< 0.1 * arc cell size), the result value is averaged with the value from the adjacent cell. + * @param point the geographical coordinates (x = longitude value (-180 to 180), y = latitude value (-90 to 90)) + * @return the altimetric/bathymetric value for the coordinate (in meters) + * @throws java.io.IOException if the netCDF variable z could not been read + */ + public short getZ(java.awt.geom.Point2D.Double point) throws java.io.IOException { + return getZ(point.getX(), point.getY()); + } + + /** + * Returns the altimetric/bathymetric values of the given array of geographical coordinates. + * The value derives from the corresponding cell in the coordinate grid (120 x 120 cells per degree). + * If a coordinate is close to the cell border (< 0.1 * arc cell size), the result value is averaged with the value from the adjacent cell. + * @param points array of geographical coordinates (x = longitude value (-180 to 180), y = latitude value (-90 to 90)) + * @return an array of the same shape as points, containing the altimetric/bathymetric value for the coordinate (in meters) + * @throws java.io.IOException if the netCDF variable z could not been read + */ + public short[] getZ(java.awt.geom.Point2D.Double[] points) throws java.io.IOException { + short[] result = new short[points.length]; + for (int i = 0; i < points.length; i++) result[i] = getZ(points[i].getX(), points[i].getY()); + return result; + } + + /** + * Returns the altimetric/bathymetric value of the given geographical coordinates. + * The value derives from the corresponding cell in the coordinate grid (120 x 120 cells per degree). + * If a coordinate is close to the cell border (< 0.1 * arc cell size), the result value is averaged with the value from the adjacent cell. + * @param x longitude coordinate (-180 to 180) + * @param y latitude coordinate (-90 to 90) + * @return the altimetric/bathymetric value for the coordinate (in meters) + * @throws NullPointerException if the netCDF file has not been opened or the variable z (containing the altimetry/bathymetry values) is not found + * @throws java.io.IOException if the netCDF variable z could not been read + */ + public short getZ(double x, double y) throws java.lang.NullPointerException, java.io.IOException { + int resultInt = 0; + + if (ncFile == null) throw new NullPointerException("No netCDF file not open"); + if (var == null) throw new NullPointerException("Variable 'z' not found"); + + double gridColD = (x + 180) * 120; // left border of cell + boolean averageWithNextCol = false, averageWithNextRow = false; + + int gridCol, gridRow; + if (gridColD < 0 || gridColD >= GRID_COLS) { + gridCol = GRID_COLS - 1; + averageWithNextCol = true; + } else { + gridCol = (int)Math.floor(gridColD); + if (gridColD - gridCol < 0.1) { + gridCol = (gridCol + GRID_COLS - 1) % GRID_COLS; + averageWithNextCol = true; + } else if (gridColD - gridCol > 0.9) { + averageWithNextCol = true; + } + } + double gridRowD = (90 - y) * 120; // upper border of cell + if (gridRowD < 0) { + gridRow = 0; + } else if (gridRowD >= GRID_ROWS) { + gridRow = GRID_ROWS - 1; + } else { + gridRow = (int)Math.floor(gridRowD); + if (gridRowD - gridRow < 0.1 && gridRow > 0) { + gridRow--; + averageWithNextRow = true; + } else if (gridRowD - gridRow > 0.9 && gridRow < GRID_ROWS - 1) { + averageWithNextRow = true; + } + } + + int[] shape = new int[1]; + if (averageWithNextCol) shape[0] = 2; + else shape[0] = 1; + int[] origin = {GRID_COLS * gridRow + gridCol}; + + short result = 0; + try { + Array arr = var.read(origin, shape); + resultInt = arr.getShort(0); + if (averageWithNextCol) resultInt += arr.getShort(1); + if (averageWithNextRow) { + origin[0] += GRID_COLS; + arr = var.read(origin, shape); + resultInt += arr.getShort(0); + if (averageWithNextCol) resultInt += arr.getShort(1); + } + result = (short)(resultInt / ((averageWithNextCol ? 2 : 1) * (averageWithNextRow ? 2 : 1))); + } catch (InvalidRangeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + if (debug) { + System.out.println("x,y --- col,row (calculated) --- col[+],row[+] (actual): " + x + "," + y + " --- " + gridColD + "," + gridRowD + " --- " + gridCol + (averageWithNextCol ? "+" : "") + "," + gridRow + (averageWithNextRow ? "+" : "")); + + System.out.print("Grid cell indexes used: " + (GRID_COLS * gridRow + gridCol)); + if (averageWithNextCol) System.out.print(" " + (GRID_COLS * gridRow + (gridCol + 1) % GRID_COLS)); + if (averageWithNextRow) { + System.out.print(" " + (GRID_COLS * (gridRow + 1) + gridCol)); + if (averageWithNextCol) System.out.print(" " + (GRID_COLS * (gridRow + 1) + (gridCol + 1) % GRID_COLS)); + } + if (averageWithNextCol) System.out.print(", eastern adjacent cell used"); + if (averageWithNextRow) System.out.print(", southern adjacent cell used"); + System.out.println(); + + /*System.out.print("Corresponding altimetry/bathymetry values: " + arr.getShort(GRID_COLS * gridRow + gridCol)); + if (averageWithNextCol) System.out.print(" " + arr.getShort(GRID_COLS * gridRow + (gridCol + 1) % GRID_COLS)); + if (averageWithNextRow) { + System.out.print(" " + arr.getShort(GRID_COLS * (gridRow + 1) + gridCol)); + if (averageWithNextCol) System.out.print(" " + arr.getShort(GRID_COLS * (gridRow + 1) + (gridCol + 1) % GRID_COLS)); + }*/ + System.out.println(); + System.out.println("Result: " + result); + } + + return result; + } + + private void draw() { + /* try { + if (arr == null) throw new Exception("Variable 'z' not found"); + + long size = arr.getSize(); + for (int i = 0; i < 30; i++) { + for (int j = 0; j < 120; j++) { + int index = 31104000 * i + 360 * j + 15552000 + 180; + if (index >= size) return; + System.out.print(arr.getShort(index) < 0 ? "~" : "@"); + } + System.out.println(); + } + + } catch (Exception e) { + System.err.println("ERROR: " + e.getClass().getName() + ": " + e.getMessage()); + e.printStackTrace(); + }*/ + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIClassificator.java b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIClassificator.java new file mode 100644 index 0000000..a68c9e4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/vti/vesselsprocessing/VTIClassificator.java @@ -0,0 +1,85 @@ +package org.gcube.dataanalysis.geo.vti.vesselsprocessing; + +import org.gcube.dataanalysis.ecoengine.utils.Tuple; + + +public class VTIClassificator { + + private static int bathymetryThr = -500; + + public static String speedClassification(int classif){ + + if (classif==1) + return "Hauling"; + else if (classif==2) + return "Fishing"; + else if (classif==3) + return "Steaming"; + else + return "Unclassified"; + } + + public static String bathymetryClassification(int classif){ + + if (classif==1) + return "Hauling"; + else if (classif==2) + return "Trawling"; + else if (classif==3) + return "Midwater trawling"; + else if (classif==4) + return "Steaming"; + else + return "Unclassified"; + } + + public static Tuple[] classify(Tuple[] pairs){ + + Tuple[] outClasses= new Tuple[pairs.length]; + int i=0; + for (Tuple pair:pairs){ + Double speed = Double.parseDouble(pair.getElements().get(0)); + Double bathymetry = Double.parseDouble(pair.getElements().get(1)); + //Hauling, dodging, shooting + Integer c1 = 1; + Integer c2 = 1; + //Fishing + if ((speed>2)&&(speed<=5)) + c1 = 2; + //Steaming + else if (speed>5) + c1 = 3; + + //midwater trawling + if ((speed>2)&&(speed<=4)&&(bathymetry4)&&(speed<=6)) + c2 = 3; + + //trawling + else if ((speed>2)&&(speed<=4)&&(bathymetry>=bathymetryThr)) + c2 = 2; + + //Steaming + else if (speed>6) + c2 = 4; + + + + //classification suggestions by Anton: +// if depth > 500 and speed [2.5, 4] -> midwater trawling +// else if depth > 500 -> trawling +// else -> bottom trawling + + + + Tuple outTuple = new Tuple(c1,c2); + outClasses[i] = outTuple; + + i++; + } + + return outClasses; + } +}