From b69f952ce6641f2ed4b0c0d58c7d08c39d4817e4 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Fri, 11 May 2018 15:50:37 +0000 Subject: [PATCH] Refactor DataHArvesterPlugin to support scheduled execution from smart-executor refs #11756 https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167434 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 32 ++ .project | 23 + .settings/org.eclipse.jdt.core.prefs | 2 + .settings/org.eclipse.m2e.core.prefs | 4 + pom.xml | 207 +++++++++ .../AccountingDataHarvesterPlugin.java | 175 ++++++++ .../DataHarvestPluginDeclaration.java | 71 ++++ .../java/org/gcube/dataharvest/dao/Dao.java | 325 +++++++++++++++ .../gcube/dataharvest/dao/DaoException.java | 13 + .../dao/DatabaseConnectionData.java | 46 ++ .../dataharvest/dao/DatabaseDataExplorer.java | 98 +++++ .../dataharvest/dao/DatabaseManager.java | 77 ++++ .../gcube/dataharvest/datamodel/Harvest.java | 85 ++++ .../dataharvest/harvester/BasicHarvester.java | 341 +++++++++++++++ .../DataMethodDownloadHarvester.java | 145 +++++++ .../harvester/MethodInvocationHarvester.java | 115 +++++ .../harvester/ResourceCatalogueHarvester.java | 200 +++++++++ .../harvester/SocialHarvester.java | 109 +++++ .../harvester/VreUsersHarvester.java | 137 ++++++ .../org/gcube/dataharvest/utils/Utils.java | 65 +++ ...nagement.executor.plugin.PluginDeclaration | 1 + .../java/org/gcube/dataharvest/Harvester.java | 230 ++++++++++ src/test/java/org/gcube/utils/DataFiller.java | 392 ++++++++++++++++++ src/test/resources/logback-test.xml | 19 + 24 files changed, 2912 insertions(+) create mode 100644 .classpath create mode 100644 .project create mode 100644 .settings/org.eclipse.jdt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs create mode 100644 pom.xml create mode 100644 src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java create mode 100644 src/main/java/org/gcube/dataharvest/DataHarvestPluginDeclaration.java create mode 100644 src/main/java/org/gcube/dataharvest/dao/Dao.java create mode 100644 src/main/java/org/gcube/dataharvest/dao/DaoException.java create mode 100644 src/main/java/org/gcube/dataharvest/dao/DatabaseConnectionData.java create mode 100644 src/main/java/org/gcube/dataharvest/dao/DatabaseDataExplorer.java create mode 100644 src/main/java/org/gcube/dataharvest/dao/DatabaseManager.java create mode 100644 src/main/java/org/gcube/dataharvest/datamodel/Harvest.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java create mode 100644 src/main/java/org/gcube/dataharvest/utils/Utils.java create mode 100644 src/main/resources/META-INF/services/org.gcube.vremanagement.executor.plugin.PluginDeclaration create mode 100644 src/test/java/org/gcube/dataharvest/Harvester.java create mode 100644 src/test/java/org/gcube/utils/DataFiller.java create mode 100644 src/test/resources/logback-test.xml diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..8dee4f3 --- /dev/null +++ b/.classpath @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..9b52a5e --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + DataHarvesterPlugin + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..4ede96d --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..3f7f075 --- /dev/null +++ b/pom.xml @@ -0,0 +1,207 @@ + + + 4.0.0 + org.gcube + DataHarvesterPlugin + 0.0.1-SNAPSHOT + SmartExecutor-DataHarvesterPlugin + Smart Executor DataHarvester Plugin + + + scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId} + scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId} + https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId} + + + + + + org.gcube.distribution + gcube-bom + LATEST + pom + import + + + + + + + org.gcube.vremanagement + smart-executor-api + [1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT) + provided + + + org.slf4j + slf4j-api + provided + + + org.gcube.vremanagement + smart-executor-client + [1.3.0-SNAPSHOT,2.0.0-SNAPSHOT) + provided + + + org.gcube.core + common-encryption + provided + + + org.gcube.common + home-library + [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + provided + + + org.gcube.common + home-library-jcr + [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + provided + + + org.gcube.resources.discovery + ic-client + provided + + + org.gcube.resources + registry-publisher + provided + + + org.gcube.resources + common-gcore-resources + + + org.gcube.information-system + information-system-model + + + org.gcube.information-system + gcube-resources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.gcube.portlets.user + gcube-url-shortener + [1.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + compile + + + org.gcube.accounting + accounting-analytics + [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + + + + org.gcube.accounting + accounting-analytics-persistence-couchbase + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + provided + + + org.gcube.accounting + accounting-lib + provided + + + + org.ancoron.postgresql + org.postgresql + 9.1.901.jdbc4.1-rc9 + compile + + + org.json + json + 20171018 + compile + + + + + + + maven-compiler-plugin + 2.3.2 + + + org.apache.maven.plugins + maven-assembly-plugin + + + servicearchive + install + + single + + + + ${distroDirectory}/descriptor.xml + + + + + package + + single + + + + jar-with-dependencies + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java new file mode 100644 index 0000000..873e030 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -0,0 +1,175 @@ +package org.gcube.dataharvest; + +import java.util.Date; +import java.util.Calendar; +import java.util.List; +import java.util.Map; + +import org.gcube.dataharvest.DataHarvestPluginDeclaration; +import org.gcube.vremanagement.executor.exception.InputsNullException; +import org.gcube.vremanagement.executor.exception.InvalidInputsException; +import org.gcube.vremanagement.executor.plugin.Plugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.gcube.dataharvest.dao.*; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester; +import org.gcube.dataharvest.harvester.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester; +import org.gcube.dataharvest.harvester.SocialHarvester; +import org.gcube.dataharvest.harvester.VreUsersHarvester; + +public class AccountingDataHarvesterPlugin extends Plugin { + + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class); + public static final String PARAMETER_FROM = "from"; + public static final String PARAMETER_TO = "to"; + public static final String TEST = "test"; + private boolean testMode = false, updateFlag = false; + private Date dateFrom, dateTo; + + public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) { + super(pluginDeclaration); + logger.debug("DataHarvestPlugin: constructor"); + } + + /** {@inheritDoc} */ + @Override + public void launch(Map inputs) throws Exception { + logger.debug("DataHarvestPlugin: launch()"); + String message = ""; + DatabaseManager dbaseManager = new DatabaseManager(testMode); + + if (inputs == null) { + message = "Invalid input parameter (null)"; + logger.error(message); + throw new Exception(message, null); + } + + Object from = inputs.get(PARAMETER_FROM); + if (!checkDate(from)) { + message = "Missing or invalid input parameter: " + PARAMETER_FROM; + logger.error(message); + throw new Exception(message, null); + } + dateFrom = (Date)from; + + Object to = inputs.get(PARAMETER_TO); + if (!checkDate(to)) { + message = "Missing or invalid input parameter: " + PARAMETER_TO; + logger.error(message); + throw new Exception(message, null); + } + dateTo = (Date)to; + + testMode = (inputs.get(TEST) == null) ? false : true; + logger.debug("TEST mode is " + testMode); + + setTimePeriod(); + logger.debug("Time period: " + dateFrom.toString() + " -> " + dateTo.toString()); + + try { + // collecting info on VRE users + VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo); + List users = vreUsersHarvester.getData(); + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, users, updateFlag); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods) + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo); + List res = resourceCatalogueHarvester.getData(); + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on Data/Method download + DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo); + List res = dataMethodDownloadHarvester.getData(); + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on social (posts, replies and likes) + SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo); + List res = socialHarvester.getData(); + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + + try { + // collecting info on method invocation + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo); + List res = methodInvocationHarvester.getData(); + //insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + } + + /** {@inheritDoc} */ + @Override + protected void onStop() throws Exception { + logger.debug("DataHarvestPlugin: onStop()"); + } + + private boolean checkDate(Object date) { + if (date == null) + return false; + return date instanceof java.util.Date; + } + +// private void insertMonthlyData(Date from, Date to, List data) { +// Dao dao = null; +// try { +// dao = dbConnect(); +// dao.insertMonthlyMeasure(data, from, to, false); +// } catch (Exception e) { +// logger.error(e.getLocalizedMessage()); +// } finally { +// if (dao != null) { +// try { +// dao.disconnect(); +// } catch (DaoException e) { +// logger.error(e.getLocalizedMessage()); +// } +// } +// } +// } +// +// private Dao dbConnect() throws DaoException { +// DatabaseDataExplorer dde = new DatabaseDataExplorer(); +// dde.setTestMode(testMode); +// DatabaseConnectionData dcd = dde.retrieveDatabaseInfo(); +// Dao dao = new Dao(); +// dao.init(); +// dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword()); +// return dao; +// } + + private void setTimePeriod() { + Calendar from = Calendar.getInstance(); + from.setTime(dateFrom); + from.set(Calendar.HOUR_OF_DAY, 0); + from.set(Calendar.MINUTE, 0); + from.set(Calendar.SECOND, 0); + dateFrom = from.getTime(); + + Calendar to = Calendar.getInstance(); + to.setTime(dateTo); + to.set(Calendar.HOUR_OF_DAY, 23); + to.set(Calendar.MINUTE, 59); + to.set(Calendar.SECOND, 59); + dateTo = to.getTime(); + } + +} diff --git a/src/main/java/org/gcube/dataharvest/DataHarvestPluginDeclaration.java b/src/main/java/org/gcube/dataharvest/DataHarvestPluginDeclaration.java new file mode 100644 index 0000000..604b010 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/DataHarvestPluginDeclaration.java @@ -0,0 +1,71 @@ +package org.gcube.dataharvest; + +import java.util.HashMap; +import java.util.Map; + +import org.gcube.dataharvest.AccountingDataHarvesterPlugin; +import org.gcube.vremanagement.executor.plugin.Plugin; +import org.gcube.vremanagement.executor.plugin.PluginDeclaration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataHarvestPluginDeclaration implements PluginDeclaration { + + private static Logger logger = LoggerFactory.getLogger(DataHarvestPluginDeclaration.class); + public static final String NAME = "AccountingDataHarvester"; + public static final String DESCRIPTION = "Data Harvest for Accounting Summary Dashboard"; + public static final String VERSION = "1.0.0"; + + /**{@inheritDoc}*/ + @Override + public void init() { + logger.debug(String.format("%s initialized", AccountingDataHarvesterPlugin.class.getSimpleName())); + } + + /**{@inheritDoc}*/ + @Override + public String getName() { + return NAME; + } + + /**{@inheritDoc}*/ + @Override + public String getDescription() { + return DESCRIPTION; + } + + /**{@inheritDoc}*/ + @Override + public String getVersion() { + return VERSION; + } + + /**{@inheritDoc}*/ + @Override + public Map getSupportedCapabilities() { + Map discoveredCapabilities = new HashMap(); + discoveredCapabilities.put("FakeKey", "FakeValue"); + return discoveredCapabilities; + } + + /**{@inheritDoc}*/ + @Override + public Class> getPluginImplementation() { + return AccountingDataHarvesterPlugin.class; + } + + @Override + public String toString(){ + return String.format("{" + + "name:%s," + + "version:%s," + + "description:%s," + + "pluginImplementation:%s," + + "}", + getName(), + getVersion(), + getDescription(), + getPluginImplementation().getClass().getSimpleName()); + } + +} diff --git a/src/main/java/org/gcube/dataharvest/dao/Dao.java b/src/main/java/org/gcube/dataharvest/dao/Dao.java new file mode 100644 index 0000000..aab5684 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/dao/Dao.java @@ -0,0 +1,325 @@ +package org.gcube.dataharvest.dao; + +import java.sql.Array; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; + +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; +import org.postgresql.util.PSQLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Dao { + private Connection conn = null; + private static Logger logger = LoggerFactory.getLogger(Dao.class); + + /** + * Class constructor. This method must be called before any other class + * method. + * + * @throws DaoException + */ + public void init() throws DaoException { + try { + Class.forName("org.postgresql.Driver"); + } catch (ClassNotFoundException ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } + } + + /** + * Connect a database + * + * @param url + * @param user + * @param password + * @throws DaoException + */ + public void connect(String url, String user, String password) throws DaoException { + try { + conn = DriverManager.getConnection(url, user, password); + logger.debug("Connected to: " + url); + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } + } + + /** + * Release a database connection + * + * @throws DaoException + */ + public void disconnect() throws DaoException { + try { + if (conn != null) { + conn.close(); + conn = null; + logger.debug("Disconnecting from database"); + } + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } + } + + /** + * Check the connection with the database. + * + * @return true|false + */ + public boolean isConnected() { + return (!(conn == null)); + } + + /** + * Getter for the database connection + * + * @return Connection class + */ + public Connection getConnection() { + return conn; + } + + /** + * Read on the database all active VRES + * + * @return + */ + public String[] getActiveVres() { + String query = "select dname from context where dname is not null and dismissed is null"; + logger.debug(query); + Statement s = null; + ResultSet rs = null; + ArrayList list = new ArrayList<>(); + try { + s = conn.createStatement(); + rs = s.executeQuery(query); + while (rs.next()) { + list.add(rs.getString("dname")); + } + + return list.toArray(new String[list.size()]); + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage()); + return null; + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException ex) { + // do nothing + } + } + if (s != null) { + try { + s.close(); + } catch (SQLException ex) { + // do nothing + } + } + } + } + + /** + * This method insert/update data in the monthly_measure database table. + * + * @param data + * @param from + * @param to + * @throws DaoException + */ + public void insertMonthlyMeasure(List data, java.util.Date from, java.util.Date to, boolean doUpdate) + throws DaoException { + // first of all: check if data of the same type are already in the + // database. + // In this case data will be updated. + Calendar cFrom = Utils.dateToCalendar(from); + Calendar cTo = Utils.dateToCalendar(to); + int monthFrom = cFrom.get(Calendar.MONTH); + int yearFrom = cFrom.get(Calendar.YEAR); + int monthTo = cTo.get(Calendar.MONTH); + int yearTo = cTo.get(Calendar.YEAR); + + if ((monthFrom != monthTo) || (yearFrom != yearTo)) { + String err = "Invalid time period. The time period MUST refer one month."; + logger.error(err); + throw new DaoException(err, null); + } + + if (data == null || data.size() <= 0) { + String err = "No data passed in input. Aborting operation."; + logger.error(err); + throw new DaoException(err, null); + } + monthFrom++; // because january = 0... + + try { + for (Harvest harvest : data) { + String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType() + + " and context_id=(select id from context where dname='" + harvest.getContext() + + "') and month=" + monthFrom + " and year=" + yearFrom; + logger.debug(query); + Statement sel = conn.createStatement(); + ResultSet rs = sel.executeQuery(query); + if (rs.next()) { + if (doUpdate) { + // record found: update it + Statement s = conn.createStatement(); + int id = rs.getInt("id"); + String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id=" + + id; + logger.debug(update); + s.execute(update); + s.close(); + } else { + logger.warn("Skipped " + harvest.getContext()); + } + } else { + // record not found: insert new record + Statement s = conn.createStatement(); + String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values ("; + insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType() + + ","; + insert += "(select id from context where dname='" + harvest.getContext() + "'),"; + insert += "'" + yearFrom + "-" + monthFrom + "-01'"; + insert += ")"; + logger.debug(insert); + s.execute(insert); + s.close(); + } + + rs.close(); + sel.close(); + + } + } catch (PSQLException x) { + // handled exception: try to iterate... + logger.error(x.getLocalizedMessage()); + } catch (Exception x) { + // not handled exception: stop + logger.error(x.getLocalizedMessage()); + throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x); + } + } + + public ArrayList getSubTree(Integer rootId) throws DaoException { + String queryBase = "select id from context where context_id in (%%)"; + ArrayList subTree = new ArrayList<>(); + ArrayList temp = new ArrayList<>(); + Statement s = null; + ResultSet rs = null; + temp.add(rootId); + subTree.add(rootId); + boolean again = true; + for (int i = 0; (i < 10) && (again); i++) { + try { + String listId = ""; + for (Integer id : temp) { + listId += "," + id; + } + listId = listId.substring(1); + + String query = queryBase.replace("%%", listId); + s = conn.createStatement(); + rs = s.executeQuery(query); + if (rs.next()) { + temp = new ArrayList<>(); + Integer dbId = rs.getInt("id"); + subTree.add(dbId); + temp.add(dbId); + while (rs.next()) { + dbId = rs.getInt("id"); + subTree.add(dbId); + temp.add(dbId); + } + } + else { + again = false; + } + rs.close(); + s.close(); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x); + } + } + + return subTree; + } + + public void createSocialReport(int contextId, int orderBy) throws DaoException { + Statement rep = null; + ResultSet rs = null; + try { + String report = "insert into report (context_id, name, orderby, chart_type, x_text, y_text) values "; + report += "(" + contextId + ", \'VRE Social\', " + orderBy + + ", \'column\', \'Period\', \'Social interaction\')"; + logger.debug(report); + rep = conn.createStatement(); + rep.execute(report); + String query = "select id from report where name=\'VRE Social\' order by id desc"; + rs = rep.executeQuery(query); + if (rs.next()) { + int id = rs.getInt("id"); + String reportItem = "insert into report_item (report_id, type_id_1, type_id_1_name) "; + reportItem += "values (" + id + ", 8, \'mt\'), (" + id + ", 9, \'mt\'), (" + id + ", 10, \'mt\')"; + rep.execute(reportItem); + } else + throw new DaoException("No report id.", null); + + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (rep != null) { + rep.close(); + } + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + } + } + } + + + /** + * Dummy tester + * + * @Deprecated + * @param originator + * @throws DaoException + * + */ + public void dummyTest(String originator) throws DaoException { + String insert = "insert into dummy (now, originator) values(current_timestamp, '" + originator + "')"; + logger.debug(insert); + Statement s = null; + try { + s = conn.createStatement(); + s.execute(insert); + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } finally { + if (s != null) + try { + s.close(); + } catch (SQLException ex) { + logger.error(ex.getLocalizedMessage()); + } + } + } + +} diff --git a/src/main/java/org/gcube/dataharvest/dao/DaoException.java b/src/main/java/org/gcube/dataharvest/dao/DaoException.java new file mode 100644 index 0000000..f46c53e --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/dao/DaoException.java @@ -0,0 +1,13 @@ +package org.gcube.dataharvest.dao; + +public class DaoException extends Exception { + + public DaoException() { + super(); + } + + public DaoException(String message, Throwable throwable) { + super(message, throwable); + } + +} diff --git a/src/main/java/org/gcube/dataharvest/dao/DatabaseConnectionData.java b/src/main/java/org/gcube/dataharvest/dao/DatabaseConnectionData.java new file mode 100644 index 0000000..f2383b1 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/dao/DatabaseConnectionData.java @@ -0,0 +1,46 @@ +package org.gcube.dataharvest.dao; + +public class DatabaseConnectionData { + String url, user, password; + + public DatabaseConnectionData() { + this.url = null; + this.user = null; + this.password = null; + } + + public DatabaseConnectionData(String url, String user, String password) { + this.url = url; + this.user = user; + this.password = password; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + @Override + public String toString() { + return "DatabaseConnectionData [url=" + url + ", user=" + user + ", password=" + password + "]"; + } +} diff --git a/src/main/java/org/gcube/dataharvest/dao/DatabaseDataExplorer.java b/src/main/java/org/gcube/dataharvest/dao/DatabaseDataExplorer.java new file mode 100644 index 0000000..0b5e8ce --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/dao/DatabaseDataExplorer.java @@ -0,0 +1,98 @@ +package org.gcube.dataharvest.dao; + +import java.util.List; + +import org.gcube.common.encryption.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatabaseDataExplorer { + private static Logger logger = LoggerFactory.getLogger(DatabaseDataExplorer.class); + private boolean testMode = false; + private boolean productionMode = false; + + public DatabaseDataExplorer() { + } + + /** + * setter method for testMode member. this member is set to TRUE if you want + * avoid to use production database during develop/test phase + * + * @param testMode + */ + public void setTestMode(boolean testMode) { + this.testMode = testMode; + } + + /** + * getter method for testMode member. + * + * @return + */ + public boolean getTestMode() { + return testMode; + } + + public DatabaseConnectionData retrieveDatabaseInfo() throws DaoException { + DatabaseConnectionData dcd = null; + if ((testMode == false) && (productionMode == false)) { + String jdbcUrl = "jdbc:postgresql://"; + dcd = new DatabaseConnectionData(); + try { + SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq 'Database'") + .addCondition("$resource/Profile/Name/text() eq 'Analytics Board'"); + DiscoveryClient client = ICFactory.client(); + List list = client.submit(query); + if (!list.isEmpty()) { + String xml = list.iterator().next(); + + String server = getItemValueFromXmlString(xml, "Endpoint"); + String db = getItemAttributeFromXmlString(xml, "Endpoint", "EntryName"); + jdbcUrl += server + "/" + db; + dcd.setUrl(jdbcUrl); + + String user = getItemValueFromXmlString(xml, "Username"); + dcd.setUser(user); + + String password = StringEncrypter.getEncrypter() + .decrypt(getItemValueFromXmlString(xml, "Password")); + dcd.setPassword(password); + logger.debug("Database data found"); + } + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage()); + throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); + } + } else if(testMode) { + logger.debug("TEST mode is ON"); + dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/analytics_b_dev", "analytics_b_dev_u", "78cb625303be21b"); + } else if(productionMode) { + logger.warn("PRODUCTION mode is FORCED ON"); + dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv.d4science.org:5432/analytics_board", "analytics_board_u", "b52b64ab07ea0b5"); + } + + return dcd; + } + + private String getItemValueFromXmlString(String xml, String item) { + int start = xml.indexOf("<" + item); + int begin = xml.indexOf(">", start); + int end = xml.indexOf("<", begin); + String value = xml.substring(begin + 1, end); + return value; + } + + private String getItemAttributeFromXmlString(String xml, String item, String attribute) { + int start = xml.indexOf("<" + item); + start = xml.indexOf(attribute, start); + int begin = xml.indexOf("\"", start); + int end = xml.indexOf("\"", begin + 1); + String attributeValue = xml.substring(begin + 1, end); + return attributeValue; + } +} diff --git a/src/main/java/org/gcube/dataharvest/dao/DatabaseManager.java b/src/main/java/org/gcube/dataharvest/dao/DatabaseManager.java new file mode 100644 index 0000000..c204fd2 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/dao/DatabaseManager.java @@ -0,0 +1,77 @@ +package org.gcube.dataharvest.dao; + +import java.util.Date; +import java.util.List; + +import org.gcube.dataharvest.datamodel.Harvest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatabaseManager { + private static Logger logger = LoggerFactory.getLogger(DatabaseManager.class); + private boolean testMode = false; + + public DatabaseManager(boolean testMode) { + this.testMode = testMode; + } + + public DatabaseManager() { + this.testMode = false; + } + + public void setTestMode(boolean testMode) { + this.testMode = testMode; + } + + public boolean isTestMode() { + return testMode; + } + + public void insertMonthlyData(Date from, Date to, List data, boolean updateFlag) { + Dao dao = null; + try { + dao = dbConnect(testMode); + dao.insertMonthlyMeasure(data, from, to, updateFlag); + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + } finally { + if (dao != null) { + try { + dao.disconnect(); + } catch (DaoException e) { + logger.error(e.getLocalizedMessage()); + } + } + } + } + + public String[] getActiveVres() { + Dao dao = null; + try { + dao = dbConnect(testMode); + return dao.getActiveVres(); + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + return null; + } finally { + if (dao != null) { + try { + dao.disconnect(); + } catch (DaoException e) { + logger.error(e.getLocalizedMessage()); + } + } + } + } + + public Dao dbConnect(boolean testMode) throws DaoException { + DatabaseDataExplorer dde = new DatabaseDataExplorer(); + dde.setTestMode(testMode); + DatabaseConnectionData dcd = dde.retrieveDatabaseInfo(); + Dao dao = new Dao(); + dao.init(); + dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword()); + return dao; + } + +} diff --git a/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java b/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java new file mode 100644 index 0000000..a8b982a --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java @@ -0,0 +1,85 @@ +package org.gcube.dataharvest.datamodel; + +import java.util.Date; +import java.io.Serializable; + +public class Harvest implements Serializable { + /** + * + */ + private static final long serialVersionUID = 1L; + public static int ACCESSESS = 1; + public static int USERS = 2; + public static int DATA_METHOD_DOWNLOAD = 3; + public static int NEW_CATALOGUE_METHODS = 4; + public static int NEW_CATALOGUE_DATASETS = 5; + public static int NEW_CATALOGUE_DELIVERABLES = 6; + public static int NEW_CATALOGUE_APPLICATIONS = 7; + public static int SOCIAL_POSTS = 8; + public static int SOCIAL_REPLIES = 9; + public static int SOCIAL_LIKES = 10; + public static int METHOD_INVOCATIONS = 11; + public static int VISUAL_TOOLS = 12; + + private int dataType; + private String context; + private long measure; + private Date day; + + public Harvest() { + + } + + public Harvest(int dataType, String context, long measure, Date day) { + super(); + this.dataType = dataType; + this.context = context; + this.measure = measure; + this.day = day; + } + + public Harvest(int dataType, String context, long measure) { + this.dataType = dataType; + this.context = context; + this.measure = measure; + } + + + public void setDataType(int dataType) { + this.dataType = dataType; + } + + public void setContext(String context) { + this.context = context; + } + + public void setMeasure(long measure) { + this.measure = measure; + } + + public void setDay(Date day) { + this.day = day; + } + + public int getDataType() { + return dataType; + } + + public String getContext() { + return context; + } + + public long getMeasure() { + return measure; + } + + public Date getDay() { + return day; + } + + @Override + public String toString() { + return "Harvest [context=" + context + ", dataType=" + dataType + ", measure=" + measure + "]"; + } + +} \ No newline at end of file diff --git a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java new file mode 100644 index 0000000..2053063 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java @@ -0,0 +1,341 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.InputStream; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Properties; + +import org.gcube.dataharvest.dao.Dao; +import org.gcube.dataharvest.dao.DatabaseManager; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.encryption.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.informationsystem.publisher.RegistryPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BasicHarvester { + private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class); + public Date startDate = null, endDate = null; + public DateFormat format; + public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester"; + public final String CATEGORY_NAME = "Accounting"; + + public BasicHarvester(String start, String end) throws ParseException { + format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH); + startDate = format.parse(start); + endDate = format.parse(end); + } + + public BasicHarvester(Date start, Date end) throws ParseException { + startDate = start; + endDate = end; + } + + public static String getCurrentContext(String token) throws Exception{ + AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); + String context = authorizationEntry.getContext(); + logger.info("Context of token {} is {}", token, context); + return context; + } + + + public static void setContext(String token) throws Exception{ + SecurityTokenProvider.instance.set(token); + ScopeProvider.instance.set(getCurrentContext(token)); + } + + public static String getCurrentContext() throws Exception { + String token = SecurityTokenProvider.instance.get(); + return getCurrentContext(token); + } + + public List getData() throws Exception { + return null; + } + + public Properties readServiceEndpoint() throws Exception { + Properties props = new Properties(); + String scope = "/d4science.research-infrastructures.eu"; + ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'"); + query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + String password = null; + try { + List list = client.submit(query); + if (list.size() > 1) { + logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME + + " in this scope having Category " + CATEGORY_NAME); + } else if (list.size() == 0) { + logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME + + " and Category " + CATEGORY_NAME + " in this scope " + scope); + } else { + for (ServiceEndpoint res : list) { + AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints() + .toArray(new AccessPoint[res.profile().accessPoints().size()]); + for (AccessPoint found : accessPoints) { + password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password()); + for (ServiceEndpoint.Property prop : found.properties()) { + props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value())); + } + } + } + } + return props; + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + throw e; + } + } + + public void updateServiceEndPoint(ArrayList newProps) throws Exception { + String scope = "/d4science.research-infrastructures.eu"; + ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'"); + query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + String password = null; + try { + List list = client.submit(query); + if (list.size() > 1) { + logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME + + " in this scope having Category " + CATEGORY_NAME); + } else if (list.size() == 0) { + logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME + + " and Category " + CATEGORY_NAME + " in this scope " + scope); + } else { + ServiceEndpoint serviceEndpoint = list.get(0); + AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0]; + Group oldProps = accessPoint.properties(); + for (Property prop : newProps) { + String propValue = prop.value(); + propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue); + String propKey = prop.name(); + org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd = + new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue); + pToAdd.encrypted(true); + accessPoint.properties().add(pToAdd); + } + + RegistryPublisher publisher = RegistryPublisherFactory.create(); + ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); + publisher.update(serviceEndpoint); + } + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + throw e; + } + } + + public String[] getActiveVREs(boolean testMode) { + DatabaseManager dbaseManager = new DatabaseManager(testMode); + return dbaseManager.getActiveVres(); + } + +// public static void main(String[] args) { +//// String[][] data = { +//// {"/d4science.research-infrastructures.eu","3fdf5a02-383c-4364-a9c6-d3e2b0a4a401-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research","1ff82b79-4042-4311-84e1-a11f955b5f7f-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","f920ca10-af4d-4b70-9fa8-75f7bb78987c-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/fosteropenscience","188fb0fa-083b-4043-94a6-62152cd7b94e-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/H2020greeneu","ed045fb9-805b-4d6c-a72a-d0a464e7a5dc-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/FAIR_DM","9429cc8d-a3d3-43f6-b793-e29be7e7bfba-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","521908fe-deab-40df-ab59-61a8a3fb7325-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","1b9caa78-004a-4173-a2f0-6f67f7904630-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","5bd19b68-aecf-4b10-8ca8-ff88f36ca350-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/AGINFRAplus","6de2ce44-7a7e-4da5-875a-5858eae8bd37-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/FisheriesandEcosystemAtMII","4928bb4b-9da5-426a-88ce-2f3a4853b665-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","8019bf10-3d31-4eaa-af48-cb327eceabac-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","bed1d5b5-106a-493e-a814-6d5cc99b4b93-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/OpenAIRE-Connect_Coordination","951c4abc-1850-49f3-a852-099d191de708-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/EuBrazilOpenBio","4449d0be-83a3-4966-bac4-beaca76577fa-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","6f66e93f-84bf-4171-83e5-a49d4d470465-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ENVRI","8194d45e-baa5-4d0a-9d27-3c81eb8b867d-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/DEMETER","975bf904-1fd7-437f-bb13-8e196ce24aab-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/InfraTraining","f392969e-d2d4-4414-9d62-fb9675ea2056-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/NERLiX","21a34e10-a4ae-4a47-8b78-2313db5a9eb2-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/E-RIHS","82ffc3ac-518f-48a7-b9c4-69f0372b49a5-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/EcoEvo","8b8e8fd9-cda5-433d-8a88-530cc26a2061-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","1c7c72a8-93ca-46d0-baee-42da39654c11-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_FIACO2017","775fb6a3-d69d-4902-b379-6d44e4d2b7dc-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/DOPA_AnalystSpatialPlanning","70a39850-264e-496b-9ce0-b764a70e4660-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/Sinay","01c6b4b5-f726-45e0-8311-a2fdbcef54ff-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/StatnMap","d71d9c9b-def6-4df3-a22f-6bcd36dce284-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ISTIOpenAccess","2c4ab315-51cd-469c-af2e-ceccd4708093-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroToREnv","b5112916-3e9d-48ab-afc0-195ceda24489-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_BNetworkAnalysis","30b7506e-a016-4438-be34-9af57af1d0c0-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/FoodborneOutbreak","b3eff03b-aac9-4bd0-a568-6870bb5559d4-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/EOSC_Services","518f0ef5-8510-46e9-9969-89d7d2c22301-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/SDG-Indicator14.4.1","1953f45d-4ff7-45ce-8232-00439f1ab48a-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/FoodSecurity","15c3a6a6-4092-473b-a296-7cd5e5eadff4-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/RAKIP_portal","39bc9d6a-39ba-4664-9c62-330f371ce073-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","a9186abb-2af0-459e-ad59-3558a8127560-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/TextCrowd","8cc715b0-36a7-480e-ad24-8ecbc6b0577d-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/EOSCPortfolio","8e499271-fe8d-4e0a-a3e2-cddef19515ef-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ParticleFormation","fec95ea3-079d-423a-8620-46a0364696fc-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICES_LogbookData","b44037d7-a4bf-4542-bc14-85c210a6087d-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","f8f18b96-8654-462e-b12d-73c265952bff-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/PerformFish","8fea1590-73f9-46e1-8303-99ef91759f58-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/EISCAT","0edb3ec5-37a1-47f2-b4be-4624a5e5f4e5-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/AgroClimaticModelling","90fa5480-de0a-48c1-83fd-2c381abbd1bf-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ENVRIPlusData4Science","8e4d9a87-0da9-45c0-9542-612f14d5b454-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/OnlineOceanography","451ce6b2-09d8-4253-ac1d-56b0c633ffa7-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/QCAPI","7a829b89-20b2-48af-8c9e-641a1acaf47a-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/SDI_Lab","76ca201a-4a89-4871-9e9d-3043ecc60be2-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ORIONKnowledgeHub","ff3228dc-4b29-4919-924d-462d9f5b9d42-843339462"}, +//// {"/d4science.research-infrastructures.eu/D4Research/ICOSEddyCovarianceProcessing","781272a4-d498-4baf-8a14-468b83839452-843339462"}, +//// {"/d4science.research-infrastructures.eu/Edison","289a3fc0-cd23-43ee-bf18-b3efc4d1a182-843339462"}, +//// {"/d4science.research-infrastructures.eu/Edison/RstudioVirtualLab","d6b795ec-2719-4ac2-a511-fe032ef43b15-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM","cb7dadd0-dbdd-46a6-8db2-1dc4ffc64759-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","f4811784-0d4a-4731-9f75-3b407544b31f-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/VME-DB","e8b93617-217e-44ac-ae02-09a45bc26886-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","17bdb8ae-cff9-44a7-ac27-c9e26a254290-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/TBTI_VRE","02c36b15-1229-47b1-a583-296ec5b404fd-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/GRSF","833d6a12-34a9-4082-9ecb-358d19244269-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/KIMAGRO_Fishfarming","b6967277-73e5-4b23-8fd6-c47e2e5a5ed2-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/D4STeam","e18b3dd5-b6fe-4865-8727-434354c24b9f-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/AquaMaps","6b08ffe9-695d-45bf-90fb-6361a4234ab5-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/iMarineBoardVRE","78f6a7be-55c5-4602-ae81-fd4955a3a72e-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","09a84b5a-b869-4552-887d-23248bc5b042-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","c6317519-b238-449c-bd3d-6c832f98a611-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","1457e707-492d-46a0-b848-6c6cd758a6ce-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","a8f98114-05b8-44b2-a89c-0f56e55412fd-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","a948e508-0ecd-4013-a094-fe3c506209e1-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/ForkysVRE","6ed093fe-f245-4638-acfe-a70a1f24146d-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","b7c16d55-8344-492b-a94b-8fbf7af27e08-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/Aquabiotech","c55477cf-8c98-4bd0-a05d-6182ab1be4e8-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/NHREUS_Aquaculture","c7e19ccc-9fc1-49c0-8351-d507830e7660-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/GALAXIDI","69e9ec43-62a7-45cb-b868-a369aeabf123-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/CFE_DATA","75d70975-bee4-4f46-94be-ff8773df6bbb-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/CWP_Secretariat","d84f0ade-be6a-40d6-a152-66e1b5d7d75d-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/SustainableBlueEconomy","19bbf2bb-58e3-4140-81cc-4043fd8755e5-843339462"}, +//// {"/d4science.research-infrastructures.eu/FARM/CollabResearch","2580b8ae-fad1-4213-8fd4-78b8a90c2c90-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps","e844592c-6428-4483-8614-e9561ad806f5-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","fe58019c-f3eb-4d3a-8094-54d18df00940-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/GEMex","afe00e12-5351-4941-aa31-1285184e2aea-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/KnowledgeBridging","cb98c3f9-0217-46f3-a74d-bda21bbf00fb-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","a563847b-bb53-4303-950f-5857c59f3217-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","dfbc4f13-874d-4e0b-99a9-b5cf1d48acef-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","8e6dcf9c-6dcc-4172-8dd6-45560af6a227-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/iSearch","df227388-3d61-4f77-97fb-4cd4fd8c6dbe-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","ce150ced-d4a3-48d2-a541-3433f26ab6d5-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","a1c1a4e5-7aea-4c12-b724-868adb8fa6ec-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","6c62b572-3b0e-43d2-b224-5c9961dfcb2e-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","44c54a21-88df-4a4b-ab1c-d9d5b636baba-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","8953528d-e990-4451-b5a3-517adca107a2-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/rScience","7bfa66d6-6c9b-448f-a11a-21d833843647-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","79fb276e-13d9-4d80-b419-56c9d64888b7-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIP","9c420378-e8f4-472e-bf7b-b5419041f6db-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ENVRIPlus","554d32ca-ae4a-44dd-b8a7-424ec7d30cba-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/TCom","aed4d51f-3dd1-4329-b622-322211ac237a-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/CNR_OpenScienceTF","6978ab04-ae3c-4cee-8b11-9f64ee733502-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","c1674036-1388-4eca-bbbb-174b9dc95dd1-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","38e4fa5a-6c50-417f-b952-1525cebd4950-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","776ea841-2205-4590-a0dc-24141138972e-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/DESCRAMBLE","2c6561bb-ee65-4537-9ccc-ec2464291630-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ARIADNE","05ffc92b-03d1-4b05-8e33-e62f795202b5-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/EcologicalModelling","2fa2177d-a2a8-48d4-898f-3f301842247c-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","eba4dc9d-b7cf-40b8-adfb-aecdc731c3de-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","9d78cbba-78e2-4efe-a38a-e14744318f2d-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","49b00009-e35f-44dc-ab0e-a63b505aca79-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","cb859658-0439-4469-8b2a-7c4516c3a659-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","9f2746e3-3e04-4f80-9dd7-e66f42ff53f9-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","8166fe91-7fe5-4636-9821-b8718aaa8500-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","867dfe04-ed24-42ad-902f-e2ee63963054-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIEngage","548ffb80-2dbe-4d0d-afb9-1fd009c2edca-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/EFG","e085b1c4-f315-4a21-88c4-46748b81b3f3-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/IGDI","95306e9b-0ce0-4b72-be18-075873fe634a-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","5cfdd223-401c-4324-9c99-9bb4462ff854-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu","6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","2856e458-7fd8-42a4-a1a9-b5566cf76ff7-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","bffc959f-5eb3-4d21-9222-27b99fa41f5b-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/gCube","9e5aa777-c700-4975-a218-6c73cc8b1c69-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","ec2d9727-2b21-4df5-823d-3f28a2d8295e-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/Parthenos","42e742e2-75eb-401e-9d07-43deb713f926-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it","8b4d9866-1d55-470a-95c3-4cdc19cecb0d-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","49c20941-27b1-4294-aa23-58809ef8616c-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/ICOS_ETC","cac5112b-1a9c-476b-bc33-3b5d3192b1d0-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","3c6dfc0b-81a3-453c-b693-fe8337a735f1-843339462"}, +//// {"/d4science.research-infrastructures.eu/gCubeApps/FrenchTropicalTunaAtlas","703fb5a1-54e2-4b1a-920c-978356d4a74a-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE","2a05405f-e878-4f80-a522-d2381b97e0df-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Technical","15ee7828-caa0-4b07-a950-9f1ffdbc812b-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Networking","156ca39a-0485-467d-aebb-59b7b49690b4-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_PSC","2d891c6a-4f6f-4281-8ffd-56270d0d0783-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_EAB","66856a20-7a58-4188-afcf-ab5fa212a732-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE_Users","64c73441-5a88-45e7-a865-dc449dadf271-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/1st_OpenAIRE_Datathon","5c6946e9-277c-4813-b949-5c5e215127b6-843339462"}, +//// {"/d4science.research-infrastructures.eu/OpenAIRE/dnet","75e2db7a-03ae-4af3-b87d-d7f1f20cc194-843339462"}, +//// {"/d4science.research-infrastructures.eu/ParthenosVO","8b8cf7cd-ff1a-4827-83cf-3933e2320d26-843339462"}, +//// {"/d4science.research-infrastructures.eu/ParthenosVO/RubRIcA","4a9e0e5a-3b9e-4293-9dc7-4a118f3a37f3-843339462"}, +//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_Registry","0cdceda3-f6e1-4bb7-aa20-c3a4b8a891ba-843339462"}, +//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_LAB","75de04c9-aa4e-4e32-9770-4094de1d0d54-843339462"}, +//// {"/d4science.research-infrastructures.eu/ParthenosVO/ACE","05f21525-b6fd-4587-ae7f-4e45a9fddfdf-843339462"}, +//// {"/d4science.research-infrastructures.eu/SmartArea","3782e93f-3d1a-4809-be0f-7c4b20eaa74b-843339462"}, +//// {"/d4science.research-infrastructures.eu/SmartArea/SmartBuilding","4216a83a-5506-43be-93b4-1464020cacce-843339462"}, +//// {"/d4science.research-infrastructures.eu/SmartArea/SmartApps","aa24a1d7-672a-4e06-b220-7d353188fc0a-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData","060a074b-bb12-4ab2-a585-cf0119f4d8ea-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens","28367906-d586-42e3-a5d8-6043a43ce54e-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue","2497dbd8-4002-4b50-b17c-ed2dc17cb0d7-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy","90ae1d9b-cfcb-4fdd-a91e-38ff6013f688-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/TagMe","9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/SMAPH","c0924749-cb6d-41e9-88a4-363e7f9e7556-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab","37d61ab1-3641-435d-8c4f-f1d37eb34ead-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/SocietalDebates","59450ce1-cf1d-4abb-9b0c-63346ad35e08-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/M-ATLAS","ace198d7-f50f-4557-9788-f7aa91c31d67-843339462"}, +//// {"/d4science.research-infrastructures.eu/SoBigData/SportsDataScience","61bf4b8d-34e2-4426-a3dd-b41a1dcbccb1-843339462"} +//// }; +//// +//// ArrayList props = new ArrayList<>(); +//// for(String[] datap : data) { +//// Property p = new Property(); +//// p = p.nameAndValue(datap[0], datap[1]); +//// props.add(p); +//// } +// +// +// try { +// //new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59").updateServiceEndPoint(props); +// BasicHarvester bh = new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59"); +// Properties props = bh.readServiceEndpoint(); +// Iterator i = props.keySet().iterator(); +// while(i.hasNext()) { +// String key = (String)i.next(); +// System.out.println(key + ":" + props.getProperty(key)); +// } +// } catch (ParseException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } catch (Exception e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// } + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java new file mode 100644 index 0000000..f5b690a --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java @@ -0,0 +1,145 @@ +package org.gcube.dataharvest.harvester; + +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; + +import org.gcube.dataharvest.AccountingDataHarvesterPlugin; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.lang.Validate; +import org.gcube.common.homelibrary.home.Home; +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.HomeManager; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException; +import org.gcube.common.homelibrary.jcr.repository.JCRRepository; +import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; +import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; +import org.gcube.common.homelibrary.model.exceptions.RepositoryException; +import org.gcube.common.scope.api.ScopeProvider; + +public class DataMethodDownloadHarvester extends BasicHarvester { + private Date startDate, endDate; + private DateFormat format; + private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; + private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class); + private int count = 0; + + public DataMethodDownloadHarvester(String start, String end) throws ParseException { + super(start, end); + //ScopeProvider.instance.set(SCOPE); + } + + public DataMethodDownloadHarvester(Date start, Date end) throws ParseException { + super(start, end); + //ScopeProvider.instance.set(SCOPE); + } + + @Override + public List getData() throws Exception { + ArrayList data = new ArrayList(); + ScopeProvider.instance.set(SCOPE); + String[] vres = getActiveVREs(true); + for (String vre : vres) { + try { + count = 0; + + logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()"); + + HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); + + String user = getVREName(vre) + "-Manager"; + + logger.error("Home home = manager.getHome(user)"); + + Home home = manager.getHome(user); + + logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()"); + + JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); + + logger.error("ws.getItemByPath..."); + + JCRWorkspaceItem item = (JCRWorkspaceItem) ws + .getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre)); + + logger.debug("Analyzing " + vre + " from " + startDate.toString() + " to " + endDate.toString()); + + logger.error("Before getStats()"); + getStats(item, startDate, endDate); + logger.error("After getStats()"); + Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, vre, count); + data.add(harvest); + logger.debug(harvest.toString()); + } catch (Exception e) { + logger.error("DataMethodDownloadHarvester: Error processing VRE: " + vre + "\n" + + e.getLocalizedMessage()); + } + } + return data; + } + + + private void getStats(WorkspaceItem root, Date start, Date end) throws InternalErrorException { + List children; + if (root.isFolder()) { + children = root.getChildren(); + for (WorkspaceItem child : children) + getStats(child, start, end); + } else { + try { + + List accounting = root.getAccounting(); + for (AccountingEntry entry : accounting) { + + switch (entry.getEntryType()) { + case CREATE: + case UPDATE: + case READ: + Calendar calendar = entry.getDate(); + if (calendar.after(Utils.dateToCalendar(start)) && calendar.before(Utils.dateToCalendar(end))) { + count++; + } + + break; + + default: + break; + } + + } + } catch (Exception e) { + logger.error("DataMethodDownloadHarvester: " + e.getLocalizedMessage()); + throw new InternalErrorException(e.getLocalizedMessage()); + } + } + } + + private static String getVREName(String vre) { + Validate.notNull(vre, "scope must be not null"); + + String newName; + if (vre.startsWith(JCRRepository.PATH_SEPARATOR)) + newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1); + else + newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-"); + return newName; + } + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java new file mode 100644 index 0000000..54fa7fe --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java @@ -0,0 +1,115 @@ +package org.gcube.dataharvest.harvester; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Calendar; + +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.accounting.analytics.Filter; +import org.gcube.accounting.analytics.Info; +import org.gcube.accounting.analytics.TemporalConstraint; +import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; +import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; + +import java.util.SortedMap; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.Iterator; + +import org.gcube.dataharvest.datamodel.Harvest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MethodInvocationHarvester extends BasicHarvester { + + private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class); + + public MethodInvocationHarvester(String start, String end) throws ParseException { + super(start, end); + } + + public MethodInvocationHarvester(Date start, Date end) throws ParseException { + super(start, end); + } + + @Override + public List getData() throws Exception { + try { + logger.debug("MethodInvocationHarvester::getData()"); + ArrayList data = new ArrayList(); + AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance(); + Calendar start = Calendar.getInstance(); + Calendar end = Calendar.getInstance(); + start.setTime(startDate); + end.setTime(endDate); + TemporalConstraint temporalConstraint = new TemporalConstraint(start.getTimeInMillis(), + end.getTimeInMillis(), AggregationMode.MONTHLY); + + List filters = new ArrayList<>(); + filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, "tag")); + + List contexts = new ArrayList<>(); + contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe"); + + logger.debug("MethodInvocationHarvester::getData()::getContextTimeSeries"); + SortedMap> result = accountingPersistenceQuery.getContextTimeSeries( + AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true); + if (result == null) { + logger.error("No data found."); + } else { + Set ks = result.keySet(); + if (ks != null) { + Iterator ksi = ks.iterator(); + while (ksi.hasNext()) { + // System.out.println("" + ksi.next().toString()); + logger.debug("Filter: " + ksi.next().toString()); + } + } + } + + return data; + } catch (Exception x) { + StackTraceElement[] ste = x.getStackTrace(); + String errorMessage = "MethodInvocationHarvester: " + x.getLocalizedMessage(); + for (StackTraceElement s : ste) { + errorMessage += "\n" + s.toString(); + } + logger.error(errorMessage); + throw x; + } + }/**/ + + /* used only for test */ + public static void main(String[] args) { + String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462"; + String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548"; + String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462"; + String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462"; + String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462"; + try { + MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59"); + MethodInvocationHarvester.setContext(SOBIGDATA_EU); + System.out.println(MethodInvocationHarvester.getCurrentContext()); + Properties props = m.readServiceEndpoint(); + System.out.println("Found " + props.size() + " properties."); + Set keys = props.keySet(); + for(Object key : keys) { + System.out.println((String)key + " :: " + props.getProperty((String) key)); + } + + m.getData(); + } catch (Exception e) { + // TODO Auto-generated catch block + System.out.println(e.getLocalizedMessage()); + e.printStackTrace(); + } + } + /**/ + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java new file mode 100644 index 0000000..d16517c --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java @@ -0,0 +1,200 @@ +package org.gcube.dataharvest.harvester; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Iterator; +import java.util.Locale; +import java.util.Properties; + +import org.apache.http.client.utils.URLEncodedUtils; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; +import org.gcube.portlets.user.urlshortener.UrlEncoderUtil; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.quartz.utils.StringKeyDirtyFlagMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResourceCatalogueHarvester extends BasicHarvester { + private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class); + private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0, + wellBeingAndEconomyCounter = 0; + + public ResourceCatalogueHarvester(String start, String end) throws ParseException { + super(start, end); + } + + public ResourceCatalogueHarvester(Date start, Date end) throws ParseException { + super(start, end); + } + + @Override + public List getData() throws Exception { + ArrayList data = new ArrayList(); + List dataDeliverable = getDataDeliverable(); + for (Harvest harvest : dataDeliverable) { + data.add(harvest); + } + + List dataMethod = getDataMethod(); + for (Harvest harvest : dataMethod) { + data.add(harvest); + } + + List dataDataset = getDataDataset(); + for (Harvest harvest : dataDataset) { + data.add(harvest); + } + + List dataApplication = getDataApplication(); + for (Harvest harvest : dataApplication) { + data.add(harvest); + } + + return data; + } + + public List getDataDeliverable() throws Exception { + String json = executeQuery("Deliverable"); + return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES); + } + + public List getDataMethod() throws Exception { + String json = executeQuery("Method"); + return buildList(json, Harvest.NEW_CATALOGUE_METHODS); + } + + public List getDataDataset() throws Exception { + String json = executeQuery("Dataset"); + return buildList(json, Harvest.NEW_CATALOGUE_DATASETS); + } + + public List getDataApplication() throws Exception { + String json = executeQuery("Application"); + return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS); + } + + private List buildList(String json, int dataType) throws Exception { + ArrayList data = new ArrayList(); + JSONObject jsonObject = new JSONObject(json); + + JSONObject responseHeader = jsonObject.getJSONObject("responseHeader"); + int status = responseHeader.getInt("status"); + if (status != 0) { + String err = "Query Deliverable in error: status " + status; + logger.error(err); + throw new Exception(err, null); + } + + JSONObject response = jsonObject.getJSONObject("response"); + + int numFound = response.getInt("numFound"); + Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue", + numFound); + logger.debug(h.toString()); + data.add(h); + if (numFound > 0) { + + JSONArray docs = response.getJSONArray("docs"); + for (Object item : docs) { + JSONObject doc = (JSONObject) item; + try { + JSONArray groups = doc.getJSONArray("groups"); + Iterator git = groups.iterator(); + while (git.hasNext()) { + String groupItem = (String) git.next(); + counterByGroup(groupItem); + } + } catch (JSONException x) { + logger.debug("Document without groups"); + } + } + + h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens", + cityOfCitizensCounter); + logger.debug(h.toString()); + data.add(h); + h = new Harvest(dataType, "/Migration Studies", migrationStudiesCounter); + logger.debug(h.toString()); + data.add(h); + h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates", + societalDebatesCounter); + logger.debug(h.toString()); + data.add(h); + h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy", + wellBeingAndEconomyCounter); + logger.debug(h.toString()); + data.add(h); + + } + + return data; + + } + + private String executeQuery(String fqSubString) throws Exception { + String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?"; + String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO " + + Utils.dateToStringWithTZ(endDate) + "]"); + query += "q=" + q; + String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\""); + query += "&fq=" + fq + "&wt=json&indent=true"; + logger.debug(query); + + String json = Utils.getJson(query); + // logger.debug(json); + + return json; + } + + private void counterByGroup(String groupName) { + cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0; + switch (groupName) { + case "city-of-citizens-group": + cityOfCitizensCounter++; + break; + case "migration-studies": + migrationStudiesCounter++; + break; + case "societal-debates-group": + societalDebatesCounter++; + break; + case "well-being-and-economy-group": + wellBeingAndEconomyCounter++; + break; + } + } + + /* + * used only for test public static void main(String[] args) { Calendar from + * = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1); + * from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR, + * 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0); + * from.set(Calendar.SECOND, 0); + * + * Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30); + * to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017); + * to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59); + * to.set(Calendar.SECOND, 59); + * + * try { ResourceCatalogueHarvester a = new + * ResourceCatalogueHarvester(from.getTime(), to.getTime()); List l + * = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); } + * // System.out.println(Utils.dateToString(a.startDate)); // + * System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch + * (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } + * } + */ +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java new file mode 100644 index 0000000..d6492c5 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java @@ -0,0 +1,109 @@ +package org.gcube.dataharvest.harvester; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SocialHarvester extends BasicHarvester { + private final String CATEGORY_NAME = "Accounting"; + private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); + private int likes, replies, posts; + private long from = 0, to = 0; + + public SocialHarvester(String start, String end) throws ParseException { + super(start, end); + this.from = startDate.getTime(); + this.to = endDate.getTime(); + } + + public SocialHarvester(Date start, Date end) throws ParseException { + super(start, end); + this.from = startDate.getTime(); + this.to = endDate.getTime(); + } + + @Override + public List getData() throws Exception { + ArrayList data = new ArrayList(); + Properties props = readServiceEndpoint(); + String[] vres = getActiveVREs(true); + for (String vre : vres) { + try { + logger.debug("Working on VRE: " + vre); + + String key = vre; // vre.substring(vre.lastIndexOf("/") + 1); + String token = props.getProperty(key); + //System.out.println(key + ":" + token); + getJson(token); + //getFinalResult(); + Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes); + data.add(harvest); + logger.debug(harvest.toString()); + + harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts); + data.add(harvest); + logger.debug(harvest.toString()); + + harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies); + data.add(harvest); + logger.debug(harvest.toString()); + } catch (Exception x) { + logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage()); + } + } + return data; + } + + private void getJson(String token) throws MalformedURLException, IOException { + likes = replies = posts = 0; + // la seguente stringa deve essere letta dinamicamente + String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token="; + JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); + + Boolean success = (Boolean) jsonObject.get("success"); + if (success == false) { + String message = "getJson() on token: " + token + " success=false"; + logger.error(message); + throw new IOException(message); + } + + JSONArray res = jsonObject.getJSONArray("result"); + int len = res.length(); + + for (int i = 0; i < len; i++) { + JSONObject item = res.getJSONObject(i); + long time = item.getLong("time"); + //System.out.println(from + " - " + time + " - " + to ); + if ((from <= time) && (time <= to)) { + posts++; + replies += item.getInt("comments_no"); + likes += item.getInt("likes_no"); + } + } + } + +/**/ + public static void main(String[] argv) { + try { + SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59"); + List list = a.getData(); + for (Harvest l : list) { + System.out.println(l.toString()); + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }/**/ +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java new file mode 100644 index 0000000..1a7a227 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java @@ -0,0 +1,137 @@ +package org.gcube.dataharvest.harvester; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Properties; +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; + +import org.json.*; + +public class VreUsersHarvester extends BasicHarvester { + // private final String CATEGORY_NAME = "Accounting"; + private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); + + public VreUsersHarvester(String start, String end) throws ParseException { + super(start, end); + } + + public VreUsersHarvester(Date start, Date end) throws ParseException { + super(start, end); + } + + @Override + public List getData() throws Exception { + ArrayList data = new ArrayList(); + Properties props = readServiceEndpoint(); + String[] vres = getActiveVREs(true); + for (String vre : vres) { + String key = vre.substring(vre.lastIndexOf("/") + 1); + String token = props.getProperty(key); + try { + int measure = get(token); + Harvest harvest = new Harvest(Harvest.USERS, vre, measure); + data.add(harvest); + logger.debug(harvest.toString()); + } catch (Exception x) { + logger.error("VreUsersHarvester::getData. " + vre + ". " + x.getLocalizedMessage()); + } + } + return data; + } + + private int get(String token) throws MalformedURLException, IOException { + int userNumber = 0; + // la seguente stringa deve essere letta dinamicamente + String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token="; + JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); + Boolean success = (Boolean) jsonObject.get("success"); + if (success == false) { + String message = "get-all-usernames returned an error. token: " + token; + logger.error(message); + throw new IOException(message); + } + userNumber = jsonObject.getJSONArray("result").length(); + return userNumber; + } + + // private Properties readServiceEndpoint() throws Exception { + // Properties props = new Properties(); + // String scope = "/d4science.research-infrastructures.eu"; + // ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); + // + // SimpleQuery query = queryFor(ServiceEndpoint.class); + // query.addCondition("$resource/Profile/Name/text() eq '" + + // RUNTIME_RESOURCE_NAME + "'"); + // query.addCondition("$resource/Profile/Category/text() eq '" + + // CATEGORY_NAME + "'"); + // + // DiscoveryClient client = + // clientFor(ServiceEndpoint.class); + // String password = null; + // try { + // List list = client.submit(query); + // if (list.size() > 1) { + // logger.error("Too many Service Endpoints having name " + + // RUNTIME_RESOURCE_NAME + // + " in this scope having Category " + CATEGORY_NAME); + // } else if (list.size() == 0) { + // logger.error("There is no Service Endpoint having name " + + // RUNTIME_RESOURCE_NAME + // + " and Category " + CATEGORY_NAME + " in this scope " + scope); + // } else { + // for (ServiceEndpoint res : list) { + // AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints() + // .toArray(new AccessPoint[res.profile().accessPoints().size()]); + // for (AccessPoint found : accessPoints) { + // password = + // org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password()); + // for (ServiceEndpoint.Property prop : found.properties()) { + // props.setProperty(prop.name(), + // org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value())); + // } + // } + // + // } + // } + // return props; + // } catch (Exception e) { + // logger.error(e.getLocalizedMessage()); + // throw e; + // } + // } + + // public static void main(String[] argv) { + // try { + // VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01"); + // List list = a.getData(); + // for(Harvest l : list) { + // System.out.println(l.toString()); + // } + // } catch (Exception e) { + // // TODO Auto-generated catch block + // e.printStackTrace(); + // } + // } +} diff --git a/src/main/java/org/gcube/dataharvest/utils/Utils.java b/src/main/java/org/gcube/dataharvest/utils/Utils.java new file mode 100644 index 0000000..5f91e90 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/utils/Utils.java @@ -0,0 +1,65 @@ +package org.gcube.dataharvest.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; + +import org.gcube.dataharvest.dao.Dao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Utils { + private static Logger logger = LoggerFactory.getLogger(Utils.class); + + public static Calendar dateToCalendar(Date date) { + Calendar cal = null; + cal = Calendar.getInstance(); + cal.setTime(date); + return cal; + } + + public static String dateToString(Date date) { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + return formatter.format(date); + } + + public static String dateToStringWithTZ(Date date) { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + return formatter.format(date) + "Z"; + } + + public static String getJson(String url) throws MalformedURLException, IOException { + URL address = new URL(url); + HttpURLConnection connection = (HttpURLConnection) address.openConnection(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String json = "", line = ""; + + while (line != null) { + line = reader.readLine(); + if (line != null) + json += line.trim(); + } + return json; + } + + // il seguente array deve essere formato dinamicamente +// public static String[] soBigDataContexts = { "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu", +// "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it", +// "/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab", +// "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue", +// "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens", +// "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates", +// "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy", +// "/d4science.research-infrastructures.eu/SoBigData/SMAPH", +// "/d4science.research-infrastructures.eu/SoBigData/TagMe" }; + +} diff --git a/src/main/resources/META-INF/services/org.gcube.vremanagement.executor.plugin.PluginDeclaration b/src/main/resources/META-INF/services/org.gcube.vremanagement.executor.plugin.PluginDeclaration new file mode 100644 index 0000000..591520e --- /dev/null +++ b/src/main/resources/META-INF/services/org.gcube.vremanagement.executor.plugin.PluginDeclaration @@ -0,0 +1 @@ +org.gcube.dataharvest.DataHarvestPluginDeclaration \ No newline at end of file diff --git a/src/test/java/org/gcube/dataharvest/Harvester.java b/src/test/java/org/gcube/dataharvest/Harvester.java new file mode 100644 index 0000000..97eb414 --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/Harvester.java @@ -0,0 +1,230 @@ +package org.gcube.dataharvest; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.gcube.dataharvest.dao.Dao; +import org.gcube.dataharvest.dao.DaoException; +import org.gcube.dataharvest.dao.DatabaseConnectionData; +import org.gcube.dataharvest.dao.DatabaseDataExplorer; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.harvester.BasicHarvester; +import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester; +import org.gcube.dataharvest.harvester.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester; +import org.gcube.dataharvest.harvester.SocialHarvester; +import org.gcube.dataharvest.harvester.VreUsersHarvester; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Harvester { + private static Logger logger = LoggerFactory.getLogger(Harvester.class); + public static final String PARAMETER_FROM = "from"; + public static final String PARAMETER_TO = "to"; + public static final String TEST = "test"; + private boolean testMode = false; + private Date dateFrom, dateTo; + + public static void main(String[] args) { + Harvester harvester = new Harvester(); + try { + harvester.processParameterArray(args); + SocialHarvester socialHarvester = new SocialHarvester(harvester.getDateFrom(), harvester.getDateTo()); + harvester.runOne(socialHarvester); + // harvester.run(); + //ArrayList list = harvester.getSubTree(17); + //harvester.createSocialReports(list); + System.out.println("End."); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public Harvester() { + testMode = false; + } + + public void setDateFrom(Date dateFrom) { + this.dateFrom = dateFrom; + } + + public void setDateTo(Date dateTo) { + this.dateTo = dateTo; + } + + public void setTestMode(boolean testMode) { + this.testMode = testMode; + } + + public Date getDateFrom() { + return dateFrom; + } + + public Date getDateTo() { + return dateTo; + } + + public void runOne(BasicHarvester harvester) { + try { + List data = harvester.getData(); + if (data != null) { + insertMonthlyData((Date) dateFrom, (Date) dateTo, data); + } + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + } + + public void runAll() { + try { + // collecting info on VRE users + VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo); + List users = vreUsersHarvester.getData(); + insertMonthlyData((Date) dateFrom, (Date) dateTo, users); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on Res. Catalogue (Dataset, Application, + // Deliverables, Methods) + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo); + List res = resourceCatalogueHarvester.getData(); + insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on Data/Method download + DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo); + List res = dataMethodDownloadHarvester.getData(); + insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on social (posts, replies and likes) + SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo); + List res = socialHarvester.getData(); + insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on method invocation + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo); + List res = methodInvocationHarvester.getData(); + // insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + } catch (Exception x) { + logger.error(x.getLocalizedMessage()); + } + } + + private void insertMonthlyData(Date from, Date to, List data) { + Dao dao = null; + try { + dao = dbConnect(); + dao.insertMonthlyMeasure(data, from, to, false); + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + } finally { + if (dao != null) { + try { + dao.disconnect(); + } catch (DaoException e) { + logger.error(e.getLocalizedMessage()); + } + } + } + } + + private boolean checkDate(Object date) { + if (date == null) + return false; + return date instanceof java.util.Date; + } + + private Dao dbConnect() throws DaoException { + DatabaseDataExplorer dde = new DatabaseDataExplorer(); + dde.setTestMode(testMode); + DatabaseConnectionData dcd = dde.retrieveDatabaseInfo(); + Dao dao = new Dao(); + dao.init(); + dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword()); + return dao; + } + + private void processParameterArray(String[] args) throws Exception { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + for (int i = 0; i < args.length; i++) { + if (args[i].startsWith(PARAMETER_FROM)) { + String from = getArgValue(args[i]); + dateFrom = formatter.parse(from); + System.out.println(dateFrom.toString()); + } else if (args[i].startsWith(PARAMETER_TO)) { + String to = getArgValue(args[i]); + dateTo = formatter.parse(to); + System.out.println(dateTo.toString()); + } else if (args[i].toLowerCase().startsWith(TEST)) { + testMode = true; + System.out.println("Test: " + testMode); + } + + } + } + + private String getArgValue(String arg) throws Exception { + String[] tokens = arg.split("="); + if (tokens.length != 2) + throw new Exception("Argument must be in the format 'name=value'."); + return tokens[1].trim(); + } + + private ArrayList getSubTree(int root) { + Dao dao = null; + try { + dao = dbConnect(); + ArrayList subTree = dao.getSubTree(root); + return subTree; + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + return null; + } finally { + if (dao != null) { + try { + dao.disconnect(); + } catch (DaoException e) { + logger.error(e.getLocalizedMessage()); + } + } + } + } + + private void createSocialReports(ArrayList ids) { + Dao dao = null; + try { + dao = dbConnect(); + for (Integer contextId : ids) + dao.createSocialReport(contextId, 2018); + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + } finally { + if (dao != null) { + try { + dao.disconnect(); + } catch (DaoException e) { + logger.error(e.getLocalizedMessage()); + } + } + } + } + +} diff --git a/src/test/java/org/gcube/utils/DataFiller.java b/src/test/java/org/gcube/utils/DataFiller.java new file mode 100644 index 0000000..276ecea --- /dev/null +++ b/src/test/java/org/gcube/utils/DataFiller.java @@ -0,0 +1,392 @@ +package org.gcube.utils; + +public class DataFiller { + private static String[] data10 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","83", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","26", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","71", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","5", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","374", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","112", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","37", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","44", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","36", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","19", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","100", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","6", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","28", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","30", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","5", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","31", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","56", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","5", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","30", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","99", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","68", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","152", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","19", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","62", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","6", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","182", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","38", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20" + }; + + private static String[] data11 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","93", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","26", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","136", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","387", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","116", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","98", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","4", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","28", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","34", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","58", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","5", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","56", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","110", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","73", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","161", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","22", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","186", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","40", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20" + }; + + private static String[] data12 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","99", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","28", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","273", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","392", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","118", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","98", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","4", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","30", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","36", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","59", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","57", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","114", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","74", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","164", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","26", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","188", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","40", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20" + }; + + public static String[] data01 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","103", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","278", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","400", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","121", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","4", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","31", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","37", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","59", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","68", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","123", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","80", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","167", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","31", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","191", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","44", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20", + }; + + public static String[] data02 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","106", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","279", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","402", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","124", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","26", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","4", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","32", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","38", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","60", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","71", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","127", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","81", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","171", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","32", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","193", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","49", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","24" + }; + + public static String[] data03 = { + "/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12", + "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","107", + "/d4science.research-infrastructures.eu/FARM/Aquabiotech","8", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35", + "/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","278", + "/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3", + "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","403", + "/d4science.research-infrastructures.eu/gCubeApps/BiOnym","124", + "/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35", + "/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45", + "/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","26", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99", + "/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10", + "/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15", + "/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27", + "/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11", + "/d4science.research-infrastructures.eu/FARM/ForkysVRE","12", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5", + "/d4science.research-infrastructures.eu/FARM/GALAXIDI","4", + "/d4science.research-infrastructures.eu/FARM/GRSF","13", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin","32", + "/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38", + "/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28", + "/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","32", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSE","29", + "/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26", + "/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29", + "/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6", + "/d4science.research-infrastructures.eu/D4Research/InfraTraining","10", + "/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","38", + "/d4science.research-infrastructures.eu/gCubeApps/iSearch","61", + "/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6", + "/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27", + "/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","71", + "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","127", + "/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","81", + "/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","171", + "/d4science.research-infrastructures.eu/gCubeApps/SIASPA","32", + "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63", + "/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26", + "/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5", + "/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","191", + "/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","49", + "/d4science.research-infrastructures.eu/FARM/VME-DB","18", + "/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","24" + }; + + + public static String[] data04 = { + "/d4science.research-infrastructures.eu/FARM/CWP_Secretariat", "8" + }; + + public static String[] vals = { + "2017", "1", "0", "2", "80", + "2017", "2", "0", "2", "80", + "2017", "3", "0", "2", "80", + "2017", "4", "0", "2", "80", + "2017", "5", "0", "2", "80", + "2017", "6", "0", "2", "80", + "2017", "7", "4", "2", "80", + "2017", "8", "4", "2", "80", + "2017", "9", "4", "2", "80", + "2017", "10", "4", "2", "80", + "2017", "11", "4", "2", "80", + "2017", "12", "4", "2", "80", + "2018", "1", "4", "2", "80", + "2018", "2", "4", "2", "80", + "2018", "3", "4", "2", "80" + }; + + public static void main(String[] args) { +// for(int i=0; i + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{0}: %msg%n + + + + + + + + + + + + \ No newline at end of file