Refactor DataHArvesterPlugin to support scheduled execution from smart-executor refs #11756
https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167434 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
commit
b69f952ce6
|
@ -0,0 +1,32 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<classpath>
|
||||||
|
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="optional" value="true"/>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="optional" value="true"/>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
||||||
|
<classpathentry kind="output" path="target/classes"/>
|
||||||
|
</classpath>
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<projectDescription>
|
||||||
|
<name>DataHarvesterPlugin</name>
|
||||||
|
<comment></comment>
|
||||||
|
<projects>
|
||||||
|
</projects>
|
||||||
|
<buildSpec>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
</buildSpec>
|
||||||
|
<natures>
|
||||||
|
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||||
|
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||||
|
</natures>
|
||||||
|
</projectDescription>
|
|
@ -0,0 +1,2 @@
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
|
@ -0,0 +1,4 @@
|
||||||
|
activeProfiles=
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
resolveWorkspaceProjects=true
|
||||||
|
version=1
|
|
@ -0,0 +1,207 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<groupId>org.gcube</groupId>
|
||||||
|
<artifactId>DataHarvesterPlugin</artifactId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
<name>SmartExecutor-DataHarvesterPlugin</name>
|
||||||
|
<description>Smart Executor DataHarvester Plugin</description>
|
||||||
|
|
||||||
|
<scm>
|
||||||
|
<connection>scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId}</connection>
|
||||||
|
<developerConnection>scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId}</developerConnection>
|
||||||
|
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/vre-management/smart-executor-${project.artifactId}</url>
|
||||||
|
</scm>
|
||||||
|
|
||||||
|
<dependencyManagement>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.distribution</groupId>
|
||||||
|
<artifactId>gcube-bom</artifactId>
|
||||||
|
<version>LATEST</version>
|
||||||
|
<type>pom</type>
|
||||||
|
<scope>import</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</dependencyManagement>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.vremanagement</groupId>
|
||||||
|
<artifactId>smart-executor-api</artifactId>
|
||||||
|
<version>[1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.vremanagement</groupId>
|
||||||
|
<artifactId>smart-executor-client</artifactId>
|
||||||
|
<version>[1.3.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.core</groupId>
|
||||||
|
<artifactId>common-encryption</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common</groupId>
|
||||||
|
<artifactId>home-library</artifactId>
|
||||||
|
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common</groupId>
|
||||||
|
<artifactId>home-library-jcr</artifactId>
|
||||||
|
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.resources.discovery</groupId>
|
||||||
|
<artifactId>ic-client</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.resources</groupId>
|
||||||
|
<artifactId>registry-publisher</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.resources</groupId>
|
||||||
|
<artifactId>common-gcore-resources</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.information-system</groupId>
|
||||||
|
<artifactId>information-system-model</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.information-system</groupId>
|
||||||
|
<artifactId>gcube-resources</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.portlets.admin</groupId> -->
|
||||||
|
<!-- <artifactId>rmp-common-library</artifactId> -->
|
||||||
|
<!-- <version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- <scope>compile</scope> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>com.sencha.gxt</groupId> -->
|
||||||
|
<!-- <artifactId>gxt2.2.5-gwt2.X</artifactId> -->
|
||||||
|
<!-- <version>2.7.0</version> -->
|
||||||
|
<!-- <scope>compile</scope> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.resourcemanagement</groupId> -->
|
||||||
|
<!-- <artifactId>ghn-manager-client</artifactId> -->
|
||||||
|
<!-- <version>[1.1.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- <scope>compile</scope> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.information-system</groupId> -->
|
||||||
|
<!-- <artifactId>resource-registry</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- <scope>compile</scope> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.portlets.admin</groupId> -->
|
||||||
|
<!-- <artifactId>rmp-common-library</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- <scope>compile</scope> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.portlets.user</groupId>
|
||||||
|
<artifactId>gcube-url-shortener</artifactId>
|
||||||
|
<version>[1.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.accounting</groupId>
|
||||||
|
<artifactId>accounting-analytics</artifactId>
|
||||||
|
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.accounting</groupId>
|
||||||
|
<artifactId>accounting-analytics-persistence-couchbase</artifactId>
|
||||||
|
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.accounting</groupId>
|
||||||
|
<artifactId>accounting-lib</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<!-- -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.ancoron.postgresql</groupId>
|
||||||
|
<artifactId>org.postgresql</artifactId>
|
||||||
|
<version>9.1.901.jdbc4.1-rc9</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.json</groupId>
|
||||||
|
<artifactId>json</artifactId>
|
||||||
|
<version>20171018</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>2.3.2</version>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>servicearchive</id>
|
||||||
|
<phase>install</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>${distroDirectory}/descriptor.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<!-- <plugin> -->
|
||||||
|
<!-- <groupId>org.apache.maven.plugins</groupId> -->
|
||||||
|
<!-- <artifactId>maven-assembly-plugin</artifactId> -->
|
||||||
|
<!-- <executions> -->
|
||||||
|
<!-- <execution> -->
|
||||||
|
<!-- <id>make-uberjar</id> -->
|
||||||
|
<!-- <phase>install</phase> -->
|
||||||
|
<!-- </execution> -->
|
||||||
|
<!-- <execution> -->
|
||||||
|
<!-- <id>make-servicearchive</id> -->
|
||||||
|
<!-- <phase>install</phase> -->
|
||||||
|
<!-- </execution> -->
|
||||||
|
<!-- </executions> -->
|
||||||
|
<!-- </plugin> -->
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
|
@ -0,0 +1,175 @@
|
||||||
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.DataHarvestPluginDeclaration;
|
||||||
|
import org.gcube.vremanagement.executor.exception.InputsNullException;
|
||||||
|
import org.gcube.vremanagement.executor.exception.InvalidInputsException;
|
||||||
|
import org.gcube.vremanagement.executor.plugin.Plugin;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.gcube.dataharvest.dao.*;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.SocialHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.VreUsersHarvester;
|
||||||
|
|
||||||
|
public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDeclaration> {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class);
|
||||||
|
public static final String PARAMETER_FROM = "from";
|
||||||
|
public static final String PARAMETER_TO = "to";
|
||||||
|
public static final String TEST = "test";
|
||||||
|
private boolean testMode = false, updateFlag = false;
|
||||||
|
private Date dateFrom, dateTo;
|
||||||
|
|
||||||
|
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
|
||||||
|
super(pluginDeclaration);
|
||||||
|
logger.debug("DataHarvestPlugin: constructor");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** {@inheritDoc} */
|
||||||
|
@Override
|
||||||
|
public void launch(Map<String, Object> inputs) throws Exception {
|
||||||
|
logger.debug("DataHarvestPlugin: launch()");
|
||||||
|
String message = "";
|
||||||
|
DatabaseManager dbaseManager = new DatabaseManager(testMode);
|
||||||
|
|
||||||
|
if (inputs == null) {
|
||||||
|
message = "Invalid input parameter (null)";
|
||||||
|
logger.error(message);
|
||||||
|
throw new Exception(message, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
Object from = inputs.get(PARAMETER_FROM);
|
||||||
|
if (!checkDate(from)) {
|
||||||
|
message = "Missing or invalid input parameter: " + PARAMETER_FROM;
|
||||||
|
logger.error(message);
|
||||||
|
throw new Exception(message, null);
|
||||||
|
}
|
||||||
|
dateFrom = (Date)from;
|
||||||
|
|
||||||
|
Object to = inputs.get(PARAMETER_TO);
|
||||||
|
if (!checkDate(to)) {
|
||||||
|
message = "Missing or invalid input parameter: " + PARAMETER_TO;
|
||||||
|
logger.error(message);
|
||||||
|
throw new Exception(message, null);
|
||||||
|
}
|
||||||
|
dateTo = (Date)to;
|
||||||
|
|
||||||
|
testMode = (inputs.get(TEST) == null) ? false : true;
|
||||||
|
logger.debug("TEST mode is " + testMode);
|
||||||
|
|
||||||
|
setTimePeriod();
|
||||||
|
logger.debug("Time period: " + dateFrom.toString() + " -> " + dateTo.toString());
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on VRE users
|
||||||
|
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> users = vreUsersHarvester.getData();
|
||||||
|
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, users, updateFlag);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
|
||||||
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = resourceCatalogueHarvester.getData();
|
||||||
|
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on Data/Method download
|
||||||
|
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
||||||
|
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on social (posts, replies and likes)
|
||||||
|
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = socialHarvester.getData();
|
||||||
|
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on method invocation
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = methodInvocationHarvester.getData();
|
||||||
|
//insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/** {@inheritDoc} */
|
||||||
|
@Override
|
||||||
|
protected void onStop() throws Exception {
|
||||||
|
logger.debug("DataHarvestPlugin: onStop()");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean checkDate(Object date) {
|
||||||
|
if (date == null)
|
||||||
|
return false;
|
||||||
|
return date instanceof java.util.Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// private void insertMonthlyData(Date from, Date to, List<Harvest> data) {
|
||||||
|
// Dao dao = null;
|
||||||
|
// try {
|
||||||
|
// dao = dbConnect();
|
||||||
|
// dao.insertMonthlyMeasure(data, from, to, false);
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// logger.error(e.getLocalizedMessage());
|
||||||
|
// } finally {
|
||||||
|
// if (dao != null) {
|
||||||
|
// try {
|
||||||
|
// dao.disconnect();
|
||||||
|
// } catch (DaoException e) {
|
||||||
|
// logger.error(e.getLocalizedMessage());
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// private Dao dbConnect() throws DaoException {
|
||||||
|
// DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
||||||
|
// dde.setTestMode(testMode);
|
||||||
|
// DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
||||||
|
// Dao dao = new Dao();
|
||||||
|
// dao.init();
|
||||||
|
// dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
||||||
|
// return dao;
|
||||||
|
// }
|
||||||
|
|
||||||
|
private void setTimePeriod() {
|
||||||
|
Calendar from = Calendar.getInstance();
|
||||||
|
from.setTime(dateFrom);
|
||||||
|
from.set(Calendar.HOUR_OF_DAY, 0);
|
||||||
|
from.set(Calendar.MINUTE, 0);
|
||||||
|
from.set(Calendar.SECOND, 0);
|
||||||
|
dateFrom = from.getTime();
|
||||||
|
|
||||||
|
Calendar to = Calendar.getInstance();
|
||||||
|
to.setTime(dateTo);
|
||||||
|
to.set(Calendar.HOUR_OF_DAY, 23);
|
||||||
|
to.set(Calendar.MINUTE, 59);
|
||||||
|
to.set(Calendar.SECOND, 59);
|
||||||
|
dateTo = to.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,71 @@
|
||||||
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
|
||||||
|
import org.gcube.vremanagement.executor.plugin.Plugin;
|
||||||
|
import org.gcube.vremanagement.executor.plugin.PluginDeclaration;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class DataHarvestPluginDeclaration implements PluginDeclaration {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DataHarvestPluginDeclaration.class);
|
||||||
|
public static final String NAME = "AccountingDataHarvester";
|
||||||
|
public static final String DESCRIPTION = "Data Harvest for Accounting Summary Dashboard";
|
||||||
|
public static final String VERSION = "1.0.0";
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public void init() {
|
||||||
|
logger.debug(String.format("%s initialized", AccountingDataHarvesterPlugin.class.getSimpleName()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public String getName() {
|
||||||
|
return NAME;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public String getDescription() {
|
||||||
|
return DESCRIPTION;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public String getVersion() {
|
||||||
|
return VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public Map<String, String> getSupportedCapabilities() {
|
||||||
|
Map<String, String> discoveredCapabilities = new HashMap<String, String>();
|
||||||
|
discoveredCapabilities.put("FakeKey", "FakeValue");
|
||||||
|
return discoveredCapabilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**{@inheritDoc}*/
|
||||||
|
@Override
|
||||||
|
public Class<? extends Plugin<? extends PluginDeclaration>> getPluginImplementation() {
|
||||||
|
return AccountingDataHarvesterPlugin.class;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString(){
|
||||||
|
return String.format("{"
|
||||||
|
+ "name:%s,"
|
||||||
|
+ "version:%s,"
|
||||||
|
+ "description:%s,"
|
||||||
|
+ "pluginImplementation:%s,"
|
||||||
|
+ "}",
|
||||||
|
getName(),
|
||||||
|
getVersion(),
|
||||||
|
getDescription(),
|
||||||
|
getPluginImplementation().getClass().getSimpleName());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,325 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
import java.sql.Array;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
import org.postgresql.util.PSQLException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class Dao {
|
||||||
|
private Connection conn = null;
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(Dao.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class constructor. This method must be called before any other class
|
||||||
|
* method.
|
||||||
|
*
|
||||||
|
* @throws DaoException
|
||||||
|
*/
|
||||||
|
public void init() throws DaoException {
|
||||||
|
try {
|
||||||
|
Class.forName("org.postgresql.Driver");
|
||||||
|
} catch (ClassNotFoundException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect a database
|
||||||
|
*
|
||||||
|
* @param url
|
||||||
|
* @param user
|
||||||
|
* @param password
|
||||||
|
* @throws DaoException
|
||||||
|
*/
|
||||||
|
public void connect(String url, String user, String password) throws DaoException {
|
||||||
|
try {
|
||||||
|
conn = DriverManager.getConnection(url, user, password);
|
||||||
|
logger.debug("Connected to: " + url);
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Release a database connection
|
||||||
|
*
|
||||||
|
* @throws DaoException
|
||||||
|
*/
|
||||||
|
public void disconnect() throws DaoException {
|
||||||
|
try {
|
||||||
|
if (conn != null) {
|
||||||
|
conn.close();
|
||||||
|
conn = null;
|
||||||
|
logger.debug("Disconnecting from database");
|
||||||
|
}
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the connection with the database.
|
||||||
|
*
|
||||||
|
* @return true|false
|
||||||
|
*/
|
||||||
|
public boolean isConnected() {
|
||||||
|
return (!(conn == null));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Getter for the database connection
|
||||||
|
*
|
||||||
|
* @return Connection class
|
||||||
|
*/
|
||||||
|
public Connection getConnection() {
|
||||||
|
return conn;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read on the database all active VRES
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public String[] getActiveVres() {
|
||||||
|
String query = "select dname from context where dname is not null and dismissed is null";
|
||||||
|
logger.debug(query);
|
||||||
|
Statement s = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
ArrayList<String> list = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
s = conn.createStatement();
|
||||||
|
rs = s.executeQuery(query);
|
||||||
|
while (rs.next()) {
|
||||||
|
list.add(rs.getString("dname"));
|
||||||
|
}
|
||||||
|
|
||||||
|
return list.toArray(new String[list.size()]);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (rs != null) {
|
||||||
|
try {
|
||||||
|
rs.close();
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
// do nothing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (s != null) {
|
||||||
|
try {
|
||||||
|
s.close();
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
// do nothing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method insert/update data in the monthly_measure database table.
|
||||||
|
*
|
||||||
|
* @param data
|
||||||
|
* @param from
|
||||||
|
* @param to
|
||||||
|
* @throws DaoException
|
||||||
|
*/
|
||||||
|
public void insertMonthlyMeasure(List<Harvest> data, java.util.Date from, java.util.Date to, boolean doUpdate)
|
||||||
|
throws DaoException {
|
||||||
|
// first of all: check if data of the same type are already in the
|
||||||
|
// database.
|
||||||
|
// In this case data will be updated.
|
||||||
|
Calendar cFrom = Utils.dateToCalendar(from);
|
||||||
|
Calendar cTo = Utils.dateToCalendar(to);
|
||||||
|
int monthFrom = cFrom.get(Calendar.MONTH);
|
||||||
|
int yearFrom = cFrom.get(Calendar.YEAR);
|
||||||
|
int monthTo = cTo.get(Calendar.MONTH);
|
||||||
|
int yearTo = cTo.get(Calendar.YEAR);
|
||||||
|
|
||||||
|
if ((monthFrom != monthTo) || (yearFrom != yearTo)) {
|
||||||
|
String err = "Invalid time period. The time period MUST refer one month.";
|
||||||
|
logger.error(err);
|
||||||
|
throw new DaoException(err, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data == null || data.size() <= 0) {
|
||||||
|
String err = "No data passed in input. Aborting operation.";
|
||||||
|
logger.error(err);
|
||||||
|
throw new DaoException(err, null);
|
||||||
|
}
|
||||||
|
monthFrom++; // because january = 0...
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (Harvest harvest : data) {
|
||||||
|
String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType()
|
||||||
|
+ " and context_id=(select id from context where dname='" + harvest.getContext()
|
||||||
|
+ "') and month=" + monthFrom + " and year=" + yearFrom;
|
||||||
|
logger.debug(query);
|
||||||
|
Statement sel = conn.createStatement();
|
||||||
|
ResultSet rs = sel.executeQuery(query);
|
||||||
|
if (rs.next()) {
|
||||||
|
if (doUpdate) {
|
||||||
|
// record found: update it
|
||||||
|
Statement s = conn.createStatement();
|
||||||
|
int id = rs.getInt("id");
|
||||||
|
String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id="
|
||||||
|
+ id;
|
||||||
|
logger.debug(update);
|
||||||
|
s.execute(update);
|
||||||
|
s.close();
|
||||||
|
} else {
|
||||||
|
logger.warn("Skipped " + harvest.getContext());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// record not found: insert new record
|
||||||
|
Statement s = conn.createStatement();
|
||||||
|
String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values (";
|
||||||
|
insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType()
|
||||||
|
+ ",";
|
||||||
|
insert += "(select id from context where dname='" + harvest.getContext() + "'),";
|
||||||
|
insert += "'" + yearFrom + "-" + monthFrom + "-01'";
|
||||||
|
insert += ")";
|
||||||
|
logger.debug(insert);
|
||||||
|
s.execute(insert);
|
||||||
|
s.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
rs.close();
|
||||||
|
sel.close();
|
||||||
|
|
||||||
|
}
|
||||||
|
} catch (PSQLException x) {
|
||||||
|
// handled exception: try to iterate...
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
} catch (Exception x) {
|
||||||
|
// not handled exception: stop
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ArrayList<Integer> getSubTree(Integer rootId) throws DaoException {
|
||||||
|
String queryBase = "select id from context where context_id in (%%)";
|
||||||
|
ArrayList<Integer> subTree = new ArrayList<>();
|
||||||
|
ArrayList<Integer> temp = new ArrayList<>();
|
||||||
|
Statement s = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
temp.add(rootId);
|
||||||
|
subTree.add(rootId);
|
||||||
|
boolean again = true;
|
||||||
|
for (int i = 0; (i < 10) && (again); i++) {
|
||||||
|
try {
|
||||||
|
String listId = "";
|
||||||
|
for (Integer id : temp) {
|
||||||
|
listId += "," + id;
|
||||||
|
}
|
||||||
|
listId = listId.substring(1);
|
||||||
|
|
||||||
|
String query = queryBase.replace("%%", listId);
|
||||||
|
s = conn.createStatement();
|
||||||
|
rs = s.executeQuery(query);
|
||||||
|
if (rs.next()) {
|
||||||
|
temp = new ArrayList<>();
|
||||||
|
Integer dbId = rs.getInt("id");
|
||||||
|
subTree.add(dbId);
|
||||||
|
temp.add(dbId);
|
||||||
|
while (rs.next()) {
|
||||||
|
dbId = rs.getInt("id");
|
||||||
|
subTree.add(dbId);
|
||||||
|
temp.add(dbId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
again = false;
|
||||||
|
}
|
||||||
|
rs.close();
|
||||||
|
s.close();
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return subTree;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void createSocialReport(int contextId, int orderBy) throws DaoException {
|
||||||
|
Statement rep = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
String report = "insert into report (context_id, name, orderby, chart_type, x_text, y_text) values ";
|
||||||
|
report += "(" + contextId + ", \'VRE Social\', " + orderBy
|
||||||
|
+ ", \'column\', \'Period\', \'Social interaction\')";
|
||||||
|
logger.debug(report);
|
||||||
|
rep = conn.createStatement();
|
||||||
|
rep.execute(report);
|
||||||
|
String query = "select id from report where name=\'VRE Social\' order by id desc";
|
||||||
|
rs = rep.executeQuery(query);
|
||||||
|
if (rs.next()) {
|
||||||
|
int id = rs.getInt("id");
|
||||||
|
String reportItem = "insert into report_item (report_id, type_id_1, type_id_1_name) ";
|
||||||
|
reportItem += "values (" + id + ", 8, \'mt\'), (" + id + ", 9, \'mt\'), (" + id + ", 10, \'mt\')";
|
||||||
|
rep.execute(reportItem);
|
||||||
|
} else
|
||||||
|
throw new DaoException("No report id.", null);
|
||||||
|
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
if (rs != null) {
|
||||||
|
rs.close();
|
||||||
|
}
|
||||||
|
if (rep != null) {
|
||||||
|
rep.close();
|
||||||
|
}
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dummy tester
|
||||||
|
*
|
||||||
|
* @Deprecated
|
||||||
|
* @param originator
|
||||||
|
* @throws DaoException
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public void dummyTest(String originator) throws DaoException {
|
||||||
|
String insert = "insert into dummy (now, originator) values(current_timestamp, '" + originator + "')";
|
||||||
|
logger.debug(insert);
|
||||||
|
Statement s = null;
|
||||||
|
try {
|
||||||
|
s = conn.createStatement();
|
||||||
|
s.execute(insert);
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
} finally {
|
||||||
|
if (s != null)
|
||||||
|
try {
|
||||||
|
s.close();
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,13 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
public class DaoException extends Exception {
|
||||||
|
|
||||||
|
public DaoException() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public DaoException(String message, Throwable throwable) {
|
||||||
|
super(message, throwable);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,46 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
public class DatabaseConnectionData {
|
||||||
|
String url, user, password;
|
||||||
|
|
||||||
|
public DatabaseConnectionData() {
|
||||||
|
this.url = null;
|
||||||
|
this.user = null;
|
||||||
|
this.password = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseConnectionData(String url, String user, String password) {
|
||||||
|
this.url = url;
|
||||||
|
this.user = user;
|
||||||
|
this.password = password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUrl() {
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUrl(String url) {
|
||||||
|
this.url = url;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUser() {
|
||||||
|
return user;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUser(String user) {
|
||||||
|
this.user = user;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPassword() {
|
||||||
|
return password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPassword(String password) {
|
||||||
|
this.password = password;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "DatabaseConnectionData [url=" + url + ", user=" + user + ", password=" + password + "]";
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,98 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.common.encryption.StringEncrypter;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class DatabaseDataExplorer {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DatabaseDataExplorer.class);
|
||||||
|
private boolean testMode = false;
|
||||||
|
private boolean productionMode = false;
|
||||||
|
|
||||||
|
public DatabaseDataExplorer() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* setter method for testMode member. this member is set to TRUE if you want
|
||||||
|
* avoid to use production database during develop/test phase
|
||||||
|
*
|
||||||
|
* @param testMode
|
||||||
|
*/
|
||||||
|
public void setTestMode(boolean testMode) {
|
||||||
|
this.testMode = testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* getter method for testMode member.
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean getTestMode() {
|
||||||
|
return testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseConnectionData retrieveDatabaseInfo() throws DaoException {
|
||||||
|
DatabaseConnectionData dcd = null;
|
||||||
|
if ((testMode == false) && (productionMode == false)) {
|
||||||
|
String jdbcUrl = "jdbc:postgresql://";
|
||||||
|
dcd = new DatabaseConnectionData();
|
||||||
|
try {
|
||||||
|
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/Category/text() eq 'Database'")
|
||||||
|
.addCondition("$resource/Profile/Name/text() eq 'Analytics Board'");
|
||||||
|
DiscoveryClient<String> client = ICFactory.client();
|
||||||
|
List<String> list = client.submit(query);
|
||||||
|
if (!list.isEmpty()) {
|
||||||
|
String xml = list.iterator().next();
|
||||||
|
|
||||||
|
String server = getItemValueFromXmlString(xml, "Endpoint");
|
||||||
|
String db = getItemAttributeFromXmlString(xml, "Endpoint", "EntryName");
|
||||||
|
jdbcUrl += server + "/" + db;
|
||||||
|
dcd.setUrl(jdbcUrl);
|
||||||
|
|
||||||
|
String user = getItemValueFromXmlString(xml, "Username");
|
||||||
|
dcd.setUser(user);
|
||||||
|
|
||||||
|
String password = StringEncrypter.getEncrypter()
|
||||||
|
.decrypt(getItemValueFromXmlString(xml, "Password"));
|
||||||
|
dcd.setPassword(password);
|
||||||
|
logger.debug("Database data found");
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
}
|
||||||
|
} else if(testMode) {
|
||||||
|
logger.debug("TEST mode is ON");
|
||||||
|
dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/analytics_b_dev", "analytics_b_dev_u", "78cb625303be21b");
|
||||||
|
} else if(productionMode) {
|
||||||
|
logger.warn("PRODUCTION mode is FORCED ON");
|
||||||
|
dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv.d4science.org:5432/analytics_board", "analytics_board_u", "b52b64ab07ea0b5");
|
||||||
|
}
|
||||||
|
|
||||||
|
return dcd;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getItemValueFromXmlString(String xml, String item) {
|
||||||
|
int start = xml.indexOf("<" + item);
|
||||||
|
int begin = xml.indexOf(">", start);
|
||||||
|
int end = xml.indexOf("<", begin);
|
||||||
|
String value = xml.substring(begin + 1, end);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getItemAttributeFromXmlString(String xml, String item, String attribute) {
|
||||||
|
int start = xml.indexOf("<" + item);
|
||||||
|
start = xml.indexOf(attribute, start);
|
||||||
|
int begin = xml.indexOf("\"", start);
|
||||||
|
int end = xml.indexOf("\"", begin + 1);
|
||||||
|
String attributeValue = xml.substring(begin + 1, end);
|
||||||
|
return attributeValue;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,77 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class DatabaseManager {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DatabaseManager.class);
|
||||||
|
private boolean testMode = false;
|
||||||
|
|
||||||
|
public DatabaseManager(boolean testMode) {
|
||||||
|
this.testMode = testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseManager() {
|
||||||
|
this.testMode = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTestMode(boolean testMode) {
|
||||||
|
this.testMode = testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isTestMode() {
|
||||||
|
return testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void insertMonthlyData(Date from, Date to, List<Harvest> data, boolean updateFlag) {
|
||||||
|
Dao dao = null;
|
||||||
|
try {
|
||||||
|
dao = dbConnect(testMode);
|
||||||
|
dao.insertMonthlyMeasure(data, from, to, updateFlag);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
} finally {
|
||||||
|
if (dao != null) {
|
||||||
|
try {
|
||||||
|
dao.disconnect();
|
||||||
|
} catch (DaoException e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] getActiveVres() {
|
||||||
|
Dao dao = null;
|
||||||
|
try {
|
||||||
|
dao = dbConnect(testMode);
|
||||||
|
return dao.getActiveVres();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (dao != null) {
|
||||||
|
try {
|
||||||
|
dao.disconnect();
|
||||||
|
} catch (DaoException e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Dao dbConnect(boolean testMode) throws DaoException {
|
||||||
|
DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
||||||
|
dde.setTestMode(testMode);
|
||||||
|
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
||||||
|
Dao dao = new Dao();
|
||||||
|
dao.init();
|
||||||
|
dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
||||||
|
return dao;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,85 @@
|
||||||
|
package org.gcube.dataharvest.datamodel;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
public class Harvest implements Serializable {
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
public static int ACCESSESS = 1;
|
||||||
|
public static int USERS = 2;
|
||||||
|
public static int DATA_METHOD_DOWNLOAD = 3;
|
||||||
|
public static int NEW_CATALOGUE_METHODS = 4;
|
||||||
|
public static int NEW_CATALOGUE_DATASETS = 5;
|
||||||
|
public static int NEW_CATALOGUE_DELIVERABLES = 6;
|
||||||
|
public static int NEW_CATALOGUE_APPLICATIONS = 7;
|
||||||
|
public static int SOCIAL_POSTS = 8;
|
||||||
|
public static int SOCIAL_REPLIES = 9;
|
||||||
|
public static int SOCIAL_LIKES = 10;
|
||||||
|
public static int METHOD_INVOCATIONS = 11;
|
||||||
|
public static int VISUAL_TOOLS = 12;
|
||||||
|
|
||||||
|
private int dataType;
|
||||||
|
private String context;
|
||||||
|
private long measure;
|
||||||
|
private Date day;
|
||||||
|
|
||||||
|
public Harvest() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public Harvest(int dataType, String context, long measure, Date day) {
|
||||||
|
super();
|
||||||
|
this.dataType = dataType;
|
||||||
|
this.context = context;
|
||||||
|
this.measure = measure;
|
||||||
|
this.day = day;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Harvest(int dataType, String context, long measure) {
|
||||||
|
this.dataType = dataType;
|
||||||
|
this.context = context;
|
||||||
|
this.measure = measure;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void setDataType(int dataType) {
|
||||||
|
this.dataType = dataType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setContext(String context) {
|
||||||
|
this.context = context;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMeasure(long measure) {
|
||||||
|
this.measure = measure;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDay(Date day) {
|
||||||
|
this.day = day;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getDataType() {
|
||||||
|
return dataType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContext() {
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getMeasure() {
|
||||||
|
return measure;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date getDay() {
|
||||||
|
return day;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "Harvest [context=" + context + ", dataType=" + dataType + ", measure=" + measure + "]";
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,341 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.dao.Dao;
|
||||||
|
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.common.authorization.client.Constants;
|
||||||
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.encryption.StringEncrypter;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
|
||||||
|
import org.gcube.common.resources.gcore.utils.Group;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.informationsystem.publisher.RegistryPublisher;
|
||||||
|
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class BasicHarvester {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class);
|
||||||
|
public Date startDate = null, endDate = null;
|
||||||
|
public DateFormat format;
|
||||||
|
public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester";
|
||||||
|
public final String CATEGORY_NAME = "Accounting";
|
||||||
|
|
||||||
|
public BasicHarvester(String start, String end) throws ParseException {
|
||||||
|
format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
|
||||||
|
startDate = format.parse(start);
|
||||||
|
endDate = format.parse(end);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BasicHarvester(Date start, Date end) throws ParseException {
|
||||||
|
startDate = start;
|
||||||
|
endDate = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getCurrentContext(String token) throws Exception{
|
||||||
|
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
|
||||||
|
String context = authorizationEntry.getContext();
|
||||||
|
logger.info("Context of token {} is {}", token, context);
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void setContext(String token) throws Exception{
|
||||||
|
SecurityTokenProvider.instance.set(token);
|
||||||
|
ScopeProvider.instance.set(getCurrentContext(token));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getCurrentContext() throws Exception {
|
||||||
|
String token = SecurityTokenProvider.instance.get();
|
||||||
|
return getCurrentContext(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Properties readServiceEndpoint() throws Exception {
|
||||||
|
Properties props = new Properties();
|
||||||
|
String scope = "/d4science.research-infrastructures.eu";
|
||||||
|
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||||
|
|
||||||
|
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
|
||||||
|
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
|
||||||
|
|
||||||
|
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||||
|
String password = null;
|
||||||
|
try {
|
||||||
|
List<ServiceEndpoint> list = client.submit(query);
|
||||||
|
if (list.size() > 1) {
|
||||||
|
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
|
||||||
|
+ " in this scope having Category " + CATEGORY_NAME);
|
||||||
|
} else if (list.size() == 0) {
|
||||||
|
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
|
||||||
|
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||||
|
} else {
|
||||||
|
for (ServiceEndpoint res : list) {
|
||||||
|
AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
|
||||||
|
.toArray(new AccessPoint[res.profile().accessPoints().size()]);
|
||||||
|
for (AccessPoint found : accessPoints) {
|
||||||
|
password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
|
||||||
|
for (ServiceEndpoint.Property prop : found.properties()) {
|
||||||
|
props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return props;
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void updateServiceEndPoint(ArrayList<Property> newProps) throws Exception {
|
||||||
|
String scope = "/d4science.research-infrastructures.eu";
|
||||||
|
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||||
|
|
||||||
|
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
|
||||||
|
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
|
||||||
|
|
||||||
|
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||||
|
String password = null;
|
||||||
|
try {
|
||||||
|
List<ServiceEndpoint> list = client.submit(query);
|
||||||
|
if (list.size() > 1) {
|
||||||
|
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
|
||||||
|
+ " in this scope having Category " + CATEGORY_NAME);
|
||||||
|
} else if (list.size() == 0) {
|
||||||
|
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
|
||||||
|
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||||
|
} else {
|
||||||
|
ServiceEndpoint serviceEndpoint = list.get(0);
|
||||||
|
AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0];
|
||||||
|
Group<Property> oldProps = accessPoint.properties();
|
||||||
|
for (Property prop : newProps) {
|
||||||
|
String propValue = prop.value();
|
||||||
|
propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue);
|
||||||
|
String propKey = prop.name();
|
||||||
|
org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd =
|
||||||
|
new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue);
|
||||||
|
pToAdd.encrypted(true);
|
||||||
|
accessPoint.properties().add(pToAdd);
|
||||||
|
}
|
||||||
|
|
||||||
|
RegistryPublisher publisher = RegistryPublisherFactory.create();
|
||||||
|
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||||
|
publisher.update(serviceEndpoint);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] getActiveVREs(boolean testMode) {
|
||||||
|
DatabaseManager dbaseManager = new DatabaseManager(testMode);
|
||||||
|
return dbaseManager.getActiveVres();
|
||||||
|
}
|
||||||
|
|
||||||
|
// public static void main(String[] args) {
|
||||||
|
//// String[][] data = {
|
||||||
|
//// {"/d4science.research-infrastructures.eu","3fdf5a02-383c-4364-a9c6-d3e2b0a4a401-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research","1ff82b79-4042-4311-84e1-a11f955b5f7f-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","f920ca10-af4d-4b70-9fa8-75f7bb78987c-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/fosteropenscience","188fb0fa-083b-4043-94a6-62152cd7b94e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/H2020greeneu","ed045fb9-805b-4d6c-a72a-d0a464e7a5dc-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/FAIR_DM","9429cc8d-a3d3-43f6-b793-e29be7e7bfba-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","521908fe-deab-40df-ab59-61a8a3fb7325-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","1b9caa78-004a-4173-a2f0-6f67f7904630-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","5bd19b68-aecf-4b10-8ca8-ff88f36ca350-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/AGINFRAplus","6de2ce44-7a7e-4da5-875a-5858eae8bd37-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/FisheriesandEcosystemAtMII","4928bb4b-9da5-426a-88ce-2f3a4853b665-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","8019bf10-3d31-4eaa-af48-cb327eceabac-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","bed1d5b5-106a-493e-a814-6d5cc99b4b93-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/OpenAIRE-Connect_Coordination","951c4abc-1850-49f3-a852-099d191de708-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/EuBrazilOpenBio","4449d0be-83a3-4966-bac4-beaca76577fa-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","6f66e93f-84bf-4171-83e5-a49d4d470465-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ENVRI","8194d45e-baa5-4d0a-9d27-3c81eb8b867d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/DEMETER","975bf904-1fd7-437f-bb13-8e196ce24aab-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/InfraTraining","f392969e-d2d4-4414-9d62-fb9675ea2056-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/NERLiX","21a34e10-a4ae-4a47-8b78-2313db5a9eb2-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/E-RIHS","82ffc3ac-518f-48a7-b9c4-69f0372b49a5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/EcoEvo","8b8e8fd9-cda5-433d-8a88-530cc26a2061-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","1c7c72a8-93ca-46d0-baee-42da39654c11-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_FIACO2017","775fb6a3-d69d-4902-b379-6d44e4d2b7dc-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/DOPA_AnalystSpatialPlanning","70a39850-264e-496b-9ce0-b764a70e4660-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/Sinay","01c6b4b5-f726-45e0-8311-a2fdbcef54ff-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/StatnMap","d71d9c9b-def6-4df3-a22f-6bcd36dce284-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ISTIOpenAccess","2c4ab315-51cd-469c-af2e-ceccd4708093-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroToREnv","b5112916-3e9d-48ab-afc0-195ceda24489-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_BNetworkAnalysis","30b7506e-a016-4438-be34-9af57af1d0c0-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/FoodborneOutbreak","b3eff03b-aac9-4bd0-a568-6870bb5559d4-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/EOSC_Services","518f0ef5-8510-46e9-9969-89d7d2c22301-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/SDG-Indicator14.4.1","1953f45d-4ff7-45ce-8232-00439f1ab48a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/FoodSecurity","15c3a6a6-4092-473b-a296-7cd5e5eadff4-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/RAKIP_portal","39bc9d6a-39ba-4664-9c62-330f371ce073-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","a9186abb-2af0-459e-ad59-3558a8127560-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/TextCrowd","8cc715b0-36a7-480e-ad24-8ecbc6b0577d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/EOSCPortfolio","8e499271-fe8d-4e0a-a3e2-cddef19515ef-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ParticleFormation","fec95ea3-079d-423a-8620-46a0364696fc-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_LogbookData","b44037d7-a4bf-4542-bc14-85c210a6087d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","f8f18b96-8654-462e-b12d-73c265952bff-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/PerformFish","8fea1590-73f9-46e1-8303-99ef91759f58-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/EISCAT","0edb3ec5-37a1-47f2-b4be-4624a5e5f4e5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/AgroClimaticModelling","90fa5480-de0a-48c1-83fd-2c381abbd1bf-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ENVRIPlusData4Science","8e4d9a87-0da9-45c0-9542-612f14d5b454-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/OnlineOceanography","451ce6b2-09d8-4253-ac1d-56b0c633ffa7-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/QCAPI","7a829b89-20b2-48af-8c9e-641a1acaf47a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/SDI_Lab","76ca201a-4a89-4871-9e9d-3043ecc60be2-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ORIONKnowledgeHub","ff3228dc-4b29-4919-924d-462d9f5b9d42-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/D4Research/ICOSEddyCovarianceProcessing","781272a4-d498-4baf-8a14-468b83839452-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/Edison","289a3fc0-cd23-43ee-bf18-b3efc4d1a182-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/Edison/RstudioVirtualLab","d6b795ec-2719-4ac2-a511-fe032ef43b15-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM","cb7dadd0-dbdd-46a6-8db2-1dc4ffc64759-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","f4811784-0d4a-4731-9f75-3b407544b31f-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/VME-DB","e8b93617-217e-44ac-ae02-09a45bc26886-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","17bdb8ae-cff9-44a7-ac27-c9e26a254290-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/TBTI_VRE","02c36b15-1229-47b1-a583-296ec5b404fd-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/GRSF","833d6a12-34a9-4082-9ecb-358d19244269-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/KIMAGRO_Fishfarming","b6967277-73e5-4b23-8fd6-c47e2e5a5ed2-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/D4STeam","e18b3dd5-b6fe-4865-8727-434354c24b9f-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/AquaMaps","6b08ffe9-695d-45bf-90fb-6361a4234ab5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/iMarineBoardVRE","78f6a7be-55c5-4602-ae81-fd4955a3a72e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","09a84b5a-b869-4552-887d-23248bc5b042-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","c6317519-b238-449c-bd3d-6c832f98a611-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","1457e707-492d-46a0-b848-6c6cd758a6ce-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","a8f98114-05b8-44b2-a89c-0f56e55412fd-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","a948e508-0ecd-4013-a094-fe3c506209e1-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/ForkysVRE","6ed093fe-f245-4638-acfe-a70a1f24146d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","b7c16d55-8344-492b-a94b-8fbf7af27e08-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/Aquabiotech","c55477cf-8c98-4bd0-a05d-6182ab1be4e8-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/NHREUS_Aquaculture","c7e19ccc-9fc1-49c0-8351-d507830e7660-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/GALAXIDI","69e9ec43-62a7-45cb-b868-a369aeabf123-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/CFE_DATA","75d70975-bee4-4f46-94be-ff8773df6bbb-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/CWP_Secretariat","d84f0ade-be6a-40d6-a152-66e1b5d7d75d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/SustainableBlueEconomy","19bbf2bb-58e3-4140-81cc-4043fd8755e5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/FARM/CollabResearch","2580b8ae-fad1-4213-8fd4-78b8a90c2c90-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps","e844592c-6428-4483-8614-e9561ad806f5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","fe58019c-f3eb-4d3a-8094-54d18df00940-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/GEMex","afe00e12-5351-4941-aa31-1285184e2aea-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/KnowledgeBridging","cb98c3f9-0217-46f3-a74d-bda21bbf00fb-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","a563847b-bb53-4303-950f-5857c59f3217-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","dfbc4f13-874d-4e0b-99a9-b5cf1d48acef-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","8e6dcf9c-6dcc-4172-8dd6-45560af6a227-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/iSearch","df227388-3d61-4f77-97fb-4cd4fd8c6dbe-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","ce150ced-d4a3-48d2-a541-3433f26ab6d5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","a1c1a4e5-7aea-4c12-b724-868adb8fa6ec-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","6c62b572-3b0e-43d2-b224-5c9961dfcb2e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","44c54a21-88df-4a4b-ab1c-d9d5b636baba-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","8953528d-e990-4451-b5a3-517adca107a2-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/rScience","7bfa66d6-6c9b-448f-a11a-21d833843647-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","79fb276e-13d9-4d80-b419-56c9d64888b7-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIP","9c420378-e8f4-472e-bf7b-b5419041f6db-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ENVRIPlus","554d32ca-ae4a-44dd-b8a7-424ec7d30cba-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/TCom","aed4d51f-3dd1-4329-b622-322211ac237a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/CNR_OpenScienceTF","6978ab04-ae3c-4cee-8b11-9f64ee733502-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","c1674036-1388-4eca-bbbb-174b9dc95dd1-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","38e4fa5a-6c50-417f-b952-1525cebd4950-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","776ea841-2205-4590-a0dc-24141138972e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/DESCRAMBLE","2c6561bb-ee65-4537-9ccc-ec2464291630-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ARIADNE","05ffc92b-03d1-4b05-8e33-e62f795202b5-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/EcologicalModelling","2fa2177d-a2a8-48d4-898f-3f301842247c-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","eba4dc9d-b7cf-40b8-adfb-aecdc731c3de-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","9d78cbba-78e2-4efe-a38a-e14744318f2d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","49b00009-e35f-44dc-ab0e-a63b505aca79-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","cb859658-0439-4469-8b2a-7c4516c3a659-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","9f2746e3-3e04-4f80-9dd7-e66f42ff53f9-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","8166fe91-7fe5-4636-9821-b8718aaa8500-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","867dfe04-ed24-42ad-902f-e2ee63963054-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIEngage","548ffb80-2dbe-4d0d-afb9-1fd009c2edca-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/EFG","e085b1c4-f315-4a21-88c4-46748b81b3f3-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/IGDI","95306e9b-0ce0-4b72-be18-075873fe634a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","5cfdd223-401c-4324-9c99-9bb4462ff854-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu","6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","2856e458-7fd8-42a4-a1a9-b5566cf76ff7-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","bffc959f-5eb3-4d21-9222-27b99fa41f5b-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/gCube","9e5aa777-c700-4975-a218-6c73cc8b1c69-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","ec2d9727-2b21-4df5-823d-3f28a2d8295e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/Parthenos","42e742e2-75eb-401e-9d07-43deb713f926-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it","8b4d9866-1d55-470a-95c3-4cdc19cecb0d-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","49c20941-27b1-4294-aa23-58809ef8616c-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICOS_ETC","cac5112b-1a9c-476b-bc33-3b5d3192b1d0-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","3c6dfc0b-81a3-453c-b693-fe8337a735f1-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/gCubeApps/FrenchTropicalTunaAtlas","703fb5a1-54e2-4b1a-920c-978356d4a74a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE","2a05405f-e878-4f80-a522-d2381b97e0df-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Technical","15ee7828-caa0-4b07-a950-9f1ffdbc812b-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Networking","156ca39a-0485-467d-aebb-59b7b49690b4-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_PSC","2d891c6a-4f6f-4281-8ffd-56270d0d0783-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_EAB","66856a20-7a58-4188-afcf-ab5fa212a732-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE_Users","64c73441-5a88-45e7-a865-dc449dadf271-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/1st_OpenAIRE_Datathon","5c6946e9-277c-4813-b949-5c5e215127b6-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/OpenAIRE/dnet","75e2db7a-03ae-4af3-b87d-d7f1f20cc194-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/ParthenosVO","8b8cf7cd-ff1a-4827-83cf-3933e2320d26-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/ParthenosVO/RubRIcA","4a9e0e5a-3b9e-4293-9dc7-4a118f3a37f3-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_Registry","0cdceda3-f6e1-4bb7-aa20-c3a4b8a891ba-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_LAB","75de04c9-aa4e-4e32-9770-4094de1d0d54-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/ParthenosVO/ACE","05f21525-b6fd-4587-ae7f-4e45a9fddfdf-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SmartArea","3782e93f-3d1a-4809-be0f-7c4b20eaa74b-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SmartArea/SmartBuilding","4216a83a-5506-43be-93b4-1464020cacce-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SmartArea/SmartApps","aa24a1d7-672a-4e06-b220-7d353188fc0a-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData","060a074b-bb12-4ab2-a585-cf0119f4d8ea-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens","28367906-d586-42e3-a5d8-6043a43ce54e-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue","2497dbd8-4002-4b50-b17c-ed2dc17cb0d7-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy","90ae1d9b-cfcb-4fdd-a91e-38ff6013f688-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/TagMe","9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/SMAPH","c0924749-cb6d-41e9-88a4-363e7f9e7556-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab","37d61ab1-3641-435d-8c4f-f1d37eb34ead-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/SocietalDebates","59450ce1-cf1d-4abb-9b0c-63346ad35e08-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/M-ATLAS","ace198d7-f50f-4557-9788-f7aa91c31d67-843339462"},
|
||||||
|
//// {"/d4science.research-infrastructures.eu/SoBigData/SportsDataScience","61bf4b8d-34e2-4426-a3dd-b41a1dcbccb1-843339462"}
|
||||||
|
//// };
|
||||||
|
////
|
||||||
|
//// ArrayList<Property> props = new ArrayList<>();
|
||||||
|
//// for(String[] datap : data) {
|
||||||
|
//// Property p = new Property();
|
||||||
|
//// p = p.nameAndValue(datap[0], datap[1]);
|
||||||
|
//// props.add(p);
|
||||||
|
//// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// try {
|
||||||
|
// //new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59").updateServiceEndPoint(props);
|
||||||
|
// BasicHarvester bh = new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
|
||||||
|
// Properties props = bh.readServiceEndpoint();
|
||||||
|
// Iterator i = props.keySet().iterator();
|
||||||
|
// while(i.hasNext()) {
|
||||||
|
// String key = (String)i.next();
|
||||||
|
// System.out.println(key + ":" + props.getProperty(key));
|
||||||
|
// }
|
||||||
|
// } catch (ParseException e) {
|
||||||
|
// // TODO Auto-generated catch block
|
||||||
|
// e.printStackTrace();
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// // TODO Auto-generated catch block
|
||||||
|
// e.printStackTrace();
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,145 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.apache.commons.lang.Validate;
|
||||||
|
import org.gcube.common.homelibrary.home.Home;
|
||||||
|
import org.gcube.common.homelibrary.home.HomeLibrary;
|
||||||
|
import org.gcube.common.homelibrary.home.HomeManager;
|
||||||
|
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
|
||||||
|
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
|
||||||
|
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
|
||||||
|
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException;
|
||||||
|
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
|
||||||
|
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
|
||||||
|
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
|
||||||
|
import org.gcube.common.homelibrary.model.exceptions.RepositoryException;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
|
||||||
|
public class DataMethodDownloadHarvester extends BasicHarvester {
|
||||||
|
private Date startDate, endDate;
|
||||||
|
private DateFormat format;
|
||||||
|
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
|
||||||
|
private int count = 0;
|
||||||
|
|
||||||
|
public DataMethodDownloadHarvester(String start, String end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
//ScopeProvider.instance.set(SCOPE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
//ScopeProvider.instance.set(SCOPE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
ScopeProvider.instance.set(SCOPE);
|
||||||
|
String[] vres = getActiveVREs(true);
|
||||||
|
for (String vre : vres) {
|
||||||
|
try {
|
||||||
|
count = 0;
|
||||||
|
|
||||||
|
logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()");
|
||||||
|
|
||||||
|
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
|
||||||
|
|
||||||
|
String user = getVREName(vre) + "-Manager";
|
||||||
|
|
||||||
|
logger.error("Home home = manager.getHome(user)");
|
||||||
|
|
||||||
|
Home home = manager.getHome(user);
|
||||||
|
|
||||||
|
logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()");
|
||||||
|
|
||||||
|
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
|
||||||
|
|
||||||
|
logger.error("ws.getItemByPath...");
|
||||||
|
|
||||||
|
JCRWorkspaceItem item = (JCRWorkspaceItem) ws
|
||||||
|
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre));
|
||||||
|
|
||||||
|
logger.debug("Analyzing " + vre + " from " + startDate.toString() + " to " + endDate.toString());
|
||||||
|
|
||||||
|
logger.error("Before getStats()");
|
||||||
|
getStats(item, startDate, endDate);
|
||||||
|
logger.error("After getStats()");
|
||||||
|
Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, vre, count);
|
||||||
|
data.add(harvest);
|
||||||
|
logger.debug(harvest.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("DataMethodDownloadHarvester: Error processing VRE: " + vre + "\n" +
|
||||||
|
e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void getStats(WorkspaceItem root, Date start, Date end) throws InternalErrorException {
|
||||||
|
List<? extends WorkspaceItem> children;
|
||||||
|
if (root.isFolder()) {
|
||||||
|
children = root.getChildren();
|
||||||
|
for (WorkspaceItem child : children)
|
||||||
|
getStats(child, start, end);
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
|
||||||
|
List<AccountingEntry> accounting = root.getAccounting();
|
||||||
|
for (AccountingEntry entry : accounting) {
|
||||||
|
|
||||||
|
switch (entry.getEntryType()) {
|
||||||
|
case CREATE:
|
||||||
|
case UPDATE:
|
||||||
|
case READ:
|
||||||
|
Calendar calendar = entry.getDate();
|
||||||
|
if (calendar.after(Utils.dateToCalendar(start)) && calendar.before(Utils.dateToCalendar(end))) {
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("DataMethodDownloadHarvester: " + e.getLocalizedMessage());
|
||||||
|
throw new InternalErrorException(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getVREName(String vre) {
|
||||||
|
Validate.notNull(vre, "scope must be not null");
|
||||||
|
|
||||||
|
String newName;
|
||||||
|
if (vre.startsWith(JCRRepository.PATH_SEPARATOR))
|
||||||
|
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1);
|
||||||
|
else
|
||||||
|
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-");
|
||||||
|
return newName;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,115 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
|
||||||
|
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||||
|
import org.gcube.accounting.analytics.Filter;
|
||||||
|
import org.gcube.accounting.analytics.Info;
|
||||||
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||||
|
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||||
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||||
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||||
|
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||||
|
|
||||||
|
import java.util.SortedMap;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class MethodInvocationHarvester extends BasicHarvester {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
|
||||||
|
|
||||||
|
public MethodInvocationHarvester(String start, String end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
try {
|
||||||
|
logger.debug("MethodInvocationHarvester::getData()");
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
|
||||||
|
Calendar start = Calendar.getInstance();
|
||||||
|
Calendar end = Calendar.getInstance();
|
||||||
|
start.setTime(startDate);
|
||||||
|
end.setTime(endDate);
|
||||||
|
TemporalConstraint temporalConstraint = new TemporalConstraint(start.getTimeInMillis(),
|
||||||
|
end.getTimeInMillis(), AggregationMode.MONTHLY);
|
||||||
|
|
||||||
|
List<Filter> filters = new ArrayList<>();
|
||||||
|
filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, "tag"));
|
||||||
|
|
||||||
|
List<String> contexts = new ArrayList<>();
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe");
|
||||||
|
|
||||||
|
logger.debug("MethodInvocationHarvester::getData()::getContextTimeSeries");
|
||||||
|
SortedMap<Filter, SortedMap<Calendar, Info>> result = accountingPersistenceQuery.getContextTimeSeries(
|
||||||
|
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||||
|
if (result == null) {
|
||||||
|
logger.error("No data found.");
|
||||||
|
} else {
|
||||||
|
Set<Filter> ks = result.keySet();
|
||||||
|
if (ks != null) {
|
||||||
|
Iterator<Filter> ksi = ks.iterator();
|
||||||
|
while (ksi.hasNext()) {
|
||||||
|
// System.out.println("" + ksi.next().toString());
|
||||||
|
logger.debug("Filter: " + ksi.next().toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
} catch (Exception x) {
|
||||||
|
StackTraceElement[] ste = x.getStackTrace();
|
||||||
|
String errorMessage = "MethodInvocationHarvester: " + x.getLocalizedMessage();
|
||||||
|
for (StackTraceElement s : ste) {
|
||||||
|
errorMessage += "\n" + s.toString();
|
||||||
|
}
|
||||||
|
logger.error(errorMessage);
|
||||||
|
throw x;
|
||||||
|
}
|
||||||
|
}/**/
|
||||||
|
|
||||||
|
/* used only for test */
|
||||||
|
public static void main(String[] args) {
|
||||||
|
String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462";
|
||||||
|
String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548";
|
||||||
|
String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462";
|
||||||
|
String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462";
|
||||||
|
String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462";
|
||||||
|
try {
|
||||||
|
MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
|
||||||
|
MethodInvocationHarvester.setContext(SOBIGDATA_EU);
|
||||||
|
System.out.println(MethodInvocationHarvester.getCurrentContext());
|
||||||
|
Properties props = m.readServiceEndpoint();
|
||||||
|
System.out.println("Found " + props.size() + " properties.");
|
||||||
|
Set keys = props.keySet();
|
||||||
|
for(Object key : keys) {
|
||||||
|
System.out.println((String)key + " :: " + props.getProperty((String) key));
|
||||||
|
}
|
||||||
|
|
||||||
|
m.getData();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
System.out.println(e.getLocalizedMessage());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**/
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,200 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.apache.http.client.utils.URLEncodedUtils;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
|
||||||
|
import org.json.JSONArray;
|
||||||
|
import org.json.JSONException;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.quartz.utils.StringKeyDirtyFlagMap;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class ResourceCatalogueHarvester extends BasicHarvester {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class);
|
||||||
|
private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0,
|
||||||
|
wellBeingAndEconomyCounter = 0;
|
||||||
|
|
||||||
|
public ResourceCatalogueHarvester(String start, String end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
List<Harvest> dataDeliverable = getDataDeliverable();
|
||||||
|
for (Harvest harvest : dataDeliverable) {
|
||||||
|
data.add(harvest);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Harvest> dataMethod = getDataMethod();
|
||||||
|
for (Harvest harvest : dataMethod) {
|
||||||
|
data.add(harvest);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Harvest> dataDataset = getDataDataset();
|
||||||
|
for (Harvest harvest : dataDataset) {
|
||||||
|
data.add(harvest);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Harvest> dataApplication = getDataApplication();
|
||||||
|
for (Harvest harvest : dataApplication) {
|
||||||
|
data.add(harvest);
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Harvest> getDataDeliverable() throws Exception {
|
||||||
|
String json = executeQuery("Deliverable");
|
||||||
|
return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Harvest> getDataMethod() throws Exception {
|
||||||
|
String json = executeQuery("Method");
|
||||||
|
return buildList(json, Harvest.NEW_CATALOGUE_METHODS);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Harvest> getDataDataset() throws Exception {
|
||||||
|
String json = executeQuery("Dataset");
|
||||||
|
return buildList(json, Harvest.NEW_CATALOGUE_DATASETS);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Harvest> getDataApplication() throws Exception {
|
||||||
|
String json = executeQuery("Application");
|
||||||
|
return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Harvest> buildList(String json, int dataType) throws Exception {
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
JSONObject jsonObject = new JSONObject(json);
|
||||||
|
|
||||||
|
JSONObject responseHeader = jsonObject.getJSONObject("responseHeader");
|
||||||
|
int status = responseHeader.getInt("status");
|
||||||
|
if (status != 0) {
|
||||||
|
String err = "Query Deliverable in error: status " + status;
|
||||||
|
logger.error(err);
|
||||||
|
throw new Exception(err, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONObject response = jsonObject.getJSONObject("response");
|
||||||
|
|
||||||
|
int numFound = response.getInt("numFound");
|
||||||
|
Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
|
||||||
|
numFound);
|
||||||
|
logger.debug(h.toString());
|
||||||
|
data.add(h);
|
||||||
|
if (numFound > 0) {
|
||||||
|
|
||||||
|
JSONArray docs = response.getJSONArray("docs");
|
||||||
|
for (Object item : docs) {
|
||||||
|
JSONObject doc = (JSONObject) item;
|
||||||
|
try {
|
||||||
|
JSONArray groups = doc.getJSONArray("groups");
|
||||||
|
Iterator<Object> git = groups.iterator();
|
||||||
|
while (git.hasNext()) {
|
||||||
|
String groupItem = (String) git.next();
|
||||||
|
counterByGroup(groupItem);
|
||||||
|
}
|
||||||
|
} catch (JSONException x) {
|
||||||
|
logger.debug("Document without groups");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
|
||||||
|
cityOfCitizensCounter);
|
||||||
|
logger.debug(h.toString());
|
||||||
|
data.add(h);
|
||||||
|
h = new Harvest(dataType, "/Migration Studies", migrationStudiesCounter);
|
||||||
|
logger.debug(h.toString());
|
||||||
|
data.add(h);
|
||||||
|
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates",
|
||||||
|
societalDebatesCounter);
|
||||||
|
logger.debug(h.toString());
|
||||||
|
data.add(h);
|
||||||
|
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy",
|
||||||
|
wellBeingAndEconomyCounter);
|
||||||
|
logger.debug(h.toString());
|
||||||
|
data.add(h);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private String executeQuery(String fqSubString) throws Exception {
|
||||||
|
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
|
||||||
|
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO "
|
||||||
|
+ Utils.dateToStringWithTZ(endDate) + "]");
|
||||||
|
query += "q=" + q;
|
||||||
|
String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\"");
|
||||||
|
query += "&fq=" + fq + "&wt=json&indent=true";
|
||||||
|
logger.debug(query);
|
||||||
|
|
||||||
|
String json = Utils.getJson(query);
|
||||||
|
// logger.debug(json);
|
||||||
|
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void counterByGroup(String groupName) {
|
||||||
|
cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0;
|
||||||
|
switch (groupName) {
|
||||||
|
case "city-of-citizens-group":
|
||||||
|
cityOfCitizensCounter++;
|
||||||
|
break;
|
||||||
|
case "migration-studies":
|
||||||
|
migrationStudiesCounter++;
|
||||||
|
break;
|
||||||
|
case "societal-debates-group":
|
||||||
|
societalDebatesCounter++;
|
||||||
|
break;
|
||||||
|
case "well-being-and-economy-group":
|
||||||
|
wellBeingAndEconomyCounter++;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* used only for test public static void main(String[] args) { Calendar from
|
||||||
|
* = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1);
|
||||||
|
* from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR,
|
||||||
|
* 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0);
|
||||||
|
* from.set(Calendar.SECOND, 0);
|
||||||
|
*
|
||||||
|
* Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30);
|
||||||
|
* to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017);
|
||||||
|
* to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59);
|
||||||
|
* to.set(Calendar.SECOND, 59);
|
||||||
|
*
|
||||||
|
* try { ResourceCatalogueHarvester a = new
|
||||||
|
* ResourceCatalogueHarvester(from.getTime(), to.getTime()); List<Harvest> l
|
||||||
|
* = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); }
|
||||||
|
* // System.out.println(Utils.dateToString(a.startDate)); //
|
||||||
|
* System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch
|
||||||
|
* (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); }
|
||||||
|
* }
|
||||||
|
*/
|
||||||
|
}
|
|
@ -0,0 +1,109 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
import org.json.JSONArray;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class SocialHarvester extends BasicHarvester {
|
||||||
|
private final String CATEGORY_NAME = "Accounting";
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
|
||||||
|
private int likes, replies, posts;
|
||||||
|
private long from = 0, to = 0;
|
||||||
|
|
||||||
|
public SocialHarvester(String start, String end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
this.from = startDate.getTime();
|
||||||
|
this.to = endDate.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
public SocialHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
this.from = startDate.getTime();
|
||||||
|
this.to = endDate.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
Properties props = readServiceEndpoint();
|
||||||
|
String[] vres = getActiveVREs(true);
|
||||||
|
for (String vre : vres) {
|
||||||
|
try {
|
||||||
|
logger.debug("Working on VRE: " + vre);
|
||||||
|
|
||||||
|
String key = vre; // vre.substring(vre.lastIndexOf("/") + 1);
|
||||||
|
String token = props.getProperty(key);
|
||||||
|
//System.out.println(key + ":" + token);
|
||||||
|
getJson(token);
|
||||||
|
//getFinalResult();
|
||||||
|
Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes);
|
||||||
|
data.add(harvest);
|
||||||
|
logger.debug(harvest.toString());
|
||||||
|
|
||||||
|
harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts);
|
||||||
|
data.add(harvest);
|
||||||
|
logger.debug(harvest.toString());
|
||||||
|
|
||||||
|
harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies);
|
||||||
|
data.add(harvest);
|
||||||
|
logger.debug(harvest.toString());
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void getJson(String token) throws MalformedURLException, IOException {
|
||||||
|
likes = replies = posts = 0;
|
||||||
|
// la seguente stringa deve essere letta dinamicamente
|
||||||
|
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token=";
|
||||||
|
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
|
||||||
|
|
||||||
|
Boolean success = (Boolean) jsonObject.get("success");
|
||||||
|
if (success == false) {
|
||||||
|
String message = "getJson() on token: " + token + " success=false";
|
||||||
|
logger.error(message);
|
||||||
|
throw new IOException(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONArray res = jsonObject.getJSONArray("result");
|
||||||
|
int len = res.length();
|
||||||
|
|
||||||
|
for (int i = 0; i < len; i++) {
|
||||||
|
JSONObject item = res.getJSONObject(i);
|
||||||
|
long time = item.getLong("time");
|
||||||
|
//System.out.println(from + " - " + time + " - " + to );
|
||||||
|
if ((from <= time) && (time <= to)) {
|
||||||
|
posts++;
|
||||||
|
replies += item.getInt("comments_no");
|
||||||
|
likes += item.getInt("likes_no");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**/
|
||||||
|
public static void main(String[] argv) {
|
||||||
|
try {
|
||||||
|
SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59");
|
||||||
|
List<Harvest> list = a.getData();
|
||||||
|
for (Harvest l : list) {
|
||||||
|
System.out.println(l.toString());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}/**/
|
||||||
|
}
|
|
@ -0,0 +1,137 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Properties;
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||||
|
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
|
||||||
|
import org.json.*;
|
||||||
|
|
||||||
|
public class VreUsersHarvester extends BasicHarvester {
|
||||||
|
// private final String CATEGORY_NAME = "Accounting";
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
|
||||||
|
|
||||||
|
public VreUsersHarvester(String start, String end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
public VreUsersHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Harvest> getData() throws Exception {
|
||||||
|
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||||
|
Properties props = readServiceEndpoint();
|
||||||
|
String[] vres = getActiveVREs(true);
|
||||||
|
for (String vre : vres) {
|
||||||
|
String key = vre.substring(vre.lastIndexOf("/") + 1);
|
||||||
|
String token = props.getProperty(key);
|
||||||
|
try {
|
||||||
|
int measure = get(token);
|
||||||
|
Harvest harvest = new Harvest(Harvest.USERS, vre, measure);
|
||||||
|
data.add(harvest);
|
||||||
|
logger.debug(harvest.toString());
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error("VreUsersHarvester::getData. " + vre + ". " + x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int get(String token) throws MalformedURLException, IOException {
|
||||||
|
int userNumber = 0;
|
||||||
|
// la seguente stringa deve essere letta dinamicamente
|
||||||
|
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token=";
|
||||||
|
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
|
||||||
|
Boolean success = (Boolean) jsonObject.get("success");
|
||||||
|
if (success == false) {
|
||||||
|
String message = "get-all-usernames returned an error. token: " + token;
|
||||||
|
logger.error(message);
|
||||||
|
throw new IOException(message);
|
||||||
|
}
|
||||||
|
userNumber = jsonObject.getJSONArray("result").length();
|
||||||
|
return userNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
// private Properties readServiceEndpoint() throws Exception {
|
||||||
|
// Properties props = new Properties();
|
||||||
|
// String scope = "/d4science.research-infrastructures.eu";
|
||||||
|
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||||
|
//
|
||||||
|
// SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||||
|
// query.addCondition("$resource/Profile/Name/text() eq '" +
|
||||||
|
// RUNTIME_RESOURCE_NAME + "'");
|
||||||
|
// query.addCondition("$resource/Profile/Category/text() eq '" +
|
||||||
|
// CATEGORY_NAME + "'");
|
||||||
|
//
|
||||||
|
// DiscoveryClient<ServiceEndpoint> client =
|
||||||
|
// clientFor(ServiceEndpoint.class);
|
||||||
|
// String password = null;
|
||||||
|
// try {
|
||||||
|
// List<ServiceEndpoint> list = client.submit(query);
|
||||||
|
// if (list.size() > 1) {
|
||||||
|
// logger.error("Too many Service Endpoints having name " +
|
||||||
|
// RUNTIME_RESOURCE_NAME
|
||||||
|
// + " in this scope having Category " + CATEGORY_NAME);
|
||||||
|
// } else if (list.size() == 0) {
|
||||||
|
// logger.error("There is no Service Endpoint having name " +
|
||||||
|
// RUNTIME_RESOURCE_NAME
|
||||||
|
// + " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||||
|
// } else {
|
||||||
|
// for (ServiceEndpoint res : list) {
|
||||||
|
// AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
|
||||||
|
// .toArray(new AccessPoint[res.profile().accessPoints().size()]);
|
||||||
|
// for (AccessPoint found : accessPoints) {
|
||||||
|
// password =
|
||||||
|
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
|
||||||
|
// for (ServiceEndpoint.Property prop : found.properties()) {
|
||||||
|
// props.setProperty(prop.name(),
|
||||||
|
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// return props;
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// logger.error(e.getLocalizedMessage());
|
||||||
|
// throw e;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public static void main(String[] argv) {
|
||||||
|
// try {
|
||||||
|
// VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01");
|
||||||
|
// List<Harvest> list = a.getData();
|
||||||
|
// for(Harvest l : list) {
|
||||||
|
// System.out.println(l.toString());
|
||||||
|
// }
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// // TODO Auto-generated catch block
|
||||||
|
// e.printStackTrace();
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
}
|
|
@ -0,0 +1,65 @@
|
||||||
|
package org.gcube.dataharvest.utils;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Locale;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.dao.Dao;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class Utils {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
||||||
|
|
||||||
|
public static Calendar dateToCalendar(Date date) {
|
||||||
|
Calendar cal = null;
|
||||||
|
cal = Calendar.getInstance();
|
||||||
|
cal.setTime(date);
|
||||||
|
return cal;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String dateToString(Date date) {
|
||||||
|
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
|
||||||
|
return formatter.format(date);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String dateToStringWithTZ(Date date) {
|
||||||
|
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
||||||
|
return formatter.format(date) + "Z";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getJson(String url) throws MalformedURLException, IOException {
|
||||||
|
URL address = new URL(url);
|
||||||
|
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
||||||
|
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||||
|
String json = "", line = "";
|
||||||
|
|
||||||
|
while (line != null) {
|
||||||
|
line = reader.readLine();
|
||||||
|
if (line != null)
|
||||||
|
json += line.trim();
|
||||||
|
}
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
// il seguente array deve essere formato dinamicamente
|
||||||
|
// public static String[] soBigDataContexts = { "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu",
|
||||||
|
// "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/SMAPH",
|
||||||
|
// "/d4science.research-infrastructures.eu/SoBigData/TagMe" };
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
org.gcube.dataharvest.DataHarvestPluginDeclaration
|
|
@ -0,0 +1,230 @@
|
||||||
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.dao.Dao;
|
||||||
|
import org.gcube.dataharvest.dao.DaoException;
|
||||||
|
import org.gcube.dataharvest.dao.DatabaseConnectionData;
|
||||||
|
import org.gcube.dataharvest.dao.DatabaseDataExplorer;
|
||||||
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.SocialHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.VreUsersHarvester;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class Harvester {
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(Harvester.class);
|
||||||
|
public static final String PARAMETER_FROM = "from";
|
||||||
|
public static final String PARAMETER_TO = "to";
|
||||||
|
public static final String TEST = "test";
|
||||||
|
private boolean testMode = false;
|
||||||
|
private Date dateFrom, dateTo;
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
Harvester harvester = new Harvester();
|
||||||
|
try {
|
||||||
|
harvester.processParameterArray(args);
|
||||||
|
SocialHarvester socialHarvester = new SocialHarvester(harvester.getDateFrom(), harvester.getDateTo());
|
||||||
|
harvester.runOne(socialHarvester);
|
||||||
|
// harvester.run();
|
||||||
|
//ArrayList<Integer> list = harvester.getSubTree(17);
|
||||||
|
//harvester.createSocialReports(list);
|
||||||
|
System.out.println("End.");
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Harvester() {
|
||||||
|
testMode = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDateFrom(Date dateFrom) {
|
||||||
|
this.dateFrom = dateFrom;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDateTo(Date dateTo) {
|
||||||
|
this.dateTo = dateTo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTestMode(boolean testMode) {
|
||||||
|
this.testMode = testMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date getDateFrom() {
|
||||||
|
return dateFrom;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date getDateTo() {
|
||||||
|
return dateTo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void runOne(BasicHarvester harvester) {
|
||||||
|
try {
|
||||||
|
List<Harvest> data = harvester.getData();
|
||||||
|
if (data != null) {
|
||||||
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, data);
|
||||||
|
}
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void runAll() {
|
||||||
|
try {
|
||||||
|
// collecting info on VRE users
|
||||||
|
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> users = vreUsersHarvester.getData();
|
||||||
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, users);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on Res. Catalogue (Dataset, Application,
|
||||||
|
// Deliverables, Methods)
|
||||||
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = resourceCatalogueHarvester.getData();
|
||||||
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on Data/Method download
|
||||||
|
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
||||||
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on social (posts, replies and likes)
|
||||||
|
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = socialHarvester.getData();
|
||||||
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// collecting info on method invocation
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
|
||||||
|
List<Harvest> res = methodInvocationHarvester.getData();
|
||||||
|
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
|
} catch (Exception x) {
|
||||||
|
logger.error(x.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void insertMonthlyData(Date from, Date to, List<Harvest> data) {
|
||||||
|
Dao dao = null;
|
||||||
|
try {
|
||||||
|
dao = dbConnect();
|
||||||
|
dao.insertMonthlyMeasure(data, from, to, false);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
} finally {
|
||||||
|
if (dao != null) {
|
||||||
|
try {
|
||||||
|
dao.disconnect();
|
||||||
|
} catch (DaoException e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean checkDate(Object date) {
|
||||||
|
if (date == null)
|
||||||
|
return false;
|
||||||
|
return date instanceof java.util.Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Dao dbConnect() throws DaoException {
|
||||||
|
DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
||||||
|
dde.setTestMode(testMode);
|
||||||
|
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
||||||
|
Dao dao = new Dao();
|
||||||
|
dao.init();
|
||||||
|
dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
||||||
|
return dao;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processParameterArray(String[] args) throws Exception {
|
||||||
|
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
||||||
|
for (int i = 0; i < args.length; i++) {
|
||||||
|
if (args[i].startsWith(PARAMETER_FROM)) {
|
||||||
|
String from = getArgValue(args[i]);
|
||||||
|
dateFrom = formatter.parse(from);
|
||||||
|
System.out.println(dateFrom.toString());
|
||||||
|
} else if (args[i].startsWith(PARAMETER_TO)) {
|
||||||
|
String to = getArgValue(args[i]);
|
||||||
|
dateTo = formatter.parse(to);
|
||||||
|
System.out.println(dateTo.toString());
|
||||||
|
} else if (args[i].toLowerCase().startsWith(TEST)) {
|
||||||
|
testMode = true;
|
||||||
|
System.out.println("Test: " + testMode);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getArgValue(String arg) throws Exception {
|
||||||
|
String[] tokens = arg.split("=");
|
||||||
|
if (tokens.length != 2)
|
||||||
|
throw new Exception("Argument must be in the format 'name=value'.");
|
||||||
|
return tokens[1].trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private ArrayList<Integer> getSubTree(int root) {
|
||||||
|
Dao dao = null;
|
||||||
|
try {
|
||||||
|
dao = dbConnect();
|
||||||
|
ArrayList<Integer> subTree = dao.getSubTree(root);
|
||||||
|
return subTree;
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (dao != null) {
|
||||||
|
try {
|
||||||
|
dao.disconnect();
|
||||||
|
} catch (DaoException e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createSocialReports(ArrayList<Integer> ids) {
|
||||||
|
Dao dao = null;
|
||||||
|
try {
|
||||||
|
dao = dbConnect();
|
||||||
|
for (Integer contextId : ids)
|
||||||
|
dao.createSocialReport(contextId, 2018);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
} finally {
|
||||||
|
if (dao != null) {
|
||||||
|
try {
|
||||||
|
dao.disconnect();
|
||||||
|
} catch (DaoException e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,392 @@
|
||||||
|
package org.gcube.utils;
|
||||||
|
|
||||||
|
public class DataFiller {
|
||||||
|
private static String[] data10 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","83",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","71",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","5",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","374",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","112",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","37",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","44",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","19",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","100",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","6",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","28",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","30",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","5",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","31",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","56",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","30",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","99",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","68",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","152",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","19",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","62",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","6",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","182",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","38",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static String[] data11 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","93",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","136",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","387",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","116",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","98",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","4",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","28",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","58",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","56",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","110",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","73",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","161",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","22",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","186",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","40",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static String[] data12 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","99",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","28",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","273",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","392",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","118",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","98",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","0",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","4",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","30",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","37",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","59",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","57",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","114",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","74",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","164",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","188",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","40",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
||||||
|
};
|
||||||
|
|
||||||
|
public static String[] data01 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","103",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","278",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","400",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","121",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","22",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","4",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","31",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","37",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","59",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","68",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","123",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","80",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","167",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","31",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","191",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","44",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20",
|
||||||
|
};
|
||||||
|
|
||||||
|
public static String[] data02 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","106",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","279",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","402",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","124",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","4",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","32",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","31",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","27",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","38",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","60",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","71",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","127",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","81",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","171",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","32",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","193",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","49",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","24"
|
||||||
|
};
|
||||||
|
|
||||||
|
public static String[] data03 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","107",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","35",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","278",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","3",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","403",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","124",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","35",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","45",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","33",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","99",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","15",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","27",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","11",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/ForkysVRE","12",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","5",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GALAXIDI","4",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF","13",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","32",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","38",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","36",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","17",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","28",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","34",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","32",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","29",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","84",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","26",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","29",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/InfraTraining","10",
|
||||||
|
"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","38",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/iSearch","61",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","6",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","27",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","71",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","127",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","81",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","171",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","32",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","63",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","26",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","5",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","191",
|
||||||
|
"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","49",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/VME-DB","18",
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","24"
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
public static String[] data04 = {
|
||||||
|
"/d4science.research-infrastructures.eu/FARM/CWP_Secretariat", "8"
|
||||||
|
};
|
||||||
|
|
||||||
|
public static String[] vals = {
|
||||||
|
"2017", "1", "0", "2", "80",
|
||||||
|
"2017", "2", "0", "2", "80",
|
||||||
|
"2017", "3", "0", "2", "80",
|
||||||
|
"2017", "4", "0", "2", "80",
|
||||||
|
"2017", "5", "0", "2", "80",
|
||||||
|
"2017", "6", "0", "2", "80",
|
||||||
|
"2017", "7", "4", "2", "80",
|
||||||
|
"2017", "8", "4", "2", "80",
|
||||||
|
"2017", "9", "4", "2", "80",
|
||||||
|
"2017", "10", "4", "2", "80",
|
||||||
|
"2017", "11", "4", "2", "80",
|
||||||
|
"2017", "12", "4", "2", "80",
|
||||||
|
"2018", "1", "4", "2", "80",
|
||||||
|
"2018", "2", "4", "2", "80",
|
||||||
|
"2018", "3", "4", "2", "80"
|
||||||
|
};
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
// for(int i=0; i<data03.length; i+=2) {
|
||||||
|
// String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) " +
|
||||||
|
// "values(2018,3," + data03[i+1] + ",2,(select id from context where dname='" + data03[i] + "'),'2018-03-01');";
|
||||||
|
// System.out.println(insert);
|
||||||
|
// }
|
||||||
|
insertMeasures();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void insertMeasures() {
|
||||||
|
String im = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values ";
|
||||||
|
String val = "(%Y, %M, %m, %T, %C, '%D')";
|
||||||
|
|
||||||
|
for(int i=0; i<vals.length; i+=5) {
|
||||||
|
String item = new String(val);
|
||||||
|
item = item.replace("%Y", vals[i]);
|
||||||
|
item = item.replace("%M", vals[i+1]);
|
||||||
|
item = item.replace("%m", vals[i+2]);
|
||||||
|
item = item.replace("%T", vals[i+3]);
|
||||||
|
item = item.replace("%C", vals[i+4]);
|
||||||
|
item = item.replace("%D", getDay(vals[i], vals[i+1]));
|
||||||
|
im += item + ",";
|
||||||
|
}
|
||||||
|
im = im.substring(0, im.length()-1);
|
||||||
|
System.out.println(im);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getDay(String year, String month) {
|
||||||
|
String day = year + "-";
|
||||||
|
if(month.length() < 2)
|
||||||
|
day += "0" + month + "-01";
|
||||||
|
else day += month + "-01";
|
||||||
|
|
||||||
|
return day;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE xml>
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{0}: %msg%n</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
|
||||||
|
<logger name="org.gcube" level="INFO" />
|
||||||
|
<!-- logger name="org.gcube.accounting.aggregator" level="TRACE" / -->
|
||||||
|
|
||||||
|
<root level="WARN">
|
||||||
|
<appender-ref ref="STDOUT" />
|
||||||
|
</root>
|
||||||
|
|
||||||
|
</configuration>
|
Loading…
Reference in New Issue