Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167567 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
1430fb676f
commit
9dec451f4c
|
@ -2,7 +2,6 @@ package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.Calendar;
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -16,29 +15,70 @@ import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester;
|
||||||
import org.gcube.dataharvest.harvester.SocialHarvester;
|
import org.gcube.dataharvest.harvester.SocialHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VreUsersHarvester;
|
import org.gcube.dataharvest.harvester.VreUsersHarvester;
|
||||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||||
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
|
import org.gcube.dataharvest.utils.MeasureType;
|
||||||
import org.gcube.vremanagement.executor.plugin.Plugin;
|
import org.gcube.vremanagement.executor.plugin.Plugin;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDeclaration> {
|
public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDeclaration> {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class);
|
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class);
|
||||||
|
|
||||||
private static final String PROPERTY_FILENAME = "config.properties";
|
private static final String PROPERTY_FILENAME = "config.properties";
|
||||||
|
|
||||||
public static final String PARAMETER_FROM = "from";
|
public static final String START_DATE_INPUT_PARAMETER = "startDate";
|
||||||
public static final String PARAMETER_TO = "to";
|
public static final String MEASURE_TYPE_INPUT_PARAMETER = "measureType";
|
||||||
public static final String TEST = "test";
|
public static final String RERUN_INPUT_PARAMETER = "reRun";
|
||||||
|
public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
|
||||||
|
|
||||||
private boolean testMode = false, updateFlag = false;
|
protected Date start;
|
||||||
private Date dateFrom, dateTo;
|
protected Date end;
|
||||||
|
|
||||||
private Properties properties;
|
private static final InheritableThreadLocal<Properties> properties = new InheritableThreadLocal<Properties>() {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Properties initialValue() {
|
||||||
|
return new Properties();
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
public static InheritableThreadLocal<Properties> getProperties() {
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
private void getConfigParameters() throws IOException {
|
private void getConfigParameters() throws IOException {
|
||||||
properties = new Properties();
|
try {
|
||||||
InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader().getResourceAsStream(PROPERTY_FILENAME);
|
Properties properties = new Properties();
|
||||||
properties.load(input);
|
InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader()
|
||||||
|
.getResourceAsStream(PROPERTY_FILENAME);
|
||||||
|
properties.load(input);
|
||||||
|
getProperties().set(properties);
|
||||||
|
} catch(Exception e) {
|
||||||
|
logger.warn(
|
||||||
|
"Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults",
|
||||||
|
PROPERTY_FILENAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final InheritableThreadLocal<ContextAuthorization> contextAuthorization = new InheritableThreadLocal<ContextAuthorization>() {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ContextAuthorization initialValue() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
public static InheritableThreadLocal<ContextAuthorization> getContextAuthorization() {
|
||||||
|
return contextAuthorization;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void retrieveAuthorizations() throws Exception {
|
||||||
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||||
|
getContextAuthorization().set(contextAuthorization);
|
||||||
}
|
}
|
||||||
|
|
||||||
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
|
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
|
||||||
|
@ -47,160 +87,116 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
|
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
@Override
|
@Override
|
||||||
public void launch(Map<String, Object> inputs) throws Exception {
|
public void launch(Map<String,Object> inputs) throws Exception {
|
||||||
logger.debug("DataHarvestPlugin: launch()");
|
logger.debug("{} is starting", this.getClass().getSimpleName());
|
||||||
try {
|
|
||||||
getConfigParameters();
|
getConfigParameters();
|
||||||
}catch (Exception e) {
|
|
||||||
logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME);
|
retrieveAuthorizations();
|
||||||
|
|
||||||
|
if(inputs == null || inputs.isEmpty()) {
|
||||||
|
throw new IllegalArgumentException("The can only be launched providing valid input parameters");
|
||||||
}
|
}
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization(properties);
|
if(!inputs.containsKey(MEASURE_TYPE_INPUT_PARAMETER)) {
|
||||||
|
throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'");
|
||||||
|
|
||||||
String message = "";
|
|
||||||
DatabaseManager dbaseManager = new DatabaseManager(testMode);
|
|
||||||
|
|
||||||
if (inputs == null) {
|
|
||||||
message = "Invalid input parameter (null)";
|
|
||||||
logger.error(message);
|
|
||||||
throw new Exception(message, null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Object from = inputs.get(PARAMETER_FROM);
|
|
||||||
if (!checkDate(from)) {
|
|
||||||
message = "Missing or invalid input parameter: " + PARAMETER_FROM;
|
|
||||||
logger.error(message);
|
|
||||||
throw new Exception(message, null);
|
|
||||||
}
|
|
||||||
dateFrom = (Date)from;
|
|
||||||
|
|
||||||
Object to = inputs.get(PARAMETER_TO);
|
|
||||||
if (!checkDate(to)) {
|
|
||||||
message = "Missing or invalid input parameter: " + PARAMETER_TO;
|
|
||||||
logger.error(message);
|
|
||||||
throw new Exception(message, null);
|
|
||||||
}
|
|
||||||
dateTo = (Date)to;
|
|
||||||
|
|
||||||
testMode = (inputs.get(TEST) == null) ? false : (boolean) inputs.get(TEST);
|
|
||||||
logger.debug("TEST mode is " + testMode);
|
|
||||||
|
|
||||||
setTimePeriod();
|
MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
|
||||||
logger.debug("Time period: " + dateFrom.toString() + " -> " + dateTo.toString());
|
|
||||||
|
boolean reRun = true;
|
||||||
|
if(inputs.containsKey(RERUN_INPUT_PARAMETER)) {
|
||||||
|
try {
|
||||||
|
reRun = (boolean) inputs.get(RERUN_INPUT_PARAMETER);
|
||||||
|
}catch (Exception e) {
|
||||||
|
throw new IllegalArgumentException("'" + RERUN_INPUT_PARAMETER + "' must be a boolean");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean dryRun = true;
|
||||||
|
if(inputs.containsKey(DRY_RUN_INPUT_PARAMETER)) {
|
||||||
|
try {
|
||||||
|
dryRun = (boolean) inputs.get(DRY_RUN_INPUT_PARAMETER);
|
||||||
|
}catch (Exception e) {
|
||||||
|
throw new IllegalArgumentException("'" + DRY_RUN_INPUT_PARAMETER + "' must be a boolean");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) {
|
||||||
|
String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
|
||||||
|
start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
|
||||||
|
} else {
|
||||||
|
start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
|
logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start),
|
||||||
|
DateUtils.LAUNCH_DATE_FORMAT.format(end));
|
||||||
|
|
||||||
|
DatabaseManager dbaseManager = new DatabaseManager();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// collecting info on VRE users
|
// collecting info on VRE users
|
||||||
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo);
|
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
|
||||||
List<Harvest> users = vreUsersHarvester.getData();
|
List<Harvest> users = vreUsersHarvester.getData();
|
||||||
if(!testMode) {
|
if(!dryRun) {
|
||||||
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, users, updateFlag);
|
dbaseManager.insertMonthlyData(start, end, users, reRun);
|
||||||
}
|
}
|
||||||
} catch (Exception x) {
|
} catch(Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
|
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
|
||||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
|
||||||
List<Harvest> res = resourceCatalogueHarvester.getData();
|
List<Harvest> res = resourceCatalogueHarvester.getData();
|
||||||
if(!testMode) {
|
if(!dryRun) {
|
||||||
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||||
}
|
}
|
||||||
} catch (Exception x) {
|
} catch(Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// collecting info on Data/Method download
|
// collecting info on Data/Method download
|
||||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
|
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
|
||||||
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
||||||
if(!testMode) {
|
if(!dryRun) {
|
||||||
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||||
}
|
}
|
||||||
} catch (Exception x) {
|
} catch(Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// collecting info on social (posts, replies and likes)
|
// collecting info on social (posts, replies and likes)
|
||||||
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
|
SocialHarvester socialHarvester = new SocialHarvester(start, end);
|
||||||
List<Harvest> res = socialHarvester.getData();
|
List<Harvest> res = socialHarvester.getData();
|
||||||
if(!testMode) {
|
if(!dryRun) {
|
||||||
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||||
}
|
}
|
||||||
} catch (Exception x) {
|
} catch(Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// collecting info on method invocation
|
// collecting info on method invocation
|
||||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
List<Harvest> res = methodInvocationHarvester.getData();
|
List<Harvest> res = methodInvocationHarvester.getData();
|
||||||
if(!testMode) {
|
if(!dryRun) {
|
||||||
dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag);
|
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||||
}
|
}
|
||||||
} catch (Exception x) {
|
} catch(Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
@Override
|
@Override
|
||||||
protected void onStop() throws Exception {
|
protected void onStop() throws Exception {
|
||||||
logger.debug("DataHarvestPlugin: onStop()");
|
logger.debug("{} is stopping", this.getClass().getSimpleName());
|
||||||
}
|
|
||||||
|
|
||||||
private boolean checkDate(Object date) {
|
|
||||||
if (date == null)
|
|
||||||
return false;
|
|
||||||
return date instanceof java.util.Date;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// private void insertMonthlyData(Date from, Date to, List<Harvest> data) {
|
|
||||||
// Dao dao = null;
|
|
||||||
// try {
|
|
||||||
// dao = dbConnect();
|
|
||||||
// dao.insertMonthlyMeasure(data, from, to, false);
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// logger.error(e.getLocalizedMessage());
|
|
||||||
// } finally {
|
|
||||||
// if (dao != null) {
|
|
||||||
// try {
|
|
||||||
// dao.disconnect();
|
|
||||||
// } catch (DaoException e) {
|
|
||||||
// logger.error(e.getLocalizedMessage());
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private Dao dbConnect() throws DaoException {
|
|
||||||
// DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
|
||||||
// dde.setTestMode(testMode);
|
|
||||||
// DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
|
||||||
// Dao dao = new Dao();
|
|
||||||
// dao.init();
|
|
||||||
// dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
|
||||||
// return dao;
|
|
||||||
// }
|
|
||||||
|
|
||||||
private void setTimePeriod() {
|
|
||||||
Calendar from = Calendar.getInstance();
|
|
||||||
from.setTime(dateFrom);
|
|
||||||
from.set(Calendar.HOUR_OF_DAY, 0);
|
|
||||||
from.set(Calendar.MINUTE, 0);
|
|
||||||
from.set(Calendar.SECOND, 0);
|
|
||||||
dateFrom = from.getTime();
|
|
||||||
|
|
||||||
Calendar to = Calendar.getInstance();
|
|
||||||
to.setTime(dateTo);
|
|
||||||
to.set(Calendar.HOUR_OF_DAY, 23);
|
|
||||||
to.set(Calendar.MINUTE, 59);
|
|
||||||
to.set(Calendar.SECOND, 59);
|
|
||||||
dateTo = to.getTime();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package org.gcube.dataharvest.dao;
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
import java.sql.Array;
|
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.DriverManager;
|
import java.sql.DriverManager;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
|
|
|
@ -1,10 +1,23 @@
|
||||||
package org.gcube.dataharvest.dao;
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Eric Perrone (ISTI - CNR)
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
public class DaoException extends Exception {
|
public class DaoException extends Exception {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generated Serial Version UID
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -6302570066137502483L;
|
||||||
|
|
||||||
public DaoException() {
|
public DaoException() {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public DaoException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
public DaoException(String message, Throwable throwable) {
|
public DaoException(String message, Throwable throwable) {
|
||||||
super(message, throwable);
|
super(message, throwable);
|
||||||
|
|
|
@ -1,26 +1,34 @@
|
||||||
package org.gcube.dataharvest.dao;
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Eric Perrone (ISTI - CNR)
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
|
||||||
public class DatabaseConnectionData {
|
public class DatabaseConnectionData {
|
||||||
String url, user, password;
|
|
||||||
|
protected String uri;
|
||||||
|
protected String user;
|
||||||
|
protected String password;
|
||||||
|
|
||||||
public DatabaseConnectionData() {
|
public DatabaseConnectionData() {
|
||||||
this.url = null;
|
this.uri = null;
|
||||||
this.user = null;
|
this.user = null;
|
||||||
this.password = null;
|
this.password = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public DatabaseConnectionData(String url, String user, String password) {
|
public DatabaseConnectionData(String uri, String user, String password) {
|
||||||
this.url = url;
|
this.uri = uri;
|
||||||
this.user = user;
|
this.user = user;
|
||||||
this.password = password;
|
this.password = password;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getUrl() {
|
public String getURI() {
|
||||||
return url;
|
return uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setUrl(String url) {
|
public void setURI(String uri) {
|
||||||
this.url = url;
|
this.uri = uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getUser() {
|
public String getUser() {
|
||||||
|
@ -41,6 +49,6 @@ public class DatabaseConnectionData {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "DatabaseConnectionData [url=" + url + ", user=" + user + ", password=" + password + "]";
|
return this.getClass().getSimpleName() + " [uri =" + uri + ", user=" + user + ", password=" + password + "]";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,99 +0,0 @@
|
||||||
package org.gcube.dataharvest.dao;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.gcube.common.encryption.StringEncrypter;
|
|
||||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
|
||||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
||||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|
||||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
public class DatabaseDataExplorer {
|
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(DatabaseDataExplorer.class);
|
|
||||||
private boolean testMode = false;
|
|
||||||
private boolean productionMode = false;
|
|
||||||
|
|
||||||
public DatabaseDataExplorer() {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* setter method for testMode member. this member is set to TRUE if you want
|
|
||||||
* avoid to use production database during develop/test phase
|
|
||||||
*
|
|
||||||
* @param testMode
|
|
||||||
*/
|
|
||||||
public void setTestMode(boolean testMode) {
|
|
||||||
this.testMode = testMode;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* getter method for testMode member.
|
|
||||||
*
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
public boolean getTestMode() {
|
|
||||||
return testMode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public DatabaseConnectionData retrieveDatabaseInfo() throws DaoException {
|
|
||||||
DatabaseConnectionData dcd = null;
|
|
||||||
if ((testMode == false) && (productionMode == false)) {
|
|
||||||
String jdbcUrl = "jdbc:postgresql://";
|
|
||||||
dcd = new DatabaseConnectionData();
|
|
||||||
try {
|
|
||||||
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
|
||||||
query.addCondition("$resource/Profile/Category/text() eq 'Database'")
|
|
||||||
.addCondition("$resource/Profile/Name/text() eq 'Analytics Board'");
|
|
||||||
DiscoveryClient<String> client = ICFactory.client();
|
|
||||||
List<String> list = client.submit(query);
|
|
||||||
if (!list.isEmpty()) {
|
|
||||||
String xml = list.iterator().next();
|
|
||||||
|
|
||||||
String server = getItemValueFromXmlString(xml, "Endpoint");
|
|
||||||
String db = getItemAttributeFromXmlString(xml, "Endpoint", "EntryName");
|
|
||||||
jdbcUrl += server + "/" + db;
|
|
||||||
dcd.setUrl(jdbcUrl);
|
|
||||||
|
|
||||||
String user = getItemValueFromXmlString(xml, "Username");
|
|
||||||
dcd.setUser(user);
|
|
||||||
|
|
||||||
String password = StringEncrypter.getEncrypter()
|
|
||||||
.decrypt(getItemValueFromXmlString(xml, "Password"));
|
|
||||||
dcd.setPassword(password);
|
|
||||||
logger.debug("Database data found");
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
logger.error(ex.getLocalizedMessage());
|
|
||||||
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
|
||||||
}
|
|
||||||
} else if(testMode) {
|
|
||||||
logger.debug("TEST mode is ON");
|
|
||||||
dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/analytics_b_dev", "analytics_b_dev_u", "78cb625303be21b");
|
|
||||||
} else if(productionMode) {
|
|
||||||
logger.warn("PRODUCTION mode is FORCED ON");
|
|
||||||
dcd = new DatabaseConnectionData("jdbc:postgresql://postgresql-srv.d4science.org:5432/analytics_board", "analytics_board_u", "b52b64ab07ea0b5");
|
|
||||||
}
|
|
||||||
|
|
||||||
return dcd;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getItemValueFromXmlString(String xml, String item) {
|
|
||||||
int start = xml.indexOf("<" + item);
|
|
||||||
int begin = xml.indexOf(">", start);
|
|
||||||
int end = xml.indexOf("<", begin);
|
|
||||||
String value = xml.substring(begin + 1, end);
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getItemAttributeFromXmlString(String xml, String item, String attribute) {
|
|
||||||
int start = xml.indexOf("<" + item);
|
|
||||||
start = xml.indexOf(attribute, start);
|
|
||||||
int begin = xml.indexOf("\"", start);
|
|
||||||
int end = xml.indexOf("\"", begin + 1);
|
|
||||||
String attributeValue = xml.substring(begin + 1, end);
|
|
||||||
return attributeValue;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -8,70 +8,57 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class DatabaseManager {
|
public class DatabaseManager {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(DatabaseManager.class);
|
private static Logger logger = LoggerFactory.getLogger(DatabaseManager.class);
|
||||||
private boolean testMode = false;
|
|
||||||
|
|
||||||
public DatabaseManager(boolean testMode) {
|
|
||||||
this.testMode = testMode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public DatabaseManager() {
|
public DatabaseManager() {
|
||||||
this.testMode = false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setTestMode(boolean testMode) {
|
|
||||||
this.testMode = testMode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isTestMode() {
|
|
||||||
return testMode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void insertMonthlyData(Date from, Date to, List<Harvest> data, boolean updateFlag) {
|
public void insertMonthlyData(Date from, Date to, List<Harvest> data, boolean updateFlag) {
|
||||||
Dao dao = null;
|
Dao dao = null;
|
||||||
try {
|
try {
|
||||||
dao = dbConnect(testMode);
|
dao = dbConnect();
|
||||||
dao.insertMonthlyMeasure(data, from, to, updateFlag);
|
dao.insertMonthlyMeasure(data, from, to, updateFlag);
|
||||||
} catch (Exception e) {
|
} catch(Exception e) {
|
||||||
logger.error(e.getLocalizedMessage());
|
logger.error(e.getLocalizedMessage());
|
||||||
} finally {
|
} finally {
|
||||||
if (dao != null) {
|
if(dao != null) {
|
||||||
try {
|
try {
|
||||||
dao.disconnect();
|
dao.disconnect();
|
||||||
} catch (DaoException e) {
|
} catch(DaoException e) {
|
||||||
logger.error(e.getLocalizedMessage());
|
logger.error(e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String[] getActiveVres() {
|
public String[] getActiveVres() {
|
||||||
Dao dao = null;
|
Dao dao = null;
|
||||||
try {
|
try {
|
||||||
dao = dbConnect(testMode);
|
dao = dbConnect();
|
||||||
return dao.getActiveVres();
|
return dao.getActiveVres();
|
||||||
} catch (Exception e) {
|
} catch(Exception e) {
|
||||||
logger.error(e.getLocalizedMessage());
|
logger.error(e.getLocalizedMessage());
|
||||||
return null;
|
return null;
|
||||||
} finally {
|
} finally {
|
||||||
if (dao != null) {
|
if(dao != null) {
|
||||||
try {
|
try {
|
||||||
dao.disconnect();
|
dao.disconnect();
|
||||||
} catch (DaoException e) {
|
} catch(DaoException e) {
|
||||||
logger.error(e.getLocalizedMessage());
|
logger.error(e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Dao dbConnect(boolean testMode) throws DaoException {
|
public Dao dbConnect() throws DaoException {
|
||||||
DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
DatabaseParameterRetriever dde = new DatabaseParameterRetriever();
|
||||||
dde.setTestMode(testMode);
|
|
||||||
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
||||||
Dao dao = new Dao();
|
Dao dao = new Dao();
|
||||||
dao.init();
|
dao.init();
|
||||||
dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword());
|
||||||
return dao;
|
return dao;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
package org.gcube.dataharvest.dao;
|
||||||
|
|
||||||
|
import java.security.Key;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.common.encryption.StringEncrypter;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
||||||
|
import org.gcube.common.resources.gcore.utils.Group;
|
||||||
|
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
public class DatabaseParameterRetriever {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DatabaseParameterRetriever.class);
|
||||||
|
|
||||||
|
public static final String LOCAL_DB = "LOCAL_DB";
|
||||||
|
|
||||||
|
public static final String SERVICE_ENDPOINT_CATEGORY = "Database";
|
||||||
|
public static final String SERVICE_ENDPOINT_NAME = "Analytics Board";
|
||||||
|
|
||||||
|
public static final String DB_URI = "DB_URI";
|
||||||
|
public static final String DB_USERNAME = "DB_USERNAME";
|
||||||
|
public static final String DB_PASSWORD = "DB_PASSWORD";
|
||||||
|
|
||||||
|
public DatabaseParameterRetriever() {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String decrypt(String encrypted, Key... key) throws Exception {
|
||||||
|
return StringEncrypter.getEncrypter().decrypt(encrypted);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void checkParameter(String parameter, String parameterName, boolean localDB) throws DaoException {
|
||||||
|
if(parameter ==null || parameter.isEmpty()) {
|
||||||
|
throw new DaoException("DB " + parameterName + " cannot be null nor empty. Please check your " + (localDB ? "local configuration." : "ServiceEndpoint"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseConnectionData retrieveDatabaseInfo() throws DaoException {
|
||||||
|
|
||||||
|
Properties properties = AccountingDataHarvesterPlugin.getProperties().get();
|
||||||
|
|
||||||
|
boolean localDB = Boolean.parseBoolean(properties.getProperty(LOCAL_DB, "true"));
|
||||||
|
|
||||||
|
String uri = "";
|
||||||
|
String username = "";
|
||||||
|
String password = "";
|
||||||
|
|
||||||
|
if(localDB) {
|
||||||
|
uri = properties.getProperty(DB_URI);
|
||||||
|
username = properties.getProperty(DB_USERNAME);
|
||||||
|
password = properties.getProperty(DB_PASSWORD);
|
||||||
|
}else {
|
||||||
|
try {
|
||||||
|
String className = this.getClass().getSimpleName();
|
||||||
|
|
||||||
|
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition(
|
||||||
|
String.format("$resource/Profile/Category/text() eq '%s'", SERVICE_ENDPOINT_CATEGORY));
|
||||||
|
query.addCondition(String.format("$resource/Profile/Name/text() eq '%s'", SERVICE_ENDPOINT_NAME));
|
||||||
|
query.addCondition(String.format("$resource/Profile/AccessPoint/Interface/Endpoint/@EntryName eq '%s'", className));
|
||||||
|
|
||||||
|
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
|
||||||
|
List<ServiceEndpoint> serviceEndpoints = client.submit(query);
|
||||||
|
if(serviceEndpoints.size() > 1) {
|
||||||
|
throw new DaoException("More than one endpoint found. Not sure which one use.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Group<AccessPoint> accessPoints = serviceEndpoints.get(0).profile().accessPoints();
|
||||||
|
|
||||||
|
for(AccessPoint accessPoint : accessPoints) {
|
||||||
|
if(accessPoint.name().compareTo(className) == 0) {
|
||||||
|
uri = accessPoint.address();
|
||||||
|
username = accessPoint.username();
|
||||||
|
|
||||||
|
String encryptedPassword = accessPoint.password();
|
||||||
|
password = decrypt(encryptedPassword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch(Exception ex) {
|
||||||
|
logger.error(ex.getLocalizedMessage());
|
||||||
|
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
checkParameter(uri, "URI", localDB);
|
||||||
|
|
||||||
|
checkParameter(username, "Username", localDB);
|
||||||
|
|
||||||
|
checkParameter(password, "Password", localDB);
|
||||||
|
|
||||||
|
return new DatabaseConnectionData(uri, username, password);
|
||||||
|
}
|
||||||
|
}
|
|
@ -150,187 +150,8 @@ public class BasicHarvester {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String[] getActiveVREs(boolean testMode) {
|
public String[] getActiveVREs(boolean testMode) {
|
||||||
DatabaseManager dbaseManager = new DatabaseManager(testMode);
|
DatabaseManager dbaseManager = new DatabaseManager();
|
||||||
return dbaseManager.getActiveVres();
|
return dbaseManager.getActiveVres();
|
||||||
}
|
}
|
||||||
|
|
||||||
// public static void main(String[] args) {
|
|
||||||
//// String[][] data = {
|
|
||||||
//// {"/d4science.research-infrastructures.eu","3fdf5a02-383c-4364-a9c6-d3e2b0a4a401-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research","1ff82b79-4042-4311-84e1-a11f955b5f7f-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSY","f920ca10-af4d-4b70-9fa8-75f7bb78987c-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/fosteropenscience","188fb0fa-083b-4043-94a6-62152cd7b94e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/H2020greeneu","ed045fb9-805b-4d6c-a72a-d0a464e7a5dc-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/FAIR_DM","9429cc8d-a3d3-43f6-b793-e29be7e7bfba-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/Blue-Datathon","521908fe-deab-40df-ab59-61a8a3fb7325-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","1b9caa78-004a-4173-a2f0-6f67f7904630-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroStockAssessment","5bd19b68-aecf-4b10-8ca8-ff88f36ca350-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/AGINFRAplus","6de2ce44-7a7e-4da5-875a-5858eae8bd37-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/FisheriesandEcosystemAtMII","4928bb4b-9da5-426a-88ce-2f3a4853b665-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/BlueBRIDGEReview","8019bf10-3d31-4eaa-af48-cb327eceabac-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/DRuMFISH","bed1d5b5-106a-493e-a814-6d5cc99b4b93-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/OpenAIRE-Connect_Coordination","951c4abc-1850-49f3-a852-099d191de708-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/EuBrazilOpenBio","4449d0be-83a3-4966-bac4-beaca76577fa-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/BOBLME_HilsaAWG","6f66e93f-84bf-4171-83e5-a49d4d470465-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ENVRI","8194d45e-baa5-4d0a-9d27-3c81eb8b867d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/DEMETER","975bf904-1fd7-437f-bb13-8e196ce24aab-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/InfraTraining","f392969e-d2d4-4414-9d62-fb9675ea2056-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/NERLiX","21a34e10-a4ae-4a47-8b78-2313db5a9eb2-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/E-RIHS","82ffc3ac-518f-48a7-b9c4-69f0372b49a5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/EcoEvo","8b8e8fd9-cda5-433d-8a88-530cc26a2061-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_AbundanceEstimationFromAcoustic","1c7c72a8-93ca-46d0-baee-42da39654c11-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_FIACO2017","775fb6a3-d69d-4902-b379-6d44e4d2b7dc-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/DOPA_AnalystSpatialPlanning","70a39850-264e-496b-9ce0-b764a70e4660-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/Sinay","01c6b4b5-f726-45e0-8311-a2fdbcef54ff-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/StatnMap","d71d9c9b-def6-4df3-a22f-6bcd36dce284-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ISTIOpenAccess","2c4ab315-51cd-469c-af2e-ceccd4708093-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_IntroToREnv","b5112916-3e9d-48ab-afc0-195ceda24489-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_BNetworkAnalysis","30b7506e-a016-4438-be34-9af57af1d0c0-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/FoodborneOutbreak","b3eff03b-aac9-4bd0-a568-6870bb5559d4-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/EOSC_Services","518f0ef5-8510-46e9-9969-89d7d2c22301-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/SDG-Indicator14.4.1","1953f45d-4ff7-45ce-8232-00439f1ab48a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/FoodSecurity","15c3a6a6-4092-473b-a296-7cd5e5eadff4-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/RAKIP_portal","39bc9d6a-39ba-4664-9c62-330f371ce073-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_MSE","a9186abb-2af0-459e-ad59-3558a8127560-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/TextCrowd","8cc715b0-36a7-480e-ad24-8ecbc6b0577d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/EOSCPortfolio","8e499271-fe8d-4e0a-a3e2-cddef19515ef-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ParticleFormation","fec95ea3-079d-423a-8620-46a0364696fc-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICES_LogbookData","b44037d7-a4bf-4542-bc14-85c210a6087d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/IOTC_SS3","f8f18b96-8654-462e-b12d-73c265952bff-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/PerformFish","8fea1590-73f9-46e1-8303-99ef91759f58-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/EISCAT","0edb3ec5-37a1-47f2-b4be-4624a5e5f4e5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/AgroClimaticModelling","90fa5480-de0a-48c1-83fd-2c381abbd1bf-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ENVRIPlusData4Science","8e4d9a87-0da9-45c0-9542-612f14d5b454-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/OnlineOceanography","451ce6b2-09d8-4253-ac1d-56b0c633ffa7-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/QCAPI","7a829b89-20b2-48af-8c9e-641a1acaf47a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/SDI_Lab","76ca201a-4a89-4871-9e9d-3043ecc60be2-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ORIONKnowledgeHub","ff3228dc-4b29-4919-924d-462d9f5b9d42-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/D4Research/ICOSEddyCovarianceProcessing","781272a4-d498-4baf-8a14-468b83839452-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/Edison","289a3fc0-cd23-43ee-bf18-b3efc4d1a182-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/Edison/RstudioVirtualLab","d6b795ec-2719-4ac2-a511-fe032ef43b15-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM","cb7dadd0-dbdd-46a6-8db2-1dc4ffc64759-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/GRSF_Admin","f4811784-0d4a-4731-9f75-3b407544b31f-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/VME-DB","e8b93617-217e-44ac-ae02-09a45bc26886-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","17bdb8ae-cff9-44a7-ac27-c9e26a254290-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/TBTI_VRE","02c36b15-1229-47b1-a583-296ec5b404fd-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/GRSF","833d6a12-34a9-4082-9ecb-358d19244269-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/KIMAGRO_Fishfarming","b6967277-73e5-4b23-8fd6-c47e2e5a5ed2-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/D4STeam","e18b3dd5-b6fe-4865-8727-434354c24b9f-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/AquaMaps","6b08ffe9-695d-45bf-90fb-6361a4234ab5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/iMarineBoardVRE","78f6a7be-55c5-4602-ae81-fd4955a3a72e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","09a84b5a-b869-4552-887d-23248bc5b042-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/STRATOS_AQUACULTURES","c6317519-b238-449c-bd3d-6c832f98a611-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/MARKELLOS_Aquaculture","1457e707-492d-46a0-b848-6c6cd758a6ce-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/iLKNAK_Aquaculture","a8f98114-05b8-44b2-a89c-0f56e55412fd-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/ARDAG_Aquaculture","a948e508-0ecd-4013-a094-fe3c506209e1-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/ForkysVRE","6ed093fe-f245-4638-acfe-a70a1f24146d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/EllinikaPsariaVRE","b7c16d55-8344-492b-a94b-8fbf7af27e08-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/Aquabiotech","c55477cf-8c98-4bd0-a05d-6182ab1be4e8-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/NHREUS_Aquaculture","c7e19ccc-9fc1-49c0-8351-d507830e7660-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/GALAXIDI","69e9ec43-62a7-45cb-b868-a369aeabf123-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/CFE_DATA","75d70975-bee4-4f46-94be-ff8773df6bbb-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/CWP_Secretariat","d84f0ade-be6a-40d6-a152-66e1b5d7d75d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/SustainableBlueEconomy","19bbf2bb-58e3-4140-81cc-4043fd8755e5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/FARM/CollabResearch","2580b8ae-fad1-4213-8fd4-78b8a90c2c90-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps","e844592c-6428-4483-8614-e9561ad806f5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/SIASPA","fe58019c-f3eb-4d3a-8094-54d18df00940-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/GEMex","afe00e12-5351-4941-aa31-1285184e2aea-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/KnowledgeBridging","cb98c3f9-0217-46f3-a74d-bda21bbf00fb-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/TabularDataLab","a563847b-bb53-4303-950f-5857c59f3217-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueUptake","dfbc4f13-874d-4e0b-99a9-b5cf1d48acef-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/RStudioLab","8e6dcf9c-6dcc-4172-8dd6-45560af6a227-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/iSearch","df227388-3d61-4f77-97fb-4cd4fd8c6dbe-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject","ce150ced-d4a3-48d2-a541-3433f26ab6d5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_StockAssessmentAdvanced","a1c1a4e5-7aea-4c12-b724-868adb8fa6ec-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab","6c62b572-3b0e-43d2-b224-5c9961dfcb2e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas","44c54a21-88df-4a4b-ab1c-d9d5b636baba-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureAtlasGeneration","8953528d-e990-4451-b5a3-517adca107a2-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/rScience","7bfa66d6-6c9b-448f-a11a-21d833843647-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-PSC","79fb276e-13d9-4d80-b419-56c9d64888b7-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIP","9c420378-e8f4-472e-bf7b-b5419041f6db-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ENVRIPlus","554d32ca-ae4a-44dd-b8a7-424ec7d30cba-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/TCom","aed4d51f-3dd1-4329-b622-322211ac237a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/CNR_OpenScienceTF","6978ab04-ae3c-4cee-8b11-9f64ee733502-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DASC","c1674036-1388-4eca-bbbb-174b9dc95dd1-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining","38e4fa5a-6c50-417f-b952-1525cebd4950-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICCAT_BFT-E","776ea841-2205-4590-a0dc-24141138972e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/DESCRAMBLE","2c6561bb-ee65-4537-9ccc-ec2464291630-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ARIADNE","05ffc92b-03d1-4b05-8e33-e62f795202b5-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/EcologicalModelling","2fa2177d-a2a8-48d4-898f-3f301842247c-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCSSM","eba4dc9d-b7cf-40b8-adfb-aecdc731c3de-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/StockAssessment","9d78cbba-78e2-4efe-a38a-e14744318f2d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/PerformanceEvaluationInAquaculture","49b00009-e35f-44dc-ab0e-a63b505aca79-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/StrategicInvestmentAnalysis","cb859658-0439-4469-8b2a-7c4516c3a659-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ProtectedAreaImpactMaps","9f2746e3-3e04-4f80-9dd7-e66f42ff53f9-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueCommons","8166fe91-7fe5-4636-9821-b8718aaa8500-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BiOnym","867dfe04-ed24-42ad-902f-e2ee63963054-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/EGIEngage","548ffb80-2dbe-4d0d-afb9-1fd009c2edca-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/EFG","e085b1c4-f315-4a21-88c4-46748b81b3f3-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/IGDI","95306e9b-0ce0-4b72-be18-075873fe634a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_TCRE","5cfdd223-401c-4324-9c99-9bb4462ff854-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu","6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab","2856e458-7fd8-42a4-a1a9-b5566cf76ff7-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_FIACO","bffc959f-5eb3-4d21-9222-27b99fa41f5b-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/gCube","9e5aa777-c700-4975-a218-6c73cc8b1c69-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICES_DALSA","ec2d9727-2b21-4df5-823d-3f28a2d8295e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/Parthenos","42e742e2-75eb-401e-9d07-43deb713f926-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it","8b4d9866-1d55-470a-95c3-4cdc19cecb0d-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/AquacultureTrainingLab","49c20941-27b1-4294-aa23-58809ef8616c-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/ICOS_ETC","cac5112b-1a9c-476b-bc33-3b5d3192b1d0-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/BlueBRIDGE-EAB","3c6dfc0b-81a3-453c-b693-fe8337a735f1-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/gCubeApps/FrenchTropicalTunaAtlas","703fb5a1-54e2-4b1a-920c-978356d4a74a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE","2a05405f-e878-4f80-a522-d2381b97e0df-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Technical","15ee7828-caa0-4b07-a950-9f1ffdbc812b-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_Networking","156ca39a-0485-467d-aebb-59b7b49690b4-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_PSC","2d891c6a-4f6f-4281-8ffd-56270d0d0783-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE-Connect_EAB","66856a20-7a58-4188-afcf-ab5fa212a732-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/OpenAIRE_Users","64c73441-5a88-45e7-a865-dc449dadf271-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/1st_OpenAIRE_Datathon","5c6946e9-277c-4813-b949-5c5e215127b6-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/OpenAIRE/dnet","75e2db7a-03ae-4af3-b87d-d7f1f20cc194-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/ParthenosVO","8b8cf7cd-ff1a-4827-83cf-3933e2320d26-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/ParthenosVO/RubRIcA","4a9e0e5a-3b9e-4293-9dc7-4a118f3a37f3-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_Registry","0cdceda3-f6e1-4bb7-aa20-c3a4b8a891ba-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_LAB","75de04c9-aa4e-4e32-9770-4094de1d0d54-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/ParthenosVO/ACE","05f21525-b6fd-4587-ae7f-4e45a9fddfdf-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SmartArea","3782e93f-3d1a-4809-be0f-7c4b20eaa74b-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SmartArea/SmartBuilding","4216a83a-5506-43be-93b4-1464020cacce-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SmartArea/SmartApps","aa24a1d7-672a-4e06-b220-7d353188fc0a-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData","060a074b-bb12-4ab2-a585-cf0119f4d8ea-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens","28367906-d586-42e3-a5d8-6043a43ce54e-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue","2497dbd8-4002-4b50-b17c-ed2dc17cb0d7-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy","90ae1d9b-cfcb-4fdd-a91e-38ff6013f688-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/TagMe","9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/SMAPH","c0924749-cb6d-41e9-88a4-363e7f9e7556-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab","37d61ab1-3641-435d-8c4f-f1d37eb34ead-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/SocietalDebates","59450ce1-cf1d-4abb-9b0c-63346ad35e08-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/M-ATLAS","ace198d7-f50f-4557-9788-f7aa91c31d67-843339462"},
|
|
||||||
//// {"/d4science.research-infrastructures.eu/SoBigData/SportsDataScience","61bf4b8d-34e2-4426-a3dd-b41a1dcbccb1-843339462"}
|
|
||||||
//// };
|
|
||||||
////
|
|
||||||
//// ArrayList<Property> props = new ArrayList<>();
|
|
||||||
//// for(String[] datap : data) {
|
|
||||||
//// Property p = new Property();
|
|
||||||
//// p = p.nameAndValue(datap[0], datap[1]);
|
|
||||||
//// props.add(p);
|
|
||||||
//// }
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// try {
|
|
||||||
// //new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59").updateServiceEndPoint(props);
|
|
||||||
// BasicHarvester bh = new BasicHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
|
|
||||||
// Properties props = bh.readServiceEndpoint();
|
|
||||||
// Iterator i = props.keySet().iterator();
|
|
||||||
// while(i.hasNext()) {
|
|
||||||
// String key = (String)i.next();
|
|
||||||
// System.out.println(key + ":" + props.getProperty(key));
|
|
||||||
// }
|
|
||||||
// } catch (ParseException e) {
|
|
||||||
// // TODO Auto-generated catch block
|
|
||||||
// e.printStackTrace();
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// // TODO Auto-generated catch block
|
|
||||||
// e.printStackTrace();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import java.util.Properties;
|
||||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
import org.gcube.common.authorization.library.provider.UserInfo;
|
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||||
import org.gcube.common.scope.impl.ScopeBean;
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
|
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
|
||||||
import org.gcube.resourcemanagement.support.server.managers.scope.ScopeManager;
|
import org.gcube.resourcemanagement.support.server.managers.scope.ScopeManager;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -21,8 +22,6 @@ public class ContextAuthorization {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(ContextAuthorization.class);
|
private static Logger logger = LoggerFactory.getLogger(ContextAuthorization.class);
|
||||||
|
|
||||||
protected Properties properties;
|
|
||||||
|
|
||||||
public static final String VO_FILE = "VO_FILE";
|
public static final String VO_FILE = "VO_FILE";
|
||||||
public static final String DEFAULT_VO_FILENAME = "scopedata.xml";
|
public static final String DEFAULT_VO_FILENAME = "scopedata.xml";
|
||||||
|
|
||||||
|
@ -46,20 +45,15 @@ public class ContextAuthorization {
|
||||||
* Contains Properties used to generate tokens
|
* Contains Properties used to generate tokens
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public ContextAuthorization(Properties properties) throws Exception {
|
public ContextAuthorization() throws Exception {
|
||||||
this.contextToToken = new HashMap<>();
|
this.contextToToken = new HashMap<>();
|
||||||
this.tokenToContext = new HashMap<>();
|
this.tokenToContext = new HashMap<>();
|
||||||
if(properties!=null) {
|
retrieveContextsAndTokens();
|
||||||
this.properties = properties;
|
|
||||||
}else {
|
|
||||||
this.properties = new Properties();
|
|
||||||
}
|
|
||||||
//retrieveContextsAndTokens();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public File getVOFile() {
|
public File getVOFile() {
|
||||||
try {
|
try {
|
||||||
String voFileName = properties.getProperty(VO_FILE, DEFAULT_VO_FILENAME);
|
String voFileName = AccountingDataHarvesterPlugin.getProperties().get().getProperty(VO_FILE, DEFAULT_VO_FILENAME);
|
||||||
URL url = ContextAuthorization.class.getClassLoader().getResource(voFileName);
|
URL url = ContextAuthorization.class.getClassLoader().getResource(voFileName);
|
||||||
File voFile = new File(url.toURI());
|
File voFile = new File(url.toURI());
|
||||||
logger.trace("VO file is {}", voFile);
|
logger.trace("VO file is {}", voFile);
|
||||||
|
@ -78,6 +72,8 @@ public class ContextAuthorization {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
Properties properties = AccountingDataHarvesterPlugin.getProperties().get();
|
||||||
|
|
||||||
LinkedHashMap<String, ScopeBean> map = ScopeManager.readScopes(getVOFile().getAbsolutePath());
|
LinkedHashMap<String, ScopeBean> map = ScopeManager.readScopes(getVOFile().getAbsolutePath());
|
||||||
for(String scope : map.keySet()) {
|
for(String scope : map.keySet()) {
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -0,0 +1,98 @@
|
||||||
|
package org.gcube.dataharvest.utils;
|
||||||
|
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.TimeZone;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class DateUtils {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(DateUtils.class);
|
||||||
|
|
||||||
|
public static TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC");
|
||||||
|
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
|
||||||
|
public static final DateFormat DEFAULT_DATE_FORMAT;
|
||||||
|
|
||||||
|
public static final String LAUNCH_DATE_FORMAT_PATTERN = "yyyy-MM-dd";
|
||||||
|
public static final DateFormat LAUNCH_DATE_FORMAT;
|
||||||
|
|
||||||
|
public static final String UTC_DATE_FORMAT_PATTERN = "yyyy-MM-dd Z";
|
||||||
|
public static final DateFormat UTC_DATE_FORMAT;
|
||||||
|
public static final String UTC = "+0000";
|
||||||
|
|
||||||
|
static {
|
||||||
|
DEFAULT_DATE_FORMAT = getUTCDateFormat(DATETIME_PATTERN);
|
||||||
|
LAUNCH_DATE_FORMAT = getUTCDateFormat(LAUNCH_DATE_FORMAT_PATTERN);
|
||||||
|
UTC_DATE_FORMAT = getUTCDateFormat(UTC_DATE_FORMAT_PATTERN);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DateFormat getUTCDateFormat(String pattern){
|
||||||
|
DateFormat dateFormat = new SimpleDateFormat(pattern);
|
||||||
|
dateFormat.setTimeZone(UTC_TIMEZONE);
|
||||||
|
return dateFormat;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Calendar getUTCCalendarInstance(){
|
||||||
|
return Calendar.getInstance(UTC_TIMEZONE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Calendar getPreviousPeriod(MeasureType measureType) {
|
||||||
|
Calendar now = Calendar.getInstance();
|
||||||
|
switch(measureType) {
|
||||||
|
|
||||||
|
case YEARLY:
|
||||||
|
now.add(Calendar.YEAR, -1);
|
||||||
|
now.set(Calendar.MONTH, Calendar.JANUARY);
|
||||||
|
now.set(Calendar.DAY_OF_MONTH,1);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case MONTHLY:
|
||||||
|
now.add(Calendar.MONTH, -1);
|
||||||
|
now.set(Calendar.DAY_OF_MONTH,1);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case DAILY:
|
||||||
|
now.add(Calendar.DAY_OF_MONTH, -1);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
now.set(Calendar.HOUR_OF_DAY, 0);
|
||||||
|
now.set(Calendar.MINUTE, 0);
|
||||||
|
now.set(Calendar.SECOND, 0);
|
||||||
|
now.set(Calendar.MILLISECOND, 0);
|
||||||
|
|
||||||
|
return now;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static Calendar getStartCalendar(int year, int month, int day){
|
||||||
|
Calendar aggregationStartCalendar = getUTCCalendarInstance();
|
||||||
|
aggregationStartCalendar.set(Calendar.YEAR, year);
|
||||||
|
aggregationStartCalendar.set(Calendar.MONTH, month);
|
||||||
|
aggregationStartCalendar.set(Calendar.DAY_OF_MONTH, day);
|
||||||
|
aggregationStartCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||||
|
aggregationStartCalendar.set(Calendar.MINUTE, 0);
|
||||||
|
aggregationStartCalendar.set(Calendar.SECOND, 0);
|
||||||
|
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
|
||||||
|
|
||||||
|
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
||||||
|
|
||||||
|
return aggregationStartCalendar;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Date getEndDateFromStartDate(MeasureType aggregationType, Date startDate, int offset) {
|
||||||
|
Calendar aggregationEndDate = getUTCCalendarInstance();
|
||||||
|
aggregationEndDate.setTimeInMillis(startDate.getTime());
|
||||||
|
aggregationEndDate.add(aggregationType.getCalendarField(), offset);
|
||||||
|
return aggregationEndDate.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,50 @@
|
||||||
|
package org.gcube.dataharvest.utils;
|
||||||
|
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.util.Calendar;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Alessandro Pieve (ISTI - CNR)
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
public enum MeasureType {
|
||||||
|
|
||||||
|
DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7),
|
||||||
|
MONTHLY(Calendar.MONTH, "yyyy-MM", 3),
|
||||||
|
YEARLY(Calendar.YEAR, "yyyy", 3);
|
||||||
|
|
||||||
|
public static final String DATE_SEPARATOR = "-";
|
||||||
|
|
||||||
|
private final int calendarField;
|
||||||
|
|
||||||
|
private final String dateFormatPattern;
|
||||||
|
private final DateFormat dateFormat;
|
||||||
|
|
||||||
|
private final int notAggregableBefore;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private MeasureType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
|
||||||
|
this.calendarField = calendarField;
|
||||||
|
this.dateFormatPattern=dateFormatPattern;
|
||||||
|
this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern);
|
||||||
|
this.notAggregableBefore = notAggregableBefore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getCalendarField() {
|
||||||
|
return calendarField;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDateFormatPattern() {
|
||||||
|
return dateFormatPattern;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DateFormat getDateFormat() {
|
||||||
|
return dateFormat;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getNotAggregableBefore(){
|
||||||
|
return notAggregableBefore;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -7,17 +7,15 @@ import java.net.HttpURLConnection;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.text.DateFormat;
|
import java.text.DateFormat;
|
||||||
import java.text.ParseException;
|
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.Locale;
|
|
||||||
|
|
||||||
import org.gcube.dataharvest.dao.Dao;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class Utils {
|
public class Utils {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
||||||
|
|
||||||
public static Calendar dateToCalendar(Date date) {
|
public static Calendar dateToCalendar(Date date) {
|
||||||
|
@ -51,15 +49,4 @@ public class Utils {
|
||||||
return json;
|
return json;
|
||||||
}
|
}
|
||||||
|
|
||||||
// il seguente array deve essere formato dinamicamente
|
|
||||||
// public static String[] soBigDataContexts = { "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu",
|
|
||||||
// "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/SoBigDataLab",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/SMAPH",
|
|
||||||
// "/d4science.research-infrastructures.eu/SoBigData/TagMe" };
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,12 +1,11 @@
|
||||||
package org.gcube.dataharvest;
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.text.DateFormat;
|
|
||||||
import java.text.SimpleDateFormat;
|
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.TimeZone;
|
|
||||||
|
|
||||||
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
|
import org.gcube.dataharvest.utils.MeasureType;
|
||||||
import org.gcube.dataharvest.utils.ScopedTest;
|
import org.gcube.dataharvest.utils.ScopedTest;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -16,63 +15,39 @@ public class AccountingDataHarvesterPluginTest extends ScopedTest {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class);
|
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class);
|
||||||
|
|
||||||
public static TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC");
|
|
||||||
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
|
|
||||||
public static final DateFormat DEFAULT_DATE_FORMAT;
|
|
||||||
|
|
||||||
static {
|
|
||||||
DEFAULT_DATE_FORMAT = getUTCDateFormat(DATETIME_PATTERN);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static DateFormat getUTCDateFormat(String pattern){
|
|
||||||
DateFormat dateFormat = new SimpleDateFormat(pattern);
|
|
||||||
dateFormat.setTimeZone(UTC_TIMEZONE);
|
|
||||||
return dateFormat;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Calendar getUTCCalendarInstance(){
|
|
||||||
return Calendar.getInstance(UTC_TIMEZONE);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Calendar getAggregationStartCalendar(int year, int month, int day){
|
|
||||||
Calendar aggregationStartCalendar = getUTCCalendarInstance();
|
|
||||||
aggregationStartCalendar.set(Calendar.YEAR, year);
|
|
||||||
aggregationStartCalendar.set(Calendar.MONTH, month);
|
|
||||||
aggregationStartCalendar.set(Calendar.DAY_OF_MONTH, day);
|
|
||||||
aggregationStartCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
|
||||||
aggregationStartCalendar.set(Calendar.MINUTE, 0);
|
|
||||||
aggregationStartCalendar.set(Calendar.SECOND, 0);
|
|
||||||
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
|
|
||||||
|
|
||||||
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
|
||||||
|
|
||||||
return aggregationStartCalendar;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void test() {
|
public void test() {
|
||||||
try {
|
try {
|
||||||
ScopedTest.setContext(ScopedTest.ROOT);
|
|
||||||
|
|
||||||
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
||||||
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(dataHarvestPluginDeclaration);
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(dataHarvestPluginDeclaration);
|
||||||
|
|
||||||
Calendar from = getAggregationStartCalendar(2018, Calendar.APRIL, 1);
|
Map<String,Object> inputs = new HashMap<>();
|
||||||
Calendar to = getAggregationStartCalendar(2018, Calendar.APRIL, 30);
|
|
||||||
|
|
||||||
Map<String,Object> map = new HashMap<>();
|
|
||||||
map.put(AccountingDataHarvesterPlugin.PARAMETER_FROM, from.getTime());
|
|
||||||
map.put(AccountingDataHarvesterPlugin.PARAMETER_TO, to.getTime());
|
|
||||||
map.put(AccountingDataHarvesterPlugin.TEST, true);
|
|
||||||
|
|
||||||
accountingDataHarvesterPlugin.launch(map);
|
MeasureType measureType = MeasureType.MONTHLY;
|
||||||
|
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType);
|
||||||
|
|
||||||
|
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
||||||
|
|
||||||
|
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
||||||
|
|
||||||
|
Calendar from
|
||||||
|
//= DateUtils.getStartCalendar(2018, Calendar.APRIL, 1);
|
||||||
|
= DateUtils.getPreviousPeriod(measureType);
|
||||||
|
|
||||||
|
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
|
||||||
|
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||||
|
// inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
accountingDataHarvesterPlugin.launch(inputs);
|
||||||
|
|
||||||
logger.info("End.");
|
logger.info("End.");
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// TODO Auto-generated catch block
|
logger.error("", e);
|
||||||
e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,14 +2,13 @@ package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Calendar;
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.dataharvest.dao.Dao;
|
import org.gcube.dataharvest.dao.Dao;
|
||||||
import org.gcube.dataharvest.dao.DaoException;
|
import org.gcube.dataharvest.dao.DaoException;
|
||||||
import org.gcube.dataharvest.dao.DatabaseConnectionData;
|
import org.gcube.dataharvest.dao.DatabaseConnectionData;
|
||||||
import org.gcube.dataharvest.dao.DatabaseDataExplorer;
|
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
|
||||||
import org.gcube.dataharvest.datamodel.Harvest;
|
import org.gcube.dataharvest.datamodel.Harvest;
|
||||||
import org.gcube.dataharvest.harvester.BasicHarvester;
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||||
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
|
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
|
||||||
|
@ -122,6 +121,7 @@ public class Harvester {
|
||||||
// collecting info on method invocation
|
// collecting info on method invocation
|
||||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
|
||||||
List<Harvest> res = methodInvocationHarvester.getData();
|
List<Harvest> res = methodInvocationHarvester.getData();
|
||||||
|
logger.debug("{}", res);
|
||||||
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
||||||
} catch (Exception x) {
|
} catch (Exception x) {
|
||||||
logger.error(x.getLocalizedMessage());
|
logger.error(x.getLocalizedMessage());
|
||||||
|
@ -146,19 +146,13 @@ public class Harvester {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean checkDate(Object date) {
|
|
||||||
if (date == null)
|
|
||||||
return false;
|
|
||||||
return date instanceof java.util.Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Dao dbConnect() throws DaoException {
|
private Dao dbConnect() throws DaoException {
|
||||||
DatabaseDataExplorer dde = new DatabaseDataExplorer();
|
DatabaseParameterRetriever dde = new DatabaseParameterRetriever();
|
||||||
dde.setTestMode(testMode);
|
//dde.setTestMode(testMode);
|
||||||
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
||||||
Dao dao = new Dao();
|
Dao dao = new Dao();
|
||||||
dao.init();
|
dao.init();
|
||||||
dao.connect(dcd.getUrl(), dcd.getUser(), dcd.getPassword());
|
dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword());
|
||||||
return dao;
|
return dao;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,7 +182,7 @@ public class Harvester {
|
||||||
return tokens[1].trim();
|
return tokens[1].trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
private ArrayList<Integer> getSubTree(int root) {
|
protected ArrayList<Integer> getSubTree(int root) {
|
||||||
Dao dao = null;
|
Dao dao = null;
|
||||||
try {
|
try {
|
||||||
dao = dbConnect();
|
dao = dbConnect();
|
||||||
|
@ -208,7 +202,7 @@ public class Harvester {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createSocialReports(ArrayList<Integer> ids) {
|
protected void createSocialReports(ArrayList<Integer> ids) {
|
||||||
Dao dao = null;
|
Dao dao = null;
|
||||||
try {
|
try {
|
||||||
dao = dbConnect();
|
dao = dbConnect();
|
||||||
|
|
|
@ -21,6 +21,7 @@ public class ContextAuthorizationTest extends ScopedTest {
|
||||||
properties = new Properties();
|
properties = new Properties();
|
||||||
InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader().getResourceAsStream(PROPERTY_FILENAME);
|
InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader().getResourceAsStream(PROPERTY_FILENAME);
|
||||||
properties.load(input);
|
properties.load(input);
|
||||||
|
AccountingDataHarvesterPlugin.getProperties().set(properties);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -30,7 +31,7 @@ public class ContextAuthorizationTest extends ScopedTest {
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME);
|
logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME);
|
||||||
}
|
}
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization(properties);
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||||
contextAuthorization.getVOFile();
|
contextAuthorization.getVOFile();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,7 +42,7 @@ public class ContextAuthorizationTest extends ScopedTest {
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME);
|
logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME);
|
||||||
}
|
}
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization(properties);
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||||
contextAuthorization.retrieveContextsAndTokens();
|
contextAuthorization.retrieveContextsAndTokens();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
package org.gcube.dataharvest.utils;
|
package org.gcube.dataharvest.utils;
|
||||||
|
|
||||||
public class DataFiller {
|
public class DataFiller {
|
||||||
private static String[] data10 = {
|
|
||||||
|
protected static String[] data10 = {
|
||||||
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","83",
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","83",
|
||||||
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
@ -56,7 +57,7 @@ public class DataFiller {
|
||||||
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
||||||
};
|
};
|
||||||
|
|
||||||
private static String[] data11 = {
|
protected static String[] data11 = {
|
||||||
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","93",
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","93",
|
||||||
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
@ -111,7 +112,7 @@ public class DataFiller {
|
||||||
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
"/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS","20"
|
||||||
};
|
};
|
||||||
|
|
||||||
private static String[] data12 = {
|
protected static String[] data12 = {
|
||||||
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
"/d4science.research-infrastructures.eu/FARM/AlieiaVRE","12",
|
||||||
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","99",
|
"/d4science.research-infrastructures.eu/D4Research/AnalyticsLab","99",
|
||||||
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
"/d4science.research-infrastructures.eu/FARM/Aquabiotech","8",
|
||||||
|
|
Loading…
Reference in New Issue