From 1753289cd556d6f3611c15e48391d6ee1a7a2559 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Wed, 16 May 2018 13:08:25 +0000 Subject: [PATCH] Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167516 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 13 +++ .../AccountingDataHarvesterPlugin.java | 37 ++++--- .../dataharvest/harvester/BasicHarvester.java | 9 +- .../harvester/VreUsersHarvester.java | 21 +--- .../AccountingDataHarvesterPluginTest.java | 80 ++++++++++++++ src/test/java/org/gcube/utils/ScopedTest.java | 103 ++++++++++++++++++ src/test/resources/logback-test.xml | 2 +- 7 files changed, 225 insertions(+), 40 deletions(-) create mode 100644 src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java create mode 100644 src/test/java/org/gcube/utils/ScopedTest.java diff --git a/pom.xml b/pom.xml index 8bb6b2b..605ed1c 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,19 @@ 20171018 compile + + + junit + junit + 4.11 + test + + + ch.qos.logback + logback-classic + 1.0.13 + test + diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index 873e030..6bc2407 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -1,30 +1,29 @@ package org.gcube.dataharvest; -import java.util.Date; import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.Map; -import org.gcube.dataharvest.DataHarvestPluginDeclaration; -import org.gcube.vremanagement.executor.exception.InputsNullException; -import org.gcube.vremanagement.executor.exception.InvalidInputsException; -import org.gcube.vremanagement.executor.plugin.Plugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.gcube.dataharvest.dao.*; +import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VreUsersHarvester; +import org.gcube.vremanagement.executor.plugin.Plugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AccountingDataHarvesterPlugin extends Plugin { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class); + public static final String PARAMETER_FROM = "from"; public static final String PARAMETER_TO = "to"; public static final String TEST = "test"; + private boolean testMode = false, updateFlag = false; private Date dateFrom, dateTo; @@ -62,7 +61,7 @@ public class AccountingDataHarvesterPlugin extends Plugin users = vreUsersHarvester.getData(); - dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, users, updateFlag); + if(!testMode) { + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, users, updateFlag); + } } catch (Exception x) { logger.error(x.getLocalizedMessage()); } @@ -81,7 +82,9 @@ public class AccountingDataHarvesterPlugin extends Plugin res = resourceCatalogueHarvester.getData(); - dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + if(!testMode) { + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } } catch (Exception x) { logger.error(x.getLocalizedMessage()); } @@ -90,7 +93,9 @@ public class AccountingDataHarvesterPlugin extends Plugin res = dataMethodDownloadHarvester.getData(); - dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + if(!testMode) { + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } } catch (Exception x) { logger.error(x.getLocalizedMessage()); } @@ -99,7 +104,9 @@ public class AccountingDataHarvesterPlugin extends Plugin res = socialHarvester.getData(); - dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + if(!testMode) { + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } } catch (Exception x) { logger.error(x.getLocalizedMessage()); } @@ -109,7 +116,9 @@ public class AccountingDataHarvesterPlugin extends Plugin res = methodInvocationHarvester.getData(); - //insertMonthlyData((Date) dateFrom, (Date) dateTo, res); + if(!testMode) { + dbaseManager.insertMonthlyData((Date) dateFrom, (Date) dateTo, res, updateFlag); + } } catch (Exception x) { logger.error(x.getLocalizedMessage()); } diff --git a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java index 2053063..be5963e 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java @@ -3,30 +3,25 @@ package org.gcube.dataharvest.harvester; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -import java.io.IOException; -import java.io.InputStream; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Properties; -import org.gcube.dataharvest.dao.Dao; -import org.gcube.dataharvest.dao.DatabaseManager; -import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.utils.Group; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataharvest.dao.DatabaseManager; +import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.informationsystem.publisher.RegistryPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; import org.gcube.resources.discovery.client.api.DiscoveryClient; diff --git a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java index 1a7a227..77c611f 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java @@ -1,33 +1,18 @@ package org.gcube.dataharvest.harvester; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; import java.net.MalformedURLException; -import java.net.URL; -import java.text.DateFormat; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Locale; import java.util.Properties; -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -import org.gcube.common.resources.gcore.ServiceEndpoint; -import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.utils.Utils; - -import org.json.*; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class VreUsersHarvester extends BasicHarvester { // private final String CATEGORY_NAME = "Accounting"; diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java new file mode 100644 index 0000000..dc5c959 --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -0,0 +1,80 @@ +package org.gcube.dataharvest; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +import org.gcube.utils.ScopedTest; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AccountingDataHarvesterPluginTest extends ScopedTest { + + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class); + + public static TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC"); + public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z"; + public static final DateFormat DEFAULT_DATE_FORMAT; + + static { + DEFAULT_DATE_FORMAT = getUTCDateFormat(DATETIME_PATTERN); + } + + public static DateFormat getUTCDateFormat(String pattern){ + DateFormat dateFormat = new SimpleDateFormat(pattern); + dateFormat.setTimeZone(UTC_TIMEZONE); + return dateFormat; + } + + public static Calendar getUTCCalendarInstance(){ + return Calendar.getInstance(UTC_TIMEZONE); + } + + public static Calendar getAggregationStartCalendar(int year, int month, int day){ + Calendar aggregationStartCalendar = getUTCCalendarInstance(); + aggregationStartCalendar.set(Calendar.YEAR, year); + aggregationStartCalendar.set(Calendar.MONTH, month); + aggregationStartCalendar.set(Calendar.DAY_OF_MONTH, day); + aggregationStartCalendar.set(Calendar.HOUR_OF_DAY, 0); + aggregationStartCalendar.set(Calendar.MINUTE, 0); + aggregationStartCalendar.set(Calendar.SECOND, 0); + aggregationStartCalendar.set(Calendar.MILLISECOND, 0); + + logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime())); + + return aggregationStartCalendar; + } + + + @Test + public void test() { + try { + ScopedTest.setContext(ScopedTest.ROOT); + + DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(dataHarvestPluginDeclaration); + + Calendar from = getAggregationStartCalendar(2018, Calendar.APRIL, 1); + Calendar to = getAggregationStartCalendar(2018, Calendar.APRIL, 30); + + Map map = new HashMap<>(); + map.put(AccountingDataHarvesterPlugin.PARAMETER_FROM, from.getTime()); + map.put(AccountingDataHarvesterPlugin.PARAMETER_TO, to.getTime()); + map.put(AccountingDataHarvesterPlugin.TEST, false); + + accountingDataHarvesterPlugin.launch(map); + + logger.info("End."); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + +} diff --git a/src/test/java/org/gcube/utils/ScopedTest.java b/src/test/java/org/gcube/utils/ScopedTest.java new file mode 100644 index 0000000..4bef19d --- /dev/null +++ b/src/test/java/org/gcube/utils/ScopedTest.java @@ -0,0 +1,103 @@ +/** + * + */ +package org.gcube.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) + * + */ +public class ScopedTest { + + private static final Logger logger = LoggerFactory.getLogger(ScopedTest.class); + + protected static final String PROPERTIES_FILENAME = "token.properties"; + + private static final String GCUBE_DEVNEXT_VARNAME = "GCUBE_DEVNEXT"; + public static final String GCUBE_DEVNEXT; + + private static final String GCUBE_DEVNEXT_NEXTNEXT_VARNAME = "GCUBE_DEVNEXT_NEXTNEXT"; + public static final String GCUBE_DEVNEXT_NEXTNEXT; + + public static final String GCUBE_DEVSEC_VARNAME = "GCUBE_DEVSEC"; + public static final String GCUBE_DEVSEC; + + public static final String GCUBE_DEVSEC_DEVVRE_VARNAME = "GCUBE_DEVSEC_DEVVRE"; + public static final String GCUBE_DEVSEC_DEVVRE; + + public static final String GCUBE_VARNAME = "GCUBE"; + public static final String GCUBE; + + public static final String DEFAULT_TEST_SCOPE; + + public static final String ROOT_VARNAME = "ROOT_ERIC"; + public static final String ROOT; + + public static final String TAGME_VARNAME = "TAGME_ERIC"; + public static final String TAGME; + + static { + Properties properties = new Properties(); + InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); + + try { + // load the properties file + properties.load(input); + } catch (IOException e) { + throw new RuntimeException(e); + } + + GCUBE = properties.getProperty(GCUBE_VARNAME); + + GCUBE_DEVNEXT = properties.getProperty(GCUBE_DEVNEXT_VARNAME); + GCUBE_DEVNEXT_NEXTNEXT = properties.getProperty(GCUBE_DEVNEXT_NEXTNEXT_VARNAME); + + GCUBE_DEVSEC = properties.getProperty(GCUBE_DEVSEC_VARNAME); + GCUBE_DEVSEC_DEVVRE = properties.getProperty(GCUBE_DEVSEC_DEVVRE_VARNAME); + + ROOT = properties.getProperty(ROOT_VARNAME); + + TAGME = properties.getProperty(TAGME_VARNAME); + + DEFAULT_TEST_SCOPE = GCUBE_DEVSEC; + } + + public static String getCurrentScope(String token) throws ObjectNotFound, Exception{ + AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); + String context = authorizationEntry.getContext(); + logger.info("Context of token {} is {}", token, context); + return context; + } + + + public static void setContext(String token) throws ObjectNotFound, Exception{ + SecurityTokenProvider.instance.set(token); + ScopeProvider.instance.set(getCurrentScope(token)); + } + + @BeforeClass + public static void beforeClass() throws Exception{ + setContext(DEFAULT_TEST_SCOPE); + } + + @AfterClass + public static void afterClass() throws Exception{ + SecurityTokenProvider.instance.reset(); + ScopeProvider.instance.reset(); + } + +} diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index 70d4803..777a7e5 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -10,7 +10,7 @@ - +