From 0f5311cb91fdb6888df5d7c98f35f2781744cdc2 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Thu, 14 Jun 2018 16:17:22 +0000 Subject: [PATCH] Added test to harvest old data git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169227 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingDataHarvesterPlugin.java | 161 +++++++++--------- ...{MeasureType.java => AggregationType.java} | 4 +- .../gcube/dataharvest/utils/DateUtils.java | 6 +- .../AccountingDataHarvesterPluginTest.java | 76 +++++++-- 4 files changed, 144 insertions(+), 103 deletions(-) rename src/main/java/org/gcube/dataharvest/utils/{MeasureType.java => AggregationType.java} (88%) diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index 896eb51..6b65896 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -24,7 +24,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.DateUtils; -import org.gcube.dataharvest.utils.MeasureType; +import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.Utils; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; @@ -101,7 +101,7 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = vreAccessesHarvester.getData(); data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Social Interactions for {}", context, e); } - } catch(Exception e) { - logger.error("Error harvesting Social Interactions for {}", context, e); - } - - try { - // Collecting info on social (posts, replies and likes) - SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); - List harvested = socialHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Social Interactions for {}", context, e); - } - - try { - // Collecting info on VRE users - if(getVREUsers) { - // Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts) - // The VREUsers can be only Harvested for the lst month - if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) { - VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); - List harvested = vreUsersHarvester.getData(); + + try { + // Collecting info on social (posts, replies and likes) + SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); + List harvested = socialHarvester.getData(); + data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Social Interactions for {}", context, e); + } + + try { + // Collecting info on VRE users + if(getVREUsers) { + // Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts) + // The VREUsers can be only Harvested for the lst month + if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) { + VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); + List harvested = vreUsersHarvester.getData(); + data.addAll(harvested); + } + } + } catch(Exception e) { + logger.error("Error harvesting Context Users for {}", context, e); + } + + if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) { + + try { + // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, + contexts); + List harvested = resourceCatalogueHarvester.getData(); data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Resource Catalogue Information for {}", context, e); + } + + try { + // Collecting info on Data/Method download + DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, + end, contexts); + List harvested = dataMethodDownloadHarvester.getData(); + data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Data Method Download for {}", context, e); + } + + } + + if(context.startsWith(TAGME_CONTEXT)) { + try { + // Collecting info on method invocation + TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester( + start, end); + List harvested = tagMeMethodInvocationHarvester.getData(); + data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Method Invocations for {}", context, e); + } + } else { + try { + // Collecting info on method invocation + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); + List harvested = methodInvocationHarvester.getData(); + data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Method Invocations for {}", context, e); } } - } catch(Exception e) { - logger.error("Error harvesting Context Users for {}", context, e); } - - if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) { - - try { - // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, - contexts); - List harvested = resourceCatalogueHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Resource Catalogue Information for {}", context, e); - } - - try { - // Collecting info on Data/Method download - DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, - end, contexts); - List harvested = dataMethodDownloadHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Data Method Download for {}", context, e); - } - - } - - if(context.startsWith(TAGME_CONTEXT)) { - try { - // Collecting info on method invocation - TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester( - start, end); - List harvested = tagMeMethodInvocationHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Method Invocations for {}", context, e); - } - } else { - try { - // Collecting info on method invocation - MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); - List harvested = methodInvocationHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Method Invocations for {}", context, e); - } - } - } Utils.setContext(initialToken); diff --git a/src/main/java/org/gcube/dataharvest/utils/MeasureType.java b/src/main/java/org/gcube/dataharvest/utils/AggregationType.java similarity index 88% rename from src/main/java/org/gcube/dataharvest/utils/MeasureType.java rename to src/main/java/org/gcube/dataharvest/utils/AggregationType.java index dbb1d30..1b0142d 100644 --- a/src/main/java/org/gcube/dataharvest/utils/MeasureType.java +++ b/src/main/java/org/gcube/dataharvest/utils/AggregationType.java @@ -6,7 +6,7 @@ import java.util.Calendar; /** * @author Luca Frosini (ISTI - CNR) */ -public enum MeasureType { +public enum AggregationType { DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7), MONTHLY(Calendar.MONTH, "yyyy-MM", 3), @@ -23,7 +23,7 @@ public enum MeasureType { - private MeasureType(int calendarField, String dateFormatPattern, int notAggregableBefore) { + private AggregationType(int calendarField, String dateFormatPattern, int notAggregableBefore) { this.calendarField = calendarField; this.dateFormatPattern=dateFormatPattern; this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern); diff --git a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java index 36ebd54..907d751 100644 --- a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java +++ b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java @@ -44,9 +44,9 @@ public class DateUtils { return Calendar.getInstance(UTC_TIMEZONE); } - public static Calendar getPreviousPeriod(MeasureType measureType) { + public static Calendar getPreviousPeriod(AggregationType aggregationType) { Calendar now = getUTCCalendarInstance(); - switch(measureType) { + switch(aggregationType) { case YEARLY: now.add(Calendar.YEAR, -1); @@ -91,7 +91,7 @@ public class DateUtils { return aggregationStartCalendar; } - public static Date getEndDateFromStartDate(MeasureType aggregationType, Date startDate, int offset) { + public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) { Calendar aggregationEndDate = getUTCCalendarInstance(); aggregationEndDate.setTimeInMillis(startDate.getTime()); aggregationEndDate.add(aggregationType.getCalendarField(), offset); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 0890577..9505ffc 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -21,7 +21,7 @@ import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; -import org.gcube.dataharvest.utils.MeasureType; +import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.Utils; import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; import org.junit.Test; @@ -56,26 +56,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(ROOT); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( dataHarvestPluginDeclaration); Map inputs = new HashMap<>(); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType aggregationType = AggregationType.MONTHLY; - inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); + inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); - /* - Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1); - String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - */ + Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - accountingDataHarvesterPlugin.launch(inputs); + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); + + while(from.before(runbeforeDate)) { + String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); + inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); + // accountingDataHarvesterPlugin.launch(inputs); + from.add(aggregationType.getCalendarField(), 1); + } logger.info("End."); @@ -84,13 +88,49 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - @Test + // @Test public void testScopeBean() throws Exception { org.gcube.dataharvest.utils.Utils.setContext(ROOT); SortedSet contexts = getContexts(); + + AggregationType aggregationType = AggregationType.MONTHLY; + + Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime(); + // start = DateUtils.getPreviousPeriod(measureType).getTime(); + Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); + + logger.info("\n\n\n"); + for(String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); - logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); + // logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); + + try { + + + if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) { + logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), + DateUtils.format(end)); + }else { + logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), + DateUtils.format(end)); + } + + if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) || + context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE) + || context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE)) + && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { + logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), + DateUtils.format(end)); + } else { + logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), + DateUtils.format(end)); + } + + } catch(Exception e) { + logger.error("Error harvesting Social Interactions for {}", context, e); + } + } } @@ -101,7 +141,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(ROOT); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); @@ -175,7 +215,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(StockAssessment); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); @@ -196,7 +236,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(TAGME); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); @@ -217,7 +257,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { try { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); @@ -243,7 +283,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); @@ -273,7 +313,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); - MeasureType measureType = MeasureType.MONTHLY; + AggregationType measureType = AggregationType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();