diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 6addb62..0e34a05 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -7,21 +7,24 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.SortedSet; import java.util.TreeSet; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; +import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; +import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; -import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.Utils; import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; import org.junit.Test; @@ -52,7 +55,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void launch() { try { - org.gcube.dataharvest.utils.Utils.setContext(ROOT); + Utils.setContext(ROOT); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); @@ -84,11 +87,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - // @Test + @Test public void launchOldData() { try { - org.gcube.dataharvest.utils.Utils.setContext(ROOT); + Utils.setContext(ROOT); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); @@ -100,9 +103,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { AggregationType aggregationType = AggregationType.MONTHLY; inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); - inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false); + inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); - inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); + inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); @@ -112,7 +115,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - // accountingDataHarvesterPlugin.launch(inputs); + accountingDataHarvesterPlugin.launch(inputs); from.add(aggregationType.getCalendarField(), 1); } @@ -123,9 +126,105 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } + + // @Test + public void launchOldDataVREAccessesHarvester() { + try { + + Utils.setContext(ROOT); + DatabaseManager dbaseManager = new DatabaseManager(); + + DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); + + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( + dataHarvestPluginDeclaration); + Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); + AccountingDataHarvesterPlugin.getProperties().set(properties); + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + + AggregationType aggregationType = AggregationType.MONTHLY; + + Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); + + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); + + while(from.before(runbeforeDate)) { + Date start = from.getTime(); + Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); + + logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); + + ArrayList data = new ArrayList(); + + VREAccessesHarvester vreAccessesHarvester = null; + + for(String context : contexts) { + // Setting the token for the context + Utils.setContext(contextAuthorization.getTokenForContext(context)); + + ScopeBean scopeBean = new ScopeBean(context); + + if(vreAccessesHarvester == null) { + + if(scopeBean.is(Type.INFRASTRUCTURE)) { + vreAccessesHarvester = new VREAccessesHarvester(start, end); + }else { + // This code should be never used because the scopes are sorted by fullname + + ScopeBean parent = scopeBean.enclosingScope(); + while(!parent.is(Type.INFRASTRUCTURE)) { + parent = scopeBean.enclosingScope(); + } + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); + + vreAccessesHarvester = new VREAccessesHarvester(start, end); + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(context)); + } + + } + + try { + if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { + logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); + } else { + // Collecting Google Analytics Data for VREs Accesses + List harvested = vreAccessesHarvester.getData(); + data.addAll(harvested); + } + } catch(Exception e) { + logger.error("Error harvesting Social Interactions for {}", context, e); + } + } + + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); + Utils.setContext(ROOT); + dbaseManager.insertMonthlyData(start, end, data, true); + + from.add(aggregationType.getCalendarField(), 1); + + } + + Utils.setContext(ROOT); + + + + } catch(Exception e) { + logger.error("", e); + } + + logger.info("End."); + + } + // @Test public void testScopeBean() throws Exception { - org.gcube.dataharvest.utils.Utils.setContext(ROOT); + Utils.setContext(ROOT); SortedSet contexts = getContexts(); AggregationType aggregationType = AggregationType.MONTHLY; @@ -174,7 +273,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testVREAccessesHarvester() { try { - org.gcube.dataharvest.utils.Utils.setContext(ROOT); + Utils.setContext(ROOT); AggregationType measureType = AggregationType.MONTHLY; @@ -248,7 +347,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testMethodInvocation() { try { - org.gcube.dataharvest.utils.Utils.setContext(StockAssessment); + Utils.setContext(StockAssessment); AggregationType measureType = AggregationType.MONTHLY; @@ -269,7 +368,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testTagMeMethodInvocation() { try { - org.gcube.dataharvest.utils.Utils.setContext(TAGME); + Utils.setContext(TAGME); AggregationType measureType = AggregationType.MONTHLY; @@ -285,12 +384,47 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } + + // @Test + public void testGetVREUsersForSpecificVRE() { + try { + Utils.setContext(ROOT); + + DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( + dataHarvestPluginDeclaration); + Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); + AccountingDataHarvesterPlugin.getProperties().set(properties); + + // DatabaseManager dbaseManager = new DatabaseManager(); + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + Utils.setContext(contextAuthorization.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); + AggregationType measureType = AggregationType.MONTHLY; + + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); + + VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); + List harvested = vreUsersHarvester.getData(); + logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); + + org.gcube.dataharvest.utils.Utils.setContext(ROOT); + // dbaseManager.insertMonthlyData(start, end, harvested, true); + + + } catch(Exception e) { + logger.error("", e); + } + + } + @Test public void testFilteringGenericResource() { try { - org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); + Utils.setContext(RESOURCE_CATALOGUE); AggregationType measureType = AggregationType.MONTHLY; @@ -316,7 +450,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testResourceCatalogueHarvester() { try { - org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); + Utils.setContext(RESOURCE_CATALOGUE); AggregationType measureType = AggregationType.MONTHLY; @@ -346,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testDataMethodDownloadHarvester() { try { - org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); + Utils.setContext(RESOURCE_CATALOGUE); AggregationType measureType = AggregationType.MONTHLY;