diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 7abf387..5f10080 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -36,29 +36,29 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AccountingDataHarvesterPluginTest extends ContextTest { - + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class); - + public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData"; - public static SortedSet getContexts() throws Exception{ + public static SortedSet getContexts() throws Exception { SortedSet contexts = new TreeSet<>(); LinkedHashMap map = ContextManager.readContexts(); for(String scope : map.keySet()) { try { String context = map.get(scope).toString(); contexts.add(context); - }catch (Exception e) { + } catch(Exception e) { throw e; } } return contexts; } - @Test + // @Test public void getDimensions() { try { - + Utils.setContext(ROOT); AccountingDao dao = AccountingDao.get(); @@ -68,35 +68,33 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel()); } - logger.info("End."); - + } catch(Exception e) { logger.error("", e); } } - // @Test public void launch() { try { - + Utils.setContext(ROOT); - + DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( dataHarvestPluginDeclaration); - + Map inputs = new HashMap<>(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); - + /* Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); @@ -107,7 +105,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { accountingDataHarvesterPlugin.launch(inputs); logger.info("End."); - + } catch(Exception e) { logger.error("", e); } @@ -116,26 +114,26 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void launchOldData() { try { - + Utils.setContext(ROOT); - + DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( dataHarvestPluginDeclaration); - + Map inputs = new HashMap<>(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); - + Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); while(from.before(runbeforeDate)) { String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); @@ -144,22 +142,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { accountingDataHarvesterPlugin.launch(inputs); from.add(aggregationType.getCalendarField(), 1); } - + logger.info("End."); - + } catch(Exception e) { logger.error("", e); } } - // @Test public void launchOldDataVREAccessesHarvester() { try { - + Utils.setContext(ROOT); - //DatabaseManager dbaseManager = new DatabaseManager(); - AccountingDao dao = AccountingDao.get(); + // AccountingDao dao = AccountingDao.get(); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); @@ -167,15 +163,15 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { dataHarvestPluginDeclaration); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDataHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1); - Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); while(from.before(runbeforeDate)) { Date start = from.getTime(); @@ -183,7 +179,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); - // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList<>(); VREAccessesHarvester vreAccessesHarvester = null; @@ -198,7 +193,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { if(scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); - }else { + } else { // This code should be never used because the scopes are sorted by fullname ScopeBean parent = scopeBean.enclosingScope(); @@ -218,28 +213,26 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } try { - if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { - logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); + if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) + && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { + logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, + DateUtils.format(start), DateUtils.format(end)); } else { // Collecting Google Analytics Data for VREs Accesses List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - /* - List harvested = vreAccessesHarvester.getData(); - data.addAll(harvested); - */ + } } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - // logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); - logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), + accountingRecords); Utils.setContext(ROOT); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - // dbaseManager.insertMonthlyData(start, end, data, true); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); Thread.sleep(TimeUnit.SECONDS.toMillis(10)); @@ -249,13 +242,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Utils.setContext(ROOT); - - } catch(Exception e) { logger.error("", e); } - - logger.info("End."); + + logger.info("End."); } @@ -278,21 +269,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { try { - if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) { logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); - }else { + } else { logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } - if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) || - context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE) + if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) + || context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE) || context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE)) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { - logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), - DateUtils.format(end)); + logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, + DateUtils.format(start), DateUtils.format(end)); } else { logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); @@ -309,26 +299,25 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void testVREAccessesHarvester() { try { - + Utils.setContext(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); VREAccessesHarvester vreAccessesHarvester = null; - // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList<>(); for(String context : contexts) { @@ -341,7 +330,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { if(scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); - }else { + } else { // This code should be never used because the scopes are sorted by fullname ScopeBean parent = scopeBean.enclosingScope(); @@ -361,18 +350,14 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } try { - if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { - logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); + if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) + && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { + logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, + DateUtils.format(start), DateUtils.format(end)); } else { // Collecting Google Analytics Data for VREs Accesses - List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - - /* - List harvested = vreAccessesHarvester.getData(); - data.addAll(harvested); - */ } } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); @@ -389,10 +374,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void testSocialInteraction() { try { - + Utils.setContext(ROOT); - //DatabaseManager dbaseManager = new DatabaseManager(); - AccountingDao dao = AccountingDao.get(); + // AccountingDao dao = AccountingDao.get(); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); @@ -400,7 +384,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { dataHarvestPluginDeclaration); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDataHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = new TreeSet<>(); @@ -412,14 +396,13 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { contexts.add("/d4science.research-infrastructures.eu/gCubeApps/gCube"); AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); Date start = from.getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); - + logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end)); - // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList<>(); for(String context : contexts) { @@ -431,78 +414,61 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); List harvested = socialHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - - /* - List harvested = socialHarvester.getData(); - data.addAll(harvested); - */ } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - - //logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); - logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); + + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), + accountingRecords); Utils.setContext(ROOT); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - // dbaseManager.insertMonthlyData(start, end, data, true); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { logger.error("", e); } - - logger.info("End."); + + logger.info("End."); } - - // @Test public void testMethodInvocation() { try { - + Utils.setContext(StockAssessment); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - - MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); - List harvestedData = methodInvocationHarvester.getAccountingRecords(); - /* - List harvested = methodInvocationHarvester.getData(); - */ - - logger.debug("{}", harvestedData); - + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); + List accountingRecords = methodInvocationHarvester.getAccountingRecords(); + + logger.debug("{}", accountingRecords); + } catch(Exception e) { logger.error("", e); } } - + // @Test public void testTagMeMethodInvocation() { try { - + Utils.setContext(TAGME); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - + TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); - List harvestedData = methodInvocationHarvester.getAccountingRecords(); - /* - List harvested = methodInvocationHarvester.getData(); - */ - - logger.debug("{}", harvestedData); - + List accountingRecords = methodInvocationHarvester.getAccountingRecords(); + + logger.debug("{}", accountingRecords); + } catch(Exception e) { logger.error("", e); } @@ -519,117 +485,110 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDataHarvesterPlugin.getProperties().set(properties); - // DatabaseManager dbaseManager = new DatabaseManager(); + // AccountingDao dao = AccountingDao.get(); ContextAuthorization contextAuthorization = new ContextAuthorization(); - Utils.setContext(contextAuthorization.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); - + Utils.setContext(contextAuthorization + .getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); List harvested = vreUsersHarvester.getAccountingRecords(); - /* - List harvested = vreUsersHarvester.getData(); - */ logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); org.gcube.dataharvest.utils.Utils.setContext(ROOT); - // dbaseManager.insertMonthlyData(start, end, harvested, true); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - } catch(Exception e) { logger.error("", e); } - + } - // @Test public void testFilteringGenericResource() { try { Utils.setContext(RESOURCE_CATALOGUE); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - + SortedSet contexts = getContexts(); - + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); - - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); - SortedSet validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_DATA_CONTEXT + "/"); + + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, + contexts); + SortedSet validContexts = resourceCatalogueHarvester.getValidContexts(contexts, + SO_BIG_DATA_CONTEXT + "/"); logger.info("Valid Contexts {}", validContexts); - + } catch(Exception e) { logger.error("", e); } - + } - + // @Test public void testResourceCatalogueHarvester() { try { - + Utils.setContext(RESOURCE_CATALOGUE); AggregationType measureType = AggregationType.MONTHLY; - + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); - + SortedSet contexts = getContexts(); - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, + contexts); List data = resourceCatalogueHarvester.getAccountingRecords(); - /* - List data = resourceCatalogueHarvester.getData(); - */ logger.debug("{}", data); - + } catch(Exception e) { logger.error("", e); } } - - + @Test public void testDataMethodDownloadHarvester() { try { - + Utils.setContext(RESOURCE_CATALOGUE); - + AggregationType measureType = AggregationType.MONTHLY; - + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); - + SortedSet contexts = getContexts(); - DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, contexts); + DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, + contexts); List data = dataMethodDownloadHarvester.getAccountingRecords(); - /* - List data = dataMethodDownloadHarvester.getData(); - */ logger.debug("{}", data); @@ -638,7 +597,4 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - - - }