package org.gcube.dataharvest; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.Utils; import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AccountingDataHarvesterPluginTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class); public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData"; public static SortedSet getContexts() throws Exception{ SortedSet contexts = new TreeSet<>(); LinkedHashMap map = ContextManager.readContexts(); for(String scope : map.keySet()) { try { String context = map.get(scope).toString(); contexts.add(context); }catch (Exception e) { throw e; } } return contexts; } @Test public void launch() { try { org.gcube.dataharvest.utils.Utils.setContext(ROOT); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( dataHarvestPluginDeclaration); Map inputs = new HashMap<>(); MeasureType measureType = MeasureType.MONTHLY; inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); /* Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1); String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); */ accountingDataHarvesterPlugin.launch(inputs); logger.info("End."); } catch(Exception e) { logger.error("", e); } } @Test public void testScopeBean() throws Exception { SortedSet contexts = getContexts(); for(String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); logger.debug("Name {}, FullName {}", scopeBean.name(), scopeBean.toString()); } } @Test public void testVREAccessesHarvester() { try { org.gcube.dataharvest.utils.Utils.setContext(ROOT); MeasureType measureType = MeasureType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); VREAccessesHarvester vreAccessesHarvester = null; ArrayList data = new ArrayList(); for(String context : contexts) { // Setting the token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); ScopeBean scopeBean = new ScopeBean(context); if(vreAccessesHarvester == null) { if(scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); }else { // This code should be never used because the scopes are sorted by fullname ScopeBean parent = scopeBean.enclosingScope(); while(!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); vreAccessesHarvester = new VREAccessesHarvester(start, end); // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); } } try { // Collecting Google Analytics Data for VREs Accesses List harvested = vreAccessesHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } logger.debug("{}", data); } catch(Exception e) { logger.error("", e); } } @Test public void testMethodInvocation() { try { org.gcube.dataharvest.utils.Utils.setContext(StockAssessment); MeasureType measureType = MeasureType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); List harvestedData = methodInvocationHarvester.getData(); logger.debug("{}", harvestedData); } catch(Exception e) { logger.error("", e); } } @Test public void testTagMeMethodInvocation() { try { org.gcube.dataharvest.utils.Utils.setContext(TAGME); MeasureType measureType = MeasureType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); List harvestedData = methodInvocationHarvester.getData(); logger.debug("{}", harvestedData); } catch(Exception e) { logger.error("", e); } } @Test public void testFilteringGenericResource() { try { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); MeasureType measureType = MeasureType.MONTHLY; Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); SortedSet contexts = getContexts(); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); SortedSet validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_DATA_CONTEXT + "/"); logger.info("Valid Contexts {}", validContexts); } catch(Exception e) { logger.error("", e); } } @Test public void testResourceCatalogueHarvester() { try { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); MeasureType measureType = MeasureType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); SortedSet contexts = getContexts(); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); List data = resourceCatalogueHarvester.getData(); logger.debug("{}", data); } catch(Exception e) { logger.error("", e); } } @Test public void testDataMethodDownloadHarvester() { try { org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); MeasureType measureType = MeasureType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); accountingDataHarvesterPlugin.getConfigParameters(); SortedSet contexts = getContexts(); DataMethodDownloadHarvester resourceCatalogueHarvester = new DataMethodDownloadHarvester(start, end, contexts); List data = resourceCatalogueHarvester.getData(); logger.debug("{}", data); } catch(Exception e) { logger.error("", e); } } }