2018-05-16 15:08:25 +02:00
|
|
|
package org.gcube.dataharvest;
|
|
|
|
|
2018-06-14 14:50:42 +02:00
|
|
|
import java.util.ArrayList;
|
2018-05-18 18:24:53 +02:00
|
|
|
import java.util.Date;
|
2018-05-16 15:08:25 +02:00
|
|
|
import java.util.HashMap;
|
2018-05-22 12:05:30 +02:00
|
|
|
import java.util.LinkedHashMap;
|
2018-05-18 18:24:53 +02:00
|
|
|
import java.util.List;
|
2018-05-16 15:08:25 +02:00
|
|
|
import java.util.Map;
|
2018-05-21 15:37:57 +02:00
|
|
|
import java.util.SortedSet;
|
2018-05-22 12:05:30 +02:00
|
|
|
import java.util.TreeSet;
|
2018-05-16 15:08:25 +02:00
|
|
|
|
2018-05-22 12:05:30 +02:00
|
|
|
import org.gcube.common.scope.impl.ScopeBean;
|
2018-06-14 14:50:42 +02:00
|
|
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
2018-05-18 18:24:53 +02:00
|
|
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
2018-05-18 18:33:28 +02:00
|
|
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
2018-06-14 14:50:42 +02:00
|
|
|
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
2018-05-24 17:10:45 +02:00
|
|
|
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
2018-05-21 12:51:17 +02:00
|
|
|
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
2018-05-18 18:24:53 +02:00
|
|
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
2018-06-14 14:50:42 +02:00
|
|
|
import org.gcube.dataharvest.utils.ContextAuthorization;
|
2018-05-18 14:17:54 +02:00
|
|
|
import org.gcube.dataharvest.utils.ContextTest;
|
2018-05-18 18:24:53 +02:00
|
|
|
import org.gcube.dataharvest.utils.DateUtils;
|
2018-05-18 17:01:49 +02:00
|
|
|
import org.gcube.dataharvest.utils.MeasureType;
|
2018-06-14 14:50:42 +02:00
|
|
|
import org.gcube.dataharvest.utils.Utils;
|
2018-05-22 12:05:30 +02:00
|
|
|
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
|
2018-05-16 15:08:25 +02:00
|
|
|
import org.junit.Test;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
2018-05-18 14:17:54 +02:00
|
|
|
public class AccountingDataHarvesterPluginTest extends ContextTest {
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-16 15:08:25 +02:00
|
|
|
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
|
|
|
|
|
|
|
|
public static SortedSet<String> getContexts() throws Exception{
|
|
|
|
SortedSet<String> contexts = new TreeSet<>();
|
|
|
|
LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
|
|
|
|
for(String scope : map.keySet()) {
|
|
|
|
try {
|
|
|
|
String context = map.get(scope).toString();
|
|
|
|
contexts.add(context);
|
|
|
|
}catch (Exception e) {
|
|
|
|
throw e;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return contexts;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-05-28 16:53:00 +02:00
|
|
|
@Test
|
2018-05-28 16:39:13 +02:00
|
|
|
public void launch() {
|
2018-05-16 15:08:25 +02:00
|
|
|
try {
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 17:39:47 +02:00
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-16 15:08:25 +02:00
|
|
|
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
2018-05-21 15:37:57 +02:00
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
|
|
|
dataHarvestPluginDeclaration);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-17 16:04:22 +02:00
|
|
|
Map<String,Object> inputs = new HashMap<>();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-17 16:04:22 +02:00
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
2018-06-04 16:35:32 +02:00
|
|
|
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
|
2018-05-17 16:04:22 +02:00
|
|
|
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
2018-06-14 14:34:11 +02:00
|
|
|
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 17:01:49 +02:00
|
|
|
/*
|
2018-06-04 16:35:32 +02:00
|
|
|
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);
|
2018-05-17 16:04:22 +02:00
|
|
|
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
|
|
|
|
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
2018-05-18 17:01:49 +02:00
|
|
|
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
|
|
|
*/
|
2018-05-29 11:08:24 +02:00
|
|
|
|
2018-05-17 16:04:22 +02:00
|
|
|
accountingDataHarvesterPlugin.launch(inputs);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-16 15:08:25 +02:00
|
|
|
logger.info("End.");
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
} catch(Exception e) {
|
2018-05-17 16:04:22 +02:00
|
|
|
logger.error("", e);
|
2018-05-16 15:08:25 +02:00
|
|
|
}
|
|
|
|
}
|
2018-06-14 14:50:42 +02:00
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testScopeBean() throws Exception {
|
|
|
|
SortedSet<String> contexts = getContexts();
|
|
|
|
for(String context : contexts) {
|
|
|
|
ScopeBean scopeBean = new ScopeBean(context);
|
|
|
|
logger.debug("Name {}, FullName {}", scopeBean.name(), scopeBean.toString());
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testVREAccessesHarvester() {
|
|
|
|
try {
|
|
|
|
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
|
|
|
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
|
|
|
|
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
|
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
|
|
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-06-14 14:50:42 +02:00
|
|
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
2018-06-14 15:30:07 +02:00
|
|
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
2018-06-14 14:50:42 +02:00
|
|
|
|
|
|
|
VREAccessesHarvester vreAccessesHarvester = null;
|
|
|
|
|
|
|
|
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
|
|
|
|
|
|
|
|
for(String context : contexts) {
|
|
|
|
// Setting the token for the context
|
|
|
|
Utils.setContext(contextAuthorization.getTokenForContext(context));
|
|
|
|
|
|
|
|
ScopeBean scopeBean = new ScopeBean(context);
|
|
|
|
|
|
|
|
if(vreAccessesHarvester == null) {
|
|
|
|
|
|
|
|
if(scopeBean.is(Type.INFRASTRUCTURE)) {
|
|
|
|
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
|
|
|
}else {
|
|
|
|
// This code should be never used because the scopes are sorted by fullname
|
|
|
|
|
|
|
|
ScopeBean parent = scopeBean.enclosingScope();
|
|
|
|
while(!parent.is(Type.INFRASTRUCTURE)) {
|
|
|
|
parent = scopeBean.enclosingScope();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Setting back token for the context
|
|
|
|
Utils.setContext(contextAuthorization.getTokenForContext(parent.toString()));
|
|
|
|
|
|
|
|
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
|
|
|
|
|
|
|
// Setting back token for the context
|
|
|
|
Utils.setContext(contextAuthorization.getTokenForContext(context));
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
try {
|
2018-06-14 15:30:46 +02:00
|
|
|
// Collecting Google Analytics Data for VREs Accesses
|
|
|
|
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
|
|
|
data.addAll(harvested);
|
2018-06-14 14:50:42 +02:00
|
|
|
} catch(Exception e) {
|
|
|
|
logger.error("Error harvesting Social Interactions for {}", context, e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.debug("{}", data);
|
|
|
|
|
|
|
|
} catch(Exception e) {
|
|
|
|
logger.error("", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
@Test
|
2018-05-18 18:33:28 +02:00
|
|
|
public void testMethodInvocation() {
|
|
|
|
try {
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 19:12:46 +02:00
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:33:28 +02:00
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:33:28 +02:00
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:33:28 +02:00
|
|
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
|
|
|
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:33:28 +02:00
|
|
|
logger.debug("{}", harvestedData);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
} catch(Exception e) {
|
2018-05-18 18:33:28 +02:00
|
|
|
logger.error("", e);
|
|
|
|
}
|
|
|
|
}
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
@Test
|
2018-05-18 18:33:28 +02:00
|
|
|
public void testTagMeMethodInvocation() {
|
2018-05-18 18:24:53 +02:00
|
|
|
try {
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
|
|
|
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-18 18:24:53 +02:00
|
|
|
logger.debug("{}", harvestedData);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
} catch(Exception e) {
|
2018-05-18 18:24:53 +02:00
|
|
|
logger.error("", e);
|
|
|
|
}
|
|
|
|
}
|
2018-05-24 12:49:53 +02:00
|
|
|
|
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
@Test
|
2018-05-24 12:49:53 +02:00
|
|
|
public void testFilteringGenericResource() {
|
2018-05-21 12:51:17 +02:00
|
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 12:51:17 +02:00
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 12:51:17 +02:00
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
SortedSet<String> contexts = getContexts();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
|
2018-05-28 15:22:59 +02:00
|
|
|
SortedSet<String> validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
|
2018-05-28 12:00:15 +02:00
|
|
|
logger.info("Valid Contexts {}", validContexts);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
} catch(Exception e) {
|
2018-05-21 12:51:17 +02:00
|
|
|
logger.error("", e);
|
|
|
|
}
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 12:51:17 +02:00
|
|
|
}
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-24 17:13:16 +02:00
|
|
|
@Test
|
2018-05-24 12:49:53 +02:00
|
|
|
public void testResourceCatalogueHarvester() {
|
2018-05-21 15:37:57 +02:00
|
|
|
try {
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
2018-05-28 12:00:15 +02:00
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-28 16:03:31 +02:00
|
|
|
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
2018-05-24 17:10:45 +02:00
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-24 17:10:45 +02:00
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
SortedSet<String> contexts = getContexts();
|
2018-05-28 12:00:15 +02:00
|
|
|
|
|
|
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
|
2018-05-24 17:10:45 +02:00
|
|
|
List<HarvestedData> data = resourceCatalogueHarvester.getData();
|
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
logger.debug("{}", data);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
2018-05-24 17:10:45 +02:00
|
|
|
} catch(Exception e) {
|
|
|
|
logger.error("", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testDataMethodDownloadHarvester() {
|
|
|
|
try {
|
|
|
|
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
|
|
|
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
|
2018-05-28 16:03:31 +02:00
|
|
|
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
2018-05-28 12:00:15 +02:00
|
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
2018-05-24 12:49:53 +02:00
|
|
|
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
|
|
|
|
2018-05-28 15:22:59 +02:00
|
|
|
SortedSet<String> contexts = getContexts();
|
2018-05-28 12:00:15 +02:00
|
|
|
|
|
|
|
DataMethodDownloadHarvester resourceCatalogueHarvester = new DataMethodDownloadHarvester(start, end, contexts);
|
2018-05-24 12:49:53 +02:00
|
|
|
List<HarvestedData> data = resourceCatalogueHarvester.getData();
|
|
|
|
|
2018-05-28 12:00:15 +02:00
|
|
|
logger.debug("{}", data);
|
|
|
|
|
2018-05-21 15:37:57 +02:00
|
|
|
} catch(Exception e) {
|
|
|
|
logger.error("", e);
|
|
|
|
}
|
|
|
|
}
|
2018-06-14 14:50:42 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-05-16 15:08:25 +02:00
|
|
|
}
|