232 lines
8.8 KiB
Java
232 lines
8.8 KiB
Java
package org.gcube.dataharvest;
|
|
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Properties;
|
|
import java.util.SortedSet;
|
|
import java.util.TreeSet;
|
|
|
|
import org.gcube.common.scope.impl.ScopeBean;
|
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
|
import org.gcube.dataharvest.utils.ContextTest;
|
|
import org.gcube.dataharvest.utils.DateUtils;
|
|
import org.gcube.dataharvest.utils.MeasureType;
|
|
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
|
|
import org.junit.Test;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|
|
|
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class);
|
|
|
|
//@Test
|
|
public void test() {
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
|
|
|
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
|
dataHarvestPluginDeclaration);
|
|
|
|
Map<String,Object> inputs = new HashMap<>();
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
|
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
|
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
|
|
|
/*
|
|
Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1);
|
|
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
|
|
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
|
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
|
*/
|
|
|
|
accountingDataHarvesterPlugin.launch(inputs);
|
|
|
|
logger.info("End.");
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
//@Test
|
|
public void testMethodInvocation() {
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
|
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
|
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
|
|
|
logger.debug("{}", harvestedData);
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
//@Test
|
|
public void testTagMeMethodInvocation() {
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
|
|
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
|
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
|
|
|
logger.debug("{}", harvestedData);
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
|
|
//@Test
|
|
public void testFilteringGenericResource() {
|
|
try {
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
|
|
|
|
LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
|
|
SortedSet<String> contexts = new TreeSet<>(map.keySet());
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
|
|
|
//Added by Francesco
|
|
Properties properties = AccountingDataHarvesterPlugin.getProperties().get();
|
|
String catalogueContext = (String) properties.get(AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT);
|
|
logger.debug("Read from properties "+AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT+" value: "+catalogueContext);
|
|
//end
|
|
|
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, catalogueContext, contexts);
|
|
// SortedSet<String> excludedContexts = resourceCatalogueHarvester.getExcludedContexts();
|
|
// logger.info("Excluded contexts {}", excludedContexts);
|
|
// SortedSet<String> validContexts = resourceCatalogueHarvester.getFilteredContexts();
|
|
//
|
|
// logger.info("Valid Contexts {}", validContexts);
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
|
|
}
|
|
|
|
//@Test
|
|
public void testResourceCatalogueHarvester() {
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
// Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
// Date start = DateUtils.getStartCalendar(2016, 12, 01).getTime();
|
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
|
|
|
//Added by Francesco
|
|
Properties properties = AccountingDataHarvesterPlugin.getProperties().get();
|
|
String catalogueContext = (String) properties.get(AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT);
|
|
logger.debug("Read from properties "+AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT+" value: "+catalogueContext);
|
|
//end
|
|
|
|
//TODO @LUCA FROSINI MUST PASS SoBigData VREs
|
|
TreeSet<String> contexts = new TreeSet<String>();
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SocietalDebates");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SMAPH");
|
|
|
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, catalogueContext, contexts);
|
|
List<HarvestedData> data = resourceCatalogueHarvester.getData();
|
|
|
|
for (HarvestedData harvestedData : data) {
|
|
System.out.println(harvestedData.toString());
|
|
}
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
|
|
@Test
|
|
public void testDataMethodDownloadHarvester() {
|
|
try {
|
|
|
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
|
|
|
MeasureType measureType = MeasureType.MONTHLY;
|
|
|
|
Date start = DateUtils.getStartCalendar(2016, 12, 01).getTime();
|
|
// Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
|
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 18);
|
|
|
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
|
|
accountingDataHarvesterPlugin.getConfigParameters();
|
|
|
|
//Added by Francesco
|
|
Properties properties = AccountingDataHarvesterPlugin.getProperties().get();
|
|
String catalogueContext = (String) properties.get(AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT);
|
|
logger.debug("Read from properties "+AccountingDataHarvesterPlugin.RESOURCE_CATALOGUE_CONTEXT+" value: "+catalogueContext);
|
|
//end
|
|
|
|
//TODO @LUCA FROSINI MUST PASS SoBigData VREs
|
|
TreeSet<String> contexts = new TreeSet<String>();
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SocietalDebates");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
|
contexts.add("/d4science.research-infrastructures.eu/SoBigData/SMAPH");
|
|
|
|
DataMethodDownloadHarvester resourceCatalogueHarvester = new DataMethodDownloadHarvester(start, end, catalogueContext, contexts);
|
|
List<HarvestedData> data = resourceCatalogueHarvester.getData();
|
|
|
|
// for (HarvestedData harvestedData : data) {
|
|
// System.out.println(harvestedData.toString());
|
|
// }
|
|
|
|
} catch(Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
}
|