gCubeSystem
/
accounting-dashboard-harvester-se-plugin
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added test to harvest old data for VREAccessesHarvester 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169295 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-06-18 13:15:12 +00:00
parent 3ba8504488
commit 87849c04b8
1 changed files with 148 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -7,21 +7,24 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
import org.junit.Test;
@ -52,7 +55,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launch() {
try {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -84,11 +87,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
// @Test
@Test
public void launchOldData() {
try {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -100,9 +103,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AggregationType aggregationType = AggregationType.MONTHLY;
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
@ -112,7 +115,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
// accountingDataHarvesterPlugin.launch(inputs);
accountingDataHarvesterPlugin.launch(inputs);
from.add(aggregationType.getCalendarField(), 1);
}
@ -123,9 +126,105 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
// @Test
public void launchOldDataVREAccessesHarvester() {
try {
Utils.setContext(ROOT);
DatabaseManager dbaseManager = new DatabaseManager();
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
dataHarvestPluginDeclaration);
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
AccountingDataHarvesterPlugin.getProperties().set(properties);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
while(from.before(runbeforeDate)) {
Date start = from.getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
VREAccessesHarvester vreAccessesHarvester = null;
for(String context : contexts) {
// Setting the token for the context
Utils.setContext(contextAuthorization.getTokenForContext(context));
ScopeBean scopeBean = new ScopeBean(context);
if(vreAccessesHarvester == null) {
if(scopeBean.is(Type.INFRASTRUCTURE)) {
vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else {
// This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope();
while(!parent.is(Type.INFRASTRUCTURE)) {
parent = scopeBean.enclosingScope();
}
// Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(parent.toString()));
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(context));
}
}
try {
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
} else {
// Collecting Google Analytics Data for VREs Accesses
List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
}
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
Utils.setContext(ROOT);
dbaseManager.insertMonthlyData(start, end, data, true);
from.add(aggregationType.getCalendarField(), 1);
}
Utils.setContext(ROOT);
} catch(Exception e) {
logger.error("", e);
}
logger.info("End.");
}
// @Test
public void testScopeBean() throws Exception {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
Utils.setContext(ROOT);
SortedSet<String> contexts = getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
@ -174,7 +273,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testVREAccessesHarvester() {
try {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
Utils.setContext(ROOT);
AggregationType measureType = AggregationType.MONTHLY;
@ -248,7 +347,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testMethodInvocation() {
try {
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
Utils.setContext(StockAssessment);
AggregationType measureType = AggregationType.MONTHLY;
@ -269,7 +368,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testTagMeMethodInvocation() {
try {
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
Utils.setContext(TAGME);
AggregationType measureType = AggregationType.MONTHLY;
@ -285,12 +384,47 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.error("", e);
}
}
// @Test
public void testGetVREUsersForSpecificVRE() {
try {
Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
dataHarvestPluginDeclaration);
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
AccountingDataHarvesterPlugin.getProperties().set(properties);
// DatabaseManager dbaseManager = new DatabaseManager();
ContextAuthorization contextAuthorization = new ContextAuthorization();
Utils.setContext(contextAuthorization.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience"));
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<HarvestedData> harvested = vreUsersHarvester.getData();
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
// dbaseManager.insertMonthlyData(start, end, harvested, true);
} catch(Exception e) {
logger.error("", e);
}
}
@Test
public void testFilteringGenericResource() {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
Utils.setContext(RESOURCE_CATALOGUE);
AggregationType measureType = AggregationType.MONTHLY;
@ -316,7 +450,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testResourceCatalogueHarvester() {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
Utils.setContext(RESOURCE_CATALOGUE);
AggregationType measureType = AggregationType.MONTHLY;
@ -346,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testDataMethodDownloadHarvester() {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
Utils.setContext(RESOURCE_CATALOGUE);
AggregationType measureType = AggregationType.MONTHLY;