gCubeSystem
/
accounting-dashboard-harvester-se-plugin
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added test to harvest old data 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169227 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-06-14 16:17:22 +00:00
parent 56d97e228b
commit 0f5311cb91
4 changed files with 144 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -24,7 +24,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.vremanagement.executor.plugin.Plugin;
import org.slf4j.Logger;
@ -101,7 +101,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'");
}
MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
AggregationType aggregationType = AggregationType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
boolean reRun = true;
if(inputs.containsKey(RERUN_INPUT_PARAMETER)) {
@ -134,10 +134,10 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
} else {
start = DateUtils.getPreviousPeriod(measureType).getTime();
start = DateUtils.getPreviousPeriod(aggregationType).getTime();
}
end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
@ -185,90 +185,91 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
}
try {
if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE)
|| context.startsWith(SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
} else {
if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE)
|| context.startsWith(SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
} else {
try {
// Collecting Google Analytics Data for VREs Accesses
List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
try {
// Collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
try {
// Collecting info on VRE users
if(getVREUsers) {
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
// The VREUsers can be only Harvested for the lst month
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) {
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<HarvestedData> harvested = vreUsersHarvester.getData();
try {
// Collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
try {
// Collecting info on VRE users
if(getVREUsers) {
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
// The VREUsers can be only Harvested for the lst month
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<HarvestedData> harvested = vreUsersHarvester.getData();
data.addAll(harvested);
}
}
} catch(Exception e) {
logger.error("Error harvesting Context Users for {}", context, e);
}
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
end, contexts);
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Data Method Download for {}", context, e);
}
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
} catch(Exception e) {
logger.error("Error harvesting Context Users for {}", context, e);
}
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
end, contexts);
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Data Method Download for {}", context, e);
}
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
}
Utils.setContext(initialToken);

4
src/main/java/org/gcube/dataharvest/utils/MeasureType.java → src/main/java/org/gcube/dataharvest/utils/AggregationType.java
View File

@ -6,7 +6,7 @@ import java.util.Calendar;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public enum MeasureType {
public enum AggregationType {
DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7),
MONTHLY(Calendar.MONTH, "yyyy-MM", 3),
@ -23,7 +23,7 @@ public enum MeasureType {
private MeasureType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
private AggregationType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
this.calendarField = calendarField;
this.dateFormatPattern=dateFormatPattern;
this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern);

6
src/main/java/org/gcube/dataharvest/utils/DateUtils.java
View File

@ -44,9 +44,9 @@ public class DateUtils {
return Calendar.getInstance(UTC_TIMEZONE);
}
public static Calendar getPreviousPeriod(MeasureType measureType) {
public static Calendar getPreviousPeriod(AggregationType aggregationType) {
Calendar now = getUTCCalendarInstance();
switch(measureType) {
switch(aggregationType) {
case YEARLY:
now.add(Calendar.YEAR, -1);
@ -91,7 +91,7 @@ public class DateUtils {
return aggregationStartCalendar;
}
public static Date getEndDateFromStartDate(MeasureType aggregationType, Date startDate, int offset) {
public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) {
Calendar aggregationEndDate = getUTCCalendarInstance();
aggregationEndDate.setTimeInMillis(startDate.getTime());
aggregationEndDate.add(aggregationType.getCalendarField(), offset);

76
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -21,7 +21,7 @@ import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
import org.junit.Test;
@ -56,26 +56,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
dataHarvestPluginDeclaration);
Map<String,Object> inputs = new HashMap<>();
MeasureType measureType = MeasureType.MONTHLY;
AggregationType aggregationType = AggregationType.MONTHLY;
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
/*
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
*/
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
accountingDataHarvesterPlugin.launch(inputs);
Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
while(from.before(runbeforeDate)) {
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
// accountingDataHarvesterPlugin.launch(inputs);
from.add(aggregationType.getCalendarField(), 1);
}
logger.info("End.");
@ -84,13 +88,49 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
@Test
// @Test
public void testScopeBean() throws Exception {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
SortedSet<String> contexts = getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime();
// start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
logger.info("\n\n\n");
for(String context : contexts) {
ScopeBean scopeBean = new ScopeBean(context);
logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
// logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
try {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}else {
logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) ||
context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
} else {
logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
}
}
@ -101,7 +141,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
@ -175,7 +215,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -196,7 +236,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -217,7 +257,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -243,7 +283,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
@ -273,7 +313,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY;
AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();