gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added test to harvest old data 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169227 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-06-14 16:17:22 +00:00
parent 56d97e228b
commit 0f5311cb91
4 changed files with 144 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -24,7 +24,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.Utils; import org.gcube.dataharvest.utils.Utils;
import org.gcube.vremanagement.executor.plugin.Plugin; import org.gcube.vremanagement.executor.plugin.Plugin;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -101,7 +101,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'"); throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'");
} }
MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER)); AggregationType aggregationType = AggregationType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
boolean reRun = true; boolean reRun = true;
if(inputs.containsKey(RERUN_INPUT_PARAMETER)) { if(inputs.containsKey(RERUN_INPUT_PARAMETER)) {
@ -134,10 +134,10 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER); String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC); start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
} else { } else {
start = DateUtils.getPreviousPeriod(measureType).getTime(); start = DateUtils.getPreviousPeriod(aggregationType).getTime();
} }
end = DateUtils.getEndDateFromStartDate(measureType, start, 1); end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
@ -185,90 +185,91 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
} }
try {
if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE) if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE)
|| context.startsWith(SO_BIG_DATA_IT_VRE)) || context.startsWith(SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), logger.info("Not Harvesting for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end)); DateUtils.format(end));
} else { } else {
try {
// Collecting Google Analytics Data for VREs Accesses // Collecting Google Analytics Data for VREs Accesses
List<HarvestedData> harvested = vreAccessesHarvester.getData(); List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested); data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
} }
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); try {
} // Collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
try { List<HarvestedData> harvested = socialHarvester.getData();
// Collecting info on social (posts, replies and likes) data.addAll(harvested);
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); } catch(Exception e) {
List<HarvestedData> harvested = socialHarvester.getData(); logger.error("Error harvesting Social Interactions for {}", context, e);
data.addAll(harvested); }
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); try {
} // Collecting info on VRE users
if(getVREUsers) {
try { // Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
// Collecting info on VRE users // The VREUsers can be only Harvested for the lst month
if(getVREUsers) { if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts) VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
// The VREUsers can be only Harvested for the lst month List<HarvestedData> harvested = vreUsersHarvester.getData();
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) { data.addAll(harvested);
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); }
List<HarvestedData> harvested = vreUsersHarvester.getData(); }
} catch(Exception e) {
logger.error("Error harvesting Context Users for {}", context, e);
}
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested); data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
end, contexts);
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Data Method Download for {}", context, e);
}
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
} }
} }
} catch(Exception e) {
logger.error("Error harvesting Context Users for {}", context, e);
} }
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
end, contexts);
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Data Method Download for {}", context, e);
}
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
} }
Utils.setContext(initialToken); Utils.setContext(initialToken);

4
src/main/java/org/gcube/dataharvest/utils/MeasureType.java → src/main/java/org/gcube/dataharvest/utils/AggregationType.java
View File

@ -6,7 +6,7 @@ import java.util.Calendar;
/** /**
* @author Luca Frosini (ISTI - CNR) * @author Luca Frosini (ISTI - CNR)
*/ */
public enum MeasureType { public enum AggregationType {
DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7), DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7),
MONTHLY(Calendar.MONTH, "yyyy-MM", 3), MONTHLY(Calendar.MONTH, "yyyy-MM", 3),
@ -23,7 +23,7 @@ public enum MeasureType {
private MeasureType(int calendarField, String dateFormatPattern, int notAggregableBefore) { private AggregationType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
this.calendarField = calendarField; this.calendarField = calendarField;
this.dateFormatPattern=dateFormatPattern; this.dateFormatPattern=dateFormatPattern;
this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern); this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern);

6
src/main/java/org/gcube/dataharvest/utils/DateUtils.java
View File

@ -44,9 +44,9 @@ public class DateUtils {
return Calendar.getInstance(UTC_TIMEZONE); return Calendar.getInstance(UTC_TIMEZONE);
} }
public static Calendar getPreviousPeriod(MeasureType measureType) { public static Calendar getPreviousPeriod(AggregationType aggregationType) {
Calendar now = getUTCCalendarInstance(); Calendar now = getUTCCalendarInstance();
switch(measureType) { switch(aggregationType) {
case YEARLY: case YEARLY:
now.add(Calendar.YEAR, -1); now.add(Calendar.YEAR, -1);
@ -91,7 +91,7 @@ public class DateUtils {
return aggregationStartCalendar; return aggregationStartCalendar;
} }
public static Date getEndDateFromStartDate(MeasureType aggregationType, Date startDate, int offset) { public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) {
Calendar aggregationEndDate = getUTCCalendarInstance(); Calendar aggregationEndDate = getUTCCalendarInstance();
aggregationEndDate.setTimeInMillis(startDate.getTime()); aggregationEndDate.setTimeInMillis(startDate.getTime());
aggregationEndDate.add(aggregationType.getCalendarField(), offset); aggregationEndDate.add(aggregationType.getCalendarField(), offset);

76
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -21,7 +21,7 @@ import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.Utils; import org.gcube.dataharvest.utils.Utils;
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
import org.junit.Test; import org.junit.Test;
@ -56,26 +56,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(ROOT); org.gcube.dataharvest.utils.Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin( AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
dataHarvestPluginDeclaration); dataHarvestPluginDeclaration);
Map<String,Object> inputs = new HashMap<>(); Map<String,Object> inputs = new HashMap<>();
MeasureType measureType = MeasureType.MONTHLY; AggregationType aggregationType = AggregationType.MONTHLY;
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
/* Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
*/
accountingDataHarvesterPlugin.launch(inputs); Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
while(from.before(runbeforeDate)) {
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
// accountingDataHarvesterPlugin.launch(inputs);
from.add(aggregationType.getCalendarField(), 1);
}
logger.info("End."); logger.info("End.");
@ -84,13 +88,49 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Test // @Test
public void testScopeBean() throws Exception { public void testScopeBean() throws Exception {
org.gcube.dataharvest.utils.Utils.setContext(ROOT); org.gcube.dataharvest.utils.Utils.setContext(ROOT);
SortedSet<String> contexts = getContexts(); SortedSet<String> contexts = getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime();
// start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
logger.info("\n\n\n");
for(String context : contexts) { for(String context : contexts) {
ScopeBean scopeBean = new ScopeBean(context); ScopeBean scopeBean = new ScopeBean(context);
logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); // logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
try {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}else {
logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) ||
context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
} else {
logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
} }
} }
@ -101,7 +141,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(ROOT); org.gcube.dataharvest.utils.Utils.setContext(ROOT);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
@ -175,7 +215,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment); org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -196,7 +236,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(TAGME); org.gcube.dataharvest.utils.Utils.setContext(TAGME);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -217,7 +257,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try { try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
@ -243,7 +283,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
@ -273,7 +313,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE); org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();