Added test to harvest old data
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169227 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
56d97e228b
commit
0f5311cb91
|
|
@ -24,7 +24,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
|||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.dataharvest.utils.MeasureType;
|
||||
import org.gcube.dataharvest.utils.AggregationType;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.gcube.vremanagement.executor.plugin.Plugin;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -101,7 +101,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'");
|
||||
}
|
||||
|
||||
MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
|
||||
AggregationType aggregationType = AggregationType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER));
|
||||
|
||||
boolean reRun = true;
|
||||
if(inputs.containsKey(RERUN_INPUT_PARAMETER)) {
|
||||
|
|
@ -134,10 +134,10 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
|
||||
start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
|
||||
} else {
|
||||
start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
start = DateUtils.getPreviousPeriod(aggregationType).getTime();
|
||||
}
|
||||
|
||||
end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||
|
||||
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
|
||||
|
||||
|
|
@ -185,90 +185,91 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
}
|
||||
|
||||
try {
|
||||
if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE)
|
||||
|| context.startsWith(SO_BIG_DATA_IT_VRE))
|
||||
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
|
||||
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
} else {
|
||||
|
||||
if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE)
|
||||
|| context.startsWith(SO_BIG_DATA_IT_VRE))
|
||||
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
|
||||
logger.info("Not Harvesting for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
} else {
|
||||
|
||||
try {
|
||||
// Collecting Google Analytics Data for VREs Accesses
|
||||
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||
List<HarvestedData> harvested = socialHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on VRE users
|
||||
if(getVREUsers) {
|
||||
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
|
||||
// The VREUsers can be only Harvested for the lst month
|
||||
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) {
|
||||
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
|
||||
List<HarvestedData> harvested = vreUsersHarvester.getData();
|
||||
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||
List<HarvestedData> harvested = socialHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on VRE users
|
||||
if(getVREUsers) {
|
||||
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
|
||||
// The VREUsers can be only Harvested for the lst month
|
||||
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
|
||||
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
|
||||
List<HarvestedData> harvested = vreUsersHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
}
|
||||
}
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Context Users for {}", context, e);
|
||||
}
|
||||
|
||||
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
|
||||
|
||||
try {
|
||||
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||
contexts);
|
||||
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on Data/Method download
|
||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
|
||||
end, contexts);
|
||||
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Data Method Download for {}", context, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(context.startsWith(TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
||||
start, end);
|
||||
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Context Users for {}", context, e);
|
||||
}
|
||||
|
||||
if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) {
|
||||
|
||||
try {
|
||||
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||
contexts);
|
||||
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on Data/Method download
|
||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
|
||||
end, contexts);
|
||||
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Data Method Download for {}", context, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(context.startsWith(TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
||||
start, end);
|
||||
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Utils.setContext(initialToken);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import java.util.Calendar;
|
|||
/**
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public enum MeasureType {
|
||||
public enum AggregationType {
|
||||
|
||||
DAILY(Calendar.DAY_OF_MONTH, "yyyy-MM-dd", 7),
|
||||
MONTHLY(Calendar.MONTH, "yyyy-MM", 3),
|
||||
|
|
@ -23,7 +23,7 @@ public enum MeasureType {
|
|||
|
||||
|
||||
|
||||
private MeasureType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
|
||||
private AggregationType(int calendarField, String dateFormatPattern, int notAggregableBefore) {
|
||||
this.calendarField = calendarField;
|
||||
this.dateFormatPattern=dateFormatPattern;
|
||||
this.dateFormat = DateUtils.getUTCDateFormat(dateFormatPattern);
|
||||
|
|
@ -44,9 +44,9 @@ public class DateUtils {
|
|||
return Calendar.getInstance(UTC_TIMEZONE);
|
||||
}
|
||||
|
||||
public static Calendar getPreviousPeriod(MeasureType measureType) {
|
||||
public static Calendar getPreviousPeriod(AggregationType aggregationType) {
|
||||
Calendar now = getUTCCalendarInstance();
|
||||
switch(measureType) {
|
||||
switch(aggregationType) {
|
||||
|
||||
case YEARLY:
|
||||
now.add(Calendar.YEAR, -1);
|
||||
|
|
@ -91,7 +91,7 @@ public class DateUtils {
|
|||
return aggregationStartCalendar;
|
||||
}
|
||||
|
||||
public static Date getEndDateFromStartDate(MeasureType aggregationType, Date startDate, int offset) {
|
||||
public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) {
|
||||
Calendar aggregationEndDate = getUTCCalendarInstance();
|
||||
aggregationEndDate.setTimeInMillis(startDate.getTime());
|
||||
aggregationEndDate.add(aggregationType.getCalendarField(), offset);
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
|||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.dataharvest.utils.MeasureType;
|
||||
import org.gcube.dataharvest.utils.AggregationType;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
|
||||
import org.junit.Test;
|
||||
|
|
@ -56,26 +56,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
||||
|
||||
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
||||
|
||||
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
||||
dataHarvestPluginDeclaration);
|
||||
|
||||
Map<String,Object> inputs = new HashMap<>();
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
|
||||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
||||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
|
||||
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
|
||||
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
||||
|
||||
/*
|
||||
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);
|
||||
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
|
||||
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||
*/
|
||||
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
|
||||
|
||||
accountingDataHarvesterPlugin.launch(inputs);
|
||||
Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
|
||||
|
||||
while(from.before(runbeforeDate)) {
|
||||
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
|
||||
logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||
inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
|
||||
// accountingDataHarvesterPlugin.launch(inputs);
|
||||
from.add(aggregationType.getCalendarField(), 1);
|
||||
}
|
||||
|
||||
logger.info("End.");
|
||||
|
||||
|
|
@ -84,13 +88,49 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void testScopeBean() throws Exception {
|
||||
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
||||
SortedSet<String> contexts = getContexts();
|
||||
|
||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime();
|
||||
// start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||
|
||||
logger.info("\n\n\n");
|
||||
|
||||
for(String context : contexts) {
|
||||
ScopeBean scopeBean = new ScopeBean(context);
|
||||
logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
|
||||
// logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
|
||||
|
||||
try {
|
||||
|
||||
|
||||
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
|
||||
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
}else {
|
||||
logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
}
|
||||
|
||||
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) ||
|
||||
context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
|
||||
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE))
|
||||
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
|
||||
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
} else {
|
||||
logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
|
||||
DateUtils.format(end));
|
||||
}
|
||||
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -101,7 +141,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
||||
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
||||
|
|
@ -175,7 +215,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(StockAssessment);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
|
@ -196,7 +236,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
|
@ -217,7 +257,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
try {
|
||||
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
|
@ -243,7 +283,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
||||
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
||||
|
|
@ -273,7 +313,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
|
||||
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
|
||||
|
|
|
|||
Loading…
Reference in New Issue