gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167629 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Luca Frosini 2018-05-18 16:24:53 +00:00
parent 17d9cd9097
commit e2a7c69e27
5 changed files with 78 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -14,7 +14,7 @@ import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
@ -179,7 +179,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {

75
src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java → src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java
View File

@ -4,9 +4,7 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import org.gcube.accounting.analytics.Filter;
@ -15,10 +13,14 @@ import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -26,58 +28,63 @@ import org.slf4j.LoggerFactory;
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class MethodInvocationHarvester extends BasicHarvester {
public class TagMeMethodInvocationHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
private static Logger logger = LoggerFactory.getLogger(TagMeMethodInvocationHarvester.class);
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
public static final String TAG_METHOD = "tag";
public TagMeMethodInvocationHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@Override
public List<HarvestedData> getData() throws Exception {
try {
logger.debug("MethodInvocationHarvester::getData()");
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
List<HarvestedData> data = new ArrayList<>();
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
Calendar start = Calendar.getInstance();
Calendar end = Calendar.getInstance();
start.setTime(startDate);
end.setTime(endDate);
TemporalConstraint temporalConstraint = new TemporalConstraint(start.getTimeInMillis(),
end.getTimeInMillis(), AggregationMode.MONTHLY);
TemporalConstraint temporalConstraint = new TemporalConstraint(startDate.getTime(), endDate.getTime(),
AggregationMode.MONTHLY);
List<Filter> filters = new ArrayList<>();
filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, "tag"));
filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, TAG_METHOD));
String context = Utils.getCurrentContext();
List<String> contexts = new ArrayList<>();
contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe");
contexts.add(context);
logger.debug("MethodInvocationHarvester::getData()::getContextTimeSeries");
SortedMap<Filter,SortedMap<Calendar,Info>> result = accountingPersistenceQuery.getContextTimeSeries(
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
if(result == null) {
logger.error("No data found.");
} else {
Set<Filter> ks = result.keySet();
if(ks != null) {
Iterator<Filter> ksi = ks.iterator();
while(ksi.hasNext()) {
// System.out.println("" + ksi.next().toString());
logger.debug("Filter: " + ksi.next().toString());
}
if(result != null) {
for(Filter filter : result.keySet()) {
SortedMap<Calendar,Info> infoMap = result.get(filter);
Calendar calendar = DateUtils.dateToCalendar(startDate);
Info info = infoMap.get(calendar);
logger.debug("{} : {}", DateUtils.LAUNCH_DATE_FORMAT.format(calendar.getTime()), info);
JSONObject jsonObject = info.getValue();
long numberOfInvocation = jsonObject.getLong(AggregatedUsageRecord.OPERATION_COUNT);
HarvestedData harvestedData = new HarvestedData(HarvestedData.METHOD_INVOCATIONS, context,
numberOfInvocation);
data.add(harvestedData);
}
} else {
logger.error("No data found.");
}
return data;
} catch(Exception x) {
StackTraceElement[] ste = x.getStackTrace();
String errorMessage = "MethodInvocationHarvester: " + x.getLocalizedMessage();
for(StackTraceElement s : ste) {
errorMessage += "\n" + s.toString();
}
logger.error(errorMessage);
throw x;
} catch(Exception e) {
throw e;
}
}

9
src/main/java/org/gcube/dataharvest/utils/DateUtils.java
View File

@ -45,7 +45,7 @@ public class DateUtils {
}
public static Calendar getPreviousPeriod(MeasureType measureType) {
Calendar now = Calendar.getInstance();
Calendar now = getUTCCalendarInstance();
switch(measureType) {
case YEARLY:
@ -101,10 +101,9 @@ public class DateUtils {
/* OLD functions of Eric Perrone (ISTI - CNR) */
public static Calendar dateToCalendar(Date date) {
Calendar cal = null;
cal = Calendar.getInstance();
cal.setTime(date);
return cal;
Calendar calendar = DateUtils.getUTCCalendarInstance();
calendar.setTime(date);
return calendar;
}
public static String dateToStringWithTZ(Date date) {

31
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -1,9 +1,14 @@
package org.gcube.dataharvest;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
import org.junit.Test;
import org.slf4j.Logger;
@ -26,10 +31,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
MeasureType measureType = MeasureType.MONTHLY;
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
/*
@ -49,4 +53,27 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
@Test
public void testTagMe() {
try {
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
MeasureType measureType = MeasureType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
logger.debug("{}", harvestedData);
}catch (Exception e) {
logger.error("", e);
}
}
}

4
src/test/java/org/gcube/dataharvest/Harvester.java
View File

@ -14,7 +14,7 @@ import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -125,7 +125,7 @@ public class Harvester {
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo);
List<HarvestedData> res = methodInvocationHarvester.getData();
logger.debug("{}", res);
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);