From e2a7c69e27d176a27db3372597e01ad332d94038 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Fri, 18 May 2018 16:24:53 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167629 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingDataHarvesterPlugin.java | 4 +- ...va => TagMeMethodInvocationHarvester.java} | 75 ++++++++++--------- .../gcube/dataharvest/utils/DateUtils.java | 9 +-- .../AccountingDataHarvesterPluginTest.java | 31 +++++++- .../java/org/gcube/dataharvest/Harvester.java | 4 +- 5 files changed, 78 insertions(+), 45 deletions(-) rename src/main/java/org/gcube/dataharvest/harvester/sobigdata/{MethodInvocationHarvester.java => TagMeMethodInvocationHarvester.java} (53%) diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index a09f9e7..e2e50b4 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -14,7 +14,7 @@ import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; -import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.DateUtils; @@ -179,7 +179,7 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = methodInvocationHarvester.getData(); data.addAll(harvested); } catch(Exception e) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java similarity index 53% rename from src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java rename to src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java index b66a529..ee77910 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java @@ -4,9 +4,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import java.util.Iterator; import java.util.List; -import java.util.Set; import java.util.SortedMap; import org.gcube.accounting.analytics.Filter; @@ -15,10 +13,14 @@ import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; +import org.gcube.accounting.datamodel.AggregatedUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.BasicHarvester; +import org.gcube.dataharvest.utils.DateUtils; +import org.gcube.dataharvest.utils.Utils; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,58 +28,63 @@ import org.slf4j.LoggerFactory; * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ -public class MethodInvocationHarvester extends BasicHarvester { +public class TagMeMethodInvocationHarvester extends BasicHarvester { - private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class); + private static Logger logger = LoggerFactory.getLogger(TagMeMethodInvocationHarvester.class); - public MethodInvocationHarvester(Date start, Date end) throws ParseException { + public static final String TAG_METHOD = "tag"; + + public TagMeMethodInvocationHarvester(Date start, Date end) throws ParseException { super(start, end); } @Override public List getData() throws Exception { try { - logger.debug("MethodInvocationHarvester::getData()"); - ArrayList data = new ArrayList(); + + List data = new ArrayList<>(); + AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance(); - Calendar start = Calendar.getInstance(); - Calendar end = Calendar.getInstance(); - start.setTime(startDate); - end.setTime(endDate); - TemporalConstraint temporalConstraint = new TemporalConstraint(start.getTimeInMillis(), - end.getTimeInMillis(), AggregationMode.MONTHLY); + TemporalConstraint temporalConstraint = new TemporalConstraint(startDate.getTime(), endDate.getTime(), + AggregationMode.MONTHLY); List filters = new ArrayList<>(); - filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, "tag")); + filters.add(new Filter(ServiceUsageRecord.CALLED_METHOD, TAG_METHOD)); + + String context = Utils.getCurrentContext(); List contexts = new ArrayList<>(); - contexts.add("/d4science.research-infrastructures.eu/SoBigData/TagMe"); + contexts.add(context); - logger.debug("MethodInvocationHarvester::getData()::getContextTimeSeries"); SortedMap> result = accountingPersistenceQuery.getContextTimeSeries( AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true); - if(result == null) { - logger.error("No data found."); - } else { - Set ks = result.keySet(); - if(ks != null) { - Iterator ksi = ks.iterator(); - while(ksi.hasNext()) { - // System.out.println("" + ksi.next().toString()); - logger.debug("Filter: " + ksi.next().toString()); - } + + if(result != null) { + for(Filter filter : result.keySet()) { + SortedMap infoMap = result.get(filter); + + Calendar calendar = DateUtils.dateToCalendar(startDate); + + Info info = infoMap.get(calendar); + logger.debug("{} : {}", DateUtils.LAUNCH_DATE_FORMAT.format(calendar.getTime()), info); + + JSONObject jsonObject = info.getValue(); + long numberOfInvocation = jsonObject.getLong(AggregatedUsageRecord.OPERATION_COUNT); + + HarvestedData harvestedData = new HarvestedData(HarvestedData.METHOD_INVOCATIONS, context, + numberOfInvocation); + data.add(harvestedData); + } + + } else { + logger.error("No data found."); } return data; - } catch(Exception x) { - StackTraceElement[] ste = x.getStackTrace(); - String errorMessage = "MethodInvocationHarvester: " + x.getLocalizedMessage(); - for(StackTraceElement s : ste) { - errorMessage += "\n" + s.toString(); - } - logger.error(errorMessage); - throw x; + + } catch(Exception e) { + throw e; } } diff --git a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java index 89bb774..8fe782b 100644 --- a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java +++ b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java @@ -45,7 +45,7 @@ public class DateUtils { } public static Calendar getPreviousPeriod(MeasureType measureType) { - Calendar now = Calendar.getInstance(); + Calendar now = getUTCCalendarInstance(); switch(measureType) { case YEARLY: @@ -101,10 +101,9 @@ public class DateUtils { /* OLD functions of Eric Perrone (ISTI - CNR) */ public static Calendar dateToCalendar(Date date) { - Calendar cal = null; - cal = Calendar.getInstance(); - cal.setTime(date); - return cal; + Calendar calendar = DateUtils.getUTCCalendarInstance(); + calendar.setTime(date); + return calendar; } public static String dateToStringWithTZ(Date date) { diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index c16256b..546a6c5 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -1,9 +1,14 @@ package org.gcube.dataharvest; +import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.gcube.dataharvest.datamodel.HarvestedData; +import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.utils.ContextTest; +import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.MeasureType; import org.junit.Test; import org.slf4j.Logger; @@ -26,10 +31,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { MeasureType measureType = MeasureType.MONTHLY; + inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); - inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); - inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); /* @@ -49,4 +53,27 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } + + @Test + public void testTagMe() { + try { + + org.gcube.dataharvest.utils.Utils.setContext(TAGME); + + MeasureType measureType = MeasureType.MONTHLY; + + Date start = DateUtils.getPreviousPeriod(measureType).getTime(); + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); + + TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); + List harvestedData = methodInvocationHarvester.getData(); + + logger.debug("{}", harvestedData); + + + }catch (Exception e) { + logger.error("", e); + } + } + } diff --git a/src/test/java/org/gcube/dataharvest/Harvester.java b/src/test/java/org/gcube/dataharvest/Harvester.java index 7668c35..1fa87cb 100644 --- a/src/test/java/org/gcube/dataharvest/Harvester.java +++ b/src/test/java/org/gcube/dataharvest/Harvester.java @@ -14,7 +14,7 @@ import org.gcube.dataharvest.harvester.BasicHarvester; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; -import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,7 +125,7 @@ public class Harvester { try { // collecting info on method invocation - MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo); + TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo); List res = methodInvocationHarvester.getData(); logger.debug("{}", res); // insertMonthlyData((Date) dateFrom, (Date) dateTo, res);