gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167631 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-18 16:33:28 +00:00
parent e2a7c69e27
commit 0bf82648db
5 changed files with 151 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -11,6 +11,7 @@ import java.util.SortedSet;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
@ -176,15 +177,25 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.error("Error harvesting Data Method Download for {}", context, e);
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
if(context.startsWith(TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}else {
try {
// Collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}

91
src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java Normal file
View File

@ -0,0 +1,91 @@
package org.gcube.dataharvest.harvester;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.SortedMap;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class MethodInvocationHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
public static final String DATAMINER_SERVICE_NAME = "dataminer";
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@Override
public List<HarvestedData> getData() throws Exception {
try {
List<HarvestedData> data = new ArrayList<>();
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
TemporalConstraint temporalConstraint = new TemporalConstraint(startDate.getTime(), endDate.getTime(),
AggregationMode.MONTHLY);
List<Filter> filters = new ArrayList<>();
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
String context = Utils.getCurrentContext();
List<String> contexts = new ArrayList<>();
contexts.add(context);
SortedMap<Filter,SortedMap<Calendar,Info>> result = accountingPersistenceQuery.getContextTimeSeries(
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
if(result != null) {
for(Filter filter : result.keySet()) {
SortedMap<Calendar,Info> infoMap = result.get(filter);
Calendar calendar = DateUtils.dateToCalendar(startDate);
Info info = infoMap.get(calendar);
logger.debug("{} : {}", DateUtils.LAUNCH_DATE_FORMAT.format(calendar.getTime()), info);
JSONObject jsonObject = info.getValue();
long numberOfInvocation = jsonObject.getLong(AggregatedUsageRecord.OPERATION_COUNT);
HarvestedData harvestedData = new HarvestedData(HarvestedData.METHOD_INVOCATIONS, context,
numberOfInvocation);
data.add(harvestedData);
}
} else {
logger.error("No data found.");
}
return data;
} catch(Exception e) {
throw e;
}
}
}

20
src/main/java/org/gcube/dataharvest/utils/Utils.java
View File

@ -22,35 +22,37 @@ import org.slf4j.LoggerFactory;
public class Utils {
private static Logger logger = LoggerFactory.getLogger(Utils.class);
public static String getJson(String url) throws MalformedURLException, IOException {
URL address = new URL(url);
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String json = "", line = "";
while (line != null) {
String json = "";
String line = "";
while(line != null) {
line = reader.readLine();
if (line != null)
if(line != null) {
json += line.trim();
}
}
return json;
}
public static String getCurrentContext() throws ObjectNotFound, Exception{
public static String getCurrentContext() throws ObjectNotFound, Exception {
return getCurrentContext(SecurityTokenProvider.instance.get());
}
public static String getCurrentContext(String token) throws ObjectNotFound, Exception{
public static String getCurrentContext(String token) throws ObjectNotFound, Exception {
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context);
return context;
}
public static void setContext(String token) throws ObjectNotFound, Exception{
public static void setContext(String token) throws ObjectNotFound, Exception {
SecurityTokenProvider.instance.set(token);
ScopeProvider.instance.set(getCurrentContext(token));
}
}

25
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import java.util.Map;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
@ -52,10 +53,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
@Test
public void testMethodInvocation() {
try {
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
MeasureType measureType = MeasureType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
logger.debug("{}", harvestedData);
}catch (Exception e) {
logger.error("", e);
}
}
@Test
public void testTagMe() {
public void testTagMeMethodInvocation() {
try {
org.gcube.dataharvest.utils.Utils.setContext(TAGME);

6
src/test/java/org/gcube/dataharvest/utils/ContextTest.java
View File

@ -46,6 +46,10 @@ public class ContextTest {
public static final String TAGME_VARNAME = "TAGME_ERIC";
public static final String TAGME;
public static final String RPrototypingLab_VARNAME = "RPrototypingLab";
public static final String RPrototypingLab;
static {
logger.trace("Retrieving Tokens from {}", PROPERTIES_FILENAME);
@ -72,6 +76,8 @@ public class ContextTest {
TAGME = properties.getProperty(TAGME_VARNAME);
RPrototypingLab = properties.getProperty(RPrototypingLab_VARNAME);
DEFAULT_TEST_SCOPE = GCUBE;
}