Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167631 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
e2a7c69e27
commit
0bf82648db
|
@ -11,6 +11,7 @@ import java.util.SortedSet;
|
||||||
|
|
||||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
import org.gcube.dataharvest.harvester.SocialHarvester;
|
import org.gcube.dataharvest.harvester.SocialHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||||
|
@ -176,15 +177,25 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
logger.error("Error harvesting Data Method Download for {}", context, e);
|
logger.error("Error harvesting Data Method Download for {}", context, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(context.startsWith(TAGME_CONTEXT)) {
|
}
|
||||||
try {
|
|
||||||
// Collecting info on method invocation
|
if(context.startsWith(TAGME_CONTEXT)) {
|
||||||
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
try {
|
||||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
// Collecting info on method invocation
|
||||||
data.addAll(harvested);
|
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
||||||
} catch(Exception e) {
|
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
data.addAll(harvested);
|
||||||
}
|
} catch(Exception e) {
|
||||||
|
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||||
|
}
|
||||||
|
}else {
|
||||||
|
try {
|
||||||
|
// Collecting info on method invocation
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
|
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||||
|
data.addAll(harvested);
|
||||||
|
} catch(Exception e) {
|
||||||
|
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,91 @@
|
||||||
|
package org.gcube.dataharvest.harvester;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.SortedMap;
|
||||||
|
|
||||||
|
import org.gcube.accounting.analytics.Filter;
|
||||||
|
import org.gcube.accounting.analytics.Info;
|
||||||
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||||
|
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||||
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||||
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||||
|
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
|
||||||
|
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||||
|
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||||
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||||
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Eric Perrone (ISTI - CNR)
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
public class MethodInvocationHarvester extends BasicHarvester {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
|
||||||
|
|
||||||
|
public static final String DATAMINER_SERVICE_NAME = "dataminer";
|
||||||
|
|
||||||
|
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
|
||||||
|
super(start, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<HarvestedData> getData() throws Exception {
|
||||||
|
try {
|
||||||
|
|
||||||
|
List<HarvestedData> data = new ArrayList<>();
|
||||||
|
|
||||||
|
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
|
||||||
|
TemporalConstraint temporalConstraint = new TemporalConstraint(startDate.getTime(), endDate.getTime(),
|
||||||
|
AggregationMode.MONTHLY);
|
||||||
|
|
||||||
|
List<Filter> filters = new ArrayList<>();
|
||||||
|
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
||||||
|
|
||||||
|
String context = Utils.getCurrentContext();
|
||||||
|
|
||||||
|
List<String> contexts = new ArrayList<>();
|
||||||
|
contexts.add(context);
|
||||||
|
|
||||||
|
SortedMap<Filter,SortedMap<Calendar,Info>> result = accountingPersistenceQuery.getContextTimeSeries(
|
||||||
|
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||||
|
|
||||||
|
if(result != null) {
|
||||||
|
for(Filter filter : result.keySet()) {
|
||||||
|
SortedMap<Calendar,Info> infoMap = result.get(filter);
|
||||||
|
|
||||||
|
Calendar calendar = DateUtils.dateToCalendar(startDate);
|
||||||
|
|
||||||
|
Info info = infoMap.get(calendar);
|
||||||
|
logger.debug("{} : {}", DateUtils.LAUNCH_DATE_FORMAT.format(calendar.getTime()), info);
|
||||||
|
|
||||||
|
JSONObject jsonObject = info.getValue();
|
||||||
|
long numberOfInvocation = jsonObject.getLong(AggregatedUsageRecord.OPERATION_COUNT);
|
||||||
|
|
||||||
|
HarvestedData harvestedData = new HarvestedData(HarvestedData.METHOD_INVOCATIONS, context,
|
||||||
|
numberOfInvocation);
|
||||||
|
data.add(harvestedData);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
logger.error("No data found.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -22,35 +22,37 @@ import org.slf4j.LoggerFactory;
|
||||||
public class Utils {
|
public class Utils {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
||||||
|
|
||||||
public static String getJson(String url) throws MalformedURLException, IOException {
|
public static String getJson(String url) throws MalformedURLException, IOException {
|
||||||
URL address = new URL(url);
|
URL address = new URL(url);
|
||||||
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
||||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||||
String json = "", line = "";
|
String json = "";
|
||||||
|
String line = "";
|
||||||
while (line != null) {
|
|
||||||
|
while(line != null) {
|
||||||
line = reader.readLine();
|
line = reader.readLine();
|
||||||
if (line != null)
|
if(line != null) {
|
||||||
json += line.trim();
|
json += line.trim();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return json;
|
return json;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getCurrentContext() throws ObjectNotFound, Exception{
|
public static String getCurrentContext() throws ObjectNotFound, Exception {
|
||||||
return getCurrentContext(SecurityTokenProvider.instance.get());
|
return getCurrentContext(SecurityTokenProvider.instance.get());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getCurrentContext(String token) throws ObjectNotFound, Exception{
|
public static String getCurrentContext(String token) throws ObjectNotFound, Exception {
|
||||||
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
|
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
|
||||||
String context = authorizationEntry.getContext();
|
String context = authorizationEntry.getContext();
|
||||||
logger.info("Context of token {} is {}", token, context);
|
logger.info("Context of token {} is {}", token, context);
|
||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void setContext(String token) throws ObjectNotFound, Exception {
|
||||||
public static void setContext(String token) throws ObjectNotFound, Exception{
|
|
||||||
SecurityTokenProvider.instance.set(token);
|
SecurityTokenProvider.instance.set(token);
|
||||||
ScopeProvider.instance.set(getCurrentContext(token));
|
ScopeProvider.instance.set(getCurrentContext(token));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||||
import org.gcube.dataharvest.utils.ContextTest;
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.gcube.dataharvest.utils.DateUtils;
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
|
@ -52,10 +53,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMethodInvocation() {
|
||||||
|
try {
|
||||||
|
|
||||||
|
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
||||||
|
|
||||||
|
MeasureType measureType = MeasureType.MONTHLY;
|
||||||
|
|
||||||
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||||
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
|
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
||||||
|
|
||||||
|
logger.debug("{}", harvestedData);
|
||||||
|
|
||||||
|
|
||||||
|
}catch (Exception e) {
|
||||||
|
logger.error("", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testTagMe() {
|
public void testTagMeMethodInvocation() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
||||||
|
|
|
@ -46,6 +46,10 @@ public class ContextTest {
|
||||||
public static final String TAGME_VARNAME = "TAGME_ERIC";
|
public static final String TAGME_VARNAME = "TAGME_ERIC";
|
||||||
public static final String TAGME;
|
public static final String TAGME;
|
||||||
|
|
||||||
|
public static final String RPrototypingLab_VARNAME = "RPrototypingLab";
|
||||||
|
public static final String RPrototypingLab;
|
||||||
|
|
||||||
|
|
||||||
static {
|
static {
|
||||||
|
|
||||||
logger.trace("Retrieving Tokens from {}", PROPERTIES_FILENAME);
|
logger.trace("Retrieving Tokens from {}", PROPERTIES_FILENAME);
|
||||||
|
@ -72,6 +76,8 @@ public class ContextTest {
|
||||||
|
|
||||||
TAGME = properties.getProperty(TAGME_VARNAME);
|
TAGME = properties.getProperty(TAGME_VARNAME);
|
||||||
|
|
||||||
|
RPrototypingLab = properties.getProperty(RPrototypingLab_VARNAME);
|
||||||
|
|
||||||
DEFAULT_TEST_SCOPE = GCUBE;
|
DEFAULT_TEST_SCOPE = GCUBE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue