Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167631 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
e2a7c69e27
commit
0bf82648db
|
@ -11,6 +11,7 @@ import java.util.SortedSet;
|
|||
|
||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.SocialHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||
|
@ -176,15 +177,25 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
logger.error("Error harvesting Data Method Download for {}", context, e);
|
||||
}
|
||||
|
||||
if(context.startsWith(TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
|
||||
if(context.startsWith(TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}else {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.SortedMap;
|
||||
|
||||
import org.gcube.accounting.analytics.Filter;
|
||||
import org.gcube.accounting.analytics.Info;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author Eric Perrone (ISTI - CNR)
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public class MethodInvocationHarvester extends BasicHarvester {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
|
||||
|
||||
public static final String DATAMINER_SERVICE_NAME = "dataminer";
|
||||
|
||||
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HarvestedData> getData() throws Exception {
|
||||
try {
|
||||
|
||||
List<HarvestedData> data = new ArrayList<>();
|
||||
|
||||
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startDate.getTime(), endDate.getTime(),
|
||||
AggregationMode.MONTHLY);
|
||||
|
||||
List<Filter> filters = new ArrayList<>();
|
||||
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
||||
|
||||
String context = Utils.getCurrentContext();
|
||||
|
||||
List<String> contexts = new ArrayList<>();
|
||||
contexts.add(context);
|
||||
|
||||
SortedMap<Filter,SortedMap<Calendar,Info>> result = accountingPersistenceQuery.getContextTimeSeries(
|
||||
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||
|
||||
if(result != null) {
|
||||
for(Filter filter : result.keySet()) {
|
||||
SortedMap<Calendar,Info> infoMap = result.get(filter);
|
||||
|
||||
Calendar calendar = DateUtils.dateToCalendar(startDate);
|
||||
|
||||
Info info = infoMap.get(calendar);
|
||||
logger.debug("{} : {}", DateUtils.LAUNCH_DATE_FORMAT.format(calendar.getTime()), info);
|
||||
|
||||
JSONObject jsonObject = info.getValue();
|
||||
long numberOfInvocation = jsonObject.getLong(AggregatedUsageRecord.OPERATION_COUNT);
|
||||
|
||||
HarvestedData harvestedData = new HarvestedData(HarvestedData.METHOD_INVOCATIONS, context,
|
||||
numberOfInvocation);
|
||||
data.add(harvestedData);
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
logger.error("No data found.");
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
} catch(Exception e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -22,35 +22,37 @@ import org.slf4j.LoggerFactory;
|
|||
public class Utils {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
||||
|
||||
|
||||
public static String getJson(String url) throws MalformedURLException, IOException {
|
||||
URL address = new URL(url);
|
||||
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
String json = "", line = "";
|
||||
|
||||
while (line != null) {
|
||||
String json = "";
|
||||
String line = "";
|
||||
|
||||
while(line != null) {
|
||||
line = reader.readLine();
|
||||
if (line != null)
|
||||
if(line != null) {
|
||||
json += line.trim();
|
||||
}
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
public static String getCurrentContext() throws ObjectNotFound, Exception{
|
||||
public static String getCurrentContext() throws ObjectNotFound, Exception {
|
||||
return getCurrentContext(SecurityTokenProvider.instance.get());
|
||||
}
|
||||
|
||||
public static String getCurrentContext(String token) throws ObjectNotFound, Exception{
|
||||
public static String getCurrentContext(String token) throws ObjectNotFound, Exception {
|
||||
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
|
||||
String context = authorizationEntry.getContext();
|
||||
logger.info("Context of token {} is {}", token, context);
|
||||
return context;
|
||||
}
|
||||
|
||||
|
||||
public static void setContext(String token) throws ObjectNotFound, Exception{
|
||||
public static void setContext(String token) throws ObjectNotFound, Exception {
|
||||
SecurityTokenProvider.instance.set(token);
|
||||
ScopeProvider.instance.set(getCurrentContext(token));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
|
@ -52,10 +53,30 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMethodInvocation() {
|
||||
try {
|
||||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
||||
|
||||
MeasureType measureType = MeasureType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvestedData = methodInvocationHarvester.getData();
|
||||
|
||||
logger.debug("{}", harvestedData);
|
||||
|
||||
|
||||
}catch (Exception e) {
|
||||
logger.error("", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTagMe() {
|
||||
public void testTagMeMethodInvocation() {
|
||||
try {
|
||||
|
||||
org.gcube.dataharvest.utils.Utils.setContext(TAGME);
|
||||
|
|
|
@ -46,6 +46,10 @@ public class ContextTest {
|
|||
public static final String TAGME_VARNAME = "TAGME_ERIC";
|
||||
public static final String TAGME;
|
||||
|
||||
public static final String RPrototypingLab_VARNAME = "RPrototypingLab";
|
||||
public static final String RPrototypingLab;
|
||||
|
||||
|
||||
static {
|
||||
|
||||
logger.trace("Retrieving Tokens from {}", PROPERTIES_FILENAME);
|
||||
|
@ -72,6 +76,8 @@ public class ContextTest {
|
|||
|
||||
TAGME = properties.getProperty(TAGME_VARNAME);
|
||||
|
||||
RPrototypingLab = properties.getProperty(RPrototypingLab_VARNAME);
|
||||
|
||||
DEFAULT_TEST_SCOPE = GCUBE;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue