gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Integrated VREAccessesHarvester 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169191 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-06-14 12:34:11 +00:00
parent af5e7cb8c8
commit 6d0c7ec0f4
3 changed files with 40 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -14,12 +14,7 @@ import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.MeasureType;
@ -150,8 +145,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
String initialToken = SecurityTokenProvider.instance.get(); String initialToken = SecurityTokenProvider.instance.get();
// GAnalytics gAnalytics = null VREAccessesHarvester vreAccessesHarvester = null;
Object gAnalytics = null;
for(String context : contexts) { for(String context : contexts) {
// Setting the token for the context // Setting the token for the context
@ -159,38 +153,42 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
ScopeBean scopeBean = new ScopeBean(context); ScopeBean scopeBean = new ScopeBean(context);
if(gAnalytics == null) { if(vreAccessesHarvester == null) {
if(scopeBean.is(Type.INFRASTRUCTURE)) { if(scopeBean.is(Type.INFRASTRUCTURE)) {
// gAnalytics = new GAnalytics(); vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else { }else {
// This code should be never used because the scopes are sorted by fullname // This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope(); ScopeBean parent = scopeBean.enclosingScope();
while(!parent.is(Type.INFRASTRUCTURE)) { while(!parent.is(Type.INFRASTRUCTURE)) {
parent = scopeBean.enclosingScope(); parent = scopeBean.enclosingScope();
} }
// Setting back token for the context // Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); Utils.setContext(contextAuthorization.getTokenForContext(parent.toString()));
// gAnalytics = new GAnalytics(); vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context // Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(context)); Utils.setContext(contextAuthorization.getTokenForContext(context));
} }
} }
try { try {
if(scopeBean.is(Type.VRE)) {
// Collecting Google Analytics Data for VREs Accesses // Collecting Google Analytics Data for VREs Accesses
// List<HarvestedData> harvested = gAnalytics.getData(); List<HarvestedData> harvested = vreAccessesHarvester.getData();
// data.addAll(harvested); data.addAll(harvested);
}
} catch(Exception e) { } catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); logger.error("Error harvesting Social Interactions for {}", context, e);
} }
/*
try { try {
// Collecting info on social (posts, replies and likes) // Collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
@ -256,11 +254,13 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.error("Error harvesting Method Invocations for {}", context, e); logger.error("Error harvesting Method Invocations for {}", context, e);
} }
} }
*/
} }
Utils.setContext(initialToken); Utils.setContext(initialToken);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
if(!dryRun) { if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, data, reRun); dbaseManager.insertMonthlyData(start, end, data, reRun);
} }

49
src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java
View File

@ -12,14 +12,12 @@ import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey; import java.security.PrivateKey;
import java.security.spec.InvalidKeySpecException; import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.PKCS8EncodedKeySpec;
import java.text.ParseException;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
@ -32,6 +30,7 @@ import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.resources.gcore.utils.Group; import org.gcube.common.resources.gcore.utils.Group;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials;
import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.datamodel.HarvestedDataKey;
@ -66,7 +65,9 @@ import com.google.api.services.analyticsreporting.v4.model.ReportRequest;
import com.google.api.services.analyticsreporting.v4.model.ReportRow; import com.google.api.services.analyticsreporting.v4.model.ReportRow;
public class VREAccessesHarvester extends BasicHarvester { public class VREAccessesHarvester extends BasicHarvester {
private static Logger _log = LoggerFactory.getLogger(VREAccessesHarvester.class);
private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class);
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService";
@ -78,16 +79,9 @@ public class VREAccessesHarvester extends BasicHarvester {
private List<VREAccessesReportRow> vreAccesses; private List<VREAccessesReportRow> vreAccesses;
public VREAccessesHarvester(Date start, Date end) throws ParseException { public VREAccessesHarvester(Date start, Date end) throws Exception {
super(start, end); super(start, end);
Calendar cal = Calendar.getInstance(); vreAccesses = getAllAccesses(start, end);
cal.add(Calendar.MONTH, -1);
String infrastructureScope = "/d4science.research-infrastructures.eu";
try {
vreAccesses = getAllAccesses(start, end, infrastructureScope);
} catch (Exception e) {
e.printStackTrace();
}
} }
@Override @Override
@ -96,6 +90,9 @@ public class VREAccessesHarvester extends BasicHarvester {
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>(); ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
int measure = 0; int measure = 0;
ScopeBean scopeBean = new ScopeBean(context);
String[] splitContext = context.split("/"); String[] splitContext = context.split("/");
if (splitContext.length > 3) { if (splitContext.length > 3) {
String lowerCasedContext = splitContext[3].toLowerCase(); String lowerCasedContext = splitContext[3].toLowerCase();
@ -106,8 +103,8 @@ public class VREAccessesHarvester extends BasicHarvester {
if (splits.length > 2 && lowerCasedContext.compareTo(splits[2]) == 0) if (splits.length > 2 && lowerCasedContext.compareTo(splits[2]) == 0)
measure++; measure++;
} }
HarvestedData harvest = new HarvestedData(HarvestedDataKey.USERS, context, measure); HarvestedData harvest = new HarvestedData(HarvestedDataKey.ACCESSES, context, measure);
_log.debug(harvest.toString()); logger.debug(harvest.toString());
data.add(harvest); data.add(harvest);
} }
return data; return data;
@ -123,11 +120,11 @@ public class VREAccessesHarvester extends BasicHarvester {
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2] * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2]
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39] * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39]
*/ */
private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end, String infrastructureScope) throws Exception { private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end) throws Exception {
DateRange dateRange = getDateRangeForAnalytics(start, end); DateRange dateRange = getDateRangeForAnalytics(start, end);
System.out.println("getting accesses in this time range: " + dateRange.toPrettyString()); System.out.println("getting accesses in this time range: " + dateRange.toPrettyString());
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(infrastructureScope); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
List<VREAccessesReportRow> totalAccesses = new ArrayList<>(); List<VREAccessesReportRow> totalAccesses = new ArrayList<>();
@ -176,7 +173,7 @@ public class VREAccessesHarvester extends BasicHarvester {
Dimension pageTitle = new Dimension().setName("ga:pagePath"); Dimension pageTitle = new Dimension().setName("ga:pagePath");
for (String view : viewIDs) { for (String view : viewIDs) {
_log.info("Getting data from Google Analytics for viewid: "+ view); logger.info("Getting data from Google Analytics for viewid: "+ view);
// Create the ReportRequest object. // Create the ReportRequest object.
ReportRequest request = new ReportRequest() ReportRequest request = new ReportRequest()
.setViewId(view) .setViewId(view)
@ -216,7 +213,7 @@ public class VREAccessesHarvester extends BasicHarvester {
List<ReportRow> rows = report.getData().getRows(); List<ReportRow> rows = report.getData().getRows();
if (rows == null) { if (rows == null) {
_log.warn("No data found for " + viewId); logger.warn("No data found for " + viewId);
} }
else else
for (ReportRow row: rows) { for (ReportRow row: rows) {
@ -241,7 +238,7 @@ public class VREAccessesHarvester extends BasicHarvester {
} }
} }
toReturn.add(var); toReturn.add(var);
_log.debug(var.toString()); logger.debug(var.toString());
} }
} }
} }
@ -304,7 +301,7 @@ public class VREAccessesHarvester extends BasicHarvester {
} catch (InvalidKeySpecException exception) { } catch (InvalidKeySpecException exception) {
unexpectedException = exception; unexpectedException = exception;
} }
throw new IOException("Unexpected exception reading PKCS data"); throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
} }
private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception {
@ -321,19 +318,19 @@ public class VREAccessesHarvester extends BasicHarvester {
} }
/** /**
* l * l
* @throws Exception
*/ */
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs(String infrastructureScope) { private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception {
AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials();
String context = infrastructureScope; String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
ScopeProvider.instance.set(context);
try { try {
List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(infrastructureScope); List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(context);
if (list.size() > 1) { if (list.size() > 1) {
_log.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
} }
else if (list.size() == 0){ else if (list.size() == 0){
_log.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + infrastructureScope); logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
} }
else { else {

2
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -60,7 +60,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false); inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
/* /*
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1); Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);