gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Integrated VREAccessesHarvester 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169191 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-06-14 12:34:11 +00:00
parent af5e7cb8c8
commit 6d0c7ec0f4
3 changed files with 40 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -14,12 +14,7 @@ import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
@ -150,8 +145,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
String initialToken = SecurityTokenProvider.instance.get();
// GAnalytics gAnalytics = null
Object gAnalytics = null;
VREAccessesHarvester vreAccessesHarvester = null;
for(String context : contexts) {
// Setting the token for the context
@ -159,38 +153,42 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
ScopeBean scopeBean = new ScopeBean(context);
if(gAnalytics == null) {
if(vreAccessesHarvester == null) {
if(scopeBean.is(Type.INFRASTRUCTURE)) {
// gAnalytics = new GAnalytics();
vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else {
// This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope();
while(!parent.is(Type.INFRASTRUCTURE)) {
parent = scopeBean.enclosingScope();
}
// Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(parent.toString()));
// gAnalytics = new GAnalytics();
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
Utils.setContext(contextAuthorization.getTokenForContext(context));
}
}
try {
// Collecting Google Analytics Data for VREs Accesses
// List<HarvestedData> harvested = gAnalytics.getData();
// data.addAll(harvested);
if(scopeBean.is(Type.VRE)) {
// Collecting Google Analytics Data for VREs Accesses
List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
/*
try {
// Collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
@ -256,11 +254,13 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
*/
}
Utils.setContext(initialToken);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, data, reRun);
}

49
src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java
View File

@ -12,14 +12,12 @@ import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.text.ParseException;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@ -32,6 +30,7 @@ import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.resources.gcore.utils.Group;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
@ -66,7 +65,9 @@ import com.google.api.services.analyticsreporting.v4.model.ReportRequest;
import com.google.api.services.analyticsreporting.v4.model.ReportRow;
public class VREAccessesHarvester extends BasicHarvester {
private static Logger _log = LoggerFactory.getLogger(VREAccessesHarvester.class);
private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class);
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService";
@ -78,16 +79,9 @@ public class VREAccessesHarvester extends BasicHarvester {
private List<VREAccessesReportRow> vreAccesses;
public VREAccessesHarvester(Date start, Date end) throws ParseException {
public VREAccessesHarvester(Date start, Date end) throws Exception {
super(start, end);
Calendar cal = Calendar.getInstance();
cal.add(Calendar.MONTH, -1);
String infrastructureScope = "/d4science.research-infrastructures.eu";
try {
vreAccesses = getAllAccesses(start, end, infrastructureScope);
} catch (Exception e) {
e.printStackTrace();
}
vreAccesses = getAllAccesses(start, end);
}
@Override
@ -96,6 +90,9 @@ public class VREAccessesHarvester extends BasicHarvester {
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
int measure = 0;
ScopeBean scopeBean = new ScopeBean(context);
String[] splitContext = context.split("/");
if (splitContext.length > 3) {
String lowerCasedContext = splitContext[3].toLowerCase();
@ -106,8 +103,8 @@ public class VREAccessesHarvester extends BasicHarvester {
if (splits.length > 2 && lowerCasedContext.compareTo(splits[2]) == 0)
measure++;
}
HarvestedData harvest = new HarvestedData(HarvestedDataKey.USERS, context, measure);
_log.debug(harvest.toString());
HarvestedData harvest = new HarvestedData(HarvestedDataKey.ACCESSES, context, measure);
logger.debug(harvest.toString());
data.add(harvest);
}
return data;
@ -123,11 +120,11 @@ public class VREAccessesHarvester extends BasicHarvester {
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2]
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39]
*/
private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end, String infrastructureScope) throws Exception {
private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end) throws Exception {
DateRange dateRange = getDateRangeForAnalytics(start, end);
System.out.println("getting accesses in this time range: " + dateRange.toPrettyString());
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(infrastructureScope);
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
List<VREAccessesReportRow> totalAccesses = new ArrayList<>();
@ -176,7 +173,7 @@ public class VREAccessesHarvester extends BasicHarvester {
Dimension pageTitle = new Dimension().setName("ga:pagePath");
for (String view : viewIDs) {
_log.info("Getting data from Google Analytics for viewid: "+ view);
logger.info("Getting data from Google Analytics for viewid: "+ view);
// Create the ReportRequest object.
ReportRequest request = new ReportRequest()
.setViewId(view)
@ -216,7 +213,7 @@ public class VREAccessesHarvester extends BasicHarvester {
List<ReportRow> rows = report.getData().getRows();
if (rows == null) {
_log.warn("No data found for " + viewId);
logger.warn("No data found for " + viewId);
}
else
for (ReportRow row: rows) {
@ -241,7 +238,7 @@ public class VREAccessesHarvester extends BasicHarvester {
}
}
toReturn.add(var);
_log.debug(var.toString());
logger.debug(var.toString());
}
}
}
@ -304,7 +301,7 @@ public class VREAccessesHarvester extends BasicHarvester {
} catch (InvalidKeySpecException exception) {
unexpectedException = exception;
}
throw new IOException("Unexpected exception reading PKCS data");
throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
}
private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception {
@ -321,19 +318,19 @@ public class VREAccessesHarvester extends BasicHarvester {
}
/**
* l
* @throws Exception
*/
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs(String infrastructureScope) {
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception {
AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials();
String context = infrastructureScope;
ScopeProvider.instance.set(context);
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
try {
List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(infrastructureScope);
List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(context);
if (list.size() > 1) {
_log.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
}
else if (list.size() == 0){
_log.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + infrastructureScope);
logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
}
else {

2
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -60,7 +60,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
/*
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);