Integrated VREAccessesHarvester
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169191 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
af5e7cb8c8
commit
6d0c7ec0f4
|
@ -14,12 +14,7 @@ import org.gcube.common.scope.impl.ScopeBean;
|
|||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.dataharvest.utils.MeasureType;
|
||||
|
@ -150,8 +145,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
String initialToken = SecurityTokenProvider.instance.get();
|
||||
|
||||
// GAnalytics gAnalytics = null
|
||||
Object gAnalytics = null;
|
||||
VREAccessesHarvester vreAccessesHarvester = null;
|
||||
|
||||
for(String context : contexts) {
|
||||
// Setting the token for the context
|
||||
|
@ -159,38 +153,42 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
ScopeBean scopeBean = new ScopeBean(context);
|
||||
|
||||
if(gAnalytics == null) {
|
||||
if(vreAccessesHarvester == null) {
|
||||
|
||||
if(scopeBean.is(Type.INFRASTRUCTURE)) {
|
||||
// gAnalytics = new GAnalytics();
|
||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||
}else {
|
||||
// This code should be never used because the scopes are sorted by fullname
|
||||
|
||||
ScopeBean parent = scopeBean.enclosingScope();
|
||||
while(!parent.is(Type.INFRASTRUCTURE)) {
|
||||
parent = scopeBean.enclosingScope();
|
||||
|
||||
}
|
||||
|
||||
// Setting back token for the context
|
||||
Utils.setContext(contextAuthorization.getTokenForContext(parent.toString()));
|
||||
|
||||
// gAnalytics = new GAnalytics();
|
||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||
|
||||
// Setting back token for the context
|
||||
Utils.setContext(contextAuthorization.getTokenForContext(context));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
if(scopeBean.is(Type.VRE)) {
|
||||
// Collecting Google Analytics Data for VREs Accesses
|
||||
// List<HarvestedData> harvested = gAnalytics.getData();
|
||||
// data.addAll(harvested);
|
||||
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
}
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||
|
@ -256,11 +254,13 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
Utils.setContext(initialToken);
|
||||
|
||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, data, reRun);
|
||||
}
|
||||
|
|
|
@ -12,14 +12,12 @@ import java.security.NoSuchAlgorithmException;
|
|||
import java.security.PrivateKey;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.security.spec.PKCS8EncodedKeySpec;
|
||||
import java.text.ParseException;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
|
@ -32,6 +30,7 @@ import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
|||
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
|
||||
import org.gcube.common.resources.gcore.utils.Group;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
||||
|
@ -66,7 +65,9 @@ import com.google.api.services.analyticsreporting.v4.model.ReportRequest;
|
|||
import com.google.api.services.analyticsreporting.v4.model.ReportRow;
|
||||
|
||||
public class VREAccessesHarvester extends BasicHarvester {
|
||||
private static Logger _log = LoggerFactory.getLogger(VREAccessesHarvester.class);
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class);
|
||||
|
||||
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
|
||||
|
||||
private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService";
|
||||
|
@ -78,16 +79,9 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
|
||||
private List<VREAccessesReportRow> vreAccesses;
|
||||
|
||||
public VREAccessesHarvester(Date start, Date end) throws ParseException {
|
||||
public VREAccessesHarvester(Date start, Date end) throws Exception {
|
||||
super(start, end);
|
||||
Calendar cal = Calendar.getInstance();
|
||||
cal.add(Calendar.MONTH, -1);
|
||||
String infrastructureScope = "/d4science.research-infrastructures.eu";
|
||||
try {
|
||||
vreAccesses = getAllAccesses(start, end, infrastructureScope);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
vreAccesses = getAllAccesses(start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -96,6 +90,9 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
|
||||
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
|
||||
int measure = 0;
|
||||
|
||||
ScopeBean scopeBean = new ScopeBean(context);
|
||||
|
||||
String[] splitContext = context.split("/");
|
||||
if (splitContext.length > 3) {
|
||||
String lowerCasedContext = splitContext[3].toLowerCase();
|
||||
|
@ -106,8 +103,8 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
if (splits.length > 2 && lowerCasedContext.compareTo(splits[2]) == 0)
|
||||
measure++;
|
||||
}
|
||||
HarvestedData harvest = new HarvestedData(HarvestedDataKey.USERS, context, measure);
|
||||
_log.debug(harvest.toString());
|
||||
HarvestedData harvest = new HarvestedData(HarvestedDataKey.ACCESSES, context, measure);
|
||||
logger.debug(harvest.toString());
|
||||
data.add(harvest);
|
||||
}
|
||||
return data;
|
||||
|
@ -123,11 +120,11 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2]
|
||||
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39]
|
||||
*/
|
||||
private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end, String infrastructureScope) throws Exception {
|
||||
private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end) throws Exception {
|
||||
DateRange dateRange = getDateRangeForAnalytics(start, end);
|
||||
System.out.println("getting accesses in this time range: " + dateRange.toPrettyString());
|
||||
|
||||
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(infrastructureScope);
|
||||
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
|
||||
AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
|
||||
HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
|
||||
List<VREAccessesReportRow> totalAccesses = new ArrayList<>();
|
||||
|
@ -176,7 +173,7 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
Dimension pageTitle = new Dimension().setName("ga:pagePath");
|
||||
|
||||
for (String view : viewIDs) {
|
||||
_log.info("Getting data from Google Analytics for viewid: "+ view);
|
||||
logger.info("Getting data from Google Analytics for viewid: "+ view);
|
||||
// Create the ReportRequest object.
|
||||
ReportRequest request = new ReportRequest()
|
||||
.setViewId(view)
|
||||
|
@ -216,7 +213,7 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
List<ReportRow> rows = report.getData().getRows();
|
||||
|
||||
if (rows == null) {
|
||||
_log.warn("No data found for " + viewId);
|
||||
logger.warn("No data found for " + viewId);
|
||||
}
|
||||
else
|
||||
for (ReportRow row: rows) {
|
||||
|
@ -241,7 +238,7 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
}
|
||||
}
|
||||
toReturn.add(var);
|
||||
_log.debug(var.toString());
|
||||
logger.debug(var.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -304,7 +301,7 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
} catch (InvalidKeySpecException exception) {
|
||||
unexpectedException = exception;
|
||||
}
|
||||
throw new IOException("Unexpected exception reading PKCS data");
|
||||
throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
|
||||
}
|
||||
|
||||
private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception {
|
||||
|
@ -321,19 +318,19 @@ public class VREAccessesHarvester extends BasicHarvester {
|
|||
}
|
||||
/**
|
||||
* l
|
||||
* @throws Exception
|
||||
*/
|
||||
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs(String infrastructureScope) {
|
||||
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception {
|
||||
AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials();
|
||||
|
||||
String context = infrastructureScope;
|
||||
ScopeProvider.instance.set(context);
|
||||
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
|
||||
try {
|
||||
List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(infrastructureScope);
|
||||
List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(context);
|
||||
if (list.size() > 1) {
|
||||
_log.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
|
||||
logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
|
||||
}
|
||||
else if (list.size() == 0){
|
||||
_log.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + infrastructureScope);
|
||||
logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
|
||||
}
|
||||
else {
|
||||
|
||||
|
|
|
@ -60,7 +60,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
||||
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
|
||||
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
|
||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
||||
|
||||
/*
|
||||
Calendar from = DateUtils.getStartCalendar(2018, Calendar.FEBRUARY, 1);
|
||||
|
|
Loading…
Reference in New Issue