From af5e7cb8c8a88bb272faaf37940edd2be0de33e9 Mon Sep 17 00:00:00 2001 From: "massimiliano.assante" Date: Thu, 14 Jun 2018 10:46:50 +0000 Subject: [PATCH] added VRE Access Harvester git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169190 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../datamodel/AnalyticsReportCredentials.java | 108 +++++ .../datamodel/HarvestedDataKey.java | 2 +- .../datamodel/VREAccessesReportRow.java | 52 +++ .../harvester/VREAccessesHarvester.java | 384 ++++++++++++++++++ 4 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java create mode 100644 src/main/java/org/gcube/dataharvest/datamodel/VREAccessesReportRow.java create mode 100644 src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java diff --git a/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java new file mode 100644 index 0000000..3dc23b8 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java @@ -0,0 +1,108 @@ +package org.gcube.dataharvest.datamodel; + +import java.util.List; + +/** + * + * @author massi + * + */ +public class AnalyticsReportCredentials { + + private List viewIds; + private String projectId; + private String clientId; + private String clientEmail; + private String privateKeyPem; + private String privateKeyId; + private String tokenUri; + + public AnalyticsReportCredentials() { + super(); + } + + public List getViewIds() { + return viewIds; + } + + public void setViewIds(List viewIds) { + this.viewIds = viewIds; + } + + public String getProjectId() { + return projectId; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public String getClientId() { + return clientId; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public String getClientEmail() { + return clientEmail; + } + + public void setClientEmail(String clientEmail) { + this.clientEmail = clientEmail; + } + + public String getPrivateKeyPem() { + return privateKeyPem; + } + /** + * Please note: + * The key is stored in the resource with blanks " " instead of "\n" as it causes issues and + * without the BEGIN and END Delimiters (e.g. -----END PRIVATE KEY-----) which myst be readded + * @param privateKeyPem + */ + public void setPrivateKeyPem(String privateKeyPem) { + privateKeyPem = privateKeyPem.replace(" ", "\n"); + this.privateKeyPem = "-----BEGIN PRIVATE KEY-----\n"+privateKeyPem+"\n-----END PRIVATE KEY-----"; + } + + public String getPrivateKeyId() { + return privateKeyId; + } + + public void setPrivateKeyId(String privateKeyId) { + this.privateKeyId = privateKeyId; + } + + public String getTokenUri() { + return tokenUri; + } + + public void setTokenUri(String tokenUri) { + this.tokenUri = tokenUri; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("AnalyticsReportCredentials [viewIds="); + builder.append(viewIds); + builder.append(", projectId="); + builder.append(projectId); + builder.append(", clientId="); + builder.append(clientId); + builder.append(", clientEmail="); + builder.append(clientEmail); + builder.append(", privateKeyPem=\n"); + builder.append(privateKeyPem); + builder.append("\n, privateKeyId="); + builder.append(privateKeyId); + builder.append(", tokenUri="); + builder.append(tokenUri); + builder.append("]"); + return builder.toString(); + } + + +} diff --git a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java index 76be82d..6bfe7ca 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java @@ -11,7 +11,7 @@ package org.gcube.dataharvest.datamodel; */ public enum HarvestedDataKey { - ACCESSESS(1), + ACCESSES(1), USERS(2), DATA_METHOD_DOWNLOAD(3), NEW_CATALOGUE_METHODS(4), diff --git a/src/main/java/org/gcube/dataharvest/datamodel/VREAccessesReportRow.java b/src/main/java/org/gcube/dataharvest/datamodel/VREAccessesReportRow.java new file mode 100644 index 0000000..223baf1 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/datamodel/VREAccessesReportRow.java @@ -0,0 +1,52 @@ +package org.gcube.dataharvest.datamodel; + +public class VREAccessesReportRow { + + private String pagePath; + private int visitNumber; + + public VREAccessesReportRow(String pagePath, int visitNumber) { + super(); + this.pagePath = pagePath; + this.visitNumber = visitNumber; + } + + + public VREAccessesReportRow() { + // TODO Auto-generated constructor stub + } + + + public String getPagePath() { + return pagePath; + } + + + public void setPagePath(String pagePath) { + this.pagePath = pagePath; + } + + + public int getVisitNumber() { + return visitNumber; + } + + + public void setVisitNumber(int visitNumber) { + this.visitNumber = visitNumber; + } + + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("VREAccessesReport [pagePath="); + builder.append(pagePath); + builder.append(", visitNumber="); + builder.append(visitNumber); + builder.append("]"); + return builder.toString(); + } + + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java new file mode 100644 index 0000000..aff6c79 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java @@ -0,0 +1,384 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.text.ParseException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import org.gcube.common.encryption.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; +import org.gcube.dataharvest.datamodel.HarvestedData; +import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.dataharvest.datamodel.VREAccessesReportRow; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.PemReader; +import com.google.api.client.util.PemReader.Section; +import com.google.api.client.util.SecurityUtils; +import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; +import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; +import com.google.api.services.analyticsreporting.v4.model.ColumnHeader; +import com.google.api.services.analyticsreporting.v4.model.DateRange; +import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; +import com.google.api.services.analyticsreporting.v4.model.Dimension; +import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; +import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; +import com.google.api.services.analyticsreporting.v4.model.Metric; +import com.google.api.services.analyticsreporting.v4.model.MetricHeaderEntry; +import com.google.api.services.analyticsreporting.v4.model.Report; +import com.google.api.services.analyticsreporting.v4.model.ReportRequest; +import com.google.api.services.analyticsreporting.v4.model.ReportRow; + +public class VREAccessesHarvester extends BasicHarvester { + private static Logger _log = LoggerFactory.getLogger(VREAccessesHarvester.class); + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; + private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String AP_VIEWS_PROPERTY = "views"; + private static final String AP_CLIENT_PROPERTY = "clientId"; + private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; + private static final String APPLICATION_NAME = "Analytics Reporting"; + + private List vreAccesses; + + public VREAccessesHarvester(Date start, Date end) throws ParseException { + super(start, end); + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.MONTH, -1); + String infrastructureScope = "/d4science.research-infrastructures.eu"; + try { + vreAccesses = getAllAccesses(start, end, infrastructureScope); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public List getData() throws Exception { + try { + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + ArrayList data = new ArrayList(); + int measure = 0; + String[] splitContext = context.split("/"); + if (splitContext.length > 3) { + String lowerCasedContext = splitContext[3].toLowerCase(); + for (VREAccessesReportRow row : vreAccesses) { + + String pagePath = row.getPagePath(); + String[] splits = pagePath.split("/"); + if (splits.length > 2 && lowerCasedContext.compareTo(splits[2]) == 0) + measure++; + } + HarvestedData harvest = new HarvestedData(HarvestedDataKey.USERS, context, measure); + _log.debug(harvest.toString()); + data.add(harvest); + } + return data; + } catch(Exception e) { + throw e; + } + } + + /** + * + * @return a list of {@link VREAccessesReportRow} objects containing the pagePath and the visit number e.g. + * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/add-new-users, visitNumber=1] + * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2] + * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39] + */ + private static List getAllAccesses(Date start, Date end, String infrastructureScope) throws Exception { + DateRange dateRange = getDateRangeForAnalytics(start, end); + System.out.println("getting accesses in this time range: " + dateRange.toPrettyString()); + + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(infrastructureScope); + AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + HashMap responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); + List totalAccesses = new ArrayList<>(); + + for (String view : responses.keySet()) { + List viewReport = parseResponse(view, responses.get(view)); + System.out.println("got " + viewReport.size() + " entries from view id= "+view); + totalAccesses.addAll(viewReport); + } + System.out.println("Merged in " + totalAccesses.size() + " toal entries from all views"); + return totalAccesses; + } + + + /** + * Initializes an Analytics Reporting API V4 service object. + * + * @return An authorized Analytics Reporting API V4 service object. + * @throws IOException + * @throws GeneralSecurityException + */ + private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws GeneralSecurityException, IOException { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); + + // Construct the Analytics Reporting service object. + return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + } + + /** + * Queries the Analytics Reporting API V4. + * + * @param service An authorized Analytics Reporting API V4 service object. + * @return GetReportResponse The Analytics Reporting API V4 response. + * @throws IOException + */ + private static HashMap getReportResponses(AnalyticsReporting service, List viewIDs, DateRange dateRange) throws IOException { + + HashMap reports = new HashMap<>(); + + // Create the Metrics object. + Metric sessions = new Metric() + .setExpression("ga:pageviews") + .setAlias("pages"); + Dimension pageTitle = new Dimension().setName("ga:pagePath"); + + for (String view : viewIDs) { + _log.info("Getting data from Google Analytics for viewid: "+ view); + // Create the ReportRequest object. + ReportRequest request = new ReportRequest() + .setViewId(view) + .setDateRanges(Arrays.asList(dateRange)) + .setMetrics(Arrays.asList(sessions)) + .setDimensions(Arrays.asList(pageTitle)); + + ArrayList requests = new ArrayList(); + requests.add(request); + + // Create the GetReportsRequest object. + GetReportsRequest getReport = new GetReportsRequest() + .setReportRequests(requests); + + // Call the batchGet method. + GetReportsResponse response = service.reports().batchGet(getReport).execute(); + reports.put(view, response); + } + // Return the response. + return reports; + } + + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response An Analytics Reporting API V4 response. + */ + private static List parseResponse(String viewId, GetReportsResponse response) { + System.out.println("\n*** parsing Response for " + viewId); + + List toReturn = new ArrayList<>(); + + for (Report report: response.getReports()) { + ColumnHeader header = report.getColumnHeader(); + List dimensionHeaders = header.getDimensions(); + List metricHeaders = header.getMetricHeader().getMetricHeaderEntries(); + List rows = report.getData().getRows(); + + if (rows == null) { + _log.warn("No data found for " + viewId); + } + else + for (ReportRow row: rows) { + List dimensions = row.getDimensions(); + List metrics = row.getMetrics(); + + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + for (int i = 0; i < dimensionHeaders.size() && i < dimensions.size(); i++) { + //System.out.println(dimensionHeaders.get(i) + ": " + dimensions.get(i)); + String pagePath = dimensions.get(i); + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimensions.get(i)); + validEntry = true; + } + } + if (validEntry) { + for (int j = 0; j < metrics.size(); j++) { + DateRangeValues values = metrics.get(j); + for (int k = 0; k < values.getValues().size() && k < metricHeaders.size(); k++) { + var.setVisitNumber(Integer.parseInt(values.getValues().get(k))); + } + } + toReturn.add(var); + _log.debug(var.toString()); + } + } + } + return toReturn; + } + private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { + + String clientId = cred.getClientId(); + String clientEmail = cred.getClientEmail(); + String privateKeyPem = cred.getPrivateKeyPem(); + String privateKeyId = cred.getPrivateKeyId(); + String tokenUri = cred.getTokenUri(); + String projectId = cred.getProjectId(); + + if (clientId == null || clientEmail == null || privateKeyPem == null + || privateKeyId == null) { + throw new IOException("Error reading service account credential from stream, " + + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); + } + + PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); + + Collection emptyScopes = Collections.emptyList(); + + Builder credentialBuilder = new GoogleCredential.Builder() + .setTransport( Utils.getDefaultTransport()) + .setJsonFactory(Utils.getDefaultJsonFactory()) + .setServiceAccountId(clientEmail) + .setServiceAccountScopes(emptyScopes) + .setServiceAccountPrivateKey(privateKey) + .setServiceAccountPrivateKeyId(privateKeyId); + + if (tokenUri != null) { + credentialBuilder.setTokenServerEncodedUrl(tokenUri); + } + + if (projectId != null) { + credentialBuilder.setServiceAccountProjectId(projectId); + } + + // Don't do a refresh at this point, as it will always fail before the scopes are added. + return credentialBuilder.build(); + } + + private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { + Reader reader = new StringReader(privateKeyPem); + Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); + if (section == null) { + throw new IOException("Invalid PKCS8 data."); + } + byte[] bytes = section.getBase64DecodedBytes(); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); + Exception unexpectedException = null; + try { + KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + return privateKey; + } catch (NoSuchAlgorithmException exception) { + unexpectedException = exception; + } catch (InvalidKeySpecException exception) { + unexpectedException = exception; + } + throw new IOException("Unexpected exception reading PKCS data"); + } + + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { + String scope = infrastructureScope; + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(scope); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '"+ SERVICE_ENDPOINT_CATEGORY +"'"); + query.addCondition("$resource/Profile/Name/text() eq '"+ SERVICE_ENDPOINT_NAME +"'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List toReturn = client.submit(query); + ScopeProvider.instance.set(currScope); + return toReturn; + } + /** + * l + */ + private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs(String infrastructureScope) { + AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); + + String context = infrastructureScope; + ScopeProvider.instance.set(context); + try { + List list = getAnalyticsReportingConfigurationFromIS(infrastructureScope); + if (list.size() > 1) { + _log.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); + } + else if (list.size() == 0){ + _log.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + infrastructureScope); + } + else { + + for (ServiceEndpoint res : list) { + reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + Group apGroup = res.profile().accessPoints(); + AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); + AccessPoint found = accessPoints[0]; + reportCredentials.setClientEmail(found.address()); + reportCredentials.setProjectId(found.username()); + reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + for (Property prop : found.properties()) { + if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + String[] views = decryptedValue.split(";"); + reportCredentials.setViewIds(Arrays.asList(views)); + } + if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setClientId(decryptedValue); + } + if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setPrivateKeyId(decryptedValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return reportCredentials; + } + private static LocalDate asLocalDate(Date date) { + return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); + } + private static DateRange getDateRangeForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics + String startDate = asLocalDate(start).format(formatter); + String endDate = asLocalDate(end).format(formatter); + DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` + dateRange.setStartDate(startDate); + dateRange.setEndDate(endDate); + return dateRange; + } + +}