diff --git a/src/main/java/org/gcube/dataharvest/datamodel/CoreServiceAccessesReportRow.java b/src/main/java/org/gcube/dataharvest/datamodel/CoreServiceAccessesReportRow.java new file mode 100644 index 0000000..c83c990 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/datamodel/CoreServiceAccessesReportRow.java @@ -0,0 +1,44 @@ +package org.gcube.dataharvest.datamodel; + +public class CoreServiceAccessesReportRow { + private String dashboardContext; + private HarvestedDataKey key; + private String pagePath; + private int visitNumber; + + public CoreServiceAccessesReportRow() { + // TODO Auto-generated constructor stub + } + public HarvestedDataKey getKey() { + return key; + } + public void setKey(HarvestedDataKey key) { + this.key = key; + } + public String getPagePath() { + return pagePath; + } + public void setPagePath(String pagePath) { + this.pagePath = pagePath; + } + public int getVisitNumber() { + return visitNumber; + } + public void setVisitNumber(int visitNumber) { + this.visitNumber = visitNumber; + } + public String getDashboardContext() { + return dashboardContext; + } + public void setDashboardContext(String dashboardContext) { + this.dashboardContext = dashboardContext; + } + @Override + public String toString() { + return "CoreServiceAccessesReportRow [dashboardContext=" + dashboardContext + ", key=" + key + ", pagePath=" + + pagePath + ", visitNumber=" + visitNumber + "]"; + } + + + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java new file mode 100644 index 0000000..eb275d9 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java @@ -0,0 +1,488 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.encryption.encrypter.StringEncrypter; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; +import org.gcube.dataharvest.datamodel.CoreServiceAccessesReportRow; +import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Node; +import org.xml.sax.InputSource; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.PemReader; +import com.google.api.client.util.PemReader.Section; +import com.google.api.client.util.SecurityUtils; +import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; +import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; +import com.google.api.services.analyticsreporting.v4.model.DateRange; +import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; +import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; +import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; +import com.google.api.services.analyticsreporting.v4.model.Metric; +import com.google.api.services.analyticsreporting.v4.model.Report; +import com.google.api.services.analyticsreporting.v4.model.ReportRequest; +import com.google.api.services.analyticsreporting.v4.model.ReportRow; + +public class CoreServicesAccessesHarvester extends BasicHarvester { + + private static Logger logger = LoggerFactory.getLogger(CoreServicesAccessesHarvester.class); + + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + + private static final String MAPPING_RESOURCE_CATEGORY = "BigGAnalyticsMapping"; + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; + private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String AP_VIEWS_PROPERTY = "views"; + private static final String AP_CLIENT_PROPERTY = "clientId"; + private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; + private static final String APPLICATION_NAME = "Analytics Reporting"; + + + private static final String PAGE_WORKSPACE_ACCESSES = "/workspace"; + private static final String PAGE_MESSAGES_ACCESSES = "/messages"; + private static final String PAGE_PROFILE_ACCESSES = "/profile"; + private static final String PAGE_NOTIFICATION_ACCESSES = "/notifications"; + + private HashMap> coreServicesAccesses; + + public CoreServicesAccessesHarvester(Date start, Date end) throws Exception { + super(start, end); + coreServicesAccesses = getAllAccesses(start, end); + } + + @Override + public List getAccountingRecords() throws Exception { + try { + ArrayList accountingRecords = new ArrayList(); + for (String dashboardContext : coreServicesAccesses.keySet()) { + int workspaceAccesses = 0; + int messagesAccesses = 0; + int notificationsAccesses = 0; + int profileAccesses = 0; + logger.debug("CoreServices accesses for {} ", dashboardContext); + for(CoreServiceAccessesReportRow row : coreServicesAccesses.get(dashboardContext)) { + // String pagePath = row.getPagePath(); + switch (row.getKey()) { + case WORKSPACE_ACCESSES: + workspaceAccesses += row.getVisitNumber(); + break; + case MESSAGES_ACCESSES: + messagesAccesses += row.getVisitNumber(); + break; + case NOTIFICATIONS_ACCESSES: + notificationsAccesses += row.getVisitNumber(); + break; + case PROFILE_ACCESSES: + profileAccesses += row.getVisitNumber(); + break; + default: + break; + } + } + ScopeDescriptor scopeDescriptor = new ScopeDescriptor(); + ScopeBean scopeBean = new ScopeBean(dashboardContext); + scopeDescriptor.setId(dashboardContext); + scopeDescriptor.setName(scopeBean.name()); + + AccountingRecord ar1 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.WORKSPACE_ACCESSES), (long) workspaceAccesses); + AccountingRecord ar2 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.MESSAGES_ACCESSES), (long) messagesAccesses); + AccountingRecord ar3 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.NOTIFICATIONS_ACCESSES), (long) notificationsAccesses); + AccountingRecord ar4 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.PROFILE_ACCESSES), (long) profileAccesses); + logger.debug("{} : {}", ar1.getDimension().getId(), ar1.getMeasure()); + accountingRecords.add(ar1); + logger.debug("{} : {}", ar2.getDimension().getId(), ar2.getMeasure()); + accountingRecords.add(ar2); + logger.debug("{} : {}", ar3.getDimension().getId(), ar3.getMeasure()); + accountingRecords.add(ar3); + logger.debug("{} : {}", ar4.getDimension().getId(), ar4.getMeasure()); + accountingRecords.add(ar4); + + } + logger.debug("Returning {} accountingRecords ", accountingRecords.size()); + return accountingRecords; + + } catch(Exception e) { + throw e; + } + } + + /** + * + */ + private static HashMap> getAllAccesses(Date start, Date end) throws Exception { + DateRange dateRange = getDateRangeForAnalytics(start, end); + logger.trace("Getting core services accesses in this time range {}", dateRange.toPrettyString()); + + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); + + logger.trace("gotten credentialsFromD4S id = {}", credentialsFromD4S.getClientId()); + + AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + + logger.trace("gotten credentialsFromD4S viewIds= {}", credentialsFromD4S.getViewIds().toString()); + + HashMap> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); + HashMap> toReturn = new HashMap<>(); + + for(String view : responses.keySet()) { + String dashboardContext = getAccountingDashboardContextGivenGAViewID(view); + if (dashboardContext != null ) { + logger.trace("Parsing responses for this Gateway view, which corresponds to Dashboard Context: " + dashboardContext); + List viewReport = parseResponse(view, responses.get(view), dashboardContext); + logger.trace("Got {} entries from view id={}", viewReport.size(), view); + toReturn.put(dashboardContext, viewReport); + } else { + logger.warn("Got entries from view id={} but cannot find Dashboard Context correspondance, I think you need to update the Generic Resource of the Mappings", view); + } + } + return toReturn; + } + + /** + * Initializes an Analytics Reporting API V4 service object. + * + * @return An authorized Analytics Reporting API V4 service object. + * @throws IOException + * @throws GeneralSecurityException + */ + private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) + throws GeneralSecurityException, IOException { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); + + // Construct the Analytics Reporting service object. + return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + } + + /** + * Queries the Analytics Reporting API V4. + * + * @param service An authorized Analytics Reporting API V4 service object. + * @return GetReportResponse The Analytics Reporting API V4 response. + * @throws IOException + */ + private static HashMap> getReportResponses(AnalyticsReporting service, + List viewIDs, DateRange dateRange) throws IOException { + + HashMap> reports = new HashMap<>(); + + // Create the Metrics object. + Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); + com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension().setName("ga:pagePath"); + + for(String view : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.info("Getting data from Google Analytics for gateway viewid: " + view); + boolean iterateMorePages = true; + String nextPageToken = null; + while (iterateMorePages) { + // Create the ReportRequest object. + ReportRequest request = new ReportRequest().setViewId(view.trim()).setDateRanges(Arrays.asList(dateRange)) + .setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle)); + request.setPageSize(1000); + request.setPageToken(nextPageToken); + ArrayList requests = new ArrayList(); + requests.add(request); + // Create the GetReportsRequest object. + GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); + // Call the batchGet method. + GetReportsResponse response = service.reports().batchGet(getReport).execute(); + nextPageToken = response.getReports().get(0).getNextPageToken(); + iterateMorePages = (nextPageToken != null); + logger.debug("got nextPageToken: "+nextPageToken); + gReportResponses.add(response); + } + reports.put(view, gReportResponses); + } + // Return the response. + return reports; + } + + /** + * Parses and prints the Analytics Reporting API V4 response. + * @param dashboardContext + * + * @param response An Analytics Reporting API V4 response. + */ + private static List parseResponse(String viewId, List responses, String dashboardContext) { + logger.debug("parsing Response for " + viewId); + + List toReturn = new ArrayList<>(); + for (GetReportsResponse response : responses) { + for (Report report: response.getReports()) { + List rows = report.getData().getRows(); + if (rows == null) { + logger.warn("No data found for " + viewId); + } + else { + for (ReportRow row: rows) { + String dimension = row.getDimensions().get(0); + DateRangeValues metric = row.getMetrics().get(0); + CoreServiceAccessesReportRow var = new CoreServiceAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + logger.trace("parsing pagepath {}: value: {}", pagePath, Integer.parseInt(metric.getValues().get(0))); + + if (!pagePath.contains("_redirect=/group")) { + if ( pagePath.contains(PAGE_WORKSPACE_ACCESSES)) { + var.setKey(HarvestedDataKey.WORKSPACE_ACCESSES); + System.out.println("\n **matched "+pagePath); + validEntry = true; + } + else if ( pagePath.contains(PAGE_MESSAGES_ACCESSES)) { + var.setKey(HarvestedDataKey.MESSAGES_ACCESSES); + System.out.println("\n **matched "+pagePath); + validEntry = true; + } + else if ( pagePath.contains(PAGE_PROFILE_ACCESSES)) { + var.setKey(HarvestedDataKey.PROFILE_ACCESSES); + System.out.println("\n **matched "+pagePath); + validEntry = true; + } + else if ( pagePath.contains(PAGE_NOTIFICATION_ACCESSES)) { + var.setKey(HarvestedDataKey.NOTIFICATIONS_ACCESSES); + System.out.println("\n **matched "+pagePath); + validEntry = true; + } + } + if (validEntry) { + var.setDashboardContext(dashboardContext); + var.setPagePath(dimension); + var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); + toReturn.add(var); + } + } + } + } + } + return toReturn; + } + + private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { + + String clientId = cred.getClientId(); + String clientEmail = cred.getClientEmail(); + String privateKeyPem = cred.getPrivateKeyPem(); + String privateKeyId = cred.getPrivateKeyId(); + String tokenUri = cred.getTokenUri(); + String projectId = cred.getProjectId(); + + if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { + throw new IOException("Error reading service account credential from stream, " + + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); + } + + PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); + + Collection emptyScopes = Collections.emptyList(); + + Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) + .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) + .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) + .setServiceAccountPrivateKeyId(privateKeyId); + + if(tokenUri != null) { + credentialBuilder.setTokenServerEncodedUrl(tokenUri); + } + + if(projectId != null) { + credentialBuilder.setServiceAccountProjectId(projectId); + } + + // Don't do a refresh at this point, as it will always fail before the scopes are added. + return credentialBuilder.build(); + } + + private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { + Reader reader = new StringReader(privateKeyPem); + Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); + if(section == null) { + throw new IOException("Invalid PKCS8 data."); + } + byte[] bytes = section.getBase64DecodedBytes(); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); + Exception unexpectedException = null; + try { + KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + return privateKey; + } catch(NoSuchAlgorithmException exception) { + unexpectedException = exception; + } catch(InvalidKeySpecException exception) { + unexpectedException = exception; + } + throw new IOException("Unexpected exception reading PKCS data", unexpectedException); + } + + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) + throws Exception { + String scope = infrastructureScope; + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(scope); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); + query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List toReturn = client.submit(query); + ScopeProvider.instance.set(currScope); + return toReturn; + } + + /** + * This method look up in the IS the Gateway which corresponds to a given Google Analytics viewId + * @param viewID + * @return the gateway name, e.g. "Blue-Cloud Gateway" or null if no correspondance was found + * @throws Exception + * @throws ObjectNotFound + */ + private static String getAccountingDashboardContextGivenGAViewID(String viewID) throws ObjectNotFound, Exception { + String toReturn = null; + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(context); + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/SecondaryType/text() eq '" + MAPPING_RESOURCE_CATEGORY + "'"); + query.addCondition("$resource/Profile/Body/Property/viewID/text() eq '" + viewID + "'"); + DiscoveryClient client = clientFor(GenericResource.class); + List list = client.submit(query); + if(list.size() > 1) { + logger.error("Too many Generic Resources having GA viewID " + viewID + + " in this scope having SecondaryType " + MAPPING_RESOURCE_CATEGORY); + } else if(list.size() == 0) { + logger.warn("There is no Generic Resources having GA viewID " + viewID + " and SecondaryType " + + MAPPING_RESOURCE_CATEGORY + " in this context: " + context); + } else { + GenericResource found = list.get(0); + String elem = new StringBuilder("").append(found.profile().bodyAsString()).append("").toString(); + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement(); + XPathHelper helper = new XPathHelper(node); + List currValue = helper.evaluate("//Property/viewID/text()"); + if (currValue != null && currValue.size() > 0) { + List contexts = currValue; + for (int i = 0; i < contexts.size(); i++) { + if (currValue.get(i).trim().compareTo(viewID) == 0) { + toReturn = helper.evaluate("//Property/DashboardContext/text()").get(i); + break; + } + } + } + logger.debug("Found DashboardContext for viewId {} : {} ", viewID, toReturn); + } + ScopeProvider.instance.set(currScope); + return toReturn; + } + + /** + * l + * @throws Exception + */ + private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { + AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); + + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + try { + List list = getAnalyticsReportingConfigurationFromIS(context); + if(list.size() > 1) { + logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); + } else if(list.size() == 0) { + logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); + } else { + + for(ServiceEndpoint res : list) { + reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + Group apGroup = res.profile().accessPoints(); + AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); + AccessPoint found = accessPoints[0]; + reportCredentials.setClientEmail(found.address()); + reportCredentials.setProjectId(found.username()); + reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + for(Property prop : found.properties()) { + if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + String[] views = decryptedValue.split(";"); + reportCredentials.setViewIds(Arrays.asList(views)); + } + if(prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setClientId(decryptedValue); + } + if(prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setPrivateKeyId(decryptedValue); + } + } + } + } + } catch(Exception e) { + e.printStackTrace(); + return null; + } + return reportCredentials; + } + + private static LocalDate asLocalDate(Date date) { + return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static DateRange getDateRangeForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics + String startDate = asLocalDate(start).format(formatter); + String endDate = asLocalDate(end).format(formatter); + DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` + dateRange.setStartDate(startDate); + dateRange.setEndDate(endDate); + return dateRange; + } + +}