package org.gcube.dataharvest.harvester; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; import java.security.GeneralSecurityException; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.common.encryption.encrypter.StringEncrypter; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.utils.Group; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.datamodel.VREAccessesReportRow; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; import com.google.analytics.data.v1beta.DateRange; import com.google.analytics.data.v1beta.DateRange.Builder; import com.google.analytics.data.v1beta.Dimension; import com.google.analytics.data.v1beta.Metric; import com.google.analytics.data.v1beta.Row; import com.google.analytics.data.v1beta.RunReportRequest; import com.google.analytics.data.v1beta.RunReportResponse; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.auth.oauth2.ServiceAccountCredentials; public class VREAccessesHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class); private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_VIEWS_PROPERTY = "views"; private static final String AP_CLIENT_ID = "client_id"; private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id"; private List vreAccesses; public VREAccessesHarvester(Date start, Date end) throws Exception { super(start, end); vreAccesses = getAllAccesses(start, end); } @Override public List getAccountingRecords() throws Exception { try { String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); ArrayList accountingRecords = new ArrayList(); int measure = 0; ScopeBean scopeBean = new ScopeBean(context); String lowerCasedContext = scopeBean.name().toLowerCase(); String case1 = lowerCasedContext + "/"; String case2 = lowerCasedContext + "?"; for(VREAccessesReportRow row : vreAccesses) { String pagePath = row.getPagePath(); if (!pagePath.contains("_redirect=/group") && !pagePath.contains("workspace")) { if(pagePath.endsWith(lowerCasedContext)) { logger.debug("Matched endsWith({}) : {}", lowerCasedContext, pagePath); measure += row.getVisitNumber(); } else if(pagePath.contains(case1) || pagePath.contains(case2)) { logger.debug("Matched contains({}) || contains({}) : {}", case1, case2, pagePath); measure += row.getVisitNumber(); } } } ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.ACCESSES), (long) measure); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); accountingRecords.add(ar); return accountingRecords; } catch(Exception e) { throw e; } } /** * * @return a list of {@link VREAccessesReportRow} objects containing the pagePath and the visit number e.g. * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/add-new-users, visitNumber=1] * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/administration, visitNumber=2] * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39] */ private static List getAllAccesses(Date start, Date end) throws Exception { Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); logger.debug("Getting accesses in this time range {}", dateRangeBuilder.toString()); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); logger.debug("Getting credentials credentialsFromD4S"); BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); logger.debug("initializeAnalyticsReporting service settings"); HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); List totalAccesses = new ArrayList<>(); for(String view : responses.keySet()) { List viewReport = parseResponse(view, responses.get(view)); logger.debug("Got {} entries from view id={}", viewReport.size(), view); totalAccesses.addAll(viewReport); } logger.debug("Merged in {} total entries from all views", totalAccesses.size()); return totalAccesses; } /** * Initializes an Google Analytics Data API service object. * * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { return BetaAnalyticsDataSettings.newBuilder() .setCredentialsProvider(FixedCredentialsProvider.create( ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) .build(); } /** * Queries Analytics Data API service * * @param service Analytics Data API service service settings. * @return Row Analytics Data API service * @throws IOException */ private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, List viewIDs, Builder dateRangeBuilder) throws IOException { HashMap> reports = new HashMap<>(); try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { for(String propertyId : viewIDs) { List gReportResponses = new ArrayList<>(); logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); RunReportRequest request = RunReportRequest.newBuilder() .setProperty("properties/" + propertyId) .addDimensions(Dimension.newBuilder().setName("pagePath")) .addMetrics(Metric.newBuilder().setName("screenPageViews")) .addDateRanges(dateRangeBuilder) .build(); // Make the request. RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); // Iterate through every row of the API response. // for (Row row : response.getRowsList()) { // System.out.printf( // "%s, %s%n", row.getDimensionValues(0).getValue(), row.getMetricValues(0).getValue()); // } reports.put(propertyId, gReportResponses); } } return reports; } /** * Parses and prints the Analytics Data API service respose * * @param response An Analytics Data API service response. */ private static List parseResponse(String viewId, List responses) { logger.debug("parsing Response for propertyID=" + viewId); List toReturn = new ArrayList<>(); for (RunReportResponse response : responses) { for (Row row: response.getRowsList()) { String dimension = row.getDimensionValues(0).getValue(); String metric = row.getMetricValues(0).getValue(); VREAccessesReportRow var = new VREAccessesReportRow(); boolean validEntry = false; String pagePath = dimension; if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { var.setPagePath(dimension); validEntry = true; } if (validEntry) { var.setVisitNumber(Integer.parseInt(metric)); toReturn.add(var); } } } return toReturn; } private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { String scope = infrastructureScope; String currScope = ScopeProvider.instance.get(); ScopeProvider.instance.set(scope); SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); DiscoveryClient client = clientFor(ServiceEndpoint.class); List toReturn = client.submit(query); ScopeProvider.instance.set(currScope); return toReturn; } /** * l * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); try { List list = getAnalyticsReportingConfigurationFromIS(context); if(list.size() > 1) { logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); } else if(list.size() == 0) { logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); } else { for(ServiceEndpoint res : list) { Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; reportCredentials.setClientEmail(found.username()); String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); for(Property prop : found.properties()) { if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); String[] views = decryptedValue.split(";"); reportCredentials.setViewIds(Arrays.asList(views)); } if(prop.name().compareTo(AP_CLIENT_ID) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setClientId(decryptedValue); } if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setPrivateKeyId(decryptedValue); } } } } } catch(Exception e) { e.printStackTrace(); return null; } return reportCredentials; } private static LocalDate asLocalDate(Date date) { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); return dateRangeBuilder; } }