From 4cbdef880e6151b3e25681d67a215bc582723db3 Mon Sep 17 00:00:00 2001 From: Massimiliano Assante Date: Fri, 30 Jun 2023 12:15:01 +0200 Subject: [PATCH] rewritten Jupyter, RStudio and Catalogue Harv., missing Core Serv one --- .../harvester/CatalogueAccessesHarvester.java | 322 +++++++----------- .../harvester/JupyterAccessesHarvester.java | 291 ++++++---------- .../harvester/RStudioAccessesHarvester.java | 292 ++++++---------- .../harvester/VREAccessesHarvester.java | 2 +- .../AccountingDataHarvesterPluginTest.java | 22 +- .../gcube/dataharvest/utils/ContextTest.java | 4 +- 6 files changed, 336 insertions(+), 597 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/harvester/CatalogueAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/CatalogueAccessesHarvester.java index 9282efb..52f9855 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/CatalogueAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/CatalogueAccessesHarvester.java @@ -4,22 +4,14 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -49,26 +41,20 @@ import org.slf4j.LoggerFactory; import org.w3c.dom.Node; import org.xml.sax.InputSource; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.util.Utils; -import com.google.api.client.http.HttpTransport; +import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; +import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; +import com.google.analytics.data.v1beta.DateRange; +import com.google.analytics.data.v1beta.DateRange.Builder; +import com.google.analytics.data.v1beta.Dimension; +import com.google.analytics.data.v1beta.Metric; +import com.google.analytics.data.v1beta.Row; +import com.google.analytics.data.v1beta.RunReportRequest; +import com.google.analytics.data.v1beta.RunReportResponse; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.PemReader; -import com.google.api.client.util.PemReader.Section; -import com.google.api.client.util.SecurityUtils; -import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; -import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; -import com.google.api.services.analyticsreporting.v4.model.DateRange; -import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; -import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; -import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; -import com.google.api.services.analyticsreporting.v4.model.Metric; -import com.google.api.services.analyticsreporting.v4.model.Report; -import com.google.api.services.analyticsreporting.v4.model.ReportRequest; -import com.google.api.services.analyticsreporting.v4.model.ReportRow; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; + public class CatalogueAccessesHarvester extends BasicHarvester { @@ -76,13 +62,13 @@ public class CatalogueAccessesHarvester extends BasicHarvester { private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + private static final String MAPPING_RESOURCE_CATEGORY = "BigGAnalyticsMapping"; private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; - private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_CATALOGUE_PAGEVIEWS_PROPERTY = "catalogue-pageviews"; - private static final String AP_CLIENT_PROPERTY = "clientId"; - private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; - private static final String APPLICATION_NAME = "Analytics Reporting"; + private static final String AP_CLIENT_PROPERTY = "client_id"; + private static final String AP_PRIVATEKEY_PROPERTY = "private_key_id"; private static final String REGEX_CATALOGUE_ACCESSES = "^\\/$"; private static final String REGEX_CATALOGUE_DATASET_LIST_ACCESSES = "^\\/dataset(\\?([a-zA-Z0-9_.-]*.+))*"; @@ -107,7 +93,7 @@ public class CatalogueAccessesHarvester extends BasicHarvester { int catalogueResourceAccesses = 0; logger.debug("Catalogue accesses for {} ", dashboardContext); for(CatalogueAccessesReportRow row : catalogueAccesses.get(dashboardContext)) { -// String pagePath = row.getPagePath(); + // String pagePath = row.getPagePath(); switch (row.getKey()) { case CATALOGUE_ACCESSES: catalogueTotalAccesses += row.getVisitNumber(); @@ -126,23 +112,29 @@ public class CatalogueAccessesHarvester extends BasicHarvester { } } ScopeDescriptor scopeDescriptor = new ScopeDescriptor(); - ScopeBean scopeBean = new ScopeBean(dashboardContext); - scopeDescriptor.setId(dashboardContext); - scopeDescriptor.setName(scopeBean.name()); - - AccountingRecord ar1 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_ACCESSES), (long) catalogueTotalAccesses); - AccountingRecord ar2 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_DATASET_LIST_ACCESSES), (long) catalogueDatasetListAccesses); - AccountingRecord ar3 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_DATASET_ACCESSES), (long) catalogueDatasetAccesses); - AccountingRecord ar4 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_RESOURCE_ACCESSES), (long) catalogueResourceAccesses); - logger.debug("{} : {}", ar1.getDimension().getId(), ar1.getMeasure()); - accountingRecords.add(ar1); - logger.debug("{} : {}", ar2.getDimension().getId(), ar2.getMeasure()); - accountingRecords.add(ar2); - logger.debug("{} : {}", ar3.getDimension().getId(), ar3.getMeasure()); - accountingRecords.add(ar3); - logger.debug("{} : {}", ar4.getDimension().getId(), ar4.getMeasure()); - accountingRecords.add(ar4); + try { + ScopeBean scopeBean = new ScopeBean(dashboardContext); + scopeDescriptor.setId(dashboardContext); + scopeDescriptor.setName(scopeBean.name()); + + AccountingRecord ar1 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_ACCESSES), (long) catalogueTotalAccesses); + AccountingRecord ar2 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_DATASET_LIST_ACCESSES), (long) catalogueDatasetListAccesses); + AccountingRecord ar3 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_DATASET_ACCESSES), (long) catalogueDatasetAccesses); + AccountingRecord ar4 = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.CATALOGUE_RESOURCE_ACCESSES), (long) catalogueResourceAccesses); + logger.debug("{} : {}", ar1.getDimension().getId(), ar1.getMeasure()); + accountingRecords.add(ar1); + logger.debug("{} : {}", ar2.getDimension().getId(), ar2.getMeasure()); + accountingRecords.add(ar2); + logger.debug("{} : {}", ar3.getDimension().getId(), ar3.getMeasure()); + accountingRecords.add(ar3); + logger.debug("{} : {}", ar4.getDimension().getId(), ar4.getMeasure()); + accountingRecords.add(ar4); + } catch (NullPointerException e) { + logger.warn("I found no correspondance in the Genereric Resource for a PropertyId you should check this, type: BigGAnalyticsMapping name: AccountingDashboardMapping"); + e.printStackTrace(); + } + } logger.debug("Returning {} accountingRecords ", accountingRecords.size()); return accountingRecords; @@ -156,17 +148,24 @@ public class CatalogueAccessesHarvester extends BasicHarvester { * */ private static HashMap> getAllAccesses(Date start, Date end) throws Exception { - DateRange dateRange = getDateRangeForAnalytics(start, end); - logger.trace("Getting catalogue accesses in this time range {}", dateRange.toPrettyString()); + Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); + logger.debug("Getting Catalogue accesses in this time range {}", dateRangeBuilder.toString()); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); - AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); - HashMap> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); + + logger.debug("Getting credentials credentialsFromD4S"); + + BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); + + logger.debug("initializeAnalyticsReporting service settings"); + + + HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); HashMap> toReturn = new HashMap<>(); for(String view : responses.keySet()) { String dashboardContext = getAccountingDashboardContextGivenGAViewID(view); - logger.trace("Parsing responses for this catalogue view, which corresponds to Dashboard Context: " + dashboardContext); + logger.info("\n\n**************** Parsing responses for this catalogue view, which corresponds to Dashboard Context: " + dashboardContext); List viewReport = parseResponse(view, responses.get(view), dashboardContext); logger.trace("Got {} entries from view id={}", viewReport.size(), view); toReturn.put(dashboardContext, viewReport); @@ -175,175 +174,102 @@ public class CatalogueAccessesHarvester extends BasicHarvester { } /** - * Initializes an Analytics Reporting API V4 service object. + * Initializes an Google Analytics Data API service object. * - * @return An authorized Analytics Reporting API V4 service object. + * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ - private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) - throws GeneralSecurityException, IOException { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); - - // Construct the Analytics Reporting service object. - return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); + private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { + return BetaAnalyticsDataSettings.newBuilder() + .setCredentialsProvider(FixedCredentialsProvider.create( + ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) + .build(); } /** - * Queries the Analytics Reporting API V4. + * Queries Analytics Data API service * - * @param service An authorized Analytics Reporting API V4 service object. - * @return GetReportResponse The Analytics Reporting API V4 response. + * @param service Analytics Data API service service settings. + * @return Row Analytics Data API service * @throws IOException */ - private static HashMap> getReportResponses(AnalyticsReporting service, - List viewIDs, DateRange dateRange) throws IOException { + private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, + List viewIDs, Builder dateRangeBuilder) throws IOException { - HashMap> reports = new HashMap<>(); + HashMap> reports = new HashMap<>(); - // Create the Metrics object. - Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); - com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension().setName("ga:pagePath"); + try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { - for(String view : viewIDs) { - List gReportResponses = new ArrayList<>(); - logger.info("Getting data from Google Analytics for catalogue viewid: " + view); - boolean iterateMorePages = true; - String nextPageToken = null; - while (iterateMorePages) { - // Create the ReportRequest object. - ReportRequest request = new ReportRequest().setViewId(view.trim()).setDateRanges(Arrays.asList(dateRange)) - .setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle)); - request.setPageSize(1000); - request.setPageToken(nextPageToken); - ArrayList requests = new ArrayList(); - requests.add(request); - // Create the GetReportsRequest object. - GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); - // Call the batchGet method. - GetReportsResponse response = service.reports().batchGet(getReport).execute(); - nextPageToken = response.getReports().get(0).getNextPageToken(); - iterateMorePages = (nextPageToken != null); - logger.debug("got nextPageToken: "+nextPageToken); + for(String propertyId : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); + RunReportRequest request = + RunReportRequest.newBuilder() + .setProperty("properties/" + propertyId) + .addDimensions(Dimension.newBuilder().setName("pagePath")) + .addMetrics(Metric.newBuilder().setName("screenPageViews")) + .addDateRanges(dateRangeBuilder) + .build(); + + // Make the request. + RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); + // Iterate through every row of the API response. + // for (Row row : response.getRowsList()) { + // System.out.printf( + // "%s, %s%n", row.getDimensionValues(0).getValue(), row.getMetricValues(0).getValue()); + // } + reports.put(propertyId, gReportResponses); } - reports.put(view, gReportResponses); } - // Return the response. return reports; } /** - * Parses and prints the Analytics Reporting API V4 response. - * @param dashboardContext + * Parses and prints the Analytics Data API service respose * - * @param response An Analytics Reporting API V4 response. + * @param dashboardContext */ - private static List parseResponse(String viewId, List responses, String dashboardContext) { + private static List parseResponse(String viewId, List responses, String dashboardContext) { logger.debug("parsing Response for " + viewId); List toReturn = new ArrayList<>(); - for (GetReportsResponse response : responses) { - for (Report report: response.getReports()) { - List rows = report.getData().getRows(); - if (rows == null) { - logger.warn("No data found for " + viewId); + for (RunReportResponse response : responses) { + for (Row row: response.getRowsList()) { + String dimension = row.getDimensionValues(0).getValue(); + String metric = row.getMetricValues(0).getValue(); + CatalogueAccessesReportRow var = new CatalogueAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + logger.trace("parsing pagepath {}: value: {}", pagePath, Integer.parseInt(metric)); + if (pagePath.matches(REGEX_CATALOGUE_RESOURCE_ACCESSES)) { + var.setKey(HarvestedDataKey.CATALOGUE_RESOURCE_ACCESSES); + validEntry = true; } - else { - for (ReportRow row: rows) { - String dimension = row.getDimensions().get(0); - DateRangeValues metric = row.getMetrics().get(0); - CatalogueAccessesReportRow var = new CatalogueAccessesReportRow(); - boolean validEntry = false; - String pagePath = dimension; - logger.trace("parsing pagepath {}: value: {}", pagePath, Integer.parseInt(metric.getValues().get(0))); - - if (pagePath.matches(REGEX_CATALOGUE_RESOURCE_ACCESSES)) { - var.setKey(HarvestedDataKey.CATALOGUE_RESOURCE_ACCESSES); - validEntry = true; - } - else if (pagePath.matches(REGEX_CATALOGUE_DATASET_ACCESSES)) { - var.setKey(HarvestedDataKey.CATALOGUE_DATASET_ACCESSES); - validEntry = true; - } - else if (pagePath.matches(REGEX_CATALOGUE_DATASET_LIST_ACCESSES)) { - var.setKey(HarvestedDataKey.CATALOGUE_DATASET_LIST_ACCESSES); - validEntry = true; - } - else if (pagePath.matches(REGEX_CATALOGUE_ACCESSES)) { - var.setKey(HarvestedDataKey.CATALOGUE_ACCESSES); - validEntry = true; - } - if (validEntry) { - var.setDashboardContext(dashboardContext); - var.setPagePath(dimension); - var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); - toReturn.add(var); - } - } + else if (pagePath.matches(REGEX_CATALOGUE_DATASET_ACCESSES)) { + var.setKey(HarvestedDataKey.CATALOGUE_DATASET_ACCESSES); + validEntry = true; + } + else if (pagePath.matches(REGEX_CATALOGUE_DATASET_LIST_ACCESSES)) { + var.setKey(HarvestedDataKey.CATALOGUE_DATASET_LIST_ACCESSES); + validEntry = true; + } + else if (pagePath.matches(REGEX_CATALOGUE_ACCESSES)) { + var.setKey(HarvestedDataKey.CATALOGUE_ACCESSES); + validEntry = true; + } + if (validEntry) { + var.setDashboardContext(dashboardContext); + var.setPagePath(dimension); + var.setVisitNumber(Integer.parseInt(metric)); + toReturn.add(var); } } } return toReturn; } - private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { - - String clientId = cred.getClientId(); - String clientEmail = cred.getClientEmail(); - String privateKeyPem = cred.getPrivateKeyPem(); - String privateKeyId = cred.getPrivateKeyId(); - String tokenUri = cred.getTokenUri(); - String projectId = cred.getProjectId(); - - if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { - throw new IOException("Error reading service account credential from stream, " - + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); - } - - PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); - - Collection emptyScopes = Collections.emptyList(); - - Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) - .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) - .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) - .setServiceAccountPrivateKeyId(privateKeyId); - - if(tokenUri != null) { - credentialBuilder.setTokenServerEncodedUrl(tokenUri); - } - - if(projectId != null) { - credentialBuilder.setServiceAccountProjectId(projectId); - } - - // Don't do a refresh at this point, as it will always fail before the scopes are added. - return credentialBuilder.build(); - } - - private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { - Reader reader = new StringReader(privateKeyPem); - Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); - if(section == null) { - throw new IOException("Invalid PKCS8 data."); - } - byte[] bytes = section.getBase64DecodedBytes(); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); - Exception unexpectedException = null; - try { - KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); - PrivateKey privateKey = keyFactory.generatePrivate(keySpec); - return privateKey; - } catch(NoSuchAlgorithmException exception) { - unexpectedException = exception; - } catch(InvalidKeySpecException exception) { - unexpectedException = exception; - } - throw new IOException("Unexpected exception reading PKCS data", unexpectedException); - } private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { @@ -405,7 +331,6 @@ public class CatalogueAccessesHarvester extends BasicHarvester { } /** - * l * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { @@ -423,13 +348,13 @@ public class CatalogueAccessesHarvester extends BasicHarvester { } else { for(ServiceEndpoint res : list) { - reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; - reportCredentials.setClientEmail(found.address()); - reportCredentials.setProjectId(found.username()); - reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + reportCredentials.setClientEmail(found.username()); + String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); + reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); + for(Property prop : found.properties()) { if(prop.name().compareTo(AP_CATALOGUE_PAGEVIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); @@ -458,14 +383,13 @@ public class CatalogueAccessesHarvester extends BasicHarvester { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } - private static DateRange getDateRangeForAnalytics(Date start, Date end) { + private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); - DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` - dateRange.setStartDate(startDate); - dateRange.setEndDate(endDate); - return dateRange; + Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); + + return dateRangeBuilder; } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index 2aba8fa..a797773 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -4,22 +4,13 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -42,26 +33,19 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.util.Utils; -import com.google.api.client.http.HttpTransport; +import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; +import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; +import com.google.analytics.data.v1beta.DateRange; +import com.google.analytics.data.v1beta.DateRange.Builder; +import com.google.analytics.data.v1beta.Dimension; +import com.google.analytics.data.v1beta.Metric; +import com.google.analytics.data.v1beta.Row; +import com.google.analytics.data.v1beta.RunReportRequest; +import com.google.analytics.data.v1beta.RunReportResponse; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.PemReader; -import com.google.api.client.util.PemReader.Section; -import com.google.api.client.util.SecurityUtils; -import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; -import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; -import com.google.api.services.analyticsreporting.v4.model.DateRange; -import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; -import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; -import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; -import com.google.api.services.analyticsreporting.v4.model.Metric; -import com.google.api.services.analyticsreporting.v4.model.Report; -import com.google.api.services.analyticsreporting.v4.model.ReportRequest; -import com.google.api.services.analyticsreporting.v4.model.ReportRow; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; /** * @@ -75,11 +59,10 @@ public class JupyterAccessesHarvester extends BasicHarvester { private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; - private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_VIEWS_PROPERTY = "views"; - private static final String AP_CLIENT_PROPERTY = "clientId"; - private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; - private static final String APPLICATION_NAME = "Analytics Reporting"; + private static final String AP_CLIENT_ID = "client_id"; + private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id"; private List vreAccesses; @@ -143,188 +126,111 @@ public class JupyterAccessesHarvester extends BasicHarvester { * visitNumber=39] */ private static List getAllAccesses(Date start, Date end) throws Exception { - DateRange dateRange = getDateRangeForAnalytics(start, end); - logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); + Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); + logger.debug("Getting accesses in this time range {}", dateRangeBuilder.toString()); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); - AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); - HashMap> responses = getReportResponses(service, - credentialsFromD4S.getViewIds(), dateRange); + + logger.debug("Getting credentials credentialsFromD4S"); + + BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); + + logger.debug("initializeAnalyticsReporting service settings"); + + + HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); List totalAccesses = new ArrayList<>(); - for (String view : responses.keySet()) { + for(String view : responses.keySet()) { List viewReport = parseResponse(view, responses.get(view)); - logger.trace("Got {} entries from view id={}", viewReport.size(), view); + logger.debug("Got {} entries from view id={}", viewReport.size(), view); totalAccesses.addAll(viewReport); } - logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + logger.debug("Merged in {} total entries from all views", totalAccesses.size()); return totalAccesses; } + + + /** - * Initializes an Analytics Reporting API V4 service object. + * Initializes an Google Analytics Data API service object. * - * @return An authorized Analytics Reporting API V4 service object. + * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ - private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) - throws GeneralSecurityException, IOException { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); - - // Construct the Analytics Reporting service object. - return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); + private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { + return BetaAnalyticsDataSettings.newBuilder() + .setCredentialsProvider(FixedCredentialsProvider.create( + ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) + .build(); } /** - * Queries the Analytics Reporting API V4. + * Queries Analytics Data API service * - * @param service - * An authorized Analytics Reporting API V4 service object. - * @return GetReportResponse The Analytics Reporting API V4 response. + * @param service Analytics Data API service service settings. + * @return Row Analytics Data API service * @throws IOException */ - private static HashMap> getReportResponses(AnalyticsReporting service, - List viewIDs, DateRange dateRange) throws IOException { + private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, + List viewIDs, Builder dateRangeBuilder) throws IOException { - HashMap> reports = new HashMap<>(); + HashMap> reports = new HashMap<>(); - // Create the Metrics object. - Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); - com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension() - .setName("ga:pagePath"); + try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { - for (String view : viewIDs) { - List gReportResponses = new ArrayList<>(); - logger.info("Getting data from Google Analytics for viewid: " + view); - boolean iterateMorePages = true; - String nextPageToken = null; - while (iterateMorePages) { - // Create the ReportRequest object. - ReportRequest request = new ReportRequest().setViewId(view.trim()) - .setDateRanges(Arrays.asList(dateRange)).setMetrics(Arrays.asList(sessions)) - .setDimensions(Arrays.asList(pageTitle)); - request.setPageSize(1000); - request.setPageToken(nextPageToken); - ArrayList requests = new ArrayList(); - requests.add(request); - // Create the GetReportsRequest object. - GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); - // Call the batchGet method. - GetReportsResponse response = service.reports().batchGet(getReport).execute(); - nextPageToken = response.getReports().get(0).getNextPageToken(); - iterateMorePages = (nextPageToken != null); - logger.debug("got nextPageToken: " + nextPageToken); + for(String propertyId : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); + RunReportRequest request = + RunReportRequest.newBuilder() + .setProperty("properties/" + propertyId) + .addDimensions(Dimension.newBuilder().setName("pagePath")) + .addMetrics(Metric.newBuilder().setName("screenPageViews")) + .addDateRanges(dateRangeBuilder) + .build(); + + // Make the request. + RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); + reports.put(propertyId, gReportResponses); } - reports.put(view, gReportResponses); } - // Return the response. return reports; } + /** - * Parses and prints the Analytics Reporting API V4 response. + * Parses and prints the Analytics Data API service respose * - * @param response - * An Analytics Reporting API V4 response. + * @param response An Analytics Data API service response. */ - /** - * Parses and prints the Analytics Reporting API V4 response. - * - * @param response - * An Analytics Reporting API V4 response. - */ - private static List parseResponse(String viewId, List responses) { - logger.debug("parsing Response for " + viewId); + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for propertyID=" + viewId); List toReturn = new ArrayList<>(); - for (GetReportsResponse response : responses) { - for (Report report : response.getReports()) { - List rows = report.getData().getRows(); - if (rows == null) { - logger.warn("No data found for " + viewId); - } else { - for (ReportRow row : rows) { - String dimension = row.getDimensions().get(0); - DateRangeValues metric = row.getMetrics().get(0); - VREAccessesReportRow var = new VREAccessesReportRow(); - boolean validEntry = false; - String pagePath = dimension; - if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { - var.setPagePath(dimension); - validEntry = true; - } - if (validEntry) { - var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); - toReturn.add(var); - } - } + for (RunReportResponse response : responses) { + for (Row row: response.getRowsList()) { + String dimension = row.getDimensionValues(0).getValue(); + String metric = row.getMetricValues(0).getValue(); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; + } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric)); + toReturn.add(var); } } } return toReturn; } - private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { - - String clientId = cred.getClientId(); - String clientEmail = cred.getClientEmail(); - String privateKeyPem = cred.getPrivateKeyPem(); - String privateKeyId = cred.getPrivateKeyId(); - String tokenUri = cred.getTokenUri(); - String projectId = cred.getProjectId(); - - if (clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { - throw new IOException("Error reading service account credential from stream, " - + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); - } - - PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); - - Collection emptyScopes = Collections.emptyList(); - - Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) - .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) - .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) - .setServiceAccountPrivateKeyId(privateKeyId); - - if (tokenUri != null) { - credentialBuilder.setTokenServerEncodedUrl(tokenUri); - } - - if (projectId != null) { - credentialBuilder.setServiceAccountProjectId(projectId); - } - - // Don't do a refresh at this point, as it will always fail before the - // scopes are added. - return credentialBuilder.build(); - } - - private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { - Reader reader = new StringReader(privateKeyPem); - Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); - if (section == null) { - throw new IOException("Invalid PKCS8 data."); - } - byte[] bytes = section.getBase64DecodedBytes(); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); - Exception unexpectedException = null; - try { - KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); - PrivateKey privateKey = keyFactory.generatePrivate(keySpec); - return privateKey; - } catch (NoSuchAlgorithmException exception) { - unexpectedException = exception; - } catch (InvalidKeySpecException exception) { - unexpectedException = exception; - } - throw new IOException("Unexpected exception reading PKCS data", unexpectedException); - } - private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { String scope = infrastructureScope; @@ -341,8 +247,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { /** * l - * - * @throws Exception + * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); @@ -350,60 +255,58 @@ public class JupyterAccessesHarvester extends BasicHarvester { String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); try { List list = getAnalyticsReportingConfigurationFromIS(context); - if (list.size() > 1) { + if(list.size() > 1) { logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); - } else if (list.size() == 0) { + } else if(list.size() == 0) { logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); } else { - for (ServiceEndpoint res : list) { - reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + for(ServiceEndpoint res : list) { Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; - reportCredentials.setClientEmail(found.address()); - reportCredentials.setProjectId(found.username()); - reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); - for (Property prop : found.properties()) { - if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + reportCredentials.setClientEmail(found.username()); + String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); + reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); + + for(Property prop : found.properties()) { + if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); String[] views = decryptedValue.split(";"); reportCredentials.setViewIds(Arrays.asList(views)); } - if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + if(prop.name().compareTo(AP_CLIENT_ID) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setClientId(decryptedValue); } - if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setPrivateKeyId(decryptedValue); } } } } - } catch (Exception e) { + } catch(Exception e) { e.printStackTrace(); return null; } return reportCredentials; } + private static LocalDate asLocalDate(Date date) { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } - private static DateRange getDateRangeForAnalytics(Date start, Date end) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // required - // by - // Analytics + private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); - DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` - dateRange.setStartDate(startDate); - dateRange.setEndDate(endDate); - return dateRange; + Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); + + return dateRangeBuilder; } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java index 9f70656..1bb94c0 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java @@ -4,22 +4,13 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -42,26 +33,20 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.util.Utils; -import com.google.api.client.http.HttpTransport; +import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; +import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; +import com.google.analytics.data.v1beta.DateRange; +import com.google.analytics.data.v1beta.DateRange.Builder; +import com.google.analytics.data.v1beta.Dimension; +import com.google.analytics.data.v1beta.Metric; +import com.google.analytics.data.v1beta.Row; +import com.google.analytics.data.v1beta.RunReportRequest; +import com.google.analytics.data.v1beta.RunReportResponse; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.PemReader; -import com.google.api.client.util.PemReader.Section; -import com.google.api.client.util.SecurityUtils; -import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; -import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; -import com.google.api.services.analyticsreporting.v4.model.DateRange; -import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; -import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; -import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; -import com.google.api.services.analyticsreporting.v4.model.Metric; -import com.google.api.services.analyticsreporting.v4.model.Report; -import com.google.api.services.analyticsreporting.v4.model.ReportRequest; -import com.google.api.services.analyticsreporting.v4.model.ReportRow; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; + /** * @@ -75,11 +60,10 @@ public class RStudioAccessesHarvester extends BasicHarvester { private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; - private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_VIEWS_PROPERTY = "views"; - private static final String AP_CLIENT_PROPERTY = "clientId"; - private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; - private static final String APPLICATION_NAME = "Analytics Reporting"; + private static final String AP_CLIENT_ID = "client_id"; + private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id"; private List vreAccesses; @@ -143,188 +127,111 @@ public class RStudioAccessesHarvester extends BasicHarvester { * visitNumber=39] */ private static List getAllAccesses(Date start, Date end) throws Exception { - DateRange dateRange = getDateRangeForAnalytics(start, end); - logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); + Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); + logger.debug("Getting accesses in this time range {}", dateRangeBuilder.toString()); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); - AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); - HashMap> responses = getReportResponses(service, - credentialsFromD4S.getViewIds(), dateRange); + + logger.debug("Getting credentials credentialsFromD4S"); + + BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); + + logger.debug("initializeAnalyticsReporting service settings"); + + + HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); List totalAccesses = new ArrayList<>(); - for (String view : responses.keySet()) { + for(String view : responses.keySet()) { List viewReport = parseResponse(view, responses.get(view)); - logger.trace("Got {} entries from view id={}", viewReport.size(), view); + logger.debug("Got {} entries from view id={}", viewReport.size(), view); totalAccesses.addAll(viewReport); } - logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + logger.debug("Merged in {} total entries from all views", totalAccesses.size()); return totalAccesses; } + + + /** - * Initializes an Analytics Reporting API V4 service object. + * Initializes an Google Analytics Data API service object. * - * @return An authorized Analytics Reporting API V4 service object. + * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ - private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) - throws GeneralSecurityException, IOException { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); - - // Construct the Analytics Reporting service object. - return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); + private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { + return BetaAnalyticsDataSettings.newBuilder() + .setCredentialsProvider(FixedCredentialsProvider.create( + ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) + .build(); } /** - * Queries the Analytics Reporting API V4. + * Queries Analytics Data API service * - * @param service - * An authorized Analytics Reporting API V4 service object. - * @return GetReportResponse The Analytics Reporting API V4 response. + * @param service Analytics Data API service service settings. + * @return Row Analytics Data API service * @throws IOException */ - private static HashMap> getReportResponses(AnalyticsReporting service, - List viewIDs, DateRange dateRange) throws IOException { + private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, + List viewIDs, Builder dateRangeBuilder) throws IOException { - HashMap> reports = new HashMap<>(); + HashMap> reports = new HashMap<>(); - // Create the Metrics object. - Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); - com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension() - .setName("ga:pagePath"); + try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { - for (String view : viewIDs) { - List gReportResponses = new ArrayList<>(); - logger.info("Getting data from Google Analytics for viewid: " + view); - boolean iterateMorePages = true; - String nextPageToken = null; - while (iterateMorePages) { - // Create the ReportRequest object. - ReportRequest request = new ReportRequest().setViewId(view.trim()) - .setDateRanges(Arrays.asList(dateRange)).setMetrics(Arrays.asList(sessions)) - .setDimensions(Arrays.asList(pageTitle)); - request.setPageSize(1000); - request.setPageToken(nextPageToken); - ArrayList requests = new ArrayList(); - requests.add(request); - // Create the GetReportsRequest object. - GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); - // Call the batchGet method. - GetReportsResponse response = service.reports().batchGet(getReport).execute(); - nextPageToken = response.getReports().get(0).getNextPageToken(); - iterateMorePages = (nextPageToken != null); - logger.debug("got nextPageToken: " + nextPageToken); + for(String propertyId : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); + RunReportRequest request = + RunReportRequest.newBuilder() + .setProperty("properties/" + propertyId) + .addDimensions(Dimension.newBuilder().setName("pagePath")) + .addMetrics(Metric.newBuilder().setName("screenPageViews")) + .addDateRanges(dateRangeBuilder) + .build(); + + // Make the request. + RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); + reports.put(propertyId, gReportResponses); } - reports.put(view, gReportResponses); } - // Return the response. return reports; } + /** - * Parses and prints the Analytics Reporting API V4 response. + * Parses and prints the Analytics Data API service respose * - * @param response - * An Analytics Reporting API V4 response. + * @param response An Analytics Data API service response. */ - /** - * Parses and prints the Analytics Reporting API V4 response. - * - * @param response - * An Analytics Reporting API V4 response. - */ - private static List parseResponse(String viewId, List responses) { - logger.debug("parsing Response for " + viewId); + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for propertyID=" + viewId); List toReturn = new ArrayList<>(); - for (GetReportsResponse response : responses) { - for (Report report : response.getReports()) { - List rows = report.getData().getRows(); - if (rows == null) { - logger.warn("No data found for " + viewId); - } else { - for (ReportRow row : rows) { - String dimension = row.getDimensions().get(0); - DateRangeValues metric = row.getMetrics().get(0); - VREAccessesReportRow var = new VREAccessesReportRow(); - boolean validEntry = false; - String pagePath = dimension; - if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { - var.setPagePath(dimension); - validEntry = true; - } - if (validEntry) { - var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); - toReturn.add(var); - } - } + for (RunReportResponse response : responses) { + for (Row row: response.getRowsList()) { + String dimension = row.getDimensionValues(0).getValue(); + String metric = row.getMetricValues(0).getValue(); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; + } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric)); + toReturn.add(var); } } } return toReturn; } - private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { - - String clientId = cred.getClientId(); - String clientEmail = cred.getClientEmail(); - String privateKeyPem = cred.getPrivateKeyPem(); - String privateKeyId = cred.getPrivateKeyId(); - String tokenUri = cred.getTokenUri(); - String projectId = cred.getProjectId(); - - if (clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { - throw new IOException("Error reading service account credential from stream, " - + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); - } - - PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); - - Collection emptyScopes = Collections.emptyList(); - - Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) - .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) - .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) - .setServiceAccountPrivateKeyId(privateKeyId); - - if (tokenUri != null) { - credentialBuilder.setTokenServerEncodedUrl(tokenUri); - } - - if (projectId != null) { - credentialBuilder.setServiceAccountProjectId(projectId); - } - - // Don't do a refresh at this point, as it will always fail before the - // scopes are added. - return credentialBuilder.build(); - } - - private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { - Reader reader = new StringReader(privateKeyPem); - Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); - if (section == null) { - throw new IOException("Invalid PKCS8 data."); - } - byte[] bytes = section.getBase64DecodedBytes(); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); - Exception unexpectedException = null; - try { - KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); - PrivateKey privateKey = keyFactory.generatePrivate(keySpec); - return privateKey; - } catch (NoSuchAlgorithmException exception) { - unexpectedException = exception; - } catch (InvalidKeySpecException exception) { - unexpectedException = exception; - } - throw new IOException("Unexpected exception reading PKCS data", unexpectedException); - } - private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { String scope = infrastructureScope; @@ -341,8 +248,7 @@ public class RStudioAccessesHarvester extends BasicHarvester { /** * l - * - * @throws Exception + * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); @@ -350,60 +256,58 @@ public class RStudioAccessesHarvester extends BasicHarvester { String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); try { List list = getAnalyticsReportingConfigurationFromIS(context); - if (list.size() > 1) { + if(list.size() > 1) { logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); - } else if (list.size() == 0) { + } else if(list.size() == 0) { logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); } else { - for (ServiceEndpoint res : list) { - reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + for(ServiceEndpoint res : list) { Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; - reportCredentials.setClientEmail(found.address()); - reportCredentials.setProjectId(found.username()); - reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); - for (Property prop : found.properties()) { - if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + reportCredentials.setClientEmail(found.username()); + String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); + reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); + + for(Property prop : found.properties()) { + if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); String[] views = decryptedValue.split(";"); reportCredentials.setViewIds(Arrays.asList(views)); } - if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + if(prop.name().compareTo(AP_CLIENT_ID) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setClientId(decryptedValue); } - if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setPrivateKeyId(decryptedValue); } } } } - } catch (Exception e) { + } catch(Exception e) { e.printStackTrace(); return null; } return reportCredentials; } + private static LocalDate asLocalDate(Date date) { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } - private static DateRange getDateRangeForAnalytics(Date start, Date end) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // required - // by - // Analytics + private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); - DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` - dateRange.setStartDate(startDate); - dateRange.setEndDate(endDate); - return dateRange; + Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); + + return dateRangeBuilder; } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java index 7eaf477..9b24ddf 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java @@ -214,6 +214,7 @@ public class VREAccessesHarvester extends BasicHarvester { var.setVisitNumber(Integer.parseInt(metric)); toReturn.add(var); } + //System.out.printf("%s, %s%n", row.getDimensionValues(0).getValue(), row.getMetricValues(0).getValue()); } } return toReturn; @@ -234,7 +235,6 @@ public class VREAccessesHarvester extends BasicHarvester { } /** - * l * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 9770759..46a7abc 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -23,7 +23,9 @@ import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; +import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.RStudioAccessesHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; @@ -244,7 +246,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { ArrayList accountingRecords = new ArrayList<>(); - VREAccessesHarvester vreAccessesHarvester = null; + //VREAccessesHarvester vreAccessesHarvester = null; + //JupyterAccessesHarvester vreAccessesHarvester = null; + RStudioAccessesHarvester vreAccessesHarvester = null; for (String context : contexts) { ContextTest.set(context); @@ -254,7 +258,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { if (vreAccessesHarvester == null) { if (scopeBean.is(Type.INFRASTRUCTURE)) { - vreAccessesHarvester = new VREAccessesHarvester(start, end); + vreAccessesHarvester = new RStudioAccessesHarvester(start, end); } else { // This code should be never used because the scopes // are sorted by fullname @@ -267,7 +271,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // Setting back token for the context ContextTest.set(parent.toString()); - vreAccessesHarvester = new VREAccessesHarvester(start, end); + vreAccessesHarvester = new RStudioAccessesHarvester(start, end); // Setting back token for the context ContextTest.set(context); @@ -741,8 +745,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - @Ignore - // @Test +// @Ignore + @Test public void testCatalogueHarvester() { try { @@ -751,8 +755,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { AccountingDao dao = getAccountingDao(); - Date start = DateUtils.getStartCalendar(2020, Calendar.JANUARY, 1).getTime(); - Date end = DateUtils.getStartCalendar(2020, Calendar.FEBRUARY, 1).getTime(); + Calendar from = DateUtils.getStartCalendar(2023, Calendar.JUNE, 1); + Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1); + AggregationType aggregationType = AggregationType.MONTHLY; + + Date start = from.getTime(); + Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); /* * Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java index 2f3f0c2..432f463 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java @@ -42,7 +42,7 @@ public class ContextTest { } } - private static void set(Secret secret) throws Exception { + public static void set(Secret secret) throws Exception { SecretManagerProvider.instance.reset(); SecretManager secretManager = new SecretManager(); SecretManagerProvider.instance.set(secretManager); @@ -70,7 +70,7 @@ public class ContextTest { return tr; } - private static Secret generateSecretByContextName(String context) throws Exception { + public static Secret generateSecretByContextName(String context) throws Exception { TokenResponse tr = getJWTAccessToken(context); Secret secret = new JWTSecret(tr.getAccessToken()); return secret;