From 284431ca8d095e93ae91693805c9a5838508c9b0 Mon Sep 17 00:00:00 2001 From: Massimiliano Assante Date: Mon, 3 Jul 2023 17:06:02 +0200 Subject: [PATCH] also core services refactored --- .../CoreServicesAccessesHarvester.java | 292 ++++++------------ .../AccountingDataHarvesterPluginTest.java | 22 +- src/test/resources/.gitignore | 1 + 3 files changed, 111 insertions(+), 204 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java index a9997a4..2823a2f 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/CoreServicesAccessesHarvester.java @@ -4,22 +4,14 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -49,26 +41,17 @@ import org.slf4j.LoggerFactory; import org.w3c.dom.Node; import org.xml.sax.InputSource; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.util.Utils; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.PemReader; -import com.google.api.client.util.PemReader.Section; -import com.google.api.client.util.SecurityUtils; -import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; -import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; -import com.google.api.services.analyticsreporting.v4.model.DateRange; -import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; -import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; -import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; -import com.google.api.services.analyticsreporting.v4.model.Metric; -import com.google.api.services.analyticsreporting.v4.model.Report; -import com.google.api.services.analyticsreporting.v4.model.ReportRequest; -import com.google.api.services.analyticsreporting.v4.model.ReportRow; +import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; +import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; +import com.google.analytics.data.v1beta.DateRange; +import com.google.analytics.data.v1beta.DateRange.Builder; +import com.google.analytics.data.v1beta.Dimension; +import com.google.analytics.data.v1beta.Metric; +import com.google.analytics.data.v1beta.Row; +import com.google.analytics.data.v1beta.RunReportRequest; +import com.google.analytics.data.v1beta.RunReportResponse; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; /** * @author Massimiliano Assante (ISTI - CNR) @@ -77,16 +60,12 @@ public class CoreServicesAccessesHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(CoreServicesAccessesHarvester.class); - private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - private static final String MAPPING_RESOURCE_CATEGORY = "BigGAnalyticsMapping"; private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; - private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_VIEWS_PROPERTY = "views"; - private static final String AP_CLIENT_PROPERTY = "clientId"; - private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; - private static final String APPLICATION_NAME = "Analytics Reporting"; - + private static final String AP_CLIENT_ID = "client_id"; + private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id"; private static final String PAGE_WORKSPACE_ACCESSES = "/workspace"; private static final String PAGE_MESSAGES_ACCESSES = "/messages"; @@ -160,18 +139,18 @@ public class CoreServicesAccessesHarvester extends BasicHarvester { * */ private static HashMap> getAllAccesses(Date start, Date end) throws Exception { - DateRange dateRange = getDateRangeForAnalytics(start, end); - logger.trace("Getting core services accesses in this time range {}", dateRange.toPrettyString()); + Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); + logger.trace("Getting core services accesses in this time range {}", dateRangeBuilder.toString()); AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); logger.trace("gotten credentialsFromD4S id = {}", credentialsFromD4S.getClientId()); - AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); logger.trace("gotten credentialsFromD4S viewIds= {}", credentialsFromD4S.getViewIds().toString()); - HashMap> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); + HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); HashMap> toReturn = new HashMap<>(); int i = 1; @@ -191,181 +170,106 @@ public class CoreServicesAccessesHarvester extends BasicHarvester { } /** - * Initializes an Analytics Reporting API V4 service object. + * Initializes an Google Analytics Data API service object. * - * @return An authorized Analytics Reporting API V4 service object. + * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ - private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) - throws GeneralSecurityException, IOException { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); - - // Construct the Analytics Reporting service object. - return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); + private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { + return BetaAnalyticsDataSettings.newBuilder() + .setCredentialsProvider(FixedCredentialsProvider.create( + ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) + .build(); } /** - * Queries the Analytics Reporting API V4. + * Queries Analytics Data API service * - * @param service An authorized Analytics Reporting API V4 service object. - * @return GetReportResponse The Analytics Reporting API V4 response. + * @param service Analytics Data API service service settings. + * @return Row Analytics Data API service * @throws IOException */ - private static HashMap> getReportResponses(AnalyticsReporting service, - List viewIDs, DateRange dateRange) throws IOException { + private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, + List viewIDs, Builder dateRangeBuilder) throws IOException { - HashMap> reports = new HashMap<>(); + HashMap> reports = new HashMap<>(); - // Create the Metrics object. - Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); - com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension().setName("ga:pagePath"); + try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { - for(String view : viewIDs) { - List gReportResponses = new ArrayList<>(); - logger.info("Getting data from Google Analytics for gateway viewid: " + view); - boolean iterateMorePages = true; - String nextPageToken = null; - while (iterateMorePages) { - // Create the ReportRequest object. - ReportRequest request = new ReportRequest().setViewId(view.trim()).setDateRanges(Arrays.asList(dateRange)) - .setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle)); - request.setPageSize(1000); - request.setPageToken(nextPageToken); - ArrayList requests = new ArrayList(); - requests.add(request); - // Create the GetReportsRequest object. - GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); - // Call the batchGet method. - GetReportsResponse response = service.reports().batchGet(getReport).execute(); - nextPageToken = response.getReports().get(0).getNextPageToken(); - iterateMorePages = (nextPageToken != null); - logger.debug("got nextPageToken: "+nextPageToken); + for(String propertyId : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); + RunReportRequest request = + RunReportRequest.newBuilder() + .setProperty("properties/" + propertyId) + .addDimensions(Dimension.newBuilder().setName("pagePath")) + .addMetrics(Metric.newBuilder().setName("screenPageViews")) + .addDateRanges(dateRangeBuilder) + .build(); + + // Make the request. + RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); + reports.put(propertyId, gReportResponses); } - reports.put(view, gReportResponses); } - // Return the response. return reports; } + /** - * Parses and prints the Analytics Reporting API V4 response. - * @param dashboardContext + * Parses and prints the Analytics Data API service respose * - * @param response An Analytics Reporting API V4 response. + * @param response An Analytics Data API service response. */ - private static List parseResponse(String viewId, List responses, String dashboardContext) { - logger.debug("parsing Response for " + viewId); - + private static List parseResponse(String viewId, List responses, String dashboardContext) { + logger.debug("parsing Response for propertyID=" + viewId); List toReturn = new ArrayList<>(); - for (GetReportsResponse response : responses) { - for (Report report: response.getReports()) { - List rows = report.getData().getRows(); - if (rows == null) { - logger.warn("No data found for " + viewId); - } - else { - for (ReportRow row: rows) { - String dimension = row.getDimensions().get(0); - DateRangeValues metric = row.getMetrics().get(0); - CoreServiceAccessesReportRow var = new CoreServiceAccessesReportRow(); - boolean validEntry = false; - String pagePath = dimension; - logger.trace("parsing pagepath {}: value: {}", pagePath, Integer.parseInt(metric.getValues().get(0))); + + for (RunReportResponse response : responses) { + for (Row row: response.getRowsList()) { + String dimension = row.getDimensionValues(0).getValue(); + String metric = row.getMetricValues(0).getValue(); + CoreServiceAccessesReportRow var = new CoreServiceAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + logger.trace("parsing pagepath {}: value: {}", pagePath, Integer.parseInt(metric)); - if (!pagePath.contains("_redirect=/group")) { - if ( pagePath.contains(PAGE_WORKSPACE_ACCESSES)) { - var.setKey(HarvestedDataKey.WORKSPACE_ACCESSES); - logger.trace("**matched "+pagePath); - validEntry = true; - } - else if ( pagePath.contains(PAGE_MESSAGES_ACCESSES)) { - var.setKey(HarvestedDataKey.MESSAGES_ACCESSES); - logger.trace("**matched "+pagePath); - validEntry = true; - } - else if ( pagePath.contains(PAGE_PROFILE_ACCESSES)) { - var.setKey(HarvestedDataKey.PROFILE_ACCESSES); - logger.trace("**matched "+pagePath); - validEntry = true; - } - else if ( pagePath.contains(PAGE_NOTIFICATION_ACCESSES)) { - var.setKey(HarvestedDataKey.NOTIFICATIONS_ACCESSES); - logger.trace("**matched "+pagePath); - validEntry = true; - } - } - if (validEntry) { - var.setDashboardContext(dashboardContext); - var.setPagePath(dimension); - var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); - toReturn.add(var); - } + if (!pagePath.contains("_redirect=/group")) { + if ( pagePath.contains(PAGE_WORKSPACE_ACCESSES)) { + var.setKey(HarvestedDataKey.WORKSPACE_ACCESSES); + logger.trace("**matched "+pagePath); + validEntry = true; } + else if ( pagePath.contains(PAGE_MESSAGES_ACCESSES)) { + var.setKey(HarvestedDataKey.MESSAGES_ACCESSES); + logger.trace("**matched "+pagePath); + validEntry = true; + } + else if ( pagePath.contains(PAGE_PROFILE_ACCESSES)) { + var.setKey(HarvestedDataKey.PROFILE_ACCESSES); + logger.trace("**matched "+pagePath); + validEntry = true; + } + else if ( pagePath.contains(PAGE_NOTIFICATION_ACCESSES)) { + var.setKey(HarvestedDataKey.NOTIFICATIONS_ACCESSES); + logger.trace("**matched "+pagePath); + validEntry = true; + } + } + if (validEntry) { + var.setDashboardContext(dashboardContext); + var.setPagePath(dimension); + var.setVisitNumber(Integer.parseInt(metric)); + toReturn.add(var); } } } return toReturn; } - private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { - - String clientId = cred.getClientId(); - String clientEmail = cred.getClientEmail(); - String privateKeyPem = cred.getPrivateKeyPem(); - String privateKeyId = cred.getPrivateKeyId(); - String tokenUri = cred.getTokenUri(); - String projectId = cred.getProjectId(); - - if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { - throw new IOException("Error reading service account credential from stream, " - + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); - } - - PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); - - Collection emptyScopes = Collections.emptyList(); - - Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) - .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) - .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) - .setServiceAccountPrivateKeyId(privateKeyId); - - if(tokenUri != null) { - credentialBuilder.setTokenServerEncodedUrl(tokenUri); - } - - if(projectId != null) { - credentialBuilder.setServiceAccountProjectId(projectId); - } - - // Don't do a refresh at this point, as it will always fail before the scopes are added. - return credentialBuilder.build(); - } - - private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { - Reader reader = new StringReader(privateKeyPem); - Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); - if(section == null) { - throw new IOException("Invalid PKCS8 data."); - } - byte[] bytes = section.getBase64DecodedBytes(); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); - Exception unexpectedException = null; - try { - KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); - PrivateKey privateKey = keyFactory.generatePrivate(keySpec); - return privateKey; - } catch(NoSuchAlgorithmException exception) { - unexpectedException = exception; - } catch(InvalidKeySpecException exception) { - unexpectedException = exception; - } - throw new IOException("Unexpected exception reading PKCS data", unexpectedException); - } + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { @@ -445,24 +349,24 @@ public class CoreServicesAccessesHarvester extends BasicHarvester { } else { for(ServiceEndpoint res : list) { - reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; - reportCredentials.setClientEmail(found.address()); - reportCredentials.setProjectId(found.username()); - reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + reportCredentials.setClientEmail(found.username()); + String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); + reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); + for(Property prop : found.properties()) { if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); String[] views = decryptedValue.split(";"); reportCredentials.setViewIds(Arrays.asList(views)); } - if(prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + if(prop.name().compareTo(AP_CLIENT_ID) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setClientId(decryptedValue); } - if(prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setPrivateKeyId(decryptedValue); } @@ -476,18 +380,18 @@ public class CoreServicesAccessesHarvester extends BasicHarvester { return reportCredentials; } + private static LocalDate asLocalDate(Date date) { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } - private static DateRange getDateRangeForAnalytics(Date start, Date end) { + private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); - DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` - dateRange.setStartDate(startDate); - dateRange.setEndDate(endDate); - return dateRange; + Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); + + return dateRangeBuilder; } } diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 46a7abc..c3e97ef 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -248,7 +248,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { //VREAccessesHarvester vreAccessesHarvester = null; //JupyterAccessesHarvester vreAccessesHarvester = null; - RStudioAccessesHarvester vreAccessesHarvester = null; + //RStudioAccessesHarvester vreAccessesHarvester = null; + CoreServicesAccessesHarvester vreAccessesHarvester = null; for (String context : contexts) { ContextTest.set(context); @@ -258,7 +259,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { if (vreAccessesHarvester == null) { if (scopeBean.is(Type.INFRASTRUCTURE)) { - vreAccessesHarvester = new RStudioAccessesHarvester(start, end); + vreAccessesHarvester = new CoreServicesAccessesHarvester(start, end); } else { // This code should be never used because the scopes // are sorted by fullname @@ -271,7 +272,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // Setting back token for the context ContextTest.set(parent.toString()); - vreAccessesHarvester = new RStudioAccessesHarvester(start, end); + vreAccessesHarvester = new CoreServicesAccessesHarvester(start, end); // Setting back token for the context ContextTest.set(context); @@ -709,8 +710,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - @Ignore - // @Test + // @Ignore + @Test public void testCoreServicesHarvester() { try { @@ -718,16 +719,17 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { ContextTest.set(context); AccountingDao dao = getAccountingDao(); + Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1); + Calendar finalEnd = DateUtils.getStartCalendar(2023, Calendar.JULY, 1); - Date start = DateUtils.getStartCalendar(2017, Calendar.MAY, 1).getTime(); - Date finalEnd = DateUtils.getStartCalendar(2020, Calendar.MAY, 1).getTime(); - - Date end = DateUtils.getEndDateFromStartDate(AggregationType.MONTHLY, start, 1); + AggregationType aggregationType = AggregationType.MONTHLY; + Date start = from.getTime(); + Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - while (end.before(finalEnd)) { + while (from.before(end)) { CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); List accountingRecords = coreServicesHarvester.getAccountingRecords(); // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); diff --git a/src/test/resources/.gitignore b/src/test/resources/.gitignore index 03235d1..85090ad 100644 --- a/src/test/resources/.gitignore +++ b/src/test/resources/.gitignore @@ -2,3 +2,4 @@ /*.key /*.properties /howto.txt +/scopedata 2.xml