From d3ad4f43aeaa69ece94fa908c5afbaf681b0b214 Mon Sep 17 00:00:00 2001 From: Massimiliano Assante Date: Wed, 28 Jun 2023 18:20:13 +0200 Subject: [PATCH] pusho tutto --- CHANGELOG.md | 4 + pom.xml | 74 +++- .../datamodel/AnalyticsReportCredentials.java | 6 +- .../harvester/VREAccessesHarvester.java | 326 +++++++----------- .../utils/ContextAuthorization.java | 4 +- .../gcube/dataharvest/utils/ContextTest.java | 80 +++++ .../gcube/dataharvest/utils/TestMassi.java | 129 +++++++ .../AccountingDataHarvesterPluginTest.java | 8 +- .../gcube/dataharvest/utils/ContextTest.java | 6 +- 9 files changed, 403 insertions(+), 234 deletions(-) create mode 100644 src/main/java/org/gcube/dataharvest/utils/ContextTest.java create mode 100644 src/main/java/org/gcube/dataharvest/utils/TestMassi.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc950e..b0ceaaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for "accounting-dashboard-harvester-se-plugin" +## [v2.3.0-SNAPSHOT] + +- Ported GA harvesters to GA4 + ## [v2.2.0] - Switching security to the new IAM [#21904] diff --git a/pom.xml b/pom.xml index ecb4d6b..ddc1c6b 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 4.0.0 org.gcube.accounting accounting-dashboard-harvester-se-plugin - 2.2.0 + 2.3.0-SNAPSHOT Accounting Dashboard Harvester Smart Executor Plugin Accounting Dashboard Harvester Smart Executor Plugin harvest accounting @@ -57,27 +57,48 @@ com.google.apis google-api-services-analyticsreporting v4-rev124-1.23.0 + + + com.google.guava + guava-jdk5 + + - com.fasterxml.jackson.core - jackson-core - 2.8.4 + com.google.analytics + google-analytics-data + 0.16.0 + + + com.google.guava + guava + 23.6-jre + + + com.google.auth + google-auth-library-oauth2-http + 1.12.1 + + + com.fasterxml.jackson.core + jackson-core + 2.8.4 com.google.http-client google-http-client-gson 1.21.0 - - - - - - - - - - + + + + + + + + + + org.gcube.data-catalogue gcat-client @@ -118,8 +139,8 @@ - org.gcube.social-networking - social-service-client + org.gcube.social-networking + social-service-client [1.0.0, 2.0.0-SNAPSHOT) @@ -134,7 +155,8 @@ [2.0.0, 3.0.0-SNAPSHOT) - + org.gcube.vremanagement smart-executor-client @@ -197,6 +219,24 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + -Xdoclint:none + -Xdoclint:none + + 3.1.0 + + + generate-doc + install + + jar + + + + diff --git a/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java index 3dc23b8..e74511c 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java @@ -59,12 +59,12 @@ public class AnalyticsReportCredentials { /** * Please note: * The key is stored in the resource with blanks " " instead of "\n" as it causes issues and - * without the BEGIN and END Delimiters (e.g. -----END PRIVATE KEY-----) which myst be readded + * without the BEGIN and END Delimiters (e.g. -----END PRIVATE KEY-----) which must be readded * @param privateKeyPem */ public void setPrivateKeyPem(String privateKeyPem) { - privateKeyPem = privateKeyPem.replace(" ", "\n"); - this.privateKeyPem = "-----BEGIN PRIVATE KEY-----\n"+privateKeyPem+"\n-----END PRIVATE KEY-----"; + this.privateKeyPem = privateKeyPem.replace(" ", "\n"); + this.privateKeyPem = "-----BEGIN PRIVATE KEY-----\n"+this.privateKeyPem+"\n-----END PRIVATE KEY-----\n"; } public String getPrivateKeyId() { diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java index 3086ba0..82013eb 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java @@ -4,22 +4,13 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import java.security.GeneralSecurityException; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -42,56 +33,50 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.util.Utils; -import com.google.api.client.http.HttpTransport; +import com.google.analytics.data.v1beta.BetaAnalyticsDataClient; +import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings; +import com.google.analytics.data.v1beta.DateRange; +import com.google.analytics.data.v1beta.DateRange.Builder; +import com.google.analytics.data.v1beta.Dimension; +import com.google.analytics.data.v1beta.Metric; +import com.google.analytics.data.v1beta.Row; +import com.google.analytics.data.v1beta.RunReportRequest; +import com.google.analytics.data.v1beta.RunReportResponse; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.PemReader; -import com.google.api.client.util.PemReader.Section; -import com.google.api.client.util.SecurityUtils; -import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; -import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; -import com.google.api.services.analyticsreporting.v4.model.DateRange; -import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; -import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; -import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; -import com.google.api.services.analyticsreporting.v4.model.Metric; -import com.google.api.services.analyticsreporting.v4.model.Report; -import com.google.api.services.analyticsreporting.v4.model.ReportRequest; -import com.google.api.services.analyticsreporting.v4.model.ReportRow; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; + public class VREAccessesHarvester extends BasicHarvester { - + private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class); - + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; - private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; private static final String AP_VIEWS_PROPERTY = "views"; - private static final String AP_CLIENT_PROPERTY = "clientId"; - private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; - private static final String APPLICATION_NAME = "Analytics Reporting"; - + private static final String AP_CLIENT_ID = "client_id"; + private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id"; + + private List vreAccesses; - + public VREAccessesHarvester(Date start, Date end) throws Exception { super(start, end); vreAccesses = getAllAccesses(start, end); } - + @Override public List getAccountingRecords() throws Exception { try { String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); - + ArrayList accountingRecords = new ArrayList(); - + int measure = 0; - + ScopeBean scopeBean = new ScopeBean(context); String lowerCasedContext = scopeBean.name().toLowerCase(); String case1 = lowerCasedContext + "/"; @@ -100,28 +85,28 @@ public class VREAccessesHarvester extends BasicHarvester { String pagePath = row.getPagePath(); if (!pagePath.contains("_redirect=/group") && !pagePath.contains("workspace")) { if(pagePath.endsWith(lowerCasedContext)) { - logger.trace("Matched endsWith({}) : {}", lowerCasedContext, pagePath); + logger.debug("Matched endsWith({}) : {}", lowerCasedContext, pagePath); measure += row.getVisitNumber(); } else if(pagePath.contains(case1) || pagePath.contains(case2)) { - logger.trace("Matched contains({}) || contains({}) : {}", case1, case2, pagePath); + logger.debug("Matched contains({}) || contains({}) : {}", case1, case2, pagePath); measure += row.getVisitNumber(); } } } - + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); - + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.ACCESSES), (long) measure); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); accountingRecords.add(ar); - + return accountingRecords; - + } catch(Exception e) { throw e; } } - + /** * * @return a list of {@link VREAccessesReportRow} objects containing the pagePath and the visit number e.g. @@ -130,182 +115,112 @@ public class VREAccessesHarvester extends BasicHarvester { * VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39] */ private static List getAllAccesses(Date start, Date end) throws Exception { - DateRange dateRange = getDateRangeForAnalytics(start, end); - logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); - + Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end); + logger.debug("Getting accesses in this time range {}", dateRangeBuilder.toString()); + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); - AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); - HashMap> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange); - List totalAccesses = new ArrayList<>(); + logger.debug("Getting credentials credentialsFromD4S"); + + BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S); + + logger.debug("initializeAnalyticsReporting service settings"); + + + HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder); + List totalAccesses = new ArrayList<>(); + for(String view : responses.keySet()) { List viewReport = parseResponse(view, responses.get(view)); - logger.trace("Got {} entries from view id={}", viewReport.size(), view); + logger.debug("Got {} entries from view id={}", viewReport.size(), view); totalAccesses.addAll(viewReport); } - logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + logger.debug("Merged in {} total entries from all views", totalAccesses.size()); return totalAccesses; } - + /** - * Initializes an Analytics Reporting API V4 service object. + * Initializes an Google Analytics Data API service object. * - * @return An authorized Analytics Reporting API V4 service object. + * @return An authorized Google Analytics Data API * @throws IOException * @throws GeneralSecurityException */ - private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) - throws GeneralSecurityException, IOException { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); - - // Construct the Analytics Reporting service object. - return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); + private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException { + return BetaAnalyticsDataSettings.newBuilder() + .setCredentialsProvider(FixedCredentialsProvider.create( + ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null))) + .build(); } - + /** - * Queries the Analytics Reporting API V4. + * Queries Analytics Data API service * - * @param service An authorized Analytics Reporting API V4 service object. - * @return GetReportResponse The Analytics Reporting API V4 response. + * @param service Analytics Data API service service settings. + * @return Row Analytics Data API service * @throws IOException */ - private static HashMap> getReportResponses(AnalyticsReporting service, - List viewIDs, DateRange dateRange) throws IOException { - - HashMap> reports = new HashMap<>(); - - // Create the Metrics object. - Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); - com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension().setName("ga:pagePath"); - - for(String view : viewIDs) { - List gReportResponses = new ArrayList<>(); - logger.info("Getting data from Google Analytics for viewid: " + view); - boolean iterateMorePages = true; - String nextPageToken = null; - while (iterateMorePages) { - // Create the ReportRequest object. - ReportRequest request = new ReportRequest().setViewId(view.trim()).setDateRanges(Arrays.asList(dateRange)) - .setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle)); - request.setPageSize(1000); - request.setPageToken(nextPageToken); - ArrayList requests = new ArrayList(); - requests.add(request); - // Create the GetReportsRequest object. - GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); - // Call the batchGet method. - GetReportsResponse response = service.reports().batchGet(getReport).execute(); - nextPageToken = response.getReports().get(0).getNextPageToken(); - iterateMorePages = (nextPageToken != null); - logger.debug("got nextPageToken: "+nextPageToken); + private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings, + List viewIDs, Builder dateRangeBuilder) throws IOException { + + HashMap> reports = new HashMap<>(); + + try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) { + + for(String propertyId : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId); + RunReportRequest request = + RunReportRequest.newBuilder() + .setProperty("properties/" + propertyId) + .addDimensions(Dimension.newBuilder().setName("pagePath")) + .addMetrics(Metric.newBuilder().setName("screenPageViews")) + .addDateRanges(dateRangeBuilder) + .build(); + + // Make the request. + RunReportResponse response = analyticsData.runReport(request); gReportResponses.add(response); + // Iterate through every row of the API response. +// for (Row row : response.getRowsList()) { +// System.out.printf( +// "%s, %s%n", row.getDimensionValues(0).getValue(), row.getMetricValues(0).getValue()); +// } + reports.put(propertyId, gReportResponses); } - reports.put(view, gReportResponses); } - // Return the response. return reports; } - + /** - * Parses and prints the Analytics Reporting API V4 response. + * Parses and prints the Analytics Data API service respose * - * @param response An Analytics Reporting API V4 response. + * @param response An Analytics Data API service response. */ - /** - * Parses and prints the Analytics Reporting API V4 response. - * - * @param response An Analytics Reporting API V4 response. - */ - private static List parseResponse(String viewId, List responses) { - logger.debug("parsing Response for " + viewId); + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for propertyID=" + viewId); List toReturn = new ArrayList<>(); - for (GetReportsResponse response : responses) { - for (Report report: response.getReports()) { - List rows = report.getData().getRows(); - if (rows == null) { - logger.warn("No data found for " + viewId); + for (RunReportResponse response : responses) { + for (Row row: response.getRowsList()) { + String dimension = row.getDimensionValues(0).getValue(); + String metric = row.getMetricValues(0).getValue(); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; } - else { - for (ReportRow row: rows) { - String dimension = row.getDimensions().get(0); - DateRangeValues metric = row.getMetrics().get(0); - VREAccessesReportRow var = new VREAccessesReportRow(); - boolean validEntry = false; - String pagePath = dimension; - if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { - var.setPagePath(dimension); - validEntry = true; - } - if (validEntry) { - var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); - toReturn.add(var); - } - } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric)); + toReturn.add(var); } } } return toReturn; } - - private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { - - String clientId = cred.getClientId(); - String clientEmail = cred.getClientEmail(); - String privateKeyPem = cred.getPrivateKeyPem(); - String privateKeyId = cred.getPrivateKeyId(); - String tokenUri = cred.getTokenUri(); - String projectId = cred.getProjectId(); - - if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { - throw new IOException("Error reading service account credential from stream, " - + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); - } - - PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); - - Collection emptyScopes = Collections.emptyList(); - - Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) - .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) - .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) - .setServiceAccountPrivateKeyId(privateKeyId); - - if(tokenUri != null) { - credentialBuilder.setTokenServerEncodedUrl(tokenUri); - } - - if(projectId != null) { - credentialBuilder.setServiceAccountProjectId(projectId); - } - - // Don't do a refresh at this point, as it will always fail before the scopes are added. - return credentialBuilder.build(); - } - - private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { - Reader reader = new StringReader(privateKeyPem); - Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); - if(section == null) { - throw new IOException("Invalid PKCS8 data."); - } - byte[] bytes = section.getBase64DecodedBytes(); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); - Exception unexpectedException = null; - try { - KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); - PrivateKey privateKey = keyFactory.generatePrivate(keySpec); - return privateKey; - } catch(NoSuchAlgorithmException exception) { - unexpectedException = exception; - } catch(InvalidKeySpecException exception) { - unexpectedException = exception; - } - throw new IOException("Unexpected exception reading PKCS data", unexpectedException); - } - + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception { String scope = infrastructureScope; @@ -319,14 +234,14 @@ public class VREAccessesHarvester extends BasicHarvester { ScopeProvider.instance.set(currScope); return toReturn; } - + /** * l * @throws Exception */ private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); - + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); try { List list = getAnalyticsReportingConfigurationFromIS(context); @@ -337,26 +252,26 @@ public class VREAccessesHarvester extends BasicHarvester { logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); } else { - + for(ServiceEndpoint res : list) { - reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); Group apGroup = res.profile().accessPoints(); AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); AccessPoint found = accessPoints[0]; - reportCredentials.setClientEmail(found.address()); - reportCredentials.setProjectId(found.username()); - reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + reportCredentials.setClientEmail(found.username()); + String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password()); + reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim()); + for(Property prop : found.properties()) { if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); String[] views = decryptedValue.split(";"); reportCredentials.setViewIds(Arrays.asList(views)); } - if(prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + if(prop.name().compareTo(AP_CLIENT_ID) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setClientId(decryptedValue); } - if(prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) { String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); reportCredentials.setPrivateKeyId(decryptedValue); } @@ -369,19 +284,18 @@ public class VREAccessesHarvester extends BasicHarvester { } return reportCredentials; } - + private static LocalDate asLocalDate(Date date) { return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); } - - private static DateRange getDateRangeForAnalytics(Date start, Date end) { + + private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics String startDate = asLocalDate(start).format(formatter); String endDate = asLocalDate(end).format(formatter); - DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` - dateRange.setStartDate(startDate); - dateRange.setEndDate(endDate); - return dateRange; + Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate); + + return dateRangeBuilder; } - + } diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java index f70e2ec..6dcdf2e 100644 --- a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java +++ b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java @@ -29,7 +29,7 @@ public class ContextAuthorization { public static final String CLIENT_ID = "accounting-dashboard-harvester-se-plugin"; - protected String clientSecret; + protected String clientSecret = "442b9d9a-1973-46f2-99f2-9a10fae94452"; /** * Contains Context full name as key and Token as Value @@ -78,7 +78,7 @@ public class ContextAuthorization { } private TokenResponse getJWTAccessToken(String context) throws Exception { - TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, getClientSecret(context), context, null); + TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, "442b9d9a-1973-46f2-99f2-9a10fae94452", context, null); return tr; } diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextTest.java b/src/main/java/org/gcube/dataharvest/utils/ContextTest.java new file mode 100644 index 0000000..1d2ece8 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/utils/ContextTest.java @@ -0,0 +1,80 @@ +/** + * + */ +package org.gcube.dataharvest.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import org.gcube.common.authorization.utils.manager.SecretManager; +import org.gcube.common.authorization.utils.manager.SecretManagerProvider; +import org.gcube.common.authorization.utils.secret.Secret; +import org.gcube.common.authorization.utils.secret.SecretUtility; + +/** + * @author Luca Frosini (ISTI - CNR) + */ +public class ContextTest { + + protected static Properties properties; + protected static final String PROPERTIES_FILENAME = "token.properties"; + + public static final String ROOT; + public static final String VO; + public static final String VRE; + + static { + properties = new Properties(); + InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); + + try { + // load the properties file + properties.load(input); + } catch(IOException e) { + throw new RuntimeException(e); + } + + // DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE"; + // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE"; + + //ROOT = "/gcube"; + ROOT = "/d4science.research-infrastructures.eu"; + VO = ROOT + "/devsec"; + VRE = VO + "/devVRE"; + // VO = ROOT + "/devNext"; + // VRE = VO + "/NextNext"; + + + } + + public static void set(Secret secret) throws Exception { + SecretManagerProvider.instance.reset(); + SecretManager secretManager = new SecretManager(); + SecretManagerProvider.instance.set(secretManager); + secretManager.addSecret(secret); + secretManager.set(); + } + + public static void setContext(String token) throws Exception { + Secret secret = getSecret(token); + set(secret); + } + + public static void setContextByName(String fullContextName) throws Exception { + Secret secret = getSecretByContextName(fullContextName); + set(secret); + } + + private static Secret getSecret(String token) throws Exception { + Secret secret = SecretUtility.getSecretByTokenString(token); + return secret; + } + + private static Secret getSecretByContextName(String fullContextName) throws Exception { + String token = ContextTest.properties.getProperty(fullContextName); + return getSecret(token); + } + + +} diff --git a/src/main/java/org/gcube/dataharvest/utils/TestMassi.java b/src/main/java/org/gcube/dataharvest/utils/TestMassi.java new file mode 100644 index 0000000..fef913b --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/utils/TestMassi.java @@ -0,0 +1,129 @@ +package org.gcube.dataharvest.utils; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; +import org.gcube.dataharvest.harvester.VREAccessesHarvester; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestMassi extends ContextTest { + private static Logger logger = LoggerFactory.getLogger(TestMassi.class); + + public static final String ROOT = "/d4science.research-infrastructures.eu"; + public TestMassi() { + // TODO Auto-generated constructor stub + } + + public static void main(String[] args) { + try { + + ContextTest.setContextByName(ROOT); + // AccountingDao dao = AccountingDao.get(); + + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); + Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); + AccountingDashboardHarvesterPlugin.getProperties().set(properties); + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + + + AggregationType aggregationType = AggregationType.MONTHLY; + + Calendar from = DateUtils.getStartCalendar(2023, Calendar.APRIL, 1); + + Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1); + + while (from.before(runbeforeDate)) { + Date start = from.getTime(); + Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); + + logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); + + ArrayList accountingRecords = new ArrayList<>(); + + VREAccessesHarvester vreAccessesHarvester = null; + + for (String context : contexts) { + // Setting the token for the context + ContextTest.set(contextAuthorization.getSecretForContext(context)); + logger.debug("Context set: {} ", context); + ScopeBean scopeBean = new ScopeBean(context); + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { + vreAccessesHarvester = new VREAccessesHarvester(start, end); + } else { + // This code should be never used because the scopes + // are sorted by fullname + + ScopeBean parent = scopeBean.enclosingScope(); + while (!parent.is(Type.INFRASTRUCTURE)) { + parent = scopeBean.enclosingScope(); + } + + // Setting back token for the context + ContextTest.set(contextAuthorization.getSecretForContext(parent.toString())); + + vreAccessesHarvester = new VREAccessesHarvester(start, end); + + // Setting back token for the context + ContextTest.set(contextAuthorization.getSecretForContext(context)); + } + + } + + try { + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { + logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, + DateUtils.format(start), DateUtils.format(end)); + } else { + // Collecting Google Analytics Data for VREs + // Accesses + List harvested = vreAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + } + } catch (Exception e) { + logger.error("Error harvesting Social Interactions for {}", context, e); + } + } + + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), + accountingRecords); + ContextTest.setContextByName(ROOT); + + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + Thread.sleep(TimeUnit.SECONDS.toMillis(10)); + + from.add(aggregationType.getCalendarField(), 1); + + } + + ContextTest.setContextByName(ROOT); + + } catch (Exception e) { + logger.error("", e); + } + + logger.info("End."); + + } +} diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index fa5275c..ea9bcc0 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -214,8 +214,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - @Ignore - // @Test + // @Ignore + @Test public void launchOldDataVREAccessesHarvester() { try { @@ -231,9 +231,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { AggregationType aggregationType = AggregationType.MONTHLY; - Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1); + Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1); - Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); + Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1); while (from.before(runbeforeDate)) { Date start = from.getTime(); diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java index a90a77f..3a7e84b 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java @@ -40,12 +40,14 @@ public class ContextTest { // DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE"; // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE"; - ROOT = "/gcube"; + //ROOT = "/gcube"; + ROOT = "/d4science.research-infrastructures.eu"; VO = ROOT + "/devsec"; VRE = VO + "/devVRE"; // VO = ROOT + "/devNext"; // VRE = VO + "/NextNext"; + } public static void set(Secret secret) throws Exception { @@ -78,7 +80,7 @@ public class ContextTest { @BeforeClass public static void beforeClass() throws Exception { - setContextByName(VRE); + setContextByName(ROOT); } @AfterClass