diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3dc950e..b0ceaaa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for "accounting-dashboard-harvester-se-plugin"
+## [v2.3.0-SNAPSHOT]
+
+- Ported GA harvesters to GA4
+
## [v2.2.0]
- Switching security to the new IAM [#21904]
diff --git a/pom.xml b/pom.xml
index ecb4d6b..ddc1c6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,7 +12,7 @@
4.0.0
org.gcube.accounting
accounting-dashboard-harvester-se-plugin
- 2.2.0
+ 2.3.0-SNAPSHOT
Accounting Dashboard Harvester Smart Executor Plugin
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
@@ -57,27 +57,48 @@
com.google.apis
google-api-services-analyticsreporting
v4-rev124-1.23.0
+
+
+ com.google.guava
+ guava-jdk5
+
+
- com.fasterxml.jackson.core
- jackson-core
- 2.8.4
+ com.google.analytics
+ google-analytics-data
+ 0.16.0
+
+
+ com.google.guava
+ guava
+ 23.6-jre
+
+
+ com.google.auth
+ google-auth-library-oauth2-http
+ 1.12.1
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+ 2.8.4
com.google.http-client
google-http-client-gson
1.21.0
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
org.gcube.data-catalogue
gcat-client
@@ -118,8 +139,8 @@
- org.gcube.social-networking
- social-service-client
+ org.gcube.social-networking
+ social-service-client
[1.0.0, 2.0.0-SNAPSHOT)
@@ -134,7 +155,8 @@
[2.0.0, 3.0.0-SNAPSHOT)
-
+
org.gcube.vremanagement
smart-executor-client
@@ -197,6 +219,24 @@
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+ -Xdoclint:none
+ -Xdoclint:none
+
+ 3.1.0
+
+
+ generate-doc
+ install
+
+ jar
+
+
+
+
diff --git a/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java
index 3dc23b8..e74511c 100644
--- a/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java
+++ b/src/main/java/org/gcube/dataharvest/datamodel/AnalyticsReportCredentials.java
@@ -59,12 +59,12 @@ public class AnalyticsReportCredentials {
/**
* Please note:
* The key is stored in the resource with blanks " " instead of "\n" as it causes issues and
- * without the BEGIN and END Delimiters (e.g. -----END PRIVATE KEY-----) which myst be readded
+ * without the BEGIN and END Delimiters (e.g. -----END PRIVATE KEY-----) which must be readded
* @param privateKeyPem
*/
public void setPrivateKeyPem(String privateKeyPem) {
- privateKeyPem = privateKeyPem.replace(" ", "\n");
- this.privateKeyPem = "-----BEGIN PRIVATE KEY-----\n"+privateKeyPem+"\n-----END PRIVATE KEY-----";
+ this.privateKeyPem = privateKeyPem.replace(" ", "\n");
+ this.privateKeyPem = "-----BEGIN PRIVATE KEY-----\n"+this.privateKeyPem+"\n-----END PRIVATE KEY-----\n";
}
public String getPrivateKeyId() {
diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java
index 3086ba0..82013eb 100644
--- a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java
+++ b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java
@@ -4,22 +4,13 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.security.GeneralSecurityException;
-import java.security.KeyFactory;
-import java.security.NoSuchAlgorithmException;
-import java.security.PrivateKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -42,56 +33,50 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.googleapis.util.Utils;
-import com.google.api.client.http.HttpTransport;
+import com.google.analytics.data.v1beta.BetaAnalyticsDataClient;
+import com.google.analytics.data.v1beta.BetaAnalyticsDataSettings;
+import com.google.analytics.data.v1beta.DateRange;
+import com.google.analytics.data.v1beta.DateRange.Builder;
+import com.google.analytics.data.v1beta.Dimension;
+import com.google.analytics.data.v1beta.Metric;
+import com.google.analytics.data.v1beta.Row;
+import com.google.analytics.data.v1beta.RunReportRequest;
+import com.google.analytics.data.v1beta.RunReportResponse;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.gson.GsonFactory;
-import com.google.api.client.util.PemReader;
-import com.google.api.client.util.PemReader.Section;
-import com.google.api.client.util.SecurityUtils;
-import com.google.api.services.analyticsreporting.v4.AnalyticsReporting;
-import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes;
-import com.google.api.services.analyticsreporting.v4.model.DateRange;
-import com.google.api.services.analyticsreporting.v4.model.DateRangeValues;
-import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest;
-import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse;
-import com.google.api.services.analyticsreporting.v4.model.Metric;
-import com.google.api.services.analyticsreporting.v4.model.Report;
-import com.google.api.services.analyticsreporting.v4.model.ReportRequest;
-import com.google.api.services.analyticsreporting.v4.model.ReportRow;
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.auth.oauth2.ServiceAccountCredentials;
+
public class VREAccessesHarvester extends BasicHarvester {
-
+
private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class);
-
+
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
-
+
private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService";
- private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService";
+ private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService";
private static final String AP_VIEWS_PROPERTY = "views";
- private static final String AP_CLIENT_PROPERTY = "clientId";
- private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId";
- private static final String APPLICATION_NAME = "Analytics Reporting";
-
+ private static final String AP_CLIENT_ID = "client_id";
+ private static final String AP_PRIVATEKEY_ID_PROPERTY = "private_key_id";
+
+
private List vreAccesses;
-
+
public VREAccessesHarvester(Date start, Date end) throws Exception {
super(start, end);
vreAccesses = getAllAccesses(start, end);
}
-
+
@Override
public List getAccountingRecords() throws Exception {
try {
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
-
+
ArrayList accountingRecords = new ArrayList();
-
+
int measure = 0;
-
+
ScopeBean scopeBean = new ScopeBean(context);
String lowerCasedContext = scopeBean.name().toLowerCase();
String case1 = lowerCasedContext + "/";
@@ -100,28 +85,28 @@ public class VREAccessesHarvester extends BasicHarvester {
String pagePath = row.getPagePath();
if (!pagePath.contains("_redirect=/group") && !pagePath.contains("workspace")) {
if(pagePath.endsWith(lowerCasedContext)) {
- logger.trace("Matched endsWith({}) : {}", lowerCasedContext, pagePath);
+ logger.debug("Matched endsWith({}) : {}", lowerCasedContext, pagePath);
measure += row.getVisitNumber();
} else if(pagePath.contains(case1) || pagePath.contains(case2)) {
- logger.trace("Matched contains({}) || contains({}) : {}", case1, case2, pagePath);
+ logger.debug("Matched contains({}) || contains({}) : {}", case1, case2, pagePath);
measure += row.getVisitNumber();
}
}
}
-
+
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor();
-
+
AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.ACCESSES), (long) measure);
logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure());
accountingRecords.add(ar);
-
+
return accountingRecords;
-
+
} catch(Exception e) {
throw e;
}
}
-
+
/**
*
* @return a list of {@link VREAccessesReportRow} objects containing the pagePath and the visit number e.g.
@@ -130,182 +115,112 @@ public class VREAccessesHarvester extends BasicHarvester {
* VREAccessesReportRow [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, visitNumber=39]
*/
private static List getAllAccesses(Date start, Date end) throws Exception {
- DateRange dateRange = getDateRangeForAnalytics(start, end);
- logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString());
-
+ Builder dateRangeBuilder = getDateRangeBuilderForAnalytics(start, end);
+ logger.debug("Getting accesses in this time range {}", dateRangeBuilder.toString());
+
AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
- AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
- HashMap> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
- List totalAccesses = new ArrayList<>();
+ logger.debug("Getting credentials credentialsFromD4S");
+
+ BetaAnalyticsDataSettings serviceSettings = initializeAnalyticsReporting(credentialsFromD4S);
+
+ logger.debug("initializeAnalyticsReporting service settings");
+
+
+ HashMap> responses = getReportResponses(serviceSettings, credentialsFromD4S.getViewIds(), dateRangeBuilder);
+ List totalAccesses = new ArrayList<>();
+
for(String view : responses.keySet()) {
List viewReport = parseResponse(view, responses.get(view));
- logger.trace("Got {} entries from view id={}", viewReport.size(), view);
+ logger.debug("Got {} entries from view id={}", viewReport.size(), view);
totalAccesses.addAll(viewReport);
}
- logger.trace("Merged in {} total entries from all views", totalAccesses.size());
+ logger.debug("Merged in {} total entries from all views", totalAccesses.size());
return totalAccesses;
}
-
+
/**
- * Initializes an Analytics Reporting API V4 service object.
+ * Initializes an Google Analytics Data API service object.
*
- * @return An authorized Analytics Reporting API V4 service object.
+ * @return An authorized Google Analytics Data API
* @throws IOException
* @throws GeneralSecurityException
*/
- private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred)
- throws GeneralSecurityException, IOException {
- HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
- GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all());
-
- // Construct the Analytics Reporting service object.
- return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential)
- .setApplicationName(APPLICATION_NAME).build();
+ private static BetaAnalyticsDataSettings initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws IOException {
+ return BetaAnalyticsDataSettings.newBuilder()
+ .setCredentialsProvider(FixedCredentialsProvider.create(
+ ServiceAccountCredentials.fromPkcs8(cred.getClientId(), cred.getClientEmail(), cred.getPrivateKeyPem(), cred.getPrivateKeyId(), null)))
+ .build();
}
-
+
/**
- * Queries the Analytics Reporting API V4.
+ * Queries Analytics Data API service
*
- * @param service An authorized Analytics Reporting API V4 service object.
- * @return GetReportResponse The Analytics Reporting API V4 response.
+ * @param service Analytics Data API service service settings.
+ * @return Row Analytics Data API service
* @throws IOException
*/
- private static HashMap> getReportResponses(AnalyticsReporting service,
- List viewIDs, DateRange dateRange) throws IOException {
-
- HashMap> reports = new HashMap<>();
-
- // Create the Metrics object.
- Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages");
- com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension().setName("ga:pagePath");
-
- for(String view : viewIDs) {
- List gReportResponses = new ArrayList<>();
- logger.info("Getting data from Google Analytics for viewid: " + view);
- boolean iterateMorePages = true;
- String nextPageToken = null;
- while (iterateMorePages) {
- // Create the ReportRequest object.
- ReportRequest request = new ReportRequest().setViewId(view.trim()).setDateRanges(Arrays.asList(dateRange))
- .setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle));
- request.setPageSize(1000);
- request.setPageToken(nextPageToken);
- ArrayList requests = new ArrayList();
- requests.add(request);
- // Create the GetReportsRequest object.
- GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests);
- // Call the batchGet method.
- GetReportsResponse response = service.reports().batchGet(getReport).execute();
- nextPageToken = response.getReports().get(0).getNextPageToken();
- iterateMorePages = (nextPageToken != null);
- logger.debug("got nextPageToken: "+nextPageToken);
+ private static HashMap> getReportResponses(BetaAnalyticsDataSettings betaAnalyticsDataSettings,
+ List viewIDs, Builder dateRangeBuilder) throws IOException {
+
+ HashMap> reports = new HashMap<>();
+
+ try (BetaAnalyticsDataClient analyticsData = BetaAnalyticsDataClient.create(betaAnalyticsDataSettings)) {
+
+ for(String propertyId : viewIDs) {
+ List gReportResponses = new ArrayList<>();
+ logger.debug("Getting data from Analytics Data API for propertyId: " + propertyId);
+ RunReportRequest request =
+ RunReportRequest.newBuilder()
+ .setProperty("properties/" + propertyId)
+ .addDimensions(Dimension.newBuilder().setName("pagePath"))
+ .addMetrics(Metric.newBuilder().setName("screenPageViews"))
+ .addDateRanges(dateRangeBuilder)
+ .build();
+
+ // Make the request.
+ RunReportResponse response = analyticsData.runReport(request);
gReportResponses.add(response);
+ // Iterate through every row of the API response.
+// for (Row row : response.getRowsList()) {
+// System.out.printf(
+// "%s, %s%n", row.getDimensionValues(0).getValue(), row.getMetricValues(0).getValue());
+// }
+ reports.put(propertyId, gReportResponses);
}
- reports.put(view, gReportResponses);
}
- // Return the response.
return reports;
}
-
+
/**
- * Parses and prints the Analytics Reporting API V4 response.
+ * Parses and prints the Analytics Data API service respose
*
- * @param response An Analytics Reporting API V4 response.
+ * @param response An Analytics Data API service response.
*/
- /**
- * Parses and prints the Analytics Reporting API V4 response.
- *
- * @param response An Analytics Reporting API V4 response.
- */
- private static List parseResponse(String viewId, List responses) {
- logger.debug("parsing Response for " + viewId);
+ private static List parseResponse(String viewId, List responses) {
+ logger.debug("parsing Response for propertyID=" + viewId);
List toReturn = new ArrayList<>();
- for (GetReportsResponse response : responses) {
- for (Report report: response.getReports()) {
- List rows = report.getData().getRows();
- if (rows == null) {
- logger.warn("No data found for " + viewId);
+ for (RunReportResponse response : responses) {
+ for (Row row: response.getRowsList()) {
+ String dimension = row.getDimensionValues(0).getValue();
+ String metric = row.getMetricValues(0).getValue();
+ VREAccessesReportRow var = new VREAccessesReportRow();
+ boolean validEntry = false;
+ String pagePath = dimension;
+ if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) {
+ var.setPagePath(dimension);
+ validEntry = true;
}
- else {
- for (ReportRow row: rows) {
- String dimension = row.getDimensions().get(0);
- DateRangeValues metric = row.getMetrics().get(0);
- VREAccessesReportRow var = new VREAccessesReportRow();
- boolean validEntry = false;
- String pagePath = dimension;
- if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) {
- var.setPagePath(dimension);
- validEntry = true;
- }
- if (validEntry) {
- var.setVisitNumber(Integer.parseInt(metric.getValues().get(0)));
- toReturn.add(var);
- }
- }
+ if (validEntry) {
+ var.setVisitNumber(Integer.parseInt(metric));
+ toReturn.add(var);
}
}
}
return toReturn;
}
-
- private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException {
-
- String clientId = cred.getClientId();
- String clientEmail = cred.getClientEmail();
- String privateKeyPem = cred.getPrivateKeyPem();
- String privateKeyId = cred.getPrivateKeyId();
- String tokenUri = cred.getTokenUri();
- String projectId = cred.getProjectId();
-
- if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) {
- throw new IOException("Error reading service account credential from stream, "
- + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'.");
- }
-
- PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem);
-
- Collection emptyScopes = Collections.emptyList();
-
- Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport())
- .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail)
- .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey)
- .setServiceAccountPrivateKeyId(privateKeyId);
-
- if(tokenUri != null) {
- credentialBuilder.setTokenServerEncodedUrl(tokenUri);
- }
-
- if(projectId != null) {
- credentialBuilder.setServiceAccountProjectId(projectId);
- }
-
- // Don't do a refresh at this point, as it will always fail before the scopes are added.
- return credentialBuilder.build();
- }
-
- private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException {
- Reader reader = new StringReader(privateKeyPem);
- Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY");
- if(section == null) {
- throw new IOException("Invalid PKCS8 data.");
- }
- byte[] bytes = section.getBase64DecodedBytes();
- PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes);
- Exception unexpectedException = null;
- try {
- KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory();
- PrivateKey privateKey = keyFactory.generatePrivate(keySpec);
- return privateKey;
- } catch(NoSuchAlgorithmException exception) {
- unexpectedException = exception;
- } catch(InvalidKeySpecException exception) {
- unexpectedException = exception;
- }
- throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
- }
-
+
private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope)
throws Exception {
String scope = infrastructureScope;
@@ -319,14 +234,14 @@ public class VREAccessesHarvester extends BasicHarvester {
ScopeProvider.instance.set(currScope);
return toReturn;
}
-
+
/**
* l
* @throws Exception
*/
private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception {
AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials();
-
+
String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
try {
List list = getAnalyticsReportingConfigurationFromIS(context);
@@ -337,26 +252,26 @@ public class VREAccessesHarvester extends BasicHarvester {
logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category "
+ SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
} else {
-
+
for(ServiceEndpoint res : list) {
- reportCredentials.setTokenUri(res.profile().runtime().hostedOn());
Group apGroup = res.profile().accessPoints();
AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]);
AccessPoint found = accessPoints[0];
- reportCredentials.setClientEmail(found.address());
- reportCredentials.setProjectId(found.username());
- reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password()));
+ reportCredentials.setClientEmail(found.username());
+ String decryptedPrivateKey = StringEncrypter.getEncrypter().decrypt(found.password());
+ reportCredentials.setPrivateKeyPem(decryptedPrivateKey.trim());
+
for(Property prop : found.properties()) {
if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) {
String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
String[] views = decryptedValue.split(";");
reportCredentials.setViewIds(Arrays.asList(views));
}
- if(prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) {
+ if(prop.name().compareTo(AP_CLIENT_ID) == 0) {
String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
reportCredentials.setClientId(decryptedValue);
}
- if(prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) {
+ if(prop.name().compareTo(AP_PRIVATEKEY_ID_PROPERTY) == 0) {
String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
reportCredentials.setPrivateKeyId(decryptedValue);
}
@@ -369,19 +284,18 @@ public class VREAccessesHarvester extends BasicHarvester {
}
return reportCredentials;
}
-
+
private static LocalDate asLocalDate(Date date) {
return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate();
}
-
- private static DateRange getDateRangeForAnalytics(Date start, Date end) {
+
+ private static Builder getDateRangeBuilderForAnalytics(Date start, Date end) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics
String startDate = asLocalDate(start).format(formatter);
String endDate = asLocalDate(end).format(formatter);
- DateRange dateRange = new DateRange();// date format `yyyy-MM-dd`
- dateRange.setStartDate(startDate);
- dateRange.setEndDate(endDate);
- return dateRange;
+ Builder dateRangeBuilder = DateRange.newBuilder().setStartDate(startDate).setEndDate(endDate);
+
+ return dateRangeBuilder;
}
-
+
}
diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
index f70e2ec..6dcdf2e 100644
--- a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
+++ b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
@@ -29,7 +29,7 @@ public class ContextAuthorization {
public static final String CLIENT_ID = "accounting-dashboard-harvester-se-plugin";
- protected String clientSecret;
+ protected String clientSecret = "442b9d9a-1973-46f2-99f2-9a10fae94452";
/**
* Contains Context full name as key and Token as Value
@@ -78,7 +78,7 @@ public class ContextAuthorization {
}
private TokenResponse getJWTAccessToken(String context) throws Exception {
- TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, getClientSecret(context), context, null);
+ TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, "442b9d9a-1973-46f2-99f2-9a10fae94452", context, null);
return tr;
}
diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextTest.java b/src/main/java/org/gcube/dataharvest/utils/ContextTest.java
new file mode 100644
index 0000000..1d2ece8
--- /dev/null
+++ b/src/main/java/org/gcube/dataharvest/utils/ContextTest.java
@@ -0,0 +1,80 @@
+/**
+ *
+ */
+package org.gcube.dataharvest.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.gcube.common.authorization.utils.manager.SecretManager;
+import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
+import org.gcube.common.authorization.utils.secret.Secret;
+import org.gcube.common.authorization.utils.secret.SecretUtility;
+
+/**
+ * @author Luca Frosini (ISTI - CNR)
+ */
+public class ContextTest {
+
+ protected static Properties properties;
+ protected static final String PROPERTIES_FILENAME = "token.properties";
+
+ public static final String ROOT;
+ public static final String VO;
+ public static final String VRE;
+
+ static {
+ properties = new Properties();
+ InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME);
+
+ try {
+ // load the properties file
+ properties.load(input);
+ } catch(IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE";
+ // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE";
+
+ //ROOT = "/gcube";
+ ROOT = "/d4science.research-infrastructures.eu";
+ VO = ROOT + "/devsec";
+ VRE = VO + "/devVRE";
+ // VO = ROOT + "/devNext";
+ // VRE = VO + "/NextNext";
+
+
+ }
+
+ public static void set(Secret secret) throws Exception {
+ SecretManagerProvider.instance.reset();
+ SecretManager secretManager = new SecretManager();
+ SecretManagerProvider.instance.set(secretManager);
+ secretManager.addSecret(secret);
+ secretManager.set();
+ }
+
+ public static void setContext(String token) throws Exception {
+ Secret secret = getSecret(token);
+ set(secret);
+ }
+
+ public static void setContextByName(String fullContextName) throws Exception {
+ Secret secret = getSecretByContextName(fullContextName);
+ set(secret);
+ }
+
+ private static Secret getSecret(String token) throws Exception {
+ Secret secret = SecretUtility.getSecretByTokenString(token);
+ return secret;
+ }
+
+ private static Secret getSecretByContextName(String fullContextName) throws Exception {
+ String token = ContextTest.properties.getProperty(fullContextName);
+ return getSecret(token);
+ }
+
+
+}
diff --git a/src/main/java/org/gcube/dataharvest/utils/TestMassi.java b/src/main/java/org/gcube/dataharvest/utils/TestMassi.java
new file mode 100644
index 0000000..fef913b
--- /dev/null
+++ b/src/main/java/org/gcube/dataharvest/utils/TestMassi.java
@@ -0,0 +1,129 @@
+package org.gcube.dataharvest.utils;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
+import org.gcube.common.scope.impl.ScopeBean;
+import org.gcube.common.scope.impl.ScopeBean.Type;
+import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
+import org.gcube.dataharvest.harvester.VREAccessesHarvester;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestMassi extends ContextTest {
+ private static Logger logger = LoggerFactory.getLogger(TestMassi.class);
+
+ public static final String ROOT = "/d4science.research-infrastructures.eu";
+ public TestMassi() {
+ // TODO Auto-generated constructor stub
+ }
+
+ public static void main(String[] args) {
+ try {
+
+ ContextTest.setContextByName(ROOT);
+ // AccountingDao dao = AccountingDao.get();
+
+ AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
+ Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
+ AccountingDashboardHarvesterPlugin.getProperties().set(properties);
+
+ ContextAuthorization contextAuthorization = new ContextAuthorization();
+ SortedSet contexts = contextAuthorization.getContexts();
+
+
+ AggregationType aggregationType = AggregationType.MONTHLY;
+
+ Calendar from = DateUtils.getStartCalendar(2023, Calendar.APRIL, 1);
+
+ Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1);
+
+ while (from.before(runbeforeDate)) {
+ Date start = from.getTime();
+ Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
+
+ logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
+
+ ArrayList accountingRecords = new ArrayList<>();
+
+ VREAccessesHarvester vreAccessesHarvester = null;
+
+ for (String context : contexts) {
+ // Setting the token for the context
+ ContextTest.set(contextAuthorization.getSecretForContext(context));
+ logger.debug("Context set: {} ", context);
+ ScopeBean scopeBean = new ScopeBean(context);
+
+ if (vreAccessesHarvester == null) {
+
+ if (scopeBean.is(Type.INFRASTRUCTURE)) {
+ vreAccessesHarvester = new VREAccessesHarvester(start, end);
+ } else {
+ // This code should be never used because the scopes
+ // are sorted by fullname
+
+ ScopeBean parent = scopeBean.enclosingScope();
+ while (!parent.is(Type.INFRASTRUCTURE)) {
+ parent = scopeBean.enclosingScope();
+ }
+
+ // Setting back token for the context
+ ContextTest.set(contextAuthorization.getSecretForContext(parent.toString()));
+
+ vreAccessesHarvester = new VREAccessesHarvester(start, end);
+
+ // Setting back token for the context
+ ContextTest.set(contextAuthorization.getSecretForContext(context));
+ }
+
+ }
+
+ try {
+ if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO)
+ && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
+ logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context,
+ DateUtils.format(start), DateUtils.format(end));
+ } else {
+ // Collecting Google Analytics Data for VREs
+ // Accesses
+ List harvested = vreAccessesHarvester.getAccountingRecords();
+ accountingRecords.addAll(harvested);
+
+ }
+ } catch (Exception e) {
+ logger.error("Error harvesting Social Interactions for {}", context, e);
+ }
+ }
+
+ logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
+ accountingRecords);
+ ContextTest.setContextByName(ROOT);
+
+ // dao.insertRecords(accountingRecords.toArray(new
+ // AccountingRecord[1]));
+
+ Thread.sleep(TimeUnit.SECONDS.toMillis(10));
+
+ from.add(aggregationType.getCalendarField(), 1);
+
+ }
+
+ ContextTest.setContextByName(ROOT);
+
+ } catch (Exception e) {
+ logger.error("", e);
+ }
+
+ logger.info("End.");
+
+ }
+}
diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
index fa5275c..ea9bcc0 100644
--- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
+++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
@@ -214,8 +214,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
- @Ignore
- // @Test
+ // @Ignore
+ @Test
public void launchOldDataVREAccessesHarvester() {
try {
@@ -231,9 +231,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AggregationType aggregationType = AggregationType.MONTHLY;
- Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1);
+ Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
- Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
+ Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1);
while (from.before(runbeforeDate)) {
Date start = from.getTime();
diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java
index a90a77f..3a7e84b 100644
--- a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java
+++ b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java
@@ -40,12 +40,14 @@ public class ContextTest {
// DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE";
// DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE";
- ROOT = "/gcube";
+ //ROOT = "/gcube";
+ ROOT = "/d4science.research-infrastructures.eu";
VO = ROOT + "/devsec";
VRE = VO + "/devVRE";
// VO = ROOT + "/devNext";
// VRE = VO + "/NextNext";
+
}
public static void set(Secret secret) throws Exception {
@@ -78,7 +80,7 @@ public class ContextTest {
@BeforeClass
public static void beforeClass() throws Exception {
- setContextByName(VRE);
+ setContextByName(ROOT);
}
@AfterClass