From 87f55941090bfbb6e269870f55f569618cfebbc0 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 3 Jun 2021 18:03:15 +0200 Subject: [PATCH 1/2] ref 21557: RStudio sessions to be published into the accounting Added R Studio harvester --- .../AccountingDashboardHarvesterPlugin.java | 50 ++- .../datamodel/HarvestedDataKey.java | 1 + .../harvester/JupyterAccessesHarvester.java | 12 +- .../harvester/RStudioAccessesHarvester.java | 409 ++++++++++++++++++ .../java/org/gcube/dataharvest/.gitignore | 1 + .../AccountingDataHarvesterJupyterTest.java | 3 +- .../AccountingDataHarvesterRStudioTest.java | 139 ++++++ 7 files changed, 599 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java create mode 100644 src/test/java/org/gcube/dataharvest/.gitignore create mode 100644 src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index 548972f..e4c33ea 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -23,6 +23,7 @@ import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.RStudioAccessesHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; @@ -241,7 +242,9 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { VREAccessesHarvester vreAccessesHarvester = null; JupyterAccessesHarvester jupyterAccessesHarvester = null; + RStudioAccessesHarvester rstudioAccessesHarvester = null; + for (String context : contexts) { // Setting the token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); @@ -296,6 +299,30 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { } + if (rstudioAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { + rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end); + } else { + // This code should be never used because the scopes are + // sorted by fullname + + ScopeBean parent = scopeBean.enclosingScope(); + while (!parent.is(Type.INFRASTRUCTURE)) { + parent = scopeBean.enclosingScope(); + } + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); + + rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end); + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(context)); + } + + } + if (jupyterAccessesHarvester == null) { if (scopeBean.is(Type.INFRASTRUCTURE)) { @@ -341,7 +368,19 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { } catch (Exception e) { logger.error("Error harvesting VRE Accesses for {}", context, e); } + + try { + // Collecting Google Analytics Data for R Studio Accesses + logger.info("Going to harvest R Studio Accesses for {}", context); + List harvested = rstudioAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + } catch (Exception e) { + logger.error("Error harvesting R Studio Accesses for {}", context, e); + } + + try { // Collecting Google Analytics Data for Jupyters Accesses logger.info("Going to harvest Jupyter Accesses for {}", context); @@ -349,13 +388,8 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { List harvested = jupyterAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - /* - * List harvested = - * jupyterAccessesHarvester.getData(); - * data.addAll(harvested); - */ } catch (Exception e) { - logger.error("Error harvesting VRE Accesses for {}", context, e); + logger.error("Error harvesting Jupyeter Accesses for {}", context, e); } try { @@ -366,10 +400,6 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { List harvested = socialHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - /* - * List harvested = - * socialHarvester.getData(); data.addAll(harvested); - */ } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } diff --git a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java index 06331ea..b5f0035 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java @@ -16,6 +16,7 @@ public enum HarvestedDataKey { NOTIFICATIONS_ACCESSES("Notifications Accesses"), PROFILE_ACCESSES("Profile Accesses"), JUPYTER_ACCESSES("Jupyter Accesses"), + RSTUDIO_ACCESSES("R Studio Accesses"), CATALOGUE_ACCESSES("Catalogue Accesses"), CATALOGUE_DATASET_LIST_ACCESSES("Item List"), diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index d2d0391..f51e7da 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -117,11 +117,13 @@ public class JupyterAccessesHarvester extends BasicHarvester { ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); - AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, - getDimension(HarvestedDataKey.JUPYTER_ACCESSES), (long) measure); - logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); - accountingRecords.add(ar); - + if (measure > 0) { + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, + getDimension(HarvestedDataKey.JUPYTER_ACCESSES), (long) measure); + logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); + accountingRecords.add(ar); + } + return accountingRecords; } catch (Exception e) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java new file mode 100644 index 0000000..c1da4f6 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java @@ -0,0 +1,409 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.encryption.encrypter.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; +import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; +import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.dataharvest.datamodel.VREAccessesReportRow; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.PemReader; +import com.google.api.client.util.PemReader.Section; +import com.google.api.client.util.SecurityUtils; +import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; +import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; +import com.google.api.services.analyticsreporting.v4.model.DateRange; +import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; +import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; +import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; +import com.google.api.services.analyticsreporting.v4.model.Metric; +import com.google.api.services.analyticsreporting.v4.model.Report; +import com.google.api.services.analyticsreporting.v4.model.ReportRequest; +import com.google.api.services.analyticsreporting.v4.model.ReportRow; + +/** + * + * @author Giancarlo Panichi (ISTI CNR) + * + */ +public class RStudioAccessesHarvester extends BasicHarvester { + + private static Logger logger = LoggerFactory.getLogger(RStudioAccessesHarvester.class); + + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; + private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String AP_VIEWS_PROPERTY = "views"; + private static final String AP_CLIENT_PROPERTY = "clientId"; + private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; + private static final String APPLICATION_NAME = "Analytics Reporting"; + + private List vreAccesses; + + public RStudioAccessesHarvester(Date start, Date end) throws Exception { + super(start, end); + logger.debug("RStudioAccessHArvester: {}, {}", start, end); + vreAccesses = getAllAccesses(start, end); + } + + @Override + public List getAccountingRecords() throws Exception { + try { + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + + ArrayList accountingRecords = new ArrayList(); + + int measure = 0; + + ScopeBean scopeBean = new ScopeBean(context); + String lowerCasedContext = scopeBean.name().toLowerCase(); + logger.debug("RStudioAccessHArvester lowerCasedContext: {}", lowerCasedContext); + for (VREAccessesReportRow row : vreAccesses) { + String pagePath = row.getPagePath().toLowerCase(); + if (pagePath != null && !pagePath.isEmpty()) { + if (pagePath.contains(lowerCasedContext)) { + if (!pagePath.contains("catalogue")) { + if (pagePath.contains("rstudio") || pagePath.contains("r-studio")) { + logger.trace("Matched rstudio or rstudio ({}) : {}", lowerCasedContext, pagePath); + measure += row.getVisitNumber(); + } + } + } + } + } + + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + + if (measure > 0) { + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, + getDimension(HarvestedDataKey.RSTUDIO_ACCESSES), (long) measure); + logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); + accountingRecords.add(ar); + } + + return accountingRecords; + + } catch (Exception e) { + throw e; + } + } + + /** + * + * @return a list of {@link VREAccessesReportRow} objects containing the + * pagePath and the visit number e.g. VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/add-new-users, + * visitNumber=1] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/administration, + * visitNumber=2] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, + * visitNumber=39] + */ + private static List getAllAccesses(Date start, Date end) throws Exception { + DateRange dateRange = getDateRangeForAnalytics(start, end); + logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); + + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); + AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + HashMap> responses = getReportResponses(service, + credentialsFromD4S.getViewIds(), dateRange); + List totalAccesses = new ArrayList<>(); + + for (String view : responses.keySet()) { + List viewReport = parseResponse(view, responses.get(view)); + logger.trace("Got {} entries from view id={}", viewReport.size(), view); + totalAccesses.addAll(viewReport); + } + logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + return totalAccesses; + } + + /** + * Initializes an Analytics Reporting API V4 service object. + * + * @return An authorized Analytics Reporting API V4 service object. + * @throws IOException + * @throws GeneralSecurityException + */ + private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) + throws GeneralSecurityException, IOException { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); + + // Construct the Analytics Reporting service object. + return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + } + + /** + * Queries the Analytics Reporting API V4. + * + * @param service + * An authorized Analytics Reporting API V4 service object. + * @return GetReportResponse The Analytics Reporting API V4 response. + * @throws IOException + */ + private static HashMap> getReportResponses(AnalyticsReporting service, + List viewIDs, DateRange dateRange) throws IOException { + + HashMap> reports = new HashMap<>(); + + // Create the Metrics object. + Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); + com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension() + .setName("ga:pagePath"); + + for (String view : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.info("Getting data from Google Analytics for viewid: " + view); + boolean iterateMorePages = true; + String nextPageToken = null; + while (iterateMorePages) { + // Create the ReportRequest object. + ReportRequest request = new ReportRequest().setViewId(view.trim()) + .setDateRanges(Arrays.asList(dateRange)).setMetrics(Arrays.asList(sessions)) + .setDimensions(Arrays.asList(pageTitle)); + request.setPageSize(1000); + request.setPageToken(nextPageToken); + ArrayList requests = new ArrayList(); + requests.add(request); + // Create the GetReportsRequest object. + GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); + // Call the batchGet method. + GetReportsResponse response = service.reports().batchGet(getReport).execute(); + nextPageToken = response.getReports().get(0).getNextPageToken(); + iterateMorePages = (nextPageToken != null); + logger.debug("got nextPageToken: " + nextPageToken); + gReportResponses.add(response); + } + reports.put(view, gReportResponses); + } + // Return the response. + return reports; + } + + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for " + viewId); + + List toReturn = new ArrayList<>(); + for (GetReportsResponse response : responses) { + for (Report report : response.getReports()) { + List rows = report.getData().getRows(); + if (rows == null) { + logger.warn("No data found for " + viewId); + } else { + for (ReportRow row : rows) { + String dimension = row.getDimensions().get(0); + DateRangeValues metric = row.getMetrics().get(0); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; + } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); + toReturn.add(var); + } + } + } + } + } + return toReturn; + } + + private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { + + String clientId = cred.getClientId(); + String clientEmail = cred.getClientEmail(); + String privateKeyPem = cred.getPrivateKeyPem(); + String privateKeyId = cred.getPrivateKeyId(); + String tokenUri = cred.getTokenUri(); + String projectId = cred.getProjectId(); + + if (clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { + throw new IOException("Error reading service account credential from stream, " + + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); + } + + PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); + + Collection emptyScopes = Collections.emptyList(); + + Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) + .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) + .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) + .setServiceAccountPrivateKeyId(privateKeyId); + + if (tokenUri != null) { + credentialBuilder.setTokenServerEncodedUrl(tokenUri); + } + + if (projectId != null) { + credentialBuilder.setServiceAccountProjectId(projectId); + } + + // Don't do a refresh at this point, as it will always fail before the + // scopes are added. + return credentialBuilder.build(); + } + + private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { + Reader reader = new StringReader(privateKeyPem); + Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); + if (section == null) { + throw new IOException("Invalid PKCS8 data."); + } + byte[] bytes = section.getBase64DecodedBytes(); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); + Exception unexpectedException = null; + try { + KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + return privateKey; + } catch (NoSuchAlgorithmException exception) { + unexpectedException = exception; + } catch (InvalidKeySpecException exception) { + unexpectedException = exception; + } + throw new IOException("Unexpected exception reading PKCS data", unexpectedException); + } + + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) + throws Exception { + String scope = infrastructureScope; + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(scope); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); + query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List toReturn = client.submit(query); + ScopeProvider.instance.set(currScope); + return toReturn; + } + + /** + * l + * + * @throws Exception + */ + private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { + AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); + + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + try { + List list = getAnalyticsReportingConfigurationFromIS(context); + if (list.size() > 1) { + logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); + } else if (list.size() == 0) { + logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); + } else { + + for (ServiceEndpoint res : list) { + reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + Group apGroup = res.profile().accessPoints(); + AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); + AccessPoint found = accessPoints[0]; + reportCredentials.setClientEmail(found.address()); + reportCredentials.setProjectId(found.username()); + reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + for (Property prop : found.properties()) { + if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + String[] views = decryptedValue.split(";"); + reportCredentials.setViewIds(Arrays.asList(views)); + } + if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setClientId(decryptedValue); + } + if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setPrivateKeyId(decryptedValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return reportCredentials; + } + + private static LocalDate asLocalDate(Date date) { + return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static DateRange getDateRangeForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // required + // by + // Analytics + String startDate = asLocalDate(start).format(formatter); + String endDate = asLocalDate(end).format(formatter); + DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` + dateRange.setStartDate(startDate); + dateRange.setEndDate(endDate); + return dateRange; + } + +} diff --git a/src/test/java/org/gcube/dataharvest/.gitignore b/src/test/java/org/gcube/dataharvest/.gitignore new file mode 100644 index 0000000..859ed2e --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/.gitignore @@ -0,0 +1 @@ +/TestDateScorro.java diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 7746460..06a0e07 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -20,6 +20,7 @@ import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +57,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { return dao; } - //@Ignore + @Ignore @Test public void testJupyterAccessesHarvester() throws Exception { try { diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java new file mode 100644 index 0000000..45b2d57 --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java @@ -0,0 +1,139 @@ +package org.gcube.dataharvest; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.stream.Stream; + +import org.gcube.accounting.accounting.summary.access.AccountingDao; +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.harvester.RStudioAccessesHarvester; +import org.gcube.dataharvest.utils.AggregationType; +import org.gcube.dataharvest.utils.ContextAuthorization; +import org.gcube.dataharvest.utils.ContextTest; +import org.gcube.dataharvest.utils.DateUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author Giancarlo Panichi (ISTI CNR) + * + */ +public class AccountingDataHarvesterRStudioTest extends ContextTest { + + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterRStudioTest.class); + public static final String ROOT = "/d4science.research-infrastructures.eu"; + // private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; + + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { + AccountingDao dao = AccountingDao.get(); + + Set scopeDescriptorSet = dao.getContexts(); + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); + } + AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); + + Set dimensionSet = dao.getDimensions(); + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { + dimensionMap.put(dimension.getId(), dimension); + } + + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); + + return dao; + } + + @Ignore + @Test + public void testJupyterAccessesHarvester() throws Exception { + try { + + ContextTest.setContextByName(ROOT); + AccountingDao dao = getAccountingDao(); + + List starts = new ArrayList<>(); + + LocalDate sdate = LocalDate.parse("2016-01-01"), edate = LocalDate.parse("2021-06-01"); + + Stream.iterate(sdate, date -> date.plusMonths(1)).limit(ChronoUnit.MONTHS.between(sdate, edate) + 1) + .forEach(dateToConvert -> starts.add(java.util.Date + .from(dateToConvert.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()))); + + AggregationType measureType = AggregationType.MONTHLY; + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + + SortedSet contexts = contextAuthorization.getContexts(); + /* + SortedSet contexts = new TreeSet<>(); + contexts.add("/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"); + contexts.add("/d4science.research-infrastructures.eu/D4OS/Zoo-Phytoplankton_EOV"); + contexts.add("/d4science.research-infrastructures.eu/D4OS/MarineEnvironmentalIndicators"); + */ + + List accountingRecords = new ArrayList<>(); + + Set scopeDescriptorSet = dao.getContexts(); + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); + } + AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); + + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + + ContextTest.setContextByName(ROOT); + RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end); + + for(String context : contexts) { + ContextTest.setContext(contextAuthorization.getTokenForContext(context)); + + ScopeBean scopeBean = new ScopeBean(context); + + ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); + if (actualScopeDescriptor == null) { + actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); + } + + AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); + + List harvested = rstudioAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + } + + } + + // logger.debug("{}", accountingRecords); + + logger.debug("Going to insert {}", accountingRecords); + + ContextTest.setContextByName(ROOT); + dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + + } catch (Throwable e) { + logger.error(e.getLocalizedMessage(), e); + throw e; + } + } + + +} From 26b11e96aff3934fb9985401972c606f8901772a Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 3 Jun 2021 18:52:07 +0200 Subject: [PATCH 2/2] ref 21557: RStudio sessions to be published into the accounting Added R Studio harvester --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e776c86..0fac4e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v2.0.0-SNAPSHOT] +- Added RStudio Harvester [#21557] - Added Jupyter Harvester [#21031] - Switched accounting JSON management to gcube-jackson [#19115] - Switched smart-executor JSON management to gcube-jackson [#19647]