From 7a335cbefdef7cf2cbf877bf72564cbc47c24878 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 25 Mar 2021 17:40:46 +0100 Subject: [PATCH 01/17] ref 21031: Add support to Jupyter Added Jupyter Accesses Harvester --- CHANGELOG.md | 1 + .../datamodel/HarvestedDataKey.java | 1 + .../harvester/JupyterAccessesHarvester.java | 400 ++++++++++ .../AccountingDataHarvesterPluginTest.java | 749 ++++++++++-------- 4 files changed, 813 insertions(+), 338 deletions(-) create mode 100644 src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4813f53..e776c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v2.0.0-SNAPSHOT] +- Added Jupyter Harvester [#21031] - Switched accounting JSON management to gcube-jackson [#19115] - Switched smart-executor JSON management to gcube-jackson [#19647] diff --git a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java index 815c6c4..06331ea 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java @@ -15,6 +15,7 @@ public enum HarvestedDataKey { MESSAGES_ACCESSES("Messages Accesses"), NOTIFICATIONS_ACCESSES("Notifications Accesses"), PROFILE_ACCESSES("Profile Accesses"), + JUPYTER_ACCESSES("Jupyter Accesses"), CATALOGUE_ACCESSES("Catalogue Accesses"), CATALOGUE_DATASET_LIST_ACCESSES("Item List"), diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java new file mode 100644 index 0000000..95fc607 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -0,0 +1,400 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.encryption.encrypter.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; +import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; +import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.dataharvest.datamodel.VREAccessesReportRow; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.PemReader; +import com.google.api.client.util.PemReader.Section; +import com.google.api.client.util.SecurityUtils; +import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; +import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; +import com.google.api.services.analyticsreporting.v4.model.DateRange; +import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; +import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; +import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; +import com.google.api.services.analyticsreporting.v4.model.Metric; +import com.google.api.services.analyticsreporting.v4.model.Report; +import com.google.api.services.analyticsreporting.v4.model.ReportRequest; +import com.google.api.services.analyticsreporting.v4.model.ReportRow; + +public class JupyterAccessesHarvester extends BasicHarvester { + + private static Logger logger = LoggerFactory.getLogger(JupyterAccessesHarvester.class); + + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; + private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String AP_VIEWS_PROPERTY = "views"; + private static final String AP_CLIENT_PROPERTY = "clientId"; + private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; + private static final String APPLICATION_NAME = "Analytics Reporting"; + + private List vreAccesses; + + public JupyterAccessesHarvester(Date start, Date end) throws Exception { + super(start, end); + vreAccesses = getAllAccesses(start, end); + } + + @Override + public List getAccountingRecords() throws Exception { + try { + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + + ArrayList accountingRecords = new ArrayList(); + + int measure = 0; + + ScopeBean scopeBean = new ScopeBean(context); + String lowerCasedContext = scopeBean.name().toLowerCase(); + + for (VREAccessesReportRow row : vreAccesses) { + String pagePath = row.getPagePath().toLowerCase(); + if (pagePath != null && !pagePath.isEmpty()) { + if (pagePath.contains(lowerCasedContext)) { + if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { + logger.trace("Matched jupyter or jupiter ({}) : {}", pagePath); + measure += row.getVisitNumber(); + } + } + } + } + + + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, + getDimension(HarvestedDataKey.ACCESSES), (long) measure); + logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); + accountingRecords.add(ar); + + return accountingRecords; + + } catch (Exception e) { + throw e; + } + } + + /** + * + * @return a list of {@link VREAccessesReportRow} objects containing the + * pagePath and the visit number e.g. VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/add-new-users, + * visitNumber=1] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/administration, + * visitNumber=2] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, + * visitNumber=39] + */ + private static List getAllAccesses(Date start, Date end) throws Exception { + DateRange dateRange = getDateRangeForAnalytics(start, end); + logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); + + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); + AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + HashMap> responses = getReportResponses(service, + credentialsFromD4S.getViewIds(), dateRange); + List totalAccesses = new ArrayList<>(); + + for (String view : responses.keySet()) { + List viewReport = parseResponse(view, responses.get(view)); + logger.trace("Got {} entries from view id={}", viewReport.size(), view); + totalAccesses.addAll(viewReport); + } + logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + return totalAccesses; + } + + /** + * Initializes an Analytics Reporting API V4 service object. + * + * @return An authorized Analytics Reporting API V4 service object. + * @throws IOException + * @throws GeneralSecurityException + */ + private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) + throws GeneralSecurityException, IOException { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); + + // Construct the Analytics Reporting service object. + return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + } + + /** + * Queries the Analytics Reporting API V4. + * + * @param service + * An authorized Analytics Reporting API V4 service object. + * @return GetReportResponse The Analytics Reporting API V4 response. + * @throws IOException + */ + private static HashMap> getReportResponses(AnalyticsReporting service, + List viewIDs, DateRange dateRange) throws IOException { + + HashMap> reports = new HashMap<>(); + + // Create the Metrics object. + Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); + com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension() + .setName("ga:pagePath"); + + for (String view : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.info("Getting data from Google Analytics for viewid: " + view); + boolean iterateMorePages = true; + String nextPageToken = null; + while (iterateMorePages) { + // Create the ReportRequest object. + ReportRequest request = new ReportRequest().setViewId(view.trim()) + .setDateRanges(Arrays.asList(dateRange)).setMetrics(Arrays.asList(sessions)) + .setDimensions(Arrays.asList(pageTitle)); + request.setPageSize(1000); + request.setPageToken(nextPageToken); + ArrayList requests = new ArrayList(); + requests.add(request); + // Create the GetReportsRequest object. + GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); + // Call the batchGet method. + GetReportsResponse response = service.reports().batchGet(getReport).execute(); + nextPageToken = response.getReports().get(0).getNextPageToken(); + iterateMorePages = (nextPageToken != null); + logger.debug("got nextPageToken: " + nextPageToken); + gReportResponses.add(response); + } + reports.put(view, gReportResponses); + } + // Return the response. + return reports; + } + + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for " + viewId); + + List toReturn = new ArrayList<>(); + for (GetReportsResponse response : responses) { + for (Report report : response.getReports()) { + List rows = report.getData().getRows(); + if (rows == null) { + logger.warn("No data found for " + viewId); + } else { + for (ReportRow row : rows) { + String dimension = row.getDimensions().get(0); + DateRangeValues metric = row.getMetrics().get(0); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; + } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); + toReturn.add(var); + } + } + } + } + } + return toReturn; + } + + private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { + + String clientId = cred.getClientId(); + String clientEmail = cred.getClientEmail(); + String privateKeyPem = cred.getPrivateKeyPem(); + String privateKeyId = cred.getPrivateKeyId(); + String tokenUri = cred.getTokenUri(); + String projectId = cred.getProjectId(); + + if (clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { + throw new IOException("Error reading service account credential from stream, " + + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); + } + + PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); + + Collection emptyScopes = Collections.emptyList(); + + Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) + .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) + .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) + .setServiceAccountPrivateKeyId(privateKeyId); + + if (tokenUri != null) { + credentialBuilder.setTokenServerEncodedUrl(tokenUri); + } + + if (projectId != null) { + credentialBuilder.setServiceAccountProjectId(projectId); + } + + // Don't do a refresh at this point, as it will always fail before the + // scopes are added. + return credentialBuilder.build(); + } + + private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { + Reader reader = new StringReader(privateKeyPem); + Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); + if (section == null) { + throw new IOException("Invalid PKCS8 data."); + } + byte[] bytes = section.getBase64DecodedBytes(); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); + Exception unexpectedException = null; + try { + KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + return privateKey; + } catch (NoSuchAlgorithmException exception) { + unexpectedException = exception; + } catch (InvalidKeySpecException exception) { + unexpectedException = exception; + } + throw new IOException("Unexpected exception reading PKCS data", unexpectedException); + } + + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) + throws Exception { + String scope = infrastructureScope; + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(scope); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); + query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List toReturn = client.submit(query); + ScopeProvider.instance.set(currScope); + return toReturn; + } + + /** + * l + * + * @throws Exception + */ + private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { + AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); + + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + try { + List list = getAnalyticsReportingConfigurationFromIS(context); + if (list.size() > 1) { + logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); + } else if (list.size() == 0) { + logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); + } else { + + for (ServiceEndpoint res : list) { + reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + Group apGroup = res.profile().accessPoints(); + AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); + AccessPoint found = accessPoints[0]; + reportCredentials.setClientEmail(found.address()); + reportCredentials.setProjectId(found.username()); + reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + for (Property prop : found.properties()) { + if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + String[] views = decryptedValue.split(";"); + reportCredentials.setViewIds(Arrays.asList(views)); + } + if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setClientId(decryptedValue); + } + if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setPrivateKeyId(decryptedValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return reportCredentials; + } + + private static LocalDate asLocalDate(Date date) { + return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static DateRange getDateRangeForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // required + // by + // Analytics + String startDate = asLocalDate(start).format(formatter); + String endDate = asLocalDate(end).format(formatter); + DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` + dateRange.setStartDate(startDate); + dateRange.setEndDate(endDate); + return dateRange; + } + +} diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index bfa4ca7..35c4dcf 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -24,6 +24,7 @@ import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; +import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; @@ -47,292 +48,305 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AccountingDataHarvesterPluginTest extends ContextTest { - + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class); - + public static final String ROOT = "/d4science.research-infrastructures.eu"; public static final String SO_BIG_VO = "/d4science.research-infrastructures.eu/SoBigData"; - + public static final String TAGME_VRE = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; public static final String STOCK_ASSESMENT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment"; - + public static SortedSet getContexts() throws Exception { SortedSet contexts = new TreeSet<>(); - LinkedHashMap map = ContextManager.readContexts(); - for(String scope : map.keySet()) { + LinkedHashMap map = ContextManager.readContexts(); + for (String scope : map.keySet()) { try { String context = map.get(scope).toString(); contexts.add(context); - } catch(Exception e) { + } catch (Exception e) { throw e; } } return contexts; } - + // @Test public void getDimensions() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDao dao = AccountingDao.get(); - + Set dimensionSet = dao.getDimensions(); - for(Dimension d : dimensionSet) { + for (Dimension d : dimensionSet) { logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel()); } - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + @Test public void launch() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); - + /* - Calendar from = DateUtils.getStartCalendar(2020, Calendar.MAY, 1); - String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - */ - + * Calendar from = DateUtils.getStartCalendar(2020, Calendar.MAY, + * 1); String fromDate = + * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + * logger.trace("{} is {}", + * AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, + * fromDate); inputs.put(AccountingDashboardHarvesterPlugin. + * START_DATE_INPUT_PARAMETER, fromDate); + */ + accountingDataHarvesterPlugin.launch(inputs); - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchPluginOnSmartExecutor() { try { - + ContextTest.setContextByName(ROOT); - + String pluginName = new AccountingDashboardHarvesterPlugin().getName(); - + SmartExecutorClient smartExecutor = SmartExecutorClientFactory.getClient(pluginName); Assert.assertNotNull(smartExecutor); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); - + /* - Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - */ - - // 3rd of the month for MONTHLY Harvesting at 10:00 - // CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 ? *"); - + * Calendar from = DateUtils.getStartCalendar(2016, + * Calendar.SEPTEMBER, 1); String fromDate = + * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + * logger.trace("{} is {}", + * AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, + * fromDate); inputs.put(AccountingDataHarvesterPlugin. + * START_DATE_INPUT_PARAMETER, fromDate); + */ + + // 3rd of the month for MONTHLY Harvesting at 10:00 + // CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 + // ? *"); + // Every day at 10:00 for partial harvesting CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 ? *"); - + Scheduling scheduling = new Scheduling(cronExpression); scheduling.setGlobal(false); LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs, scheduling); - //LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs); - + // LaunchParameter launchParameter = new LaunchParameter(pluginName, + // inputs); + smartExecutor.launch(launchParameter); - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchOldData() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); - + Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); - - while(from.before(runbeforeDate)) { + + while (from.before(runbeforeDate)) { String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); accountingDataHarvesterPlugin.launch(inputs); from.add(aggregationType.getCalendarField(), 1); } - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchOldDataVREAccessesHarvester() { try { - + ContextTest.setContextByName(ROOT); // AccountingDao dao = AccountingDao.get(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1); - + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); - - while(from.before(runbeforeDate)) { + + while (from.before(runbeforeDate)) { Date start = from.getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); - + ArrayList accountingRecords = new ArrayList<>(); - + VREAccessesHarvester vreAccessesHarvester = null; - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); - - if(vreAccessesHarvester == null) { - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); } else { - // This code should be never used because the scopes are sorted by fullname - + // This code should be never used because the scopes + // are sorted by fullname + ScopeBean parent = scopeBean.enclosingScope(); - while(!parent.is(Type.INFRASTRUCTURE)) { + while (!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(parent.toString())); - + vreAccessesHarvester = new VREAccessesHarvester(start, end); - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); } - + } - + try { - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { - // Collecting Google Analytics Data for VREs Accesses + // Collecting Google Analytics Data for VREs + // Accesses List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); ContextTest.setContextByName(ROOT); - - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - + + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + Thread.sleep(TimeUnit.SECONDS.toMillis(10)); - + from.add(aggregationType.getCalendarField(), 1); - + } - + ContextTest.setContextByName(ROOT); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } - + logger.info("End."); - + } - + // @Test public void testScopeBean() throws Exception { ContextTest.setContextByName(ROOT); SortedSet contexts = getContexts(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime(); // start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.info("\n\n\n"); - - for(String context : contexts) { + + for (String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); - // logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - + // logger.debug("FullName {} - Name {}", scopeBean.toString(), + // scopeBean.name()); + try { - - if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) { + + if (scopeBean.is(Type.VRE) + && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) { logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } - - if((context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + + if ((context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_EU_VRE) || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_IT_VRE)) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { @@ -342,20 +356,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } - - } catch(Exception e) { + + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } - + } - + } - + // @Test public void testVREAccessesHarvester() throws Exception { try { // AccountingDao dao = getAccountingDao(); - + List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2018, Calendar.OCTOBER, 1).getTime()); @@ -364,98 +378,102 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); - + AggregationType measureType = AggregationType.MONTHLY; - - String[] contextFullNames = new String[] {"/d4science.research-infrastructures.eu/FARM/GRSF", - "/d4science.research-infrastructures.eu/FARM/GRSF_Admin"}; - + + String[] contextFullNames = new String[] { "/d4science.research-infrastructures.eu/FARM/GRSF", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin" }; + List accountingRecords = new ArrayList<>(); - - for(Date start : starts) { + + for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + ContextTest.setContextByName(ROOT); VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end); - - for(String contextFullname : contextFullNames) { - + + for (String contextFullname : contextFullNames) { + setContextByNameAndScopeDescriptor(contextFullname); - + List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + logger.debug("{} - {}", contextFullname, accountingRecords); - + } } - + logger.debug("{}", accountingRecords); - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); throw e; } } - + // @Test public void testVREAccessesHarvesterAll() { try { - + ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - + VREAccessesHarvester vreAccessesHarvester = null; - + ArrayList accountingRecords = new ArrayList<>(); - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); - - if(vreAccessesHarvester == null) { - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); } else { - // This code should be never used because the scopes are sorted by fullname - + // This code should be never used because the scopes are + // sorted by fullname + ScopeBean parent = scopeBean.enclosingScope(); - while(!parent.is(Type.INFRASTRUCTURE)) { + while (!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(parent.toString())); - + vreAccessesHarvester = new VREAccessesHarvester(start, end); - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); } - + } - + try { - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); @@ -464,31 +482,81 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + + logger.debug("{}", accountingRecords); + + } catch (Exception e) { + logger.error("", e); + } + } + + // @Test + public void testJupyterccessesHarvester() throws Exception { + try { + // AccountingDao dao = getAccountingDao(); + + List starts = new ArrayList<>(); + starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.OCTOBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.NOVEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.DECEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); + + AggregationType measureType = AggregationType.MONTHLY; + + String[] contextFullNames = new String[] { "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab" }; + + List accountingRecords = new ArrayList<>(); + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + + ContextTest.setContextByName(ROOT); + JupyterAccessesHarvester vreAccessesHarvester = new JupyterAccessesHarvester(start, end); + + for (String contextFullname : contextFullNames) { + + setContextByNameAndScopeDescriptor(contextFullname); + + List harvested = vreAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + logger.debug("{} - {}", contextFullname, accountingRecords); + + } + } + logger.debug("{}", accountingRecords); - - } catch(Exception e) { + + ContextTest.setContextByName(ROOT); + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); + throw e; } } - + // @Test public void testSocialInteraction() { try { - + ContextTest.setContextByName(ROOT); // AccountingDao dao = AccountingDao.get(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); - + SortedSet contexts = new TreeSet<>(); contexts.add("/d4science.research-infrastructures.eu/D4Research"); contexts.add("/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS"); @@ -496,18 +564,18 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { contexts.add("/d4science.research-infrastructures.eu/gCubeApps/Parthenos"); contexts.add("/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining"); contexts.add("/d4science.research-infrastructures.eu/gCubeApps/gCube"); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); Date start = from.getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end)); - + ArrayList accountingRecords = new ArrayList<>(); - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); try { @@ -516,273 +584,277 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); List harvested = socialHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); } - + logger.info("End."); - + } - + // @Test public void testMethodInvocation() { try { - + ContextTest.setContextByName(ROOT); ContextAuthorization contextAuthorization = new ContextAuthorization(); String stockAssessmentToken = contextAuthorization.generateTokenForContext(STOCK_ASSESMENT_VRE, null); - + ContextTest.setContext(stockAssessmentToken); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); List accountingRecords = methodInvocationHarvester.getAccountingRecords(); - + logger.debug("{}", accountingRecords); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { AccountingDao dao = AccountingDao.get(); - + Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for(ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - + Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for(Dimension dimension : dimensionSet) { + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { dimensionMap.put(dimension.getId(), dimension); } - + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - + return dao; } - + protected void setContextByNameAndScopeDescriptor(String contextFullName) throws ObjectNotFound, Exception { ContextAuthorization contextAuthorization = new ContextAuthorization(); String tagMeToken = contextAuthorization.getTokenForContext(contextFullName); - + ContextTest.setContext(tagMeToken); - + ScopeBean scopeBean = new ScopeBean(contextFullName); ScopeDescriptor actualScopeDescriptor = AccountingDashboardHarvesterPlugin.scopeDescriptors.get() .get(contextFullName); - if(actualScopeDescriptor == null) { + if (actualScopeDescriptor == null) { actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), contextFullName); } - + AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); } - + // @Test public void testTagMeMethodInvocation() throws Exception { try { // AccountingDao dao = getAccountingDao(); setContextByNameAndScopeDescriptor(TAGME_VRE); - + List accountingRecords = new ArrayList<>(); - + AggregationType measureType = AggregationType.MONTHLY; - + List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2018, Calendar.NOVEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2018, Calendar.DECEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); - - for(Date start : starts) { - + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); accountingRecords.addAll(methodInvocationHarvester.getAccountingRecords()); - + logger.debug("{}", accountingRecords); } - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); throw e; } } - + // @Test public void testGetVREUsersForSpecificVRE() { try { ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + // AccountingDao dao = AccountingDao.get(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); ContextTest.setContext(contextAuthorization .getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); List harvested = vreUsersHarvester.getAccountingRecords(); - + logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); } - + } - + // @Test public void testFilteringGenericResource() { try { ContextTest.setContextByName(ROOT); - //Utils.setContext(RESOURCE_CATALOGUE); - + // Utils.setContext(RESOURCE_CATALOGUE); + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + SortedSet contexts = getContexts(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); SortedSet validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_VO + "/"); logger.info("Valid Contexts {}", validContexts); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } - + } - + // @Test public void testResourceCatalogueHarvester() { try { - - //Utils.setContext(RESOURCE_CATALOGUE); + + // Utils.setContext(RESOURCE_CATALOGUE); ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + SortedSet contexts = getContexts(); - + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); List data = resourceCatalogueHarvester.getAccountingRecords(); - + logger.debug("{}", data); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + @Test public void testCoreServicesHarvester() { try { - String context = ROOT; //"/gcube"; + String context = ROOT; // "/gcube"; ContextTest.setContextByName(context); - + AccountingDao dao = getAccountingDao(); - + Date start = DateUtils.getStartCalendar(2017, Calendar.MAY, 1).getTime(); Date finalEnd = DateUtils.getStartCalendar(2020, Calendar.MAY, 1).getTime(); Date end = DateUtils.getEndDateFromStartDate(AggregationType.MONTHLY, start, 1, false); - + ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - - while(end.before(finalEnd)) { - CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); + + while (end.before(finalEnd)) { + CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); List accountingRecords = coreServicesHarvester.getAccountingRecords(); dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - + logger.debug("{} -> {} Data Inserted", DateUtils.format(start), DateUtils.format(end)); logger.debug("---------------------------------------------------------------------------------------"); - + Thread.sleep(TimeUnit.SECONDS.toMillis(90)); start = end; end = DateUtils.getEndDateFromStartDate(AggregationType.MONTHLY, start, 1, false); } - - } catch(Exception e) { + } catch (Exception e) { logger.error("", e); } } - @Test public void testCatalogueHarvester() { try { - String context = ROOT; //"/gcube"; + String context = ROOT; // "/gcube"; ContextTest.setContextByName(context); - + AccountingDao dao = getAccountingDao(); - + Date start = DateUtils.getStartCalendar(2020, Calendar.JANUARY, 1).getTime(); Date end = DateUtils.getStartCalendar(2020, Calendar.FEBRUARY, 1).getTime(); /* - * Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); - * Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + * Date start = DateUtils.getPreviousPeriod(measureType, + * false).getTime(); Date end = + * DateUtils.getEndDateFromStartDate(measureType, start, 1, false); */ - + ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); + CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); List accountingRecords = catalogueHarvester.getAccountingRecords(); for (AccountingRecord accountingRecord : accountingRecords) { logger.debug("{}", accountingRecord); @@ -791,40 +863,41 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - } catch(Exception e) { + } catch (Exception e) { logger.error("", e); } } - // @Test public void testDataMethodDownloadHarvester() { try { - - //Utils.setContext(RESOURCE_CATALOGUE); + + // Utils.setContext(RESOURCE_CATALOGUE); ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - - for(String context : contexts) { + + for (String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO)) { - if(scopeBean.is(Type.VRE)) { - if(context.startsWith(TAGME_VRE)) { + + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO)) { + if (scopeBean.is(Type.VRE)) { + if (context.startsWith(TAGME_VRE)) { continue; } ContextTest.setContext(contextAuthorization.getTokenForContext(context)); @@ -834,82 +907,82 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("{}", data); } } - + } - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + public static final String E_LEARNING_AREA_VRE = "/d4science.research-infrastructures.eu/SoBigData/E-Learning_Area"; - + // @Test public void addMissingVREAccesses() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin adhp = new AccountingDashboardHarvesterPlugin(); Properties properties = adhp.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); - + // DatabaseManager dbaseManager = new DatabaseManager(); AccountingDao dao = AccountingDao.get(); - + Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for(ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - + Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for(Dimension dimension : dimensionSet) { + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { dimensionMap.put(dimension.getId(), dimension); } - + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - + // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList(); - + String context = E_LEARNING_AREA_VRE; - + // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); ScopeDescriptor scopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); - + Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey()); - + Calendar calendar = DateUtils.getStartCalendar(2018, Calendar.JULY, 1); calendar.set(Calendar.DAY_OF_MONTH, 15); - - Map monthValues = new HashMap<>(); + + Map monthValues = new HashMap<>(); monthValues.put(Calendar.JULY, 54); monthValues.put(Calendar.AUGUST, 23); monthValues.put(Calendar.SEPTEMBER, 127); monthValues.put(Calendar.OCTOBER, 192); - - for(Integer month : monthValues.keySet()) { + + for (Integer month : monthValues.keySet()) { calendar.set(Calendar.MONTH, month); Instant instant = calendar.toInstant(); - + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, dimension, (long) monthValues.get(month)); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); accountingRecords.add(ar); } - + logger.trace("{}", accountingRecords); dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } From 13481c35a527fa139566f49cdffbc6e42a69c406 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Thu, 25 Mar 2021 17:48:55 +0100 Subject: [PATCH 02/17] ref 21031: Add support to Jupyter Added Jupyter Accesses Harvester --- .../gcube/dataharvest/harvester/JupyterAccessesHarvester.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index 95fc607..a297842 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -100,7 +100,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { if (pagePath != null && !pagePath.isEmpty()) { if (pagePath.contains(lowerCasedContext)) { if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { - logger.trace("Matched jupyter or jupiter ({}) : {}", pagePath); + logger.trace("Matched jupyter or jupiter ({}) : {}", lowerCasedContext, pagePath); measure += row.getVisitNumber(); } } From 38ec08e0a3aeacd280570c5bc7dbf4f929c07cc2 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 26 Mar 2021 13:24:45 +0100 Subject: [PATCH 03/17] ref 21031: Add support to Jupyter Updated Jupyter Accesses Harvester --- .../AccountingDashboardHarvesterPlugin.java | 391 ++++++++++-------- .../harvester/JupyterAccessesHarvester.java | 6 +- .../AccountingDataHarvesterPluginTest.java | 51 +-- .../AccountingDataHarvesterJupyterTest.java | 65 +++ src/test/resources/.gitignore | 1 + 5 files changed, 290 insertions(+), 224 deletions(-) create mode 100644 src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index 32e9661..340c90c 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -21,6 +21,7 @@ import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; +import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; @@ -41,99 +42,98 @@ import org.slf4j.LoggerFactory; * @author Luca Frosini (ISTI - CNR) */ public class AccountingDashboardHarvesterPlugin extends Plugin { - + private static Logger logger = LoggerFactory.getLogger(AccountingDashboardHarvesterPlugin.class); - + private static final String PROPERTY_FILENAME = "config.properties"; - + public static final String START_DATE_INPUT_PARAMETER = "startDate"; public static final String MEASURE_TYPE_INPUT_PARAMETER = "measureType"; public static final String RERUN_INPUT_PARAMETER = "reRun"; public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers"; public static final String DRY_RUN_INPUT_PARAMETER = "dryRun"; - + /** - * Allows partial harvesting of data of the current period. - * This means that in MONTHLY aggregation type the current month is harvested instead of the previous month which - * is done when the month is completed. - * This allow the portlet to display monthly data in the current moth even the data is partial (till the current day). + * Allows partial harvesting of data of the current period. This means that + * in MONTHLY aggregation type the current month is harvested instead of the + * previous month which is done when the month is completed. This allow the + * portlet to display monthly data in the current moth even the data is + * partial (till the current day). */ public static final String PARTIAL_HARVESTING = "partialHarvesting"; - + public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData"; public static final String SO_BIG_DATA_EU_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; public static final String SO_BIG_DATA_IT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it"; public static final String SO_BIG_DATA_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue"; - + public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; - + public static final String TO_BE_SET = "TO BE SET"; - + protected Date start; protected Date end; - + public AccountingDashboardHarvesterPlugin() { super(); } - + private static final InheritableThreadLocal properties = new InheritableThreadLocal() { - + @Override protected Properties initialValue() { return new Properties(); } - + }; - - + public static InheritableThreadLocal getProperties() { return properties; } - + public static Dimension getDimension(String key) { Dimension dimension = dimensions.get().get(key); - if(dimension == null) { + if (dimension == null) { dimension = new Dimension(key, key, null, key); } return dimension; } - + protected static final InheritableThreadLocal> dimensions = new InheritableThreadLocal>() { - + @Override protected Map initialValue() { return new HashMap<>(); } - + }; - - + public static ScopeDescriptor getScopeDescriptor(String context) { return scopeDescriptors.get().get(context); } - + protected static final InheritableThreadLocal> scopeDescriptors = new InheritableThreadLocal>() { - + @Override protected Map initialValue() { return new HashMap<>(); } - + }; - + public static ScopeDescriptor getScopeDescriptor() { return scopeDescriptor.get(); } - + public static final InheritableThreadLocal scopeDescriptor = new InheritableThreadLocal() { - + @Override protected ScopeDescriptor initialValue() { - return new ScopeDescriptor("",""); + return new ScopeDescriptor("", ""); } - + }; - + public Properties getConfigParameters() throws IOException { Properties properties = new Properties(); try { @@ -141,272 +141,318 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { .getResourceAsStream(PROPERTY_FILENAME); properties.load(input); return properties; - } catch(Exception e) { + } catch (Exception e) { logger.warn( "Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME); } return properties; } - + /** {@inheritDoc} */ @Override - public void launch(Map inputs) throws Exception { + public void launch(Map inputs) throws Exception { logger.debug("{} is starting", this.getClass().getSimpleName()); - - if(inputs == null || inputs.isEmpty()) { + + if (inputs == null || inputs.isEmpty()) { throw new IllegalArgumentException("The can only be launched providing valid input parameters"); } - - if(!inputs.containsKey(MEASURE_TYPE_INPUT_PARAMETER)) { + + if (!inputs.containsKey(MEASURE_TYPE_INPUT_PARAMETER)) { throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'"); } - + AggregationType aggregationType = AggregationType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER)); - + boolean reRun = true; - if(inputs.containsKey(RERUN_INPUT_PARAMETER)) { + if (inputs.containsKey(RERUN_INPUT_PARAMETER)) { try { reRun = (boolean) inputs.get(RERUN_INPUT_PARAMETER); - } catch(Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("'" + RERUN_INPUT_PARAMETER + "' must be a boolean"); } } - + boolean getVREUsers = true; - if(inputs.containsKey(GET_VRE_USERS_INPUT_PARAMETER)) { + if (inputs.containsKey(GET_VRE_USERS_INPUT_PARAMETER)) { try { reRun = (boolean) inputs.get(GET_VRE_USERS_INPUT_PARAMETER); - } catch(Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("'" + GET_VRE_USERS_INPUT_PARAMETER + "' must be a boolean"); } } - + boolean dryRun = true; - if(inputs.containsKey(DRY_RUN_INPUT_PARAMETER)) { + if (inputs.containsKey(DRY_RUN_INPUT_PARAMETER)) { try { dryRun = (boolean) inputs.get(DRY_RUN_INPUT_PARAMETER); - } catch(Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("'" + DRY_RUN_INPUT_PARAMETER + "' must be a boolean"); } } - + boolean partialHarvesting = false; - if(inputs.containsKey(PARTIAL_HARVESTING)) { + if (inputs.containsKey(PARTIAL_HARVESTING)) { partialHarvesting = (boolean) inputs.get(PARTIAL_HARVESTING); } - - if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) { + + if (inputs.containsKey(START_DATE_INPUT_PARAMETER)) { String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER); start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC); } else { start = DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime(); } - + end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, partialHarvesting); - - logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})", - DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun); - + + logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})", DateUtils.format(start), + DateUtils.format(end), reRun, getVREUsers, dryRun); + Properties properties = getConfigParameters(); getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); - + SortedSet contexts = contextAuthorization.getContexts(); - + String root = contexts.first(); Utils.setContext(contextAuthorization.getTokenForContext(root)); - + AccountingDao dao = AccountingDao.get(); - + Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for(ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } scopeDescriptors.set(scopeDescriptorMap); - - + Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for(Dimension dimension : dimensionSet) { + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { dimensionMap.put(dimension.getId(), dimension); } - + dimensions.set(dimensionMap); - + ArrayList accountingRecords = new ArrayList(); - + String initialToken = SecurityTokenProvider.instance.get(); - + VREAccessesHarvester vreAccessesHarvester = null; - - for(String context : contexts) { + JupyterAccessesHarvester jupyterAccessesHarvester = null; + + for (String context : contexts) { // Setting the token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); - + ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); - if(actualScopeDescriptor==null) { + if (actualScopeDescriptor == null) { actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); } - + scopeDescriptor.set(actualScopeDescriptor); - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { try { - - CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); + + CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); List harvested = catalogueHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - - CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); + + CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); List records = coreServicesHarvester.getAccountingRecords(); accountingRecords.addAll(records); - - }catch (Exception e) { - logger.error("Error harvesting {} for {}", CatalogueAccessesHarvester.class.getSimpleName(), context, e); + + } catch (Exception e) { + logger.error("Error harvesting {} for {}", CatalogueAccessesHarvester.class.getSimpleName(), + context, e); } } - - - if(vreAccessesHarvester == null) { - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); } else { - // This code should be never used because the scopes are sorted by fullname - + // This code should be never used because the scopes are + // sorted by fullname + ScopeBean parent = scopeBean.enclosingScope(); - while(!parent.is(Type.INFRASTRUCTURE)) { + while (!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } - + // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); - + vreAccessesHarvester = new VREAccessesHarvester(start, end); - + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(context)); + } + + } + + if (jupyterAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { + jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); + } else { + // This code should be never used because the scopes are + // sorted by fullname + + ScopeBean parent = scopeBean.enclosingScope(); + while (!parent.is(Type.INFRASTRUCTURE)) { + parent = scopeBean.enclosingScope(); + } + + // Setting back token for the context + Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); + + jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); + // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); } - + } - - - if((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE) + + if ((context.startsWith(SO_BIG_DATA_VO) || context.startsWith(SO_BIG_DATA_EU_VRE) || context.startsWith(SO_BIG_DATA_IT_VRE)) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { - + try { // Collecting Google Analytics Data for VREs Accesses logger.info("Going to harvest VRE Accesses for {}", context); - + List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + + /* + * List harvested = + * vreAccessesHarvester.getData(); data.addAll(harvested); + */ + } catch (Exception e) { + logger.error("Error harvesting VRE Accesses for {}", context, e); + } + + try { + // Collecting Google Analytics Data for Jupyters Accesses + logger.info("Going to harvest Jupyter Accesses for {}", context); + + List harvested = jupyterAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + /* - List harvested = vreAccessesHarvester.getData(); - data.addAll(harvested); - */ - } catch(Exception e) { + * List harvested = + * jupyterAccessesHarvester.getData(); + * data.addAll(harvested); + */ + } catch (Exception e) { logger.error("Error harvesting VRE Accesses for {}", context, e); } - + try { // Collecting info on social (posts, replies and likes) logger.info("Going to harvest Social Interactions for {}", context); SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); - + List harvested = socialHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + /* - List harvested = socialHarvester.getData(); - data.addAll(harvested); - */ - } catch(Exception e) { + * List harvested = + * socialHarvester.getData(); data.addAll(harvested); + */ + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } - + try { // Collecting info on VRE users - if(getVREUsers) { - // Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts) + if (getVREUsers) { + // Harvesting Users only for VREs (not for VO and ROOT + // which is the sum of the children contexts) // The VREUsers can be only Harvested for the last month - if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime())) { + if (scopeBean.is(Type.VRE) && start + .equals(DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime())) { logger.info("Going to harvest Context Users for {}", context); VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); - + List harvested = vreUsersHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + /* - List harvested = vreUsersHarvester.getData(); - data.addAll(harvested); - */ + * List harvested = + * vreUsersHarvester.getData(); + * data.addAll(harvested); + */ } } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Context Users for {}", context, e); } - - if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) { - + + if (context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) { + try { - // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) + // Collecting info on Resource Catalogue (Dataset, + // Application, Deliverables, Methods) logger.info("Going to harvest Resource Catalogue Information for {}", context); - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, - contexts); - + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, + end, contexts); + List harvested = resourceCatalogueHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + /* - List harvested = resourceCatalogueHarvester.getData(); - data.addAll(harvested); - */ - - } catch(Exception e) { + * List harvested = + * resourceCatalogueHarvester.getData(); + * data.addAll(harvested); + */ + + } catch (Exception e) { logger.error("Error harvesting Resource Catalogue Information for {}", context, e); } - + try { // Collecting info on Data/Method download logger.info("Going to harvest Data Method Download for {}", context); DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, contexts); - + List harvested = dataMethodDownloadHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("Error harvesting Data Method Download for {}", context, e); } - + } - - if(context.startsWith(TAGME_CONTEXT)) { + + if (context.startsWith(TAGME_CONTEXT)) { try { // Collecting info on method invocation logger.info("Going to harvest Method Invocations for {}", context); TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester( start, end); - + List harvested = tagMeMethodInvocationHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + /* - List harvested = tagMeMethodInvocationHarvester.getData(); - data.addAll(harvested); - */ - - } catch(Exception e) { + * List harvested = + * tagMeMethodInvocationHarvester.getData(); + * data.addAll(harvested); + */ + + } catch (Exception e) { logger.error("Error harvesting Method Invocations for {}", context, e); } } else { @@ -414,38 +460,39 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { // Collecting info on method invocation logger.info("Going to harvest Method Invocations for {}", context); MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); - - + List harvested = methodInvocationHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + /* - List harvested = methodInvocationHarvester.getData(); - data.addAll(harvested); - */ - } catch(Exception e) { + * List harvested = + * methodInvocationHarvester.getData(); + * data.addAll(harvested); + */ + } catch (Exception e) { logger.error("Error harvesting Method Invocations for {}", context, e); } } } } - + Utils.setContext(initialToken); - - logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); - if(!dryRun) { + + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), + accountingRecords); + if (!dryRun) { dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - //dbaseManager.insertMonthlyData(start, end, data, reRun); - }else { + // dbaseManager.insertMonthlyData(start, end, data, reRun); + } else { logger.debug("Harvested measures are {}", accountingRecords); } - + } - + /** {@inheritDoc} */ @Override protected void onStop() throws Exception { logger.debug("{} is stopping", this.getClass().getSimpleName()); } - + } diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index a297842..6458be6 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -80,6 +80,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { public JupyterAccessesHarvester(Date start, Date end) throws Exception { super(start, end); + logger.debug("JupyerAccessHArvester: {}, {}",start,end); vreAccesses = getAllAccesses(start, end); } @@ -94,9 +95,10 @@ public class JupyterAccessesHarvester extends BasicHarvester { ScopeBean scopeBean = new ScopeBean(context); String lowerCasedContext = scopeBean.name().toLowerCase(); - + logger.debug("JupyerAccessHArvester lowerCasedContext: {}",lowerCasedContext); for (VREAccessesReportRow row : vreAccesses) { String pagePath = row.getPagePath().toLowerCase(); + //logger.debug("JupyerAccessHArvester pagePath: {}",lowerCasedContext); if (pagePath != null && !pagePath.isEmpty()) { if (pagePath.contains(lowerCasedContext)) { if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { @@ -111,7 +113,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, - getDimension(HarvestedDataKey.ACCESSES), (long) measure); + getDimension(HarvestedDataKey.JUPYTER_ACCESSES), (long) measure); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); accountingRecords.add(ar); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 35c4dcf..1d27558 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -494,56 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - // @Test - public void testJupyterccessesHarvester() throws Exception { - try { - // AccountingDao dao = getAccountingDao(); - - List starts = new ArrayList<>(); - starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2018, Calendar.OCTOBER, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2018, Calendar.NOVEMBER, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2018, Calendar.DECEMBER, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); - - AggregationType measureType = AggregationType.MONTHLY; - - String[] contextFullNames = new String[] { "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab" }; - - List accountingRecords = new ArrayList<>(); - - for (Date start : starts) { - Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - - ContextTest.setContextByName(ROOT); - JupyterAccessesHarvester vreAccessesHarvester = new JupyterAccessesHarvester(start, end); - - for (String contextFullname : contextFullNames) { - - setContextByNameAndScopeDescriptor(contextFullname); - - List harvested = vreAccessesHarvester.getAccountingRecords(); - accountingRecords.addAll(harvested); - - logger.debug("{} - {}", contextFullname, accountingRecords); - - } - } - - logger.debug("{}", accountingRecords); - - ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new - // AccountingRecord[1])); - - } catch (Exception e) { - logger.error("", e); - throw e; - } - } - + // @Test public void testSocialInteraction() { try { diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java new file mode 100644 index 0000000..1adeacf --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -0,0 +1,65 @@ +package org.gcube.dataharvest.jupyter; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; +import org.gcube.dataharvest.utils.AggregationType; +import org.gcube.dataharvest.utils.ContextTest; +import org.gcube.dataharvest.utils.DateUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AccountingDataHarvesterJupyterTest extends ContextTest { + + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); + public static final String ROOT = "/d4science.research-infrastructures.eu"; + private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; + + @Test + public void testJupyterccessesHarvester() throws Exception { + try { + // AccountingDao dao = getAccountingDao(); + + List starts = new ArrayList<>(); + starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); + + AggregationType measureType = AggregationType.MONTHLY; + + List accountingRecords = new ArrayList<>(); + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + + ContextTest.setContextByName(ROOT); + JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); + + ContextTest.setContextByName(SCOPE); + + List harvested = jupyterAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + logger.debug("{} - {}", SCOPE, accountingRecords); + + } + + logger.debug("{}", accountingRecords); + + ContextTest.setContextByName(ROOT); + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Throwable e) { + logger.error(e.getLocalizedMessage(), e); + throw e; + } + } + + +} diff --git a/src/test/resources/.gitignore b/src/test/resources/.gitignore index 31a87d1..03235d1 100644 --- a/src/test/resources/.gitignore +++ b/src/test/resources/.gitignore @@ -1,3 +1,4 @@ /*.gcubekey /*.key /*.properties +/howto.txt From 24f2409df7aeabb63e66f546af8227129cabe1e9 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 26 Mar 2021 16:04:07 +0100 Subject: [PATCH 04/17] ref 21031: Add support to Jupyter Updated Jupyter Accesses Harvester --- .../harvester/JupyterAccessesHarvester.java | 14 ++++---- .../AccountingDataHarvesterPluginTest.java | 35 ++++++++++++++----- .../sobigdata/SoBigDataHarvesterTest.java | 2 ++ .../AccountingDataHarvesterJupyterTest.java | 8 +++-- .../utils/ContextAuthorizationTest.java | 2 ++ 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index 6458be6..a2beed8 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -80,7 +80,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { public JupyterAccessesHarvester(Date start, Date end) throws Exception { super(start, end); - logger.debug("JupyerAccessHArvester: {}, {}",start,end); + logger.debug("JupyerAccessHArvester: {}, {}", start, end); vreAccesses = getAllAccesses(start, end); } @@ -95,21 +95,21 @@ public class JupyterAccessesHarvester extends BasicHarvester { ScopeBean scopeBean = new ScopeBean(context); String lowerCasedContext = scopeBean.name().toLowerCase(); - logger.debug("JupyerAccessHArvester lowerCasedContext: {}",lowerCasedContext); + logger.debug("JupyerAccessHArvester lowerCasedContext: {}", lowerCasedContext); for (VREAccessesReportRow row : vreAccesses) { String pagePath = row.getPagePath().toLowerCase(); - //logger.debug("JupyerAccessHArvester pagePath: {}",lowerCasedContext); if (pagePath != null && !pagePath.isEmpty()) { if (pagePath.contains(lowerCasedContext)) { - if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { - logger.trace("Matched jupyter or jupiter ({}) : {}", lowerCasedContext, pagePath); - measure += row.getVisitNumber(); + if (!pagePath.contains("catalogue")) { + if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { + logger.trace("Matched jupyter or jupiter ({}) : {}", lowerCasedContext, pagePath); + measure += row.getVisitNumber(); + } } } } } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 1d27558..5a5bc86 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -24,7 +24,6 @@ import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; -import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; @@ -42,6 +41,7 @@ import org.gcube.vremanagement.executor.api.types.Scheduling; import org.gcube.vremanagement.executor.client.SmartExecutorClient; import org.gcube.vremanagement.executor.client.SmartExecutorClientFactory; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.quartz.CronExpression; import org.slf4j.Logger; @@ -71,7 +71,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { return contexts; } - // @Test + @Ignore + @Test public void getDimensions() { try { @@ -90,7 +91,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - + + @Ignore @Test public void launch() { try { @@ -128,7 +130,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - // @Test + @Ignore + @Test public void launchPluginOnSmartExecutor() { try { @@ -181,7 +184,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - // @Test + @Ignore + //@Test public void launchOldData() { try { @@ -217,6 +221,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } + @Ignore // @Test public void launchOldDataVREAccessesHarvester() { try { @@ -317,6 +322,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } + @Ignore // @Test public void testScopeBean() throws Exception { ContextTest.setContextByName(ROOT); @@ -365,7 +371,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } - // @Test + @Ignore + @Test public void testVREAccessesHarvester() throws Exception { try { // AccountingDao dao = getAccountingDao(); @@ -416,6 +423,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } + @Ignore // @Test public void testVREAccessesHarvesterAll() { try { @@ -494,7 +502,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - + @Ignore // @Test public void testSocialInteraction() { try { @@ -554,6 +562,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } + @Ignore // @Test public void testMethodInvocation() { try { @@ -616,6 +625,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); } + @Ignore // @Test public void testTagMeMethodInvocation() throws Exception { try { @@ -654,6 +664,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } + @Ignore // @Test public void testGetVREUsersForSpecificVRE() { try { @@ -689,6 +700,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } + @Ignore // @Test public void testFilteringGenericResource() { try { @@ -716,6 +728,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } + @Ignore // @Test public void testResourceCatalogueHarvester() { try { @@ -748,7 +761,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - + + @Ignore @Test public void testCoreServicesHarvester() { try { @@ -783,7 +797,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - + + @Ignore @Test public void testCatalogueHarvester() { try { @@ -819,6 +834,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } + @Ignore // @Test public void testDataMethodDownloadHarvester() { try { @@ -868,6 +884,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public static final String E_LEARNING_AREA_VRE = "/d4science.research-infrastructures.eu/SoBigData/E-Learning_Area"; + @Ignore // @Test public void addMissingVREAccesses() { try { diff --git a/src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java b/src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java index 2c6b4d2..d48c029 100644 --- a/src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java +++ b/src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java @@ -3,6 +3,7 @@ package org.gcube.dataharvest.harvester.sobigdata; import java.util.List; import org.gcube.dataharvest.utils.ContextTest; +import org.junit.Ignore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +11,7 @@ public class SoBigDataHarvesterTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(SoBigDataHarvesterTest.class); + @Ignore // @Test public void testGroupList() throws Exception { // ContextTest.setContextByName("/d4science.research-infrastructures.eu/D4Research/AGINFRAplusDev"); diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index 1adeacf..13d61da 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -10,6 +10,7 @@ import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,15 +20,16 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); public static final String ROOT = "/d4science.research-infrastructures.eu"; private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; - + + @Ignore @Test public void testJupyterccessesHarvester() throws Exception { try { // AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); - starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); - starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); + //starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); + //starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); AggregationType measureType = AggregationType.MONTHLY; diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java index 106bc3b..d58b959 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.util.Properties; import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; +import org.junit.Ignore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +27,7 @@ public class ContextAuthorizationTest extends ContextTest { AccountingDashboardHarvesterPlugin.getProperties().set(properties); } + @Ignore // @Test public void testRetrieveContextsAndTokens() throws Exception { try { From 2bd73c2caaea3e4cffd6104b02cd0f5397dbd46c Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 26 Mar 2021 16:35:15 +0100 Subject: [PATCH 05/17] ref 21031: Add support to Jupyter Updated Jupyter Accesses Harvester --- .../dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index 13d61da..ae47719 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -23,7 +23,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { @Ignore @Test - public void testJupyterccessesHarvester() throws Exception { + public void testJupyterAccessesHarvester() throws Exception { try { // AccountingDao dao = getAccountingDao(); From 9b5d0874ece20ab56e992accc7f898d3710601ad Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Wed, 31 Mar 2021 15:21:45 +0200 Subject: [PATCH 06/17] ref 21031: Add support to Jupyter Updated Jupyter Accesses Harvester --- .../dataharvest/harvester/JupyterAccessesHarvester.java | 5 +++++ .../jupyter/AccountingDataHarvesterJupyterTest.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index a2beed8..d2d0391 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -63,6 +63,11 @@ import com.google.api.services.analyticsreporting.v4.model.Report; import com.google.api.services.analyticsreporting.v4.model.ReportRequest; import com.google.api.services.analyticsreporting.v4.model.ReportRow; +/** + * + * @author Giancarlo Panichi (ISTI CNR) + * + */ public class JupyterAccessesHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(JupyterAccessesHarvester.class); diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index ae47719..4f2055b 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -15,6 +15,11 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * + * @author Giancarlo Panichi (ISTI CNR) + * + */ public class AccountingDataHarvesterJupyterTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); From f2b37893a14dadfe9a9c686165258ead5e34ae76 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 10:14:53 +0200 Subject: [PATCH 07/17] set dry run as default in test for safety --- .../gcube/dataharvest/AccountingDataHarvesterPluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 5a5bc86..e4ee32f 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -108,7 +108,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); - inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); + inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); /* From 42527a425ad0550b483c03f8375b9126ab00d311 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 10:18:19 +0200 Subject: [PATCH 08/17] added property file required for tests --- src/test/resources/META-INF/plugin.properties | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/test/resources/META-INF/plugin.properties diff --git a/src/test/resources/META-INF/plugin.properties b/src/test/resources/META-INF/plugin.properties new file mode 100644 index 0000000..c9eb97c --- /dev/null +++ b/src/test/resources/META-INF/plugin.properties @@ -0,0 +1,4 @@ +groupId=org.gcube.accounting +artifactId=accounting-dashboard-harvester-se-plugin +version=2.0.0-SNAPSHOT +description=Accounting Dashboard Harvester Smart Executor Plugin. \ No newline at end of file From ac305c0a326424a30ad71abea7b2515f8de55c49 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 10:35:08 +0200 Subject: [PATCH 09/17] Improved test --- .../AccountingDataHarvesterJupyterTest.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index 4f2055b..ff22057 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -4,10 +4,12 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.SortedSet; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; +import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; import org.junit.Ignore; @@ -24,7 +26,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); public static final String ROOT = "/d4science.research-infrastructures.eu"; - private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; + // private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; @Ignore @Test @@ -47,12 +49,18 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { ContextTest.setContextByName(ROOT); JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); - ContextTest.setContextByName(SCOPE); - - List harvested = jupyterAccessesHarvester.getAccountingRecords(); - accountingRecords.addAll(harvested); - - logger.debug("{} - {}", SCOPE, accountingRecords); + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + + for(String context : contexts) { + ContextTest.setContext(contextAuthorization.getTokenForContext(context)); + + List harvested = jupyterAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + logger.debug("{} - {}", context, accountingRecords); + } } From fff61014913b643eb053c33607dbf68e751f0bb7 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 10:38:25 +0200 Subject: [PATCH 10/17] fixed test log --- .../dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index ff22057..f667d98 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -58,8 +58,6 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { List harvested = jupyterAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - - logger.debug("{} - {}", context, accountingRecords); } } From 72b7aeccf2028a0ab8de60980ae887affc382e05 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 10:39:08 +0200 Subject: [PATCH 11/17] fixed test log --- .../dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java index f667d98..4b73fcd 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java @@ -62,7 +62,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { } - logger.debug("{}", accountingRecords); + // logger.debug("{}", accountingRecords); ContextTest.setContextByName(ROOT); // dao.insertRecords(accountingRecords.toArray(new From ba158f3187e7b4564d99f0b6e5829d6b17ba9965 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 18:11:05 +0200 Subject: [PATCH 12/17] Fixed test --- .../AccountingDataHarvesterJupyterTest.java | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) rename src/test/java/org/gcube/dataharvest/{jupyter => }/AccountingDataHarvesterJupyterTest.java (54%) diff --git a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java similarity index 54% rename from src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java rename to src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 4b73fcd..daa1834 100644 --- a/src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -1,18 +1,24 @@ -package org.gcube.dataharvest.jupyter; +package org.gcube.dataharvest; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.SortedSet; +import org.gcube.accounting.accounting.summary.access.AccountingDao; +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,19 +34,45 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { public static final String ROOT = "/d4science.research-infrastructures.eu"; // private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { + AccountingDao dao = AccountingDao.get(); + + Set scopeDescriptorSet = dao.getContexts(); + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); + } + AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); + + Set dimensionSet = dao.getDimensions(); + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { + dimensionMap.put(dimension.getId(), dimension); + } + + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); + + return dao; + } + @Ignore @Test public void testJupyterAccessesHarvester() throws Exception { try { - // AccountingDao dao = getAccountingDao(); + + ContextTest.setContextByName(ROOT); + AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); - //starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); - //starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); AggregationType measureType = AggregationType.MONTHLY; + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + List accountingRecords = new ArrayList<>(); for (Date start : starts) { @@ -49,10 +81,6 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { ContextTest.setContextByName(ROOT); JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); - - ContextAuthorization contextAuthorization = new ContextAuthorization(); - SortedSet contexts = contextAuthorization.getContexts(); - for(String context : contexts) { ContextTest.setContext(contextAuthorization.getTokenForContext(context)); @@ -64,9 +92,10 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { // logger.debug("{}", accountingRecords); + logger.debug("Going to insert {}", accountingRecords); + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new - // AccountingRecord[1])); + dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); From 1b500a2f3d8fb094269ef8d489cea55aee2bf632 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 12 Apr 2021 18:11:36 +0200 Subject: [PATCH 13/17] fixed test --- .../dataharvest/AccountingDataHarvesterJupyterTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index daa1834..464a63a 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -19,6 +19,7 @@ import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,8 +61,8 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { public void testJupyterAccessesHarvester() throws Exception { try { - ContextTest.setContextByName(ROOT); - AccountingDao dao = getAccountingDao(); +// ContextTest.setContextByName(ROOT); +// AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); @@ -95,7 +96,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { logger.debug("Going to insert {}", accountingRecords); ContextTest.setContextByName(ROOT); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); From 695bf798f98f44573e0cbb50eb8aa8f28b310e7f Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 21 May 2021 15:24:13 +0200 Subject: [PATCH 14/17] fixed tests --- .../AccountingDataHarvesterJupyterTest.java | 11 ++- .../AccountingDataHarvesterPluginTest.java | 88 +++++++++++++------ 2 files changed, 70 insertions(+), 29 deletions(-) diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 464a63a..342128e 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -56,18 +56,21 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { return dao; } - @Ignore + //@Ignore @Test public void testJupyterAccessesHarvester() throws Exception { try { -// ContextTest.setContextByName(ROOT); -// AccountingDao dao = getAccountingDao(); + ContextTest.setContextByName(ROOT); + AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.APRIL, 1).getTime()); + //starts.add(DateUtils.getStartCalendar(2021, Calendar.MAY, 1).getTime()); + AggregationType measureType = AggregationType.MONTHLY; @@ -96,7 +99,7 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { logger.debug("Going to insert {}", accountingRecords); ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index e4ee32f..18c46a3 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -91,8 +91,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - - @Ignore + + //@Ignore @Test public void launch() { try { @@ -109,17 +109,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); - inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); + inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, true); - /* - * Calendar from = DateUtils.getStartCalendar(2020, Calendar.MAY, - * 1); String fromDate = - * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - * logger.trace("{} is {}", - * AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, - * fromDate); inputs.put(AccountingDashboardHarvesterPlugin. - * START_DATE_INPUT_PARAMETER, fromDate); - */ + Calendar from = DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1); + String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); + inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); accountingDataHarvesterPlugin.launch(inputs); @@ -153,13 +148,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); /* - * Calendar from = DateUtils.getStartCalendar(2016, - * Calendar.SEPTEMBER, 1); String fromDate = - * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + * Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); + * String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); * logger.trace("{} is {}", - * AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, - * fromDate); inputs.put(AccountingDataHarvesterPlugin. - * START_DATE_INPUT_PARAMETER, fromDate); + * AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); + * inputs.put(AccountingDataHarvesterPlugin. START_DATE_INPUT_PARAMETER, + * fromDate); */ // 3rd of the month for MONTHLY Harvesting at 10:00 @@ -184,8 +178,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - @Ignore - //@Test + //@Ignore + @Test public void launchOldData() { try { @@ -587,7 +581,52 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } + + @Test + public void testMethodInvocationOldData() { + try { + + ContextTest.setContextByName(ROOT); + AccountingDao dao = getAccountingDao(); + + List starts = new ArrayList<>(); + starts.add(DateUtils.getStartCalendar(2021, Calendar.APRIL, 1).getTime()); + + AggregationType measureType = AggregationType.MONTHLY; + + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + + List accountingRecords = new ArrayList<>(); + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); + + for (String context : contexts) { + // Setting the token for the context + ContextTest.setContext(contextAuthorization.getTokenForContext(context)); + + + List harvested = methodInvocationHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + logger.debug("{} - {}", context, harvested); + + } + } + + logger.debug("Going to insert {}", accountingRecords); + + ContextTest.setContextByName(ROOT); + dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + + } catch (Exception e) { + logger.error("", e); + } + } + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { AccountingDao dao = AccountingDao.get(); @@ -761,7 +800,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - + @Ignore @Test public void testCoreServicesHarvester() { @@ -797,7 +836,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.error("", e); } } - + @Ignore @Test public void testCatalogueHarvester() { @@ -812,9 +851,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Date end = DateUtils.getStartCalendar(2020, Calendar.FEBRUARY, 1).getTime(); /* - * Date start = DateUtils.getPreviousPeriod(measureType, - * false).getTime(); Date end = - * DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + * Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date + * end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); */ ScopeBean scopeBean = new ScopeBean(context); From 8c7bf2c22b480099b76e6137f06388dc9af60ee8 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 21 May 2021 15:39:53 +0200 Subject: [PATCH 15/17] fixed bug on VRE users --- .../gcube/dataharvest/AccountingDashboardHarvesterPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index 340c90c..2f1627d 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -176,7 +176,7 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { boolean getVREUsers = true; if (inputs.containsKey(GET_VRE_USERS_INPUT_PARAMETER)) { try { - reRun = (boolean) inputs.get(GET_VRE_USERS_INPUT_PARAMETER); + getVREUsers = (boolean) inputs.get(GET_VRE_USERS_INPUT_PARAMETER); } catch (Exception e) { throw new IllegalArgumentException("'" + GET_VRE_USERS_INPUT_PARAMETER + "' must be a boolean"); } @@ -482,7 +482,6 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { accountingRecords); if (!dryRun) { dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - // dbaseManager.insertMonthlyData(start, end, data, reRun); } else { logger.debug("Harvested measures are {}", accountingRecords); } From 77311be1aa2af916a2a391786b3246f78822fe9a Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 31 May 2021 17:39:27 +0200 Subject: [PATCH 16/17] fixed test --- .../AccountingDashboardHarvesterPlugin.java | 2 +- .../AccountingDataHarvesterJupyterTest.java | 30 +++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index 2f1627d..548972f 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -112,7 +112,7 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { return scopeDescriptors.get().get(context); } - protected static final InheritableThreadLocal> scopeDescriptors = new InheritableThreadLocal>() { + public static final InheritableThreadLocal> scopeDescriptors = new InheritableThreadLocal>() { @Override protected Map initialValue() { diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 342128e..6ab1b6d 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -8,18 +8,19 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; +import java.util.TreeSet; import org.gcube.accounting.accounting.summary.access.AccountingDao; import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.scope.impl.ScopeBean; import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,16 +70,30 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.APRIL, 1).getTime()); - //starts.add(DateUtils.getStartCalendar(2021, Calendar.MAY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2021, Calendar.MAY, 1).getTime()); - AggregationType measureType = AggregationType.MONTHLY; ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + /* + SortedSet contexts = new TreeSet<>(); + contexts.add("/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"); + contexts.add("/d4science.research-infrastructures.eu/D4OS/Zoo-Phytoplankton_EOV"); + contexts.add("/d4science.research-infrastructures.eu/D4OS/MarineEnvironmentalIndicators"); + */ List accountingRecords = new ArrayList<>(); + Set scopeDescriptorSet = dao.getContexts(); + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); + } + AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); + + for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); @@ -88,6 +103,15 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { for(String context : contexts) { ContextTest.setContext(contextAuthorization.getTokenForContext(context)); + ScopeBean scopeBean = new ScopeBean(context); + + ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); + if (actualScopeDescriptor == null) { + actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); + } + + AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); + List harvested = jupyterAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); } From 94a558d3c1962fe1ba36d853c584e078b5551f1d Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Mon, 31 May 2021 17:50:23 +0200 Subject: [PATCH 17/17] Updated to fix Jupyter test --- .../gcube/dataharvest/AccountingDataHarvesterJupyterTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 6ab1b6d..7746460 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; -import java.util.TreeSet; import org.gcube.accounting.accounting.summary.access.AccountingDao; import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;