diff --git a/CHANGELOG.md b/CHANGELOG.md index 4813f53..e776c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v2.0.0-SNAPSHOT] +- Added Jupyter Harvester [#21031] - Switched accounting JSON management to gcube-jackson [#19115] - Switched smart-executor JSON management to gcube-jackson [#19647] diff --git a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java index 815c6c4..06331ea 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/HarvestedDataKey.java @@ -15,6 +15,7 @@ public enum HarvestedDataKey { MESSAGES_ACCESSES("Messages Accesses"), NOTIFICATIONS_ACCESSES("Notifications Accesses"), PROFILE_ACCESSES("Profile Accesses"), + JUPYTER_ACCESSES("Jupyter Accesses"), CATALOGUE_ACCESSES("Catalogue Accesses"), CATALOGUE_DATASET_LIST_ACCESSES("Item List"), diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java new file mode 100644 index 0000000..95fc607 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -0,0 +1,400 @@ +package org.gcube.dataharvest.harvester; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; +import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; +import org.gcube.common.encryption.encrypter.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; +import org.gcube.dataharvest.datamodel.AnalyticsReportCredentials; +import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.dataharvest.datamodel.VREAccessesReportRow; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.PemReader; +import com.google.api.client.util.PemReader.Section; +import com.google.api.client.util.SecurityUtils; +import com.google.api.services.analyticsreporting.v4.AnalyticsReporting; +import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes; +import com.google.api.services.analyticsreporting.v4.model.DateRange; +import com.google.api.services.analyticsreporting.v4.model.DateRangeValues; +import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest; +import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse; +import com.google.api.services.analyticsreporting.v4.model.Metric; +import com.google.api.services.analyticsreporting.v4.model.Report; +import com.google.api.services.analyticsreporting.v4.model.ReportRequest; +import com.google.api.services.analyticsreporting.v4.model.ReportRow; + +public class JupyterAccessesHarvester extends BasicHarvester { + + private static Logger logger = LoggerFactory.getLogger(JupyterAccessesHarvester.class); + + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + + private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; + private static final String SERVICE_ENDPOINT_NAME = "BigGAnalyticsReportService"; + private static final String AP_VIEWS_PROPERTY = "views"; + private static final String AP_CLIENT_PROPERTY = "clientId"; + private static final String AP_PRIVATEKEY_PROPERTY = "privateKeyId"; + private static final String APPLICATION_NAME = "Analytics Reporting"; + + private List vreAccesses; + + public JupyterAccessesHarvester(Date start, Date end) throws Exception { + super(start, end); + vreAccesses = getAllAccesses(start, end); + } + + @Override + public List getAccountingRecords() throws Exception { + try { + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + + ArrayList accountingRecords = new ArrayList(); + + int measure = 0; + + ScopeBean scopeBean = new ScopeBean(context); + String lowerCasedContext = scopeBean.name().toLowerCase(); + + for (VREAccessesReportRow row : vreAccesses) { + String pagePath = row.getPagePath().toLowerCase(); + if (pagePath != null && !pagePath.isEmpty()) { + if (pagePath.contains(lowerCasedContext)) { + if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { + logger.trace("Matched jupyter or jupiter ({}) : {}", pagePath); + measure += row.getVisitNumber(); + } + } + } + } + + + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, + getDimension(HarvestedDataKey.ACCESSES), (long) measure); + logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); + accountingRecords.add(ar); + + return accountingRecords; + + } catch (Exception e) { + throw e; + } + } + + /** + * + * @return a list of {@link VREAccessesReportRow} objects containing the + * pagePath and the visit number e.g. VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/add-new-users, + * visitNumber=1] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/administration, + * visitNumber=2] VREAccessesReportRow + * [pagePath=/group/agroclimaticmodelling/agroclimaticmodelling, + * visitNumber=39] + */ + private static List getAllAccesses(Date start, Date end) throws Exception { + DateRange dateRange = getDateRangeForAnalytics(start, end); + logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString()); + + AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs(); + AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S); + HashMap> responses = getReportResponses(service, + credentialsFromD4S.getViewIds(), dateRange); + List totalAccesses = new ArrayList<>(); + + for (String view : responses.keySet()) { + List viewReport = parseResponse(view, responses.get(view)); + logger.trace("Got {} entries from view id={}", viewReport.size(), view); + totalAccesses.addAll(viewReport); + } + logger.trace("Merged in {} total entries from all views", totalAccesses.size()); + return totalAccesses; + } + + /** + * Initializes an Analytics Reporting API V4 service object. + * + * @return An authorized Analytics Reporting API V4 service object. + * @throws IOException + * @throws GeneralSecurityException + */ + private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) + throws GeneralSecurityException, IOException { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all()); + + // Construct the Analytics Reporting service object. + return new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + } + + /** + * Queries the Analytics Reporting API V4. + * + * @param service + * An authorized Analytics Reporting API V4 service object. + * @return GetReportResponse The Analytics Reporting API V4 response. + * @throws IOException + */ + private static HashMap> getReportResponses(AnalyticsReporting service, + List viewIDs, DateRange dateRange) throws IOException { + + HashMap> reports = new HashMap<>(); + + // Create the Metrics object. + Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages"); + com.google.api.services.analyticsreporting.v4.model.Dimension pageTitle = new com.google.api.services.analyticsreporting.v4.model.Dimension() + .setName("ga:pagePath"); + + for (String view : viewIDs) { + List gReportResponses = new ArrayList<>(); + logger.info("Getting data from Google Analytics for viewid: " + view); + boolean iterateMorePages = true; + String nextPageToken = null; + while (iterateMorePages) { + // Create the ReportRequest object. + ReportRequest request = new ReportRequest().setViewId(view.trim()) + .setDateRanges(Arrays.asList(dateRange)).setMetrics(Arrays.asList(sessions)) + .setDimensions(Arrays.asList(pageTitle)); + request.setPageSize(1000); + request.setPageToken(nextPageToken); + ArrayList requests = new ArrayList(); + requests.add(request); + // Create the GetReportsRequest object. + GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests); + // Call the batchGet method. + GetReportsResponse response = service.reports().batchGet(getReport).execute(); + nextPageToken = response.getReports().get(0).getNextPageToken(); + iterateMorePages = (nextPageToken != null); + logger.debug("got nextPageToken: " + nextPageToken); + gReportResponses.add(response); + } + reports.put(view, gReportResponses); + } + // Return the response. + return reports; + } + + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + /** + * Parses and prints the Analytics Reporting API V4 response. + * + * @param response + * An Analytics Reporting API V4 response. + */ + private static List parseResponse(String viewId, List responses) { + logger.debug("parsing Response for " + viewId); + + List toReturn = new ArrayList<>(); + for (GetReportsResponse response : responses) { + for (Report report : response.getReports()) { + List rows = report.getData().getRows(); + if (rows == null) { + logger.warn("No data found for " + viewId); + } else { + for (ReportRow row : rows) { + String dimension = row.getDimensions().get(0); + DateRangeValues metric = row.getMetrics().get(0); + VREAccessesReportRow var = new VREAccessesReportRow(); + boolean validEntry = false; + String pagePath = dimension; + if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) { + var.setPagePath(dimension); + validEntry = true; + } + if (validEntry) { + var.setVisitNumber(Integer.parseInt(metric.getValues().get(0))); + toReturn.add(var); + } + } + } + } + } + return toReturn; + } + + private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException { + + String clientId = cred.getClientId(); + String clientEmail = cred.getClientEmail(); + String privateKeyPem = cred.getPrivateKeyPem(); + String privateKeyId = cred.getPrivateKeyId(); + String tokenUri = cred.getTokenUri(); + String projectId = cred.getProjectId(); + + if (clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) { + throw new IOException("Error reading service account credential from stream, " + + "expecting 'client_id', 'client_email', 'private_key' and 'private_key_id'."); + } + + PrivateKey privateKey = privateKeyFromPkcs8(privateKeyPem); + + Collection emptyScopes = Collections.emptyList(); + + Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport()) + .setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail) + .setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey) + .setServiceAccountPrivateKeyId(privateKeyId); + + if (tokenUri != null) { + credentialBuilder.setTokenServerEncodedUrl(tokenUri); + } + + if (projectId != null) { + credentialBuilder.setServiceAccountProjectId(projectId); + } + + // Don't do a refresh at this point, as it will always fail before the + // scopes are added. + return credentialBuilder.build(); + } + + private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException { + Reader reader = new StringReader(privateKeyPem); + Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY"); + if (section == null) { + throw new IOException("Invalid PKCS8 data."); + } + byte[] bytes = section.getBase64DecodedBytes(); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(bytes); + Exception unexpectedException = null; + try { + KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory(); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + return privateKey; + } catch (NoSuchAlgorithmException exception) { + unexpectedException = exception; + } catch (InvalidKeySpecException exception) { + unexpectedException = exception; + } + throw new IOException("Unexpected exception reading PKCS data", unexpectedException); + } + + private static List getAnalyticsReportingConfigurationFromIS(String infrastructureScope) + throws Exception { + String scope = infrastructureScope; + String currScope = ScopeProvider.instance.get(); + ScopeProvider.instance.set(scope); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); + query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List toReturn = client.submit(query); + ScopeProvider.instance.set(currScope); + return toReturn; + } + + /** + * l + * + * @throws Exception + */ + private static AnalyticsReportCredentials getAuthorisedApplicationInfoFromIs() throws Exception { + AnalyticsReportCredentials reportCredentials = new AnalyticsReportCredentials(); + + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + try { + List list = getAnalyticsReportingConfigurationFromIS(context); + if (list.size() > 1) { + logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME + + " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY); + } else if (list.size() == 0) { + logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category " + + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context); + } else { + + for (ServiceEndpoint res : list) { + reportCredentials.setTokenUri(res.profile().runtime().hostedOn()); + Group apGroup = res.profile().accessPoints(); + AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]); + AccessPoint found = accessPoints[0]; + reportCredentials.setClientEmail(found.address()); + reportCredentials.setProjectId(found.username()); + reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password())); + for (Property prop : found.properties()) { + if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + String[] views = decryptedValue.split(";"); + reportCredentials.setViewIds(Arrays.asList(views)); + } + if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setClientId(decryptedValue); + } + if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) { + String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value()); + reportCredentials.setPrivateKeyId(decryptedValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return reportCredentials; + } + + private static LocalDate asLocalDate(Date date) { + return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static DateRange getDateRangeForAnalytics(Date start, Date end) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // required + // by + // Analytics + String startDate = asLocalDate(start).format(formatter); + String endDate = asLocalDate(end).format(formatter); + DateRange dateRange = new DateRange();// date format `yyyy-MM-dd` + dateRange.setStartDate(startDate); + dateRange.setEndDate(endDate); + return dateRange; + } + +} diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index bfa4ca7..35c4dcf 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -24,6 +24,7 @@ import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; +import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; @@ -47,292 +48,305 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AccountingDataHarvesterPluginTest extends ContextTest { - + private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPluginTest.class); - + public static final String ROOT = "/d4science.research-infrastructures.eu"; public static final String SO_BIG_VO = "/d4science.research-infrastructures.eu/SoBigData"; - + public static final String TAGME_VRE = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; public static final String STOCK_ASSESMENT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment"; - + public static SortedSet getContexts() throws Exception { SortedSet contexts = new TreeSet<>(); - LinkedHashMap map = ContextManager.readContexts(); - for(String scope : map.keySet()) { + LinkedHashMap map = ContextManager.readContexts(); + for (String scope : map.keySet()) { try { String context = map.get(scope).toString(); contexts.add(context); - } catch(Exception e) { + } catch (Exception e) { throw e; } } return contexts; } - + // @Test public void getDimensions() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDao dao = AccountingDao.get(); - + Set dimensionSet = dao.getDimensions(); - for(Dimension d : dimensionSet) { + for (Dimension d : dimensionSet) { logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel()); } - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + @Test public void launch() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); - + /* - Calendar from = DateUtils.getStartCalendar(2020, Calendar.MAY, 1); - String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - */ - + * Calendar from = DateUtils.getStartCalendar(2020, Calendar.MAY, + * 1); String fromDate = + * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + * logger.trace("{} is {}", + * AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, + * fromDate); inputs.put(AccountingDashboardHarvesterPlugin. + * START_DATE_INPUT_PARAMETER, fromDate); + */ + accountingDataHarvesterPlugin.launch(inputs); - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchPluginOnSmartExecutor() { try { - + ContextTest.setContextByName(ROOT); - + String pluginName = new AccountingDashboardHarvesterPlugin().getName(); - + SmartExecutorClient smartExecutor = SmartExecutorClientFactory.getClient(pluginName); Assert.assertNotNull(smartExecutor); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); - + /* - Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); - logger.trace("{} is {}", AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - inputs.put(AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); - */ - - // 3rd of the month for MONTHLY Harvesting at 10:00 - // CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 ? *"); - + * Calendar from = DateUtils.getStartCalendar(2016, + * Calendar.SEPTEMBER, 1); String fromDate = + * DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); + * logger.trace("{} is {}", + * AccountingDataHarvesterPlugin.START_DATE_INPUT_PARAMETER, + * fromDate); inputs.put(AccountingDataHarvesterPlugin. + * START_DATE_INPUT_PARAMETER, fromDate); + */ + + // 3rd of the month for MONTHLY Harvesting at 10:00 + // CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 + // ? *"); + // Every day at 10:00 for partial harvesting CronExpression cronExpression = new CronExpression("0 0 10 3 1/1 ? *"); - + Scheduling scheduling = new Scheduling(cronExpression); scheduling.setGlobal(false); LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs, scheduling); - //LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs); - + // LaunchParameter launchParameter = new LaunchParameter(pluginName, + // inputs); + smartExecutor.launch(launchParameter); - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchOldData() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - - Map inputs = new HashMap<>(); - + + Map inputs = new HashMap<>(); + AggregationType aggregationType = AggregationType.MONTHLY; - + inputs.put(AccountingDashboardHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); - + Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); - + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); - - while(from.before(runbeforeDate)) { + + while (from.before(runbeforeDate)) { String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); accountingDataHarvesterPlugin.launch(inputs); from.add(aggregationType.getCalendarField(), 1); } - + logger.info("End."); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + // @Test public void launchOldDataVREAccessesHarvester() { try { - + ContextTest.setContextByName(ROOT); // AccountingDao dao = AccountingDao.get(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1); - + Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); - - while(from.before(runbeforeDate)) { + + while (from.before(runbeforeDate)) { Date start = from.getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); - + ArrayList accountingRecords = new ArrayList<>(); - + VREAccessesHarvester vreAccessesHarvester = null; - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); - - if(vreAccessesHarvester == null) { - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); } else { - // This code should be never used because the scopes are sorted by fullname - + // This code should be never used because the scopes + // are sorted by fullname + ScopeBean parent = scopeBean.enclosingScope(); - while(!parent.is(Type.INFRASTRUCTURE)) { + while (!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(parent.toString())); - + vreAccessesHarvester = new VREAccessesHarvester(start, end); - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); } - + } - + try { - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { - // Collecting Google Analytics Data for VREs Accesses + // Collecting Google Analytics Data for VREs + // Accesses List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); ContextTest.setContextByName(ROOT); - - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - + + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + Thread.sleep(TimeUnit.SECONDS.toMillis(10)); - + from.add(aggregationType.getCalendarField(), 1); - + } - + ContextTest.setContextByName(ROOT); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } - + logger.info("End."); - + } - + // @Test public void testScopeBean() throws Exception { ContextTest.setContextByName(ROOT); SortedSet contexts = getContexts(); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime(); // start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.info("\n\n\n"); - - for(String context : contexts) { + + for (String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); - // logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - + // logger.debug("FullName {} - Name {}", scopeBean.toString(), + // scopeBean.name()); + try { - - if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) { + + if (scopeBean.is(Type.VRE) + && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) { logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } - - if((context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + + if ((context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_EU_VRE) || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_IT_VRE)) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { @@ -342,20 +356,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } - - } catch(Exception e) { + + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } - + } - + } - + // @Test public void testVREAccessesHarvester() throws Exception { try { // AccountingDao dao = getAccountingDao(); - + List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2018, Calendar.OCTOBER, 1).getTime()); @@ -364,98 +378,102 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); - + AggregationType measureType = AggregationType.MONTHLY; - - String[] contextFullNames = new String[] {"/d4science.research-infrastructures.eu/FARM/GRSF", - "/d4science.research-infrastructures.eu/FARM/GRSF_Admin"}; - + + String[] contextFullNames = new String[] { "/d4science.research-infrastructures.eu/FARM/GRSF", + "/d4science.research-infrastructures.eu/FARM/GRSF_Admin" }; + List accountingRecords = new ArrayList<>(); - - for(Date start : starts) { + + for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + ContextTest.setContextByName(ROOT); VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end); - - for(String contextFullname : contextFullNames) { - + + for (String contextFullname : contextFullNames) { + setContextByNameAndScopeDescriptor(contextFullname); - + List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - + logger.debug("{} - {}", contextFullname, accountingRecords); - + } } - + logger.debug("{}", accountingRecords); - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); throw e; } } - + // @Test public void testVREAccessesHarvesterAll() { try { - + ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - + VREAccessesHarvester vreAccessesHarvester = null; - + ArrayList accountingRecords = new ArrayList<>(); - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); - - if(vreAccessesHarvester == null) { - - if(scopeBean.is(Type.INFRASTRUCTURE)) { + + if (vreAccessesHarvester == null) { + + if (scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); } else { - // This code should be never used because the scopes are sorted by fullname - + // This code should be never used because the scopes are + // sorted by fullname + ScopeBean parent = scopeBean.enclosingScope(); - while(!parent.is(Type.INFRASTRUCTURE)) { + while (!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(parent.toString())); - + vreAccessesHarvester = new VREAccessesHarvester(start, end); - + // Setting back token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); } - + } - + try { - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); @@ -464,31 +482,81 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + + logger.debug("{}", accountingRecords); + + } catch (Exception e) { + logger.error("", e); + } + } + + // @Test + public void testJupyterccessesHarvester() throws Exception { + try { + // AccountingDao dao = getAccountingDao(); + + List starts = new ArrayList<>(); + starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.OCTOBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.NOVEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2018, Calendar.DECEMBER, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); + starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); + + AggregationType measureType = AggregationType.MONTHLY; + + String[] contextFullNames = new String[] { "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab" }; + + List accountingRecords = new ArrayList<>(); + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + + ContextTest.setContextByName(ROOT); + JupyterAccessesHarvester vreAccessesHarvester = new JupyterAccessesHarvester(start, end); + + for (String contextFullname : contextFullNames) { + + setContextByNameAndScopeDescriptor(contextFullname); + + List harvested = vreAccessesHarvester.getAccountingRecords(); + accountingRecords.addAll(harvested); + + logger.debug("{} - {}", contextFullname, accountingRecords); + + } + } + logger.debug("{}", accountingRecords); - - } catch(Exception e) { + + ContextTest.setContextByName(ROOT); + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); + throw e; } } - + // @Test public void testSocialInteraction() { try { - + ContextTest.setContextByName(ROOT); // AccountingDao dao = AccountingDao.get(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); - + SortedSet contexts = new TreeSet<>(); contexts.add("/d4science.research-infrastructures.eu/D4Research"); contexts.add("/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS"); @@ -496,18 +564,18 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { contexts.add("/d4science.research-infrastructures.eu/gCubeApps/Parthenos"); contexts.add("/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining"); contexts.add("/d4science.research-infrastructures.eu/gCubeApps/gCube"); - + AggregationType aggregationType = AggregationType.MONTHLY; - + Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); Date start = from.getTime(); Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false); - + logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end)); - + ArrayList accountingRecords = new ArrayList<>(); - - for(String context : contexts) { + + for (String context : contexts) { // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); try { @@ -516,273 +584,277 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); List harvested = socialHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); - } catch(Exception e) { + } catch (Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } } - + logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); } - + logger.info("End."); - + } - + // @Test public void testMethodInvocation() { try { - + ContextTest.setContextByName(ROOT); ContextAuthorization contextAuthorization = new ContextAuthorization(); String stockAssessmentToken = contextAuthorization.generateTokenForContext(STOCK_ASSESMENT_VRE, null); - + ContextTest.setContext(stockAssessmentToken); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); List accountingRecords = methodInvocationHarvester.getAccountingRecords(); - + logger.debug("{}", accountingRecords); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { AccountingDao dao = AccountingDao.get(); - + Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for(ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - + Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for(Dimension dimension : dimensionSet) { + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { dimensionMap.put(dimension.getId(), dimension); } - + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - + return dao; } - + protected void setContextByNameAndScopeDescriptor(String contextFullName) throws ObjectNotFound, Exception { ContextAuthorization contextAuthorization = new ContextAuthorization(); String tagMeToken = contextAuthorization.getTokenForContext(contextFullName); - + ContextTest.setContext(tagMeToken); - + ScopeBean scopeBean = new ScopeBean(contextFullName); ScopeDescriptor actualScopeDescriptor = AccountingDashboardHarvesterPlugin.scopeDescriptors.get() .get(contextFullName); - if(actualScopeDescriptor == null) { + if (actualScopeDescriptor == null) { actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), contextFullName); } - + AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); } - + // @Test public void testTagMeMethodInvocation() throws Exception { try { // AccountingDao dao = getAccountingDao(); setContextByNameAndScopeDescriptor(TAGME_VRE); - + List accountingRecords = new ArrayList<>(); - + AggregationType measureType = AggregationType.MONTHLY; - + List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2018, Calendar.NOVEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2018, Calendar.DECEMBER, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.JANUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime()); starts.add(DateUtils.getStartCalendar(2019, Calendar.MARCH, 1).getTime()); - - for(Date start : starts) { - + + for (Date start : starts) { + Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); accountingRecords.addAll(methodInvocationHarvester.getAccountingRecords()); - + logger.debug("{}", accountingRecords); } - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); throw e; } } - + // @Test public void testGetVREUsersForSpecificVRE() { try { ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + // AccountingDao dao = AccountingDao.get(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); ContextTest.setContext(contextAuthorization .getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); - + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); List harvested = vreUsersHarvester.getAccountingRecords(); - + logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); - + ContextTest.setContextByName(ROOT); - // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + // dao.insertRecords(accountingRecords.toArray(new + // AccountingRecord[1])); + + } catch (Exception e) { logger.error("", e); } - + } - + // @Test public void testFilteringGenericResource() { try { ContextTest.setContextByName(ROOT); - //Utils.setContext(RESOURCE_CATALOGUE); - + // Utils.setContext(RESOURCE_CATALOGUE); + AggregationType measureType = AggregationType.MONTHLY; - + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + SortedSet contexts = getContexts(); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); SortedSet validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_VO + "/"); logger.info("Valid Contexts {}", validContexts); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } - + } - + // @Test public void testResourceCatalogueHarvester() { try { - - //Utils.setContext(RESOURCE_CATALOGUE); + + // Utils.setContext(RESOURCE_CATALOGUE); ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + SortedSet contexts = getContexts(); - + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); List data = resourceCatalogueHarvester.getAccountingRecords(); - + logger.debug("{}", data); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + @Test public void testCoreServicesHarvester() { try { - String context = ROOT; //"/gcube"; + String context = ROOT; // "/gcube"; ContextTest.setContextByName(context); - + AccountingDao dao = getAccountingDao(); - + Date start = DateUtils.getStartCalendar(2017, Calendar.MAY, 1).getTime(); Date finalEnd = DateUtils.getStartCalendar(2020, Calendar.MAY, 1).getTime(); Date end = DateUtils.getEndDateFromStartDate(AggregationType.MONTHLY, start, 1, false); - + ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - - while(end.before(finalEnd)) { - CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); + + while (end.before(finalEnd)) { + CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); List accountingRecords = coreServicesHarvester.getAccountingRecords(); dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - + logger.debug("{} -> {} Data Inserted", DateUtils.format(start), DateUtils.format(end)); logger.debug("---------------------------------------------------------------------------------------"); - + Thread.sleep(TimeUnit.SECONDS.toMillis(90)); start = end; end = DateUtils.getEndDateFromStartDate(AggregationType.MONTHLY, start, 1, false); } - - } catch(Exception e) { + } catch (Exception e) { logger.error("", e); } } - @Test public void testCatalogueHarvester() { try { - String context = ROOT; //"/gcube"; + String context = ROOT; // "/gcube"; ContextTest.setContextByName(context); - + AccountingDao dao = getAccountingDao(); - + Date start = DateUtils.getStartCalendar(2020, Calendar.JANUARY, 1).getTime(); Date end = DateUtils.getStartCalendar(2020, Calendar.FEBRUARY, 1).getTime(); /* - * Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); - * Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); + * Date start = DateUtils.getPreviousPeriod(measureType, + * false).getTime(); Date end = + * DateUtils.getEndDateFromStartDate(measureType, start, 1, false); */ - + ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); + CatalogueAccessesHarvester catalogueHarvester = new CatalogueAccessesHarvester(start, end); List accountingRecords = catalogueHarvester.getAccountingRecords(); for (AccountingRecord accountingRecord : accountingRecords) { logger.debug("{}", accountingRecord); @@ -791,40 +863,41 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - } catch(Exception e) { + } catch (Exception e) { logger.error("", e); } } - // @Test public void testDataMethodDownloadHarvester() { try { - - //Utils.setContext(RESOURCE_CATALOGUE); + + // Utils.setContext(RESOURCE_CATALOGUE); ContextTest.setContextByName(ROOT); - + AggregationType measureType = AggregationType.MONTHLY; - - // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime(); - // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime(); - + + // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, + // 1).getTime(); + // Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, + // 1).getTime(); + Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false); - + AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - - for(String context : contexts) { + + for (String context : contexts) { ScopeBean scopeBean = new ScopeBean(context); logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name()); - - if(context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO)) { - if(scopeBean.is(Type.VRE)) { - if(context.startsWith(TAGME_VRE)) { + + if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO)) { + if (scopeBean.is(Type.VRE)) { + if (context.startsWith(TAGME_VRE)) { continue; } ContextTest.setContext(contextAuthorization.getTokenForContext(context)); @@ -834,82 +907,82 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("{}", data); } } - + } - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } } - + public static final String E_LEARNING_AREA_VRE = "/d4science.research-infrastructures.eu/SoBigData/E-Learning_Area"; - + // @Test public void addMissingVREAccesses() { try { - + ContextTest.setContextByName(ROOT); - + AccountingDashboardHarvesterPlugin adhp = new AccountingDashboardHarvesterPlugin(); Properties properties = adhp.getConfigParameters(); AccountingDashboardHarvesterPlugin.getProperties().set(properties); - + ContextAuthorization contextAuthorization = new ContextAuthorization(); - + // DatabaseManager dbaseManager = new DatabaseManager(); AccountingDao dao = AccountingDao.get(); - + Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for(ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - + Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for(Dimension dimension : dimensionSet) { + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { dimensionMap.put(dimension.getId(), dimension); } - + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - + // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList(); - + String context = E_LEARNING_AREA_VRE; - + // Setting the token for the context ContextTest.setContext(contextAuthorization.getTokenForContext(context)); - + ScopeBean scopeBean = new ScopeBean(context); ScopeDescriptor scopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); - + Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey()); - + Calendar calendar = DateUtils.getStartCalendar(2018, Calendar.JULY, 1); calendar.set(Calendar.DAY_OF_MONTH, 15); - - Map monthValues = new HashMap<>(); + + Map monthValues = new HashMap<>(); monthValues.put(Calendar.JULY, 54); monthValues.put(Calendar.AUGUST, 23); monthValues.put(Calendar.SEPTEMBER, 127); monthValues.put(Calendar.OCTOBER, 192); - - for(Integer month : monthValues.keySet()) { + + for (Integer month : monthValues.keySet()) { calendar.set(Calendar.MONTH, month); Instant instant = calendar.toInstant(); - + AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, dimension, (long) monthValues.get(month)); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); accountingRecords.add(ar); } - + logger.trace("{}", accountingRecords); dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); - - } catch(Exception e) { + + } catch (Exception e) { logger.error("", e); } }