From e0fd599c80befbacaf8d5ae93347e79c82a3142b Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Thu, 29 Jun 2023 16:08:01 +0200 Subject: [PATCH] Cleaning code --- .../AccountingDashboardHarvesterPlugin.java | 35 +-- .../harvester/JupyterAccessesHarvester.java | 2 +- .../harvester/MethodInvocationHarvester.java | 2 +- .../harvester/RStudioAccessesHarvester.java | 2 +- .../SocialInteractionsHarvester.java | 2 +- .../harvester/VREAccessesHarvester.java | 6 +- .../harvester/VREUsersHarvester.java | 7 +- .../sobigdata/SoBigDataHarvester.java | 2 +- .../TagMeMethodInvocationHarvester.java | 2 +- .../utils/ContextAuthorization.java | 6 +- .../gcube/dataharvest/utils/ContextTest.java | 80 ----- .../gcube/dataharvest/utils/TestMassi.java | 129 -------- .../org/gcube/dataharvest/utils/Utils.java | 7 - .../AccountingDataHarvesterJupyterTest.java | 56 +--- .../AccountingDataHarvesterPluginTest.java | 292 +++++------------- .../AccountingDataHarvesterRStudioTest.java | 64 +--- .../utils/ContextAuthorizationTest.java | 3 +- .../gcube/dataharvest/utils/ContextTest.java | 54 +--- 18 files changed, 117 insertions(+), 634 deletions(-) delete mode 100644 src/main/java/org/gcube/dataharvest/utils/ContextTest.java delete mode 100644 src/main/java/org/gcube/dataharvest/utils/TestMassi.java diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index 3bc043d..3d467df 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -78,20 +78,7 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { public AccountingDashboardHarvesterPlugin() { super(); } - - private static final InheritableThreadLocal properties = new InheritableThreadLocal() { - - @Override - protected Properties initialValue() { - return new Properties(); - } - - }; - - public static InheritableThreadLocal getProperties() { - return properties; - } - + public static Dimension getDimension(String key) { Dimension dimension = dimensions.get().get(key); if (dimension == null) { @@ -122,21 +109,8 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { }; - public static ScopeDescriptor getScopeDescriptor() { - return scopeDescriptor.get(); - } - - public static final InheritableThreadLocal scopeDescriptor = new InheritableThreadLocal() { - - @Override - protected ScopeDescriptor initialValue() { - return new ScopeDescriptor("", ""); - } - - }; - @JsonIgnore - public Properties getConfigParameters() throws IOException { + public static Properties getConfigParameters() throws IOException { Properties properties = new Properties(); try { InputStream input = AccountingDashboardHarvesterPlugin.class.getClassLoader() @@ -210,9 +184,6 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})", DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun); - Properties properties = getConfigParameters(); - getProperties().set(properties); - ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); String root = contexts.first(); @@ -255,8 +226,6 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); } - scopeDescriptor.set(actualScopeDescriptor); - if (scopeBean.is(Type.INFRASTRUCTURE)) { try { diff --git a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java index f51e7da..2aba8fa 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java @@ -115,7 +115,7 @@ public class JupyterAccessesHarvester extends BasicHarvester { } } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); if (measure > 0) { AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, diff --git a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java index b7a5044..254222c 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java @@ -72,7 +72,7 @@ public class MethodInvocationHarvester extends BasicHarvester { AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true); } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); Dimension dimension = getDimension(HarvestedDataKey.METHOD_INVOCATIONS); if(result != null) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java index c1da4f6..9f70656 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/RStudioAccessesHarvester.java @@ -115,7 +115,7 @@ public class RStudioAccessesHarvester extends BasicHarvester { } } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); if (measure > 0) { AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java index 3c2357a..5c05199 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java @@ -43,7 +43,7 @@ public class SocialInteractionsHarvester extends BasicHarvester { getJson(); - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); AccountingRecord likesAR = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.SOCIAL_LIKES), (long) likes); logger.debug("{} : {}", likesAR.getDimension().getId(), likesAR.getMeasure()); diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java index 82013eb..7eaf477 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java @@ -42,8 +42,6 @@ import com.google.analytics.data.v1beta.Metric; import com.google.analytics.data.v1beta.Row; import com.google.analytics.data.v1beta.RunReportRequest; import com.google.analytics.data.v1beta.RunReportResponse; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.gson.GsonFactory; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.auth.oauth2.ServiceAccountCredentials; @@ -52,7 +50,7 @@ public class VREAccessesHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(VREAccessesHarvester.class); - private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + // private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private static final String SERVICE_ENDPOINT_CATEGORY = "OnlineService"; private static final String SERVICE_ENDPOINT_NAME = "GA4AnalyticsDataService"; @@ -94,7 +92,7 @@ public class VREAccessesHarvester extends BasicHarvester { } } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.ACCESSES), (long) measure); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java index b36270d..ac3aa46 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java @@ -1,6 +1,5 @@ package org.gcube.dataharvest.harvester; -import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -10,7 +9,6 @@ import org.gcube.accounting.accounting.summary.access.model.update.AccountingRec import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.social_networking.social_networking_client_library.UserClient; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +30,13 @@ public class VREUsersHarvester extends BasicHarvester { @Override public List getAccountingRecords() throws Exception { try { - // String context = Utils.getCurrentContext(); + String context = org.gcube.dataharvest.utils.Utils.getCurrentContext(); + int measure = get(); ArrayList accountingRecords = new ArrayList(); - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, getDimension(HarvestedDataKey.USERS), (long) measure); logger.debug("{} : {}", ar.getDimension().getId(), ar.getMeasure()); diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java index a99eb52..e72e181 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java @@ -77,7 +77,7 @@ public abstract class SoBigDataHarvester extends BasicHarvester { * @throws ObjectNotFound */ protected void initMappingMaps() throws ObjectNotFound, Exception { - Properties properties = AccountingDashboardHarvesterPlugin.getProperties().get(); + Properties properties = AccountingDashboardHarvesterPlugin.getConfigParameters(); Set keys = properties.stringPropertyNames(); mapSystemTypeToDBEntry = new HashMap(); diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java index dcca39c..8863f7b 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/TagMeMethodInvocationHarvester.java @@ -109,7 +109,7 @@ public class TagMeMethodInvocationHarvester extends BasicHarvester { } - ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); Dimension dimension = getDimension(HarvestedDataKey.METHOD_INVOCATIONS); AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, dimension, numberOfInvocation); diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java index 6dcdf2e..0fa3e2b 100644 --- a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java +++ b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java @@ -29,7 +29,7 @@ public class ContextAuthorization { public static final String CLIENT_ID = "accounting-dashboard-harvester-se-plugin"; - protected String clientSecret = "442b9d9a-1973-46f2-99f2-9a10fae94452"; + protected String clientSecret; /** * Contains Context full name as key and Token as Value @@ -57,7 +57,7 @@ public class ContextAuthorization { * Contains Properties used to generate tokens */ public ContextAuthorization() throws Exception { - this.properties = AccountingDashboardHarvesterPlugin.getProperties().get(); + this.properties = AccountingDashboardHarvesterPlugin.getConfigParameters(); this.contextToToken = new HashMap<>(); this.tokenToContext = new HashMap<>(); retrieveContextsAndTokens(); @@ -78,7 +78,7 @@ public class ContextAuthorization { } private TokenResponse getJWTAccessToken(String context) throws Exception { - TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, "442b9d9a-1973-46f2-99f2-9a10fae94452", context, null); + TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(CLIENT_ID, getClientSecret(context), context, null); return tr; } diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextTest.java b/src/main/java/org/gcube/dataharvest/utils/ContextTest.java deleted file mode 100644 index 1d2ece8..0000000 --- a/src/main/java/org/gcube/dataharvest/utils/ContextTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * - */ -package org.gcube.dataharvest.utils; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -import org.gcube.common.authorization.utils.manager.SecretManager; -import org.gcube.common.authorization.utils.manager.SecretManagerProvider; -import org.gcube.common.authorization.utils.secret.Secret; -import org.gcube.common.authorization.utils.secret.SecretUtility; - -/** - * @author Luca Frosini (ISTI - CNR) - */ -public class ContextTest { - - protected static Properties properties; - protected static final String PROPERTIES_FILENAME = "token.properties"; - - public static final String ROOT; - public static final String VO; - public static final String VRE; - - static { - properties = new Properties(); - InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); - - try { - // load the properties file - properties.load(input); - } catch(IOException e) { - throw new RuntimeException(e); - } - - // DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE"; - // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE"; - - //ROOT = "/gcube"; - ROOT = "/d4science.research-infrastructures.eu"; - VO = ROOT + "/devsec"; - VRE = VO + "/devVRE"; - // VO = ROOT + "/devNext"; - // VRE = VO + "/NextNext"; - - - } - - public static void set(Secret secret) throws Exception { - SecretManagerProvider.instance.reset(); - SecretManager secretManager = new SecretManager(); - SecretManagerProvider.instance.set(secretManager); - secretManager.addSecret(secret); - secretManager.set(); - } - - public static void setContext(String token) throws Exception { - Secret secret = getSecret(token); - set(secret); - } - - public static void setContextByName(String fullContextName) throws Exception { - Secret secret = getSecretByContextName(fullContextName); - set(secret); - } - - private static Secret getSecret(String token) throws Exception { - Secret secret = SecretUtility.getSecretByTokenString(token); - return secret; - } - - private static Secret getSecretByContextName(String fullContextName) throws Exception { - String token = ContextTest.properties.getProperty(fullContextName); - return getSecret(token); - } - - -} diff --git a/src/main/java/org/gcube/dataharvest/utils/TestMassi.java b/src/main/java/org/gcube/dataharvest/utils/TestMassi.java deleted file mode 100644 index fef913b..0000000 --- a/src/main/java/org/gcube/dataharvest/utils/TestMassi.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.gcube.dataharvest.utils; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; - -import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; -import org.gcube.common.scope.impl.ScopeBean; -import org.gcube.common.scope.impl.ScopeBean.Type; -import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; -import org.gcube.dataharvest.harvester.VREAccessesHarvester; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestMassi extends ContextTest { - private static Logger logger = LoggerFactory.getLogger(TestMassi.class); - - public static final String ROOT = "/d4science.research-infrastructures.eu"; - public TestMassi() { - // TODO Auto-generated constructor stub - } - - public static void main(String[] args) { - try { - - ContextTest.setContextByName(ROOT); - // AccountingDao dao = AccountingDao.get(); - - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); - - ContextAuthorization contextAuthorization = new ContextAuthorization(); - SortedSet contexts = contextAuthorization.getContexts(); - - - AggregationType aggregationType = AggregationType.MONTHLY; - - Calendar from = DateUtils.getStartCalendar(2023, Calendar.APRIL, 1); - - Calendar runbeforeDate = DateUtils.getStartCalendar(2023, Calendar.JULY, 1); - - while (from.before(runbeforeDate)) { - Date start = from.getTime(); - Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); - - logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); - - ArrayList accountingRecords = new ArrayList<>(); - - VREAccessesHarvester vreAccessesHarvester = null; - - for (String context : contexts) { - // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); - logger.debug("Context set: {} ", context); - ScopeBean scopeBean = new ScopeBean(context); - - if (vreAccessesHarvester == null) { - - if (scopeBean.is(Type.INFRASTRUCTURE)) { - vreAccessesHarvester = new VREAccessesHarvester(start, end); - } else { - // This code should be never used because the scopes - // are sorted by fullname - - ScopeBean parent = scopeBean.enclosingScope(); - while (!parent.is(Type.INFRASTRUCTURE)) { - parent = scopeBean.enclosingScope(); - } - - // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(parent.toString())); - - vreAccessesHarvester = new VREAccessesHarvester(start, end); - - // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); - } - - } - - try { - if (context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) - && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { - logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, - DateUtils.format(start), DateUtils.format(end)); - } else { - // Collecting Google Analytics Data for VREs - // Accesses - List harvested = vreAccessesHarvester.getAccountingRecords(); - accountingRecords.addAll(harvested); - - } - } catch (Exception e) { - logger.error("Error harvesting Social Interactions for {}", context, e); - } - } - - logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), - accountingRecords); - ContextTest.setContextByName(ROOT); - - // dao.insertRecords(accountingRecords.toArray(new - // AccountingRecord[1])); - - Thread.sleep(TimeUnit.SECONDS.toMillis(10)); - - from.add(aggregationType.getCalendarField(), 1); - - } - - ContextTest.setContextByName(ROOT); - - } catch (Exception e) { - logger.error("", e); - } - - logger.info("End."); - - } -} diff --git a/src/main/java/org/gcube/dataharvest/utils/Utils.java b/src/main/java/org/gcube/dataharvest/utils/Utils.java index 2b06581..2a3481a 100644 --- a/src/main/java/org/gcube/dataharvest/utils/Utils.java +++ b/src/main/java/org/gcube/dataharvest/utils/Utils.java @@ -1,12 +1,5 @@ package org.gcube.dataharvest.utils; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; - import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.utils.manager.SecretManager; import org.gcube.common.authorization.utils.manager.SecretManagerProvider; diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java index 989e301..1404aaa 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java @@ -14,7 +14,6 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; -import org.gcube.common.scope.impl.ScopeBean; import org.gcube.dataharvest.harvester.JupyterAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; @@ -30,39 +29,16 @@ import org.slf4j.LoggerFactory; * @author Giancarlo Panichi (ISTI CNR) * */ -public class AccountingDataHarvesterJupyterTest extends ContextTest { +public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterPluginTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); - public static final String ROOT = "/d4science.research-infrastructures.eu"; - // private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; - - protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { - AccountingDao dao = AccountingDao.get(); - - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - - Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for (Dimension dimension : dimensionSet) { - dimensionMap.put(dimension.getId(), dimension); - } - - AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - - return dao; - } @Ignore @Test public void testJupyterAccessesHarvester() throws Exception { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); @@ -74,8 +50,6 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { AggregationType measureType = AggregationType.MONTHLY; - ContextAuthorization contextAuthorization = new ContextAuthorization(); - SortedSet contexts = contextAuthorization.getContexts(); /* SortedSet contexts = new TreeSet<>(); @@ -85,32 +59,15 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { */ List accountingRecords = new ArrayList<>(); - - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end); for(String context : contexts) { - ContextTest.set(contextAuthorization.getSecretForContext(context)); - - ScopeBean scopeBean = new ScopeBean(context); - - ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); - if (actualScopeDescriptor == null) { - actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); - } - - AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); + ContextTest.set(context); List harvested = jupyterAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); @@ -122,8 +79,9 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest { logger.debug("Going to insert {}", accountingRecords); - ContextTest.setContextByName(ROOT); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + ContextTest.set(ROOT); + + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index ea9bcc0..c3addd9 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -34,7 +33,6 @@ import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; -import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; import org.gcube.vremanagement.executor.api.types.LaunchParameter; import org.gcube.vremanagement.executor.api.types.Scheduling; import org.gcube.vremanagement.executor.client.SmartExecutorClient; @@ -55,27 +53,35 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public static final String TAGME_VRE = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; public static final String STOCK_ASSESMENT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/StockAssessment"; + + public static final String BLUE_CLOUD_LAB = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; + + protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { + AccountingDao dao = AccountingDao.get(); - public static SortedSet getContexts() throws Exception { - SortedSet contexts = new TreeSet<>(); - LinkedHashMap map = ContextManager.readContexts(); - for (String scope : map.keySet()) { - try { - String context = map.get(scope).toString(); - contexts.add(context); - } catch (Exception e) { - throw e; - } + Set scopeDescriptorSet = dao.getContexts(); + Map scopeDescriptorMap = new HashMap<>(); + for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { + scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } - return contexts; - } + AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); + Set dimensionSet = dao.getDimensions(); + Map dimensionMap = new HashMap<>(); + for (Dimension dimension : dimensionSet) { + dimensionMap.put(dimension.getId(), dimension); + } + + AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); + + return dao; + } + @Ignore @Test public void getDimensions() { try { - - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDao dao = AccountingDao.get(); @@ -92,11 +98,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } @Ignore - //@Test + // @Test public void launch() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); @@ -125,11 +131,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } @Ignore - @Test + // @Test public void launchPluginOnSmartExecutor() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); String pluginName = new AccountingDashboardHarvesterPlugin().getName(); @@ -182,7 +188,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void launchOldData() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); @@ -219,14 +225,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void launchOldDataVREAccessesHarvester() { try { - ContextTest.setContextByName(ROOT); - // AccountingDao dao = AccountingDao.get(); + ContextTest.set(ROOT); + + AccountingDao dao = getAccountingDao(); - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); - - ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); AggregationType aggregationType = AggregationType.MONTHLY; @@ -246,8 +248,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { VREAccessesHarvester vreAccessesHarvester = null; for (String context : contexts) { - // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); + ContextTest.set(context); ScopeBean scopeBean = new ScopeBean(context); @@ -265,12 +266,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(parent.toString())); + ContextTest.set(parent.toString()); vreAccessesHarvester = new VREAccessesHarvester(start, end); // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); + ContextTest.set(context); } } @@ -294,7 +295,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new // AccountingRecord[1])); @@ -305,7 +306,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); } catch (Exception e) { logger.error("", e); @@ -315,60 +316,13 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } - @Ignore - // @Test - public void testScopeBean() throws Exception { - ContextTest.setContextByName(ROOT); - SortedSet contexts = getContexts(); - - AggregationType aggregationType = AggregationType.MONTHLY; - - Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime(); - // start = DateUtils.getPreviousPeriod(measureType).getTime(); - Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1); - - logger.info("\n\n\n"); - - for (String context : contexts) { - ScopeBean scopeBean = new ScopeBean(context); - // logger.debug("FullName {} - Name {}", scopeBean.toString(), - // scopeBean.name()); - - try { - - if (scopeBean.is(Type.VRE) - && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) { - logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), - DateUtils.format(end)); - } else { - logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), - DateUtils.format(end)); - } - - if ((context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_VO) - || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_EU_VRE) - || context.startsWith(AccountingDashboardHarvesterPlugin.SO_BIG_DATA_IT_VRE)) - && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { - logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, - DateUtils.format(start), DateUtils.format(end)); - } else { - logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), - DateUtils.format(end)); - } - - } catch (Exception e) { - logger.error("Error harvesting Social Interactions for {}", context, e); - } - - } - - } - @Ignore // @Test public void testVREAccessesHarvester() throws Exception { try { - // AccountingDao dao = getAccountingDao(); + ContextTest.set(ROOT); + + AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); starts.add(DateUtils.getStartCalendar(2018, Calendar.SEPTEMBER, 1).getTime()); @@ -389,12 +343,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end); for (String contextFullname : contextFullNames) { - setContextByNameAndScopeDescriptor(contextFullname); + ContextTest.set(contextFullname); List harvested = vreAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); @@ -406,7 +360,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("{}", accountingRecords); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new // AccountingRecord[1])); @@ -421,7 +375,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testVREAccessesHarvesterAll() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AggregationType measureType = AggregationType.MONTHLY; @@ -436,7 +390,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); accountingDataHarvesterPlugin.getConfigParameters(); - ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); VREAccessesHarvester vreAccessesHarvester = null; @@ -445,7 +398,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { for (String context : contexts) { // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); + ContextTest.set(context); ScopeBean scopeBean = new ScopeBean(context); @@ -462,13 +415,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { parent = scopeBean.enclosingScope(); } - // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(parent.toString())); + ContextTest.set(parent.toString()); vreAccessesHarvester = new VREAccessesHarvester(start, end); // Setting back token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); + ContextTest.set(context); } } @@ -500,14 +452,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testSocialInteraction() { try { - ContextTest.setContextByName(ROOT); - // AccountingDao dao = AccountingDao.get(); - - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); - - ContextAuthorization contextAuthorization = new ContextAuthorization(); + ContextTest.set(ROOT); + + AccountingDao dao = getAccountingDao(); SortedSet contexts = new TreeSet<>(); contexts.add("/d4science.research-infrastructures.eu/D4Research"); @@ -529,7 +476,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { for (String context : contexts) { // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); + ContextTest.set(context); try { // Collecting info on social (posts, replies and likes) logger.info("Going to harvest Social Interactions for {}", context); @@ -543,7 +490,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new // AccountingRecord[1])); @@ -559,10 +506,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void testMethodInvocation() { try { - - ContextTest.setContextByName(ROOT); - ContextAuthorization contextAuthorization = new ContextAuthorization(); - ContextTest.set(contextAuthorization.getSecretForContext(STOCK_ASSESMENT_VRE)); + ContextTest.set(STOCK_ASSESMENT_VRE); AggregationType measureType = AggregationType.MONTHLY; @@ -584,7 +528,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testMethodInvocationOldData() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); @@ -612,16 +556,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { for (String context : contexts) { // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); - - ScopeBean scopeBean = new ScopeBean(context); - - ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); - if (actualScopeDescriptor == null) { - actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); - } - - AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); + ContextTest.set(context); List harvested = methodInvocationHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); @@ -633,7 +568,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("Going to insert {}", accountingRecords); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Exception e) { @@ -641,48 +576,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } } - protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { - AccountingDao dao = AccountingDao.get(); - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for (Dimension dimension : dimensionSet) { - dimensionMap.put(dimension.getId(), dimension); - } - - AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - - return dao; - } - - protected void setContextByNameAndScopeDescriptor(String contextFullName) throws ObjectNotFound, Exception { - ContextAuthorization contextAuthorization = new ContextAuthorization(); - ContextTest.set(contextAuthorization.getSecretForContext(contextFullName)); - - ScopeBean scopeBean = new ScopeBean(contextFullName); - ScopeDescriptor actualScopeDescriptor = AccountingDashboardHarvesterPlugin.scopeDescriptors.get() - .get(contextFullName); - if (actualScopeDescriptor == null) { - actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), contextFullName); - } - - AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); - } + @Ignore // @Test public void testTagMeMethodInvocation() throws Exception { try { - // AccountingDao dao = getAccountingDao(); - setContextByNameAndScopeDescriptor(TAGME_VRE); - + + ContextTest.set(ROOT); + AccountingDao dao = getAccountingDao(); + + ContextTest.set(TAGME_VRE); + List accountingRecords = new ArrayList<>(); AggregationType measureType = AggregationType.MONTHLY; @@ -705,7 +612,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.debug("{}", accountingRecords); } - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new // AccountingRecord[1])); @@ -719,16 +626,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void testGetVREUsersForSpecificVRE() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); + AccountingDao dao = getAccountingDao(); - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); - - // AccountingDao dao = AccountingDao.get(); - - ContextAuthorization contextAuthorization = new ContextAuthorization(); - ContextTest.set(contextAuthorization.getSecretForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); + ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience"); AggregationType measureType = AggregationType.MONTHLY; @@ -740,7 +641,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // dao.insertRecords(accountingRecords.toArray(new // AccountingRecord[1])); @@ -754,7 +655,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { // @Test public void testFilteringGenericResource() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); // Utils.setContext(RESOURCE_CATALOGUE); AggregationType measureType = AggregationType.MONTHLY; @@ -762,10 +663,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - SortedSet contexts = getContexts(); - - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - accountingDataHarvesterPlugin.getConfigParameters(); + SortedSet contexts = contextAuthorization.getContexts(); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); @@ -783,9 +681,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testResourceCatalogueHarvester() { try { - // Utils.setContext(RESOURCE_CATALOGUE); - ContextTest.setContextByName(ROOT); - + ContextTest.set(ROOT); + AggregationType measureType = AggregationType.MONTHLY; // Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, @@ -796,10 +693,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Date start = DateUtils.getPreviousPeriod(measureType, false).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin(); - accountingDataHarvesterPlugin.getConfigParameters(); - - SortedSet contexts = getContexts(); + SortedSet contexts = contextAuthorization.getContexts(); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); @@ -817,8 +711,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testCoreServicesHarvester() { try { - String context = ROOT; // "/gcube"; - ContextTest.setContextByName(context); + String context = ROOT; + ContextTest.set(context); AccountingDao dao = getAccountingDao(); @@ -833,7 +727,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { while (end.before(finalEnd)) { CoreServicesAccessesHarvester coreServicesHarvester = new CoreServicesAccessesHarvester(start, end); List accountingRecords = coreServicesHarvester.getAccountingRecords(); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); logger.debug("{} -> {} Data Inserted", DateUtils.format(start), DateUtils.format(end)); logger.debug("---------------------------------------------------------------------------------------"); @@ -853,8 +747,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void testCatalogueHarvester() { try { - String context = ROOT; // "/gcube"; - ContextTest.setContextByName(context); + String context = ROOT; + ContextTest.set(context); AccountingDao dao = getAccountingDao(); @@ -876,7 +770,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } logger.debug("{}", accountingRecords); - dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); + // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Exception e) { logger.error("", e); @@ -890,43 +784,19 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { public void addMissingVREAccesses() { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); - AccountingDashboardHarvesterPlugin adhp = new AccountingDashboardHarvesterPlugin(); - Properties properties = adhp.getConfigParameters(); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); + AccountingDao dao = getAccountingDao(); - ContextAuthorization contextAuthorization = new ContextAuthorization(); - // DatabaseManager dbaseManager = new DatabaseManager(); - AccountingDao dao = AccountingDao.get(); - - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - - Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for (Dimension dimension : dimensionSet) { - dimensionMap.put(dimension.getId(), dimension); - } - - AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - - // ArrayList data = new ArrayList(); ArrayList accountingRecords = new ArrayList(); String context = E_LEARNING_AREA_VRE; // Setting the token for the context - ContextTest.set(contextAuthorization.getSecretForContext(context)); - - ScopeBean scopeBean = new ScopeBean(context); - ScopeDescriptor scopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); + ContextTest.set(context); + ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context); Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey()); Calendar calendar = DateUtils.getStartCalendar(2018, Calendar.JULY, 1); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java index e8e1b7f..4f3cb70 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java @@ -5,22 +5,14 @@ import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.SortedSet; import java.util.stream.Stream; import org.gcube.accounting.accounting.summary.access.AccountingDao; -import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; -import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; -import org.gcube.common.authorization.client.exceptions.ObjectNotFound; -import org.gcube.common.scope.impl.ScopeBean; import org.gcube.dataharvest.harvester.RStudioAccessesHarvester; import org.gcube.dataharvest.utils.AggregationType; -import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.DateUtils; import org.junit.Ignore; @@ -33,39 +25,17 @@ import org.slf4j.LoggerFactory; * @author Giancarlo Panichi (ISTI CNR) * */ -public class AccountingDataHarvesterRStudioTest extends ContextTest { +public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterPluginTest { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterRStudioTest.class); - public static final String ROOT = "/d4science.research-infrastructures.eu"; - // private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; - - protected AccountingDao getAccountingDao() throws ObjectNotFound, Exception { - AccountingDao dao = AccountingDao.get(); - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); - - Set dimensionSet = dao.getDimensions(); - Map dimensionMap = new HashMap<>(); - for (Dimension dimension : dimensionSet) { - dimensionMap.put(dimension.getId(), dimension); - } - - AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap); - - return dao; - } @Ignore @Test public void testJupyterAccessesHarvester() throws Exception { try { - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); AccountingDao dao = getAccountingDao(); List starts = new ArrayList<>(); @@ -77,44 +47,20 @@ public class AccountingDataHarvesterRStudioTest extends ContextTest { .from(dateToConvert.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()))); AggregationType measureType = AggregationType.MONTHLY; - - ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - /* - SortedSet contexts = new TreeSet<>(); - contexts.add("/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"); - contexts.add("/d4science.research-infrastructures.eu/D4OS/Zoo-Phytoplankton_EOV"); - contexts.add("/d4science.research-infrastructures.eu/D4OS/MarineEnvironmentalIndicators"); - */ List accountingRecords = new ArrayList<>(); - - Set scopeDescriptorSet = dao.getContexts(); - Map scopeDescriptorMap = new HashMap<>(); - for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { - scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); - } - AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap); for (Date start : starts) { Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end); for(String context : contexts) { - ContextTest.set(contextAuthorization.getSecretForContext(context)); - - ScopeBean scopeBean = new ScopeBean(context); - - ScopeDescriptor actualScopeDescriptor = scopeDescriptorMap.get(context); - if (actualScopeDescriptor == null) { - actualScopeDescriptor = new ScopeDescriptor(scopeBean.name(), context); - } - - AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); + ContextTest.set(context); List harvested = rstudioAccessesHarvester.getAccountingRecords(); accountingRecords.addAll(harvested); @@ -126,7 +72,7 @@ public class AccountingDataHarvesterRStudioTest extends ContextTest { logger.debug("Going to insert {}", accountingRecords); - ContextTest.setContextByName(ROOT); + ContextTest.set(ROOT); dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); } catch (Throwable e) { diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java index d58b959..9f23a05 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java @@ -24,7 +24,6 @@ public class ContextAuthorizationTest extends ContextTest { properties = new Properties(); InputStream input = AccountingDashboardHarvesterPlugin.class.getClassLoader().getResourceAsStream(PROPERTY_FILENAME); properties.load(input); - AccountingDashboardHarvesterPlugin.getProperties().set(properties); } @Ignore @@ -35,7 +34,7 @@ public class ContextAuthorizationTest extends ContextTest { }catch (Exception e) { logger.warn("Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME); } - ContextAuthorization contextAuthorization = new ContextAuthorization(); + ContextAuthorization contextAuthorization = new ContextAuthorization(properties); contextAuthorization.retrieveContextsAndTokens(); } diff --git a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java index 3a7e84b..984dc3f 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ContextTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ContextTest.java @@ -3,14 +3,11 @@ */ package org.gcube.dataharvest.utils; -import java.io.IOException; -import java.io.InputStream; import java.util.Properties; import org.gcube.common.authorization.utils.manager.SecretManager; import org.gcube.common.authorization.utils.manager.SecretManagerProvider; import org.gcube.common.authorization.utils.secret.Secret; -import org.gcube.common.authorization.utils.secret.SecretUtility; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -22,35 +19,9 @@ public class ContextTest { protected static Properties properties; protected static final String PROPERTIES_FILENAME = "token.properties"; - public static final String ROOT; - public static final String VO; - public static final String VRE; - - static { - properties = new Properties(); - InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); - - try { - // load the properties file - properties.load(input); - } catch(IOException e) { - throw new RuntimeException(e); - } - - // DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE"; - // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE"; - - //ROOT = "/gcube"; - ROOT = "/d4science.research-infrastructures.eu"; - VO = ROOT + "/devsec"; - VRE = VO + "/devVRE"; - // VO = ROOT + "/devNext"; - // VRE = VO + "/NextNext"; - - - } - - public static void set(Secret secret) throws Exception { + protected static ContextAuthorization contextAuthorization; + + private static void set(Secret secret) throws Exception { SecretManagerProvider.instance.reset(); SecretManager secretManager = new SecretManager(); SecretManagerProvider.instance.set(secretManager); @@ -58,29 +29,18 @@ public class ContextTest { secretManager.set(); } - public static void setContext(String token) throws Exception { - Secret secret = getSecret(token); - set(secret); - } - - public static void setContextByName(String fullContextName) throws Exception { + public static void set(String fullContextName) throws Exception { Secret secret = getSecretByContextName(fullContextName); set(secret); } - - private static Secret getSecret(String token) throws Exception { - Secret secret = SecretUtility.getSecretByTokenString(token); - return secret; - } - + private static Secret getSecretByContextName(String fullContextName) throws Exception { - String token = ContextTest.properties.getProperty(fullContextName); - return getSecret(token); + return contextAuthorization.getSecretForContext(fullContextName); } @BeforeClass public static void beforeClass() throws Exception { - setContextByName(ROOT); + contextAuthorization = new ContextAuthorization(); } @AfterClass