Improved Method Harvester

pom.xml

@@ -12,7 +12,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.gcube.accounting</groupId>
 	<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
-	<version>2.3.1</version>
+	<version>2.4.0-SNAPSHOT</version>
 	<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
 	<description>
 		Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
@@ -141,7 +141,7 @@
 		<dependency>
 			<groupId>org.gcube.common</groupId>
 			<artifactId>authorization-utils</artifactId>
-			<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
+			<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<!-- Test Dependencies. Setting scope to provided to allow proper creation 

src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java

@@ -28,7 +28,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * @author Eric Perrone (ISTI - CNR)
  * @author Luca Frosini (ISTI - CNR)
  */
 public class MethodInvocationHarvester extends BasicHarvester {
@@ -58,7 +57,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
 			
 			SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
 			List<Filter> filters = new ArrayList<>();
-			filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
 			
 			Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
 			
@@ -68,6 +66,7 @@ public class MethodInvocationHarvester extends BasicHarvester {
 					AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
 			} else {
 				// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
+				filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
 				result = accountingPersistenceQuery.getContextTimeSeries(
 						AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
 			}

src/main/java/org/gcube/dataharvest/utils/DateUtils.java

@@ -91,9 +91,7 @@ public class DateUtils {
 		aggregationStartCalendar.set(Calendar.MINUTE, 0);
 		aggregationStartCalendar.set(Calendar.SECOND, 0);
 		aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
-		
+//		logger.trace("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
-		logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
-		
 		return aggregationStartCalendar;
 	}
 	

src/test/java/org/gcube/dataharvest/ContextTest.java

@@ -0,0 +1,173 @@
+/**
+ * 
+ */
+package org.gcube.dataharvest;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.gcube.common.authorization.utils.manager.SecretManager;
+import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
+import org.gcube.common.authorization.utils.secret.JWTSecret;
+import org.gcube.common.authorization.utils.secret.Secret;
+import org.gcube.common.authorization.utils.secret.SecretUtility;
+import org.gcube.common.keycloak.KeycloakClientFactory;
+import org.gcube.common.keycloak.KeycloakClientHelper;
+import org.gcube.common.keycloak.model.TokenResponse;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author Luca Frosini (ISTI - CNR)
+ */
+public class ContextTest {
+	
+	private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
+	
+	protected static final String CONFIG_INI_FILENAME = "config.ini";
+	
+	public static final String DEFAULT_TEST_SCOPE;
+	
+	public static final String GCUBE;
+	public static final String DEVNEXT;
+	public static final String NEXTNEXT;
+	public static final String DEVSEC;
+	public static final String DEVVRE;
+	
+	public static final String ROOT_PROD;
+	
+	protected static final Properties properties;
+	
+	public static final String TYPE_PROPERTY_KEY = "type";
+	public static final String USERNAME_PROPERTY_KEY = "username"; 
+	public static final String PASSWORD_PROPERTY_KEY = "password"; 
+	public static final String CLIENT_ID_PROPERTY_KEY = "clientId"; 
+	
+	static {
+		GCUBE = "/gcube";
+		DEVNEXT = GCUBE + "/devNext";
+		NEXTNEXT = DEVNEXT + "/NextNext";
+		DEVSEC = GCUBE + "/devsec";
+		DEVVRE = DEVSEC + "/devVRE";
+		
+		ROOT_PROD = "/d4science.research-infrastructures.eu";
+		
+		DEFAULT_TEST_SCOPE = GCUBE;
+
+		
+		properties = new Properties();
+		InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
+		try {
+			// load the properties file
+			properties.load(input);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+		
+	}
+	
+	private enum Type{
+		USER, CLIENT_ID
+	};
+	
+	public static void set(Secret secret) throws Exception {
+		SecretManagerProvider.instance.reset();
+		SecretManager secretManager = new SecretManager();
+		secretManager.addSecret(secret);
+		SecretManagerProvider.instance.set(secretManager);
+		SecretManagerProvider.instance.get().set();
+	}
+	
+	public static void setContextByName(String fullContextName) throws Exception {
+		logger.debug("Going to set credentials for context {}", fullContextName);
+		Secret secret = getSecretByContextName(fullContextName);
+		set(secret);
+	}
+	
+	
+	private static TokenResponse getJWTAccessToken(String context) throws Exception {
+		Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
+		
+		TokenResponse tr = null;
+		
+		int index = context.indexOf('/', 1);
+		String root = context.substring(0, index == -1 ? context.length() : index);
+		
+		switch (type) {
+			case CLIENT_ID:
+				String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
+				String clientSecret = properties.getProperty(root);
+				
+				tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
+				break;
+		
+			case USER:
+			default:
+				String username = properties.getProperty(USERNAME_PROPERTY_KEY);
+				String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
+				
+				switch (root) {
+					case "/gcube":
+					default:
+						clientId = "next.d4science.org";
+						break;
+					
+					case "/pred4s":
+						clientId = "pre.d4science.org";
+						break;
+					
+					case "/d4science.research-infrastructures.eu":
+						clientId = "services.d4science.org";
+						break;
+				}
+				clientSecret = null;
+				
+				tr = KeycloakClientHelper.getTokenForUser(context, username, password);
+				break;
+				
+		}
+
+		return tr;
+		
+	}
+	
+	public static Secret getSecretByContextName(String context) throws Exception {
+		TokenResponse tr = getJWTAccessToken(context);
+		Secret secret = new JWTSecret(tr.getAccessToken());
+		return secret;
+	}
+	
+	public static void setContext(String token) throws Exception {
+		Secret secret = getSecret(token);
+		set(secret);
+	}
+	
+	private static Secret getSecret(String token) throws Exception {
+		Secret secret = SecretUtility.getSecretByTokenString(token);
+		return secret;
+	}
+	
+	public static String getUser() {
+		String user = "UNKNOWN";
+		try {
+			user = SecretManagerProvider.instance.get().getUser().getUsername();
+		} catch(Exception e) {
+			logger.error("Unable to retrieve user. {} will be used", user);
+		}
+		return user;
+	}
+	
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		setContextByName(ROOT_PROD);
+	}
+	
+	@AfterClass
+	public static void afterClass() throws Exception {
+		SecretManagerProvider.instance.reset();
+	}
+	
+}

src/test/java/org/gcube/dataharvest/harvester/AccountingDataHarvesterJupyterTest.java

@@ -1,23 +1,17 @@
-package org.gcube.dataharvest;
+package org.gcube.dataharvest.harvester;
 
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 import java.util.SortedSet;
 
 import org.gcube.accounting.accounting.summary.access.AccountingDao;
-import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
-import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
 import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
-import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
+import org.gcube.dataharvest.ContextTest;
-import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
+import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
 import org.gcube.dataharvest.utils.AggregationType;
 import org.gcube.dataharvest.utils.ContextAuthorization;
-import org.gcube.dataharvest.utils.ContextTest;
 import org.gcube.dataharvest.utils.DateUtils;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -38,7 +32,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
 	public void testJupyterAccessesHarvester() throws Exception {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			AccountingDao dao = getAccountingDao();
 
 			List<Date> starts = new ArrayList<>();
@@ -50,6 +44,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 			/*
 			SortedSet<String> contexts = new TreeSet<>();
@@ -63,11 +58,11 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
 			for (Date start : starts) {
 				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 
-				ContextTest.set(ROOT);
+				ContextTest.setContextByName(ROOT_PROD);
 				JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
 
 				for(String context : contexts) {
-					ContextTest.set(context);
+					ContextTest.setContextByName(context);
 					
 					List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
 					accountingRecords.addAll(harvested);
@@ -79,7 +74,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
 
 			logger.debug("Going to insert {}", accountingRecords);
 			
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			
 			// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
 

src/test/java/org/gcube/dataharvest/harvester/AccountingDataHarvesterRStudioTest.java

@@ -1,4 +1,4 @@
-package org.gcube.dataharvest;
+package org.gcube.dataharvest.harvester;
 
 import java.time.LocalDate;
 import java.time.ZoneId;
@@ -11,9 +11,10 @@ import java.util.stream.Stream;
 
 import org.gcube.accounting.accounting.summary.access.AccountingDao;
 import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
-import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
+import org.gcube.dataharvest.ContextTest;
+import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
 import org.gcube.dataharvest.utils.AggregationType;
-import org.gcube.dataharvest.utils.ContextTest;
+import org.gcube.dataharvest.utils.ContextAuthorization;
 import org.gcube.dataharvest.utils.DateUtils;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -35,7 +36,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
 	public void testJupyterAccessesHarvester() throws Exception {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			AccountingDao dao = getAccountingDao();
 
 			List<Date> starts = new ArrayList<>();
@@ -48,6 +49,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 			
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 			
 			List<AccountingRecord> accountingRecords = new ArrayList<>();
@@ -56,11 +58,11 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
 			for (Date start : starts) {
 				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 
-				ContextTest.set(ROOT);
+				ContextTest.setContextByName(ROOT_PROD);
 				RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
 
 				for(String context : contexts) {
-					ContextTest.set(context);
+					ContextTest.setContextByName(context);
 					
 					List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
 					accountingRecords.addAll(harvested);
@@ -72,7 +74,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
 
 			logger.debug("Going to insert {}", accountingRecords);
 			
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
 
 		} catch (Throwable e) {

src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java

@@ -2,7 +2,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
 
 import java.util.List;
 
-import org.gcube.dataharvest.utils.ContextTest;
+import org.gcube.dataharvest.ContextTest;
 import org.junit.Ignore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/test/java/org/gcube/dataharvest/plugin/AccountingDataHarvesterPluginTest.java

@@ -1,4 +1,4 @@
-package org.gcube.dataharvest;
+package org.gcube.dataharvest.plugin;
 
 import java.time.Instant;
 import java.util.ArrayList;
@@ -7,7 +7,6 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -18,14 +17,15 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
 import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
 import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
 import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
+import org.gcube.common.authorization.utils.secret.Secret;
 import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.common.scope.impl.ScopeBean.Type;
+import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
+import org.gcube.dataharvest.ContextTest;
 import org.gcube.dataharvest.datamodel.HarvestedDataKey;
 import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
 import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
-import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
 import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
-import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
 import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
 import org.gcube.dataharvest.harvester.VREAccessesHarvester;
 import org.gcube.dataharvest.harvester.VREUsersHarvester;
@@ -33,7 +33,6 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
 import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
 import org.gcube.dataharvest.utils.AggregationType;
 import org.gcube.dataharvest.utils.ContextAuthorization;
-import org.gcube.dataharvest.utils.ContextTest;
 import org.gcube.dataharvest.utils.DateUtils;
 import org.gcube.vremanagement.executor.api.types.LaunchParameter;
 import org.gcube.vremanagement.executor.api.types.Scheduling;
@@ -73,7 +72,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			dimensionMap.put(dimension.getId(), dimension);
 		}
 
-		AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap);
 
 		return dao;
 	}
@@ -82,7 +80,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	@Test
 	public void getDimensions() {
 		try {
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			AccountingDao dao = AccountingDao.get();
 
@@ -103,7 +101,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void launch() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
 
@@ -136,7 +134,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void launchPluginOnSmartExecutor() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			String pluginName = new AccountingDashboardHarvesterPlugin().getName();
 
@@ -189,7 +187,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void launchOldData() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
 
@@ -226,10 +224,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void launchOldDataVREAccessesHarvester() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			
 			AccountingDao dao = getAccountingDao();
 
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 
 			AggregationType aggregationType = AggregationType.MONTHLY;
@@ -252,7 +251,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 				//CoreServicesAccessesHarvester vreAccessesHarvester = null;
 
 				for (String context : contexts) {
-					ContextTest.set(context);
+					ContextTest.setContextByName(context);
 
 					ScopeBean scopeBean = new ScopeBean(context);
 
@@ -270,12 +269,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 							}
 
 							// Setting back token for the context
-							ContextTest.set(parent.toString());
+							ContextTest.setContextByName(parent.toString());
 
 							vreAccessesHarvester = new VREAccessesHarvester(start, end);
 
 							// Setting back token for the context
-							ContextTest.set(context);
+							ContextTest.setContextByName(context);
 						}
 
 					}
@@ -299,7 +298,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 				logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
 						accountingRecords);
-				ContextTest.set(ROOT);
+				ContextTest.setContextByName(ROOT_PROD);
 
 				// dao.insertRecords(accountingRecords.toArray(new
 				// AccountingRecord[1]));
@@ -310,7 +309,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			}
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 		} catch (Exception e) {
 			logger.error("", e);
@@ -324,7 +323,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	// @Test
 	public void testVREAccessesHarvester() throws Exception {
 		try {
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			
 			AccountingDao dao = getAccountingDao();
 
@@ -347,12 +346,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			for (Date start : starts) {
 				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 
-				ContextTest.set(ROOT);
+				ContextTest.setContextByName(ROOT_PROD);
 				VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
 
 				for (String contextFullname : contextFullNames) {
 
-					ContextTest.set(contextFullname);
+					ContextTest.setContextByName(contextFullname);
 
 					List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
 					accountingRecords.addAll(harvested);
@@ -364,7 +363,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			logger.debug("{}", accountingRecords);
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			// dao.insertRecords(accountingRecords.toArray(new
 			// AccountingRecord[1]));
 
@@ -379,7 +378,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testVREAccessesHarvesterAll() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			AggregationType measureType = AggregationType.MONTHLY;
 
@@ -394,6 +393,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
 			accountingDataHarvesterPlugin.getConfigParameters();
 
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 
 			VREAccessesHarvester vreAccessesHarvester = null;
@@ -402,7 +402,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			for (String context : contexts) {
 				// Setting the token for the context
-				ContextTest.set(context);
+				ContextTest.setContextByName(context);
 
 				ScopeBean scopeBean = new ScopeBean(context);
 
@@ -419,12 +419,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 							parent = scopeBean.enclosingScope();
 						}
 
-						ContextTest.set(parent.toString());
+						ContextTest.setContextByName(parent.toString());
 
 						vreAccessesHarvester = new VREAccessesHarvester(start, end);
 
 						// Setting back token for the context
-						ContextTest.set(context);
+						ContextTest.setContextByName(context);
 					}
 
 				}
@@ -456,7 +456,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testSocialInteraction() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			
 			AccountingDao dao = getAccountingDao();
 
@@ -480,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			for (String context : contexts) {
 				// Setting the token for the context
-				ContextTest.set(context);
+				ContextTest.setContextByName(context);
 				try {
 					// Collecting info on social (posts, replies and likes)
 					logger.info("Going to harvest Social Interactions for {}", context);
@@ -494,7 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
 					accountingRecords);
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			// dao.insertRecords(accountingRecords.toArray(new
 			// AccountingRecord[1]));
 
@@ -506,37 +506,37 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 	}
 
-	@Ignore
+//	@Ignore
-	// @Test
+//	// @Test
-	public void testMethodInvocation() {
+//	public void testMethodInvocation() {
-		try {
+//		try {
-			ContextTest.set(STOCK_ASSESMENT_VRE);
+//			ContextTest.setContextByName(STOCK_ASSESMENT_VRE);
+//
+//			AggregationType measureType = AggregationType.MONTHLY;
+//
+//			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+//			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+//
+//			MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
+//			List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
+//
+//			logger.debug("{}", accountingRecords);
+//
+//		} catch (Exception e) {
+//			logger.error("", e);
+//		}
+//	}
 	
-			AggregationType measureType = AggregationType.MONTHLY;
+	//@Ignore
-
+	@Test
-			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
-
-			MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
-			List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
-
-			logger.debug("{}", accountingRecords);
-
-		} catch (Exception e) {
-			logger.error("", e);
-		}
-	}
-	
-	@Ignore
-	//@Test
 	public void testMethodInvocationOldData() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			AccountingDao dao = getAccountingDao();
 			
-			List<Date> starts = new ArrayList<>();
+			Date start = DateUtils.getStartCalendar(2023, Calendar.JANUARY, 1).getTime();
-			starts.add(DateUtils.getStartCalendar(2021, Calendar.JULY, 1).getTime());
+			Date last = DateUtils.getStartCalendar(2024, Calendar.FEBRUARY, 1).getTime();
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 
@@ -548,32 +548,61 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
 				scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
 			}
+			
 			AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
 			
 			
 			List<AccountingRecord> accountingRecords = new ArrayList<>();
 			
-			for (Date start : starts) {
+			while(start.before(last)) {
 				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 				
-				MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
 				
 				for (String context : contexts) {
 					// Setting the token for the context
-					ContextTest.set(context);
+					Secret s = contextAuthorization.getCatalogueSecretForContext(context);
+					ContextTest.set(s);
+					
+					
+					if (context.startsWith(AccountingDashboardHarvesterPlugin.TAGME_CONTEXT)) {
+						try {
+							// Collecting info on method invocation
+							logger.info("Going to harvest Method Invocations for {}", context);
+							TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
+									start, end);
+
+							List<AccountingRecord> harvested = tagMeMethodInvocationHarvester.getAccountingRecords();
+							logger.debug("{} - {}", context, harvested);
+							accountingRecords.addAll(harvested);
+
+						} catch (Exception e) {
+							logger.error("Error harvesting Method Invocations for {}", context, e);
+						}
+					} else {
+						try {
+							// Collecting info on method invocation
+							logger.info("Going to harvest Method Invocations for {}", context);
+							MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
+
+							List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
+							logger.debug("{} - {}", context, harvested);
+							accountingRecords.addAll(harvested);
+						} catch (Exception e) {
+							logger.error("Error harvesting Method Invocations for {}", context, e);
+						}
+					}
 
-					List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
-					accountingRecords.addAll(harvested);
 					
-					logger.debug("{} - {}", context, harvested);
 
 				}
+				
+				start = end;
 			}
 
 			logger.debug("Going to insert {}", accountingRecords);
 			
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
-			// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
+			dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
 			
 		} catch (Exception e) {
 			logger.error("", e);
@@ -589,10 +618,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testTagMeMethodInvocation() throws Exception {
 		try {
 			
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			AccountingDao dao = getAccountingDao();
 			
-			ContextTest.set(TAGME_VRE);
+			ContextTest.setContextByName(TAGME_VRE);
 			
 			List<AccountingRecord> accountingRecords = new ArrayList<>();
 
@@ -616,7 +645,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 				logger.debug("{}", accountingRecords);
 			}
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			// dao.insertRecords(accountingRecords.toArray(new
 			// AccountingRecord[1]));
 
@@ -630,10 +659,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	// @Test
 	public void testGetVREUsersForSpecificVRE() {
 		try {
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			AccountingDao dao = getAccountingDao();
 
-			ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
+			ContextTest.setContextByName("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
 
 			AggregationType measureType = AggregationType.MONTHLY;
 
@@ -645,7 +674,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 
 			logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			// dao.insertRecords(accountingRecords.toArray(new
 			// AccountingRecord[1]));
 
@@ -659,7 +688,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	// @Test
 	public void testFilteringGenericResource() {
 		try {
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			// Utils.setContext(RESOURCE_CATALOGUE);
 
 			AggregationType measureType = AggregationType.MONTHLY;
@@ -667,6 +696,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
 			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 
 			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@@ -685,7 +715,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testResourceCatalogueHarvester() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 
@@ -697,6 +727,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
 			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 
 			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@@ -715,8 +746,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testCoreServicesHarvester() {
 		try {
 
-			String context = ROOT;
+			String context = ROOT_PROD;
-			ContextTest.set(context);
+			ContextTest.setContextByName(context);
 
 			AccountingDao dao = getAccountingDao();
 			Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
@@ -752,8 +783,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void testCatalogueHarvester() {
 		try {
 
-			String context = ROOT;
+			String context = ROOT_PROD;
-			ContextTest.set(context);
+			ContextTest.setContextByName(context);
 
 			AccountingDao dao = getAccountingDao();
 
@@ -793,7 +824,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	public void addMissingVREAccesses() {
 		try {
 
-			ContextTest.set(ROOT);
+			ContextTest.setContextByName(ROOT_PROD);
 
 			AccountingDao dao = getAccountingDao();
 
@@ -803,7 +834,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			String context = E_LEARNING_AREA_VRE;
 
 			// Setting the token for the context
-			ContextTest.set(context);
+			ContextTest.setContextByName(context);
 
 			ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
 			Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());

src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java

@@ -5,6 +5,7 @@ import java.io.InputStream;
 import java.util.Properties;
 
 import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
+import org.gcube.dataharvest.ContextTest;
 import org.junit.Ignore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/test/java/org/gcube/dataharvest/utils/ContextTest.java

@@ -1,92 +0,0 @@
-/**
- * 
- */
-package org.gcube.dataharvest.utils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-
-import org.gcube.common.authorization.utils.manager.SecretManager;
-import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
-import org.gcube.common.authorization.utils.secret.JWTSecret;
-import org.gcube.common.authorization.utils.secret.Secret;
-import org.gcube.common.keycloak.KeycloakClientFactory;
-import org.gcube.common.keycloak.model.TokenResponse;
-import org.gcube.common.scope.api.ScopeProvider;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/**
- * @author Luca Frosini (ISTI - CNR)
- */
-public class ContextTest {
-	
-	public static final String ROOT = "/d4science.research-infrastructures.eu";
-	
-	protected static final String CONFIG_PROPERTIES_FILENAME = "config.properties";
-	
-	protected static ContextAuthorization contextAuthorization;
-	
-	private static Properties getProperties(){
-		try {
-			Properties properties = new Properties();
-			InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_PROPERTIES_FILENAME);
-			
-			// load the properties file
-			properties.load(input);
-		
-			return properties;
-		} catch (IOException e) {
-			throw new RuntimeException(e);
-		}
-	}
-	
-	public static void set(Secret secret) throws Exception {
-		SecretManagerProvider.instance.reset();
-		SecretManager secretManager = new SecretManager(); 
-		SecretManagerProvider.instance.set(secretManager);
-		secretManager.addSecret(secret);
-		secretManager.set();
-	}
-	
-	public static void set(String fullContextName) throws Exception {
-		Secret secret = getSecret(fullContextName);
-		set(secret);
-	}
-
-	private static Secret getSecret(String fullContextName) throws Exception {
-		return contextAuthorization.getSecretForContext(fullContextName);
-	}
-	
-	private static TokenResponse getJWTAccessToken(String context) throws Exception {
-		ScopeProvider.instance.set(context);
-		
-		int index = context.indexOf('/', 1);
-		String root = context.substring(0, index == -1 ? context.length() : index);
-		String clientSecret = getProperties().getProperty(root);
-		
-		TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(ContextAuthorization.CLIENT_ID, clientSecret, context, null);
-	    return tr;
-	}
-	
-	public static Secret generateSecretByContextName(String context) throws Exception {
-		TokenResponse tr = getJWTAccessToken(context);
-		Secret secret = new JWTSecret(tr.getAccessToken());
-		return secret;
-	}
-	
-	@BeforeClass
-	public static void beforeClass() throws Exception {
-		Secret secret = generateSecretByContextName(ROOT);
-		set(secret);
-		contextAuthorization = new ContextAuthorization();
-		SecretManagerProvider.instance.reset();
-	}
-	
-	@AfterClass
-	public static void afterClass() throws Exception {
-		SecretManagerProvider.instance.reset();
-	}
-	
-}

src/test/resources/.gitignore

@@ -3,3 +3,4 @@
 /*.properties
 /howto.txt
 /scopedata 2.xml
+/config.ini