gCubeSystem
/
accounting-dashboard-harvester-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved Method Harvester

This commit is contained in:
Luca Frosini 2024-02-26 18:03:07 +01:00
parent e76b1c3af3
commit 77cfbf6a8a
11 changed files with 302 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -12,7 +12,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
<version>2.3.1</version>
<version>2.4.0-SNAPSHOT</version>
<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
<description>
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
@ -141,7 +141,7 @@
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>authorization-utils</artifactId>
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
</dependency>
<!-- Test Dependencies. Setting scope to provided to allow proper creation

3
src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java
View File

@ -28,7 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class MethodInvocationHarvester extends BasicHarvester {
@ -58,7 +57,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
List<Filter> filters = new ArrayList<>();
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
@ -68,6 +66,7 @@ public class MethodInvocationHarvester extends BasicHarvester {
AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
} else {
// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
result = accountingPersistenceQuery.getContextTimeSeries(
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
}

4
src/main/java/org/gcube/dataharvest/utils/DateUtils.java
View File

@ -91,9 +91,7 @@ public class DateUtils {
aggregationStartCalendar.set(Calendar.MINUTE, 0);
aggregationStartCalendar.set(Calendar.SECOND, 0);
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
// logger.trace("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
return aggregationStartCalendar;
}

173
src/test/java/org/gcube/dataharvest/ContextTest.java Normal file
View File

@ -0,0 +1,173 @@
/**
*
*/
package org.gcube.dataharvest;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.JWTSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.authorization.utils.secret.SecretUtility;
import org.gcube.common.keycloak.KeycloakClientFactory;
import org.gcube.common.keycloak.KeycloakClientHelper;
import org.gcube.common.keycloak.model.TokenResponse;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class ContextTest {
private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
protected static final String CONFIG_INI_FILENAME = "config.ini";
public static final String DEFAULT_TEST_SCOPE;
public static final String GCUBE;
public static final String DEVNEXT;
public static final String NEXTNEXT;
public static final String DEVSEC;
public static final String DEVVRE;
public static final String ROOT_PROD;
protected static final Properties properties;
public static final String TYPE_PROPERTY_KEY = "type";
public static final String USERNAME_PROPERTY_KEY = "username";
public static final String PASSWORD_PROPERTY_KEY = "password";
public static final String CLIENT_ID_PROPERTY_KEY = "clientId";
static {
GCUBE = "/gcube";
DEVNEXT = GCUBE + "/devNext";
NEXTNEXT = DEVNEXT + "/NextNext";
DEVSEC = GCUBE + "/devsec";
DEVVRE = DEVSEC + "/devVRE";
ROOT_PROD = "/d4science.research-infrastructures.eu";
DEFAULT_TEST_SCOPE = GCUBE;
properties = new Properties();
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
try {
// load the properties file
properties.load(input);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private enum Type{
USER, CLIENT_ID
};
public static void set(Secret secret) throws Exception {
SecretManagerProvider.instance.reset();
SecretManager secretManager = new SecretManager();
secretManager.addSecret(secret);
SecretManagerProvider.instance.set(secretManager);
SecretManagerProvider.instance.get().set();
}
public static void setContextByName(String fullContextName) throws Exception {
logger.debug("Going to set credentials for context {}", fullContextName);
Secret secret = getSecretByContextName(fullContextName);
set(secret);
}
private static TokenResponse getJWTAccessToken(String context) throws Exception {
Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
TokenResponse tr = null;
int index = context.indexOf('/', 1);
String root = context.substring(0, index == -1 ? context.length() : index);
switch (type) {
case CLIENT_ID:
String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
String clientSecret = properties.getProperty(root);
tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
break;
case USER:
default:
String username = properties.getProperty(USERNAME_PROPERTY_KEY);
String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
switch (root) {
case "/gcube":
default:
clientId = "next.d4science.org";
break;
case "/pred4s":
clientId = "pre.d4science.org";
break;
case "/d4science.research-infrastructures.eu":
clientId = "services.d4science.org";
break;
}
clientSecret = null;
tr = KeycloakClientHelper.getTokenForUser(context, username, password);
break;
}
return tr;
}
public static Secret getSecretByContextName(String context) throws Exception {
TokenResponse tr = getJWTAccessToken(context);
Secret secret = new JWTSecret(tr.getAccessToken());
return secret;
}
public static void setContext(String token) throws Exception {
Secret secret = getSecret(token);
set(secret);
}
private static Secret getSecret(String token) throws Exception {
Secret secret = SecretUtility.getSecretByTokenString(token);
return secret;
}
public static String getUser() {
String user = "UNKNOWN";
try {
user = SecretManagerProvider.instance.get().getUser().getUsername();
} catch(Exception e) {
logger.error("Unable to retrieve user. {} will be used", user);
}
return user;
}
@BeforeClass
public static void beforeClass() throws Exception {
setContextByName(ROOT_PROD);
}
@AfterClass
public static void afterClass() throws Exception {
SecretManagerProvider.instance.reset();
}
}

21
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterJupyterTest.java → src/test/java/org/gcube/dataharvest/harvester/AccountingDataHarvesterJupyterTest.java
View File

@ -1,23 +1,17 @@
package org.gcube.dataharvest;
package org.gcube.dataharvest.harvester;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.junit.Ignore;
import org.junit.Test;
@ -38,7 +32,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
public void testJupyterAccessesHarvester() throws Exception {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>();
@ -50,6 +44,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
AggregationType measureType = AggregationType.MONTHLY;
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
/*
SortedSet<String> contexts = new TreeSet<>();
@ -63,11 +58,11 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
for(String context : contexts) {
ContextTest.set(context);
ContextTest.setContextByName(context);
List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -79,7 +74,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
logger.debug("Going to insert {}", accountingRecords);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));

16
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterRStudioTest.java → src/test/java/org/gcube/dataharvest/harvester/AccountingDataHarvesterRStudioTest.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest;
package org.gcube.dataharvest.harvester;
import java.time.LocalDate;
import java.time.ZoneId;
@ -11,9 +11,10 @@ import java.util.stream.Stream;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
import org.junit.Ignore;
import org.junit.Test;
@ -35,7 +36,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
public void testJupyterAccessesHarvester() throws Exception {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>();
@ -48,6 +49,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
AggregationType measureType = AggregationType.MONTHLY;
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
List<AccountingRecord> accountingRecords = new ArrayList<>();
@ -56,11 +58,11 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
for(String context : contexts) {
ContextTest.set(context);
ContextTest.setContextByName(context);
List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -72,7 +74,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
logger.debug("Going to insert {}", accountingRecords);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch (Throwable e) {

2
src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java
View File

@ -2,7 +2,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
import java.util.List;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.ContextTest;
import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

179
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java → src/test/java/org/gcube/dataharvest/plugin/AccountingDataHarvesterPluginTest.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest;
package org.gcube.dataharvest.plugin;
import java.time.Instant;
import java.util.ArrayList;
@ -7,7 +7,6 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@ -18,14 +17,15 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
@ -33,7 +33,6 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
import org.gcube.vremanagement.executor.api.types.Scheduling;
@ -73,7 +72,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
dimensionMap.put(dimension.getId(), dimension);
}
AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap);
return dao;
}
@ -82,7 +80,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
@Test
public void getDimensions() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = AccountingDao.get();
@ -103,7 +101,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launch() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
@ -136,7 +134,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchPluginOnSmartExecutor() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
String pluginName = new AccountingDashboardHarvesterPlugin().getName();
@ -189,7 +187,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchOldData() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
@ -226,10 +224,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchOldDataVREAccessesHarvester() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
@ -252,7 +251,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
//CoreServicesAccessesHarvester vreAccessesHarvester = null;
for (String context : contexts) {
ContextTest.set(context);
ContextTest.setContextByName(context);
ScopeBean scopeBean = new ScopeBean(context);
@ -270,12 +269,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
// Setting back token for the context
ContextTest.set(parent.toString());
ContextTest.setContextByName(parent.toString());
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
ContextTest.set(context);
ContextTest.setContextByName(context);
}
}
@ -299,7 +298,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -310,7 +309,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
} catch (Exception e) {
logger.error("", e);
@ -324,7 +323,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testVREAccessesHarvester() throws Exception {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
@ -347,12 +346,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
for (String contextFullname : contextFullNames) {
ContextTest.set(contextFullname);
ContextTest.setContextByName(contextFullname);
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -364,7 +363,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", accountingRecords);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -379,7 +378,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testVREAccessesHarvesterAll() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AggregationType measureType = AggregationType.MONTHLY;
@ -394,6 +393,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
accountingDataHarvesterPlugin.getConfigParameters();
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
VREAccessesHarvester vreAccessesHarvester = null;
@ -402,7 +402,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (String context : contexts) {
// Setting the token for the context
ContextTest.set(context);
ContextTest.setContextByName(context);
ScopeBean scopeBean = new ScopeBean(context);
@ -419,12 +419,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
parent = scopeBean.enclosingScope();
}
ContextTest.set(parent.toString());
ContextTest.setContextByName(parent.toString());
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
ContextTest.set(context);
ContextTest.setContextByName(context);
}
}
@ -456,7 +456,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testSocialInteraction() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
@ -480,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (String context : contexts) {
// Setting the token for the context
ContextTest.set(context);
ContextTest.setContextByName(context);
try {
// Collecting info on social (posts, replies and likes)
logger.info("Going to harvest Social Interactions for {}", context);
@ -494,7 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -506,37 +506,37 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
@Ignore
// @Test
public void testMethodInvocation() {
try {
ContextTest.set(STOCK_ASSESMENT_VRE);
// @Ignore
// // @Test
// public void testMethodInvocation() {
// try {
// ContextTest.setContextByName(STOCK_ASSESMENT_VRE);
//
// AggregationType measureType = AggregationType.MONTHLY;
//
// Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
//
// MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
// List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
//
// logger.debug("{}", accountingRecords);
//
// } catch (Exception e) {
// logger.error("", e);
// }
// }
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
logger.debug("{}", accountingRecords);
} catch (Exception e) {
logger.error("", e);
}
}
@Ignore
//@Test
//@Ignore
@Test
public void testMethodInvocationOldData() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>();
starts.add(DateUtils.getStartCalendar(2021, Calendar.JULY, 1).getTime());
Date start = DateUtils.getStartCalendar(2023, Calendar.JANUARY, 1).getTime();
Date last = DateUtils.getStartCalendar(2024, Calendar.FEBRUARY, 1).getTime();
AggregationType measureType = AggregationType.MONTHLY;
@ -548,32 +548,61 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
}
AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
List<AccountingRecord> accountingRecords = new ArrayList<>();
for (Date start : starts) {
while(start.before(last)) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
for (String context : contexts) {
// Setting the token for the context
ContextTest.set(context);
Secret s = contextAuthorization.getCatalogueSecretForContext(context);
ContextTest.set(s);
if (context.startsWith(AccountingDashboardHarvesterPlugin.TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
logger.info("Going to harvest Method Invocations for {}", context);
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<AccountingRecord> harvested = tagMeMethodInvocationHarvester.getAccountingRecords();
logger.debug("{} - {}", context, harvested);
accountingRecords.addAll(harvested);
} catch (Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
logger.info("Going to harvest Method Invocations for {}", context);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
logger.debug("{} - {}", context, harvested);
accountingRecords.addAll(harvested);
} catch (Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
logger.debug("{} - {}", context, harvested);
}
start = end;
}
logger.debug("Going to insert {}", accountingRecords);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
ContextTest.setContextByName(ROOT_PROD);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch (Exception e) {
logger.error("", e);
@ -589,10 +618,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testTagMeMethodInvocation() throws Exception {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
ContextTest.set(TAGME_VRE);
ContextTest.setContextByName(TAGME_VRE);
List<AccountingRecord> accountingRecords = new ArrayList<>();
@ -616,7 +645,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", accountingRecords);
}
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -630,10 +659,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testGetVREUsersForSpecificVRE() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
ContextTest.setContextByName("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
AggregationType measureType = AggregationType.MONTHLY;
@ -645,7 +674,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -659,7 +688,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testFilteringGenericResource() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
// Utils.setContext(RESOURCE_CATALOGUE);
AggregationType measureType = AggregationType.MONTHLY;
@ -667,6 +696,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@ -685,7 +715,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testResourceCatalogueHarvester() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AggregationType measureType = AggregationType.MONTHLY;
@ -697,6 +727,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@ -715,8 +746,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testCoreServicesHarvester() {
try {
String context = ROOT;
ContextTest.set(context);
String context = ROOT_PROD;
ContextTest.setContextByName(context);
AccountingDao dao = getAccountingDao();
Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
@ -752,8 +783,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testCatalogueHarvester() {
try {
String context = ROOT;
ContextTest.set(context);
String context = ROOT_PROD;
ContextTest.setContextByName(context);
AccountingDao dao = getAccountingDao();
@ -793,7 +824,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void addMissingVREAccesses() {
try {
ContextTest.set(ROOT);
ContextTest.setContextByName(ROOT_PROD);
AccountingDao dao = getAccountingDao();
@ -803,7 +834,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
String context = E_LEARNING_AREA_VRE;
// Setting the token for the context
ContextTest.set(context);
ContextTest.setContextByName(context);
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());

1
src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java
View File

@ -5,6 +5,7 @@ import java.io.InputStream;
import java.util.Properties;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.ContextTest;
import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

92
src/test/java/org/gcube/dataharvest/utils/ContextTest.java
View File

@ -1,92 +0,0 @@
/**
*
*/
package org.gcube.dataharvest.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.JWTSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.keycloak.KeycloakClientFactory;
import org.gcube.common.keycloak.model.TokenResponse;
import org.gcube.common.scope.api.ScopeProvider;
import org.junit.AfterClass;
import org.junit.BeforeClass;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class ContextTest {
public static final String ROOT = "/d4science.research-infrastructures.eu";
protected static final String CONFIG_PROPERTIES_FILENAME = "config.properties";
protected static ContextAuthorization contextAuthorization;
private static Properties getProperties(){
try {
Properties properties = new Properties();
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_PROPERTIES_FILENAME);
// load the properties file
properties.load(input);
return properties;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void set(Secret secret) throws Exception {
SecretManagerProvider.instance.reset();
SecretManager secretManager = new SecretManager();
SecretManagerProvider.instance.set(secretManager);
secretManager.addSecret(secret);
secretManager.set();
}
public static void set(String fullContextName) throws Exception {
Secret secret = getSecret(fullContextName);
set(secret);
}
private static Secret getSecret(String fullContextName) throws Exception {
return contextAuthorization.getSecretForContext(fullContextName);
}
private static TokenResponse getJWTAccessToken(String context) throws Exception {
ScopeProvider.instance.set(context);
int index = context.indexOf('/', 1);
String root = context.substring(0, index == -1 ? context.length() : index);
String clientSecret = getProperties().getProperty(root);
TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(ContextAuthorization.CLIENT_ID, clientSecret, context, null);
return tr;
}
public static Secret generateSecretByContextName(String context) throws Exception {
TokenResponse tr = getJWTAccessToken(context);
Secret secret = new JWTSecret(tr.getAccessToken());
return secret;
}
@BeforeClass
public static void beforeClass() throws Exception {
Secret secret = generateSecretByContextName(ROOT);
set(secret);
contextAuthorization = new ContextAuthorization();
SecretManagerProvider.instance.reset();
}
@AfterClass
public static void afterClass() throws Exception {
SecretManagerProvider.instance.reset();
}
}

1
src/test/resources/.gitignore vendored
View File

@ -3,3 +3,4 @@
/*.properties
/howto.txt
/scopedata 2.xml
/config.ini