Improved Method Harvester
This commit is contained in:
parent
e76b1c3af3
commit
77cfbf6a8a
4
pom.xml
4
pom.xml
|
|
@ -12,7 +12,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.gcube.accounting</groupId>
|
||||
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
|
||||
<version>2.3.1</version>
|
||||
<version>2.4.0-SNAPSHOT</version>
|
||||
<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
|
||||
<description>
|
||||
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>authorization-utils</artifactId>
|
||||
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
|
||||
<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Test Dependencies. Setting scope to provided to allow proper creation
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author Eric Perrone (ISTI - CNR)
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public class MethodInvocationHarvester extends BasicHarvester {
|
||||
|
|
@ -58,7 +57,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
|||
|
||||
SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
|
||||
List<Filter> filters = new ArrayList<>();
|
||||
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
||||
|
||||
Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
|
||||
|
||||
|
|
@ -68,6 +66,7 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
|||
AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||
} else {
|
||||
// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
|
||||
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
||||
result = accountingPersistenceQuery.getContextTimeSeries(
|
||||
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,9 +91,7 @@ public class DateUtils {
|
|||
aggregationStartCalendar.set(Calendar.MINUTE, 0);
|
||||
aggregationStartCalendar.set(Calendar.SECOND, 0);
|
||||
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
|
||||
|
||||
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
||||
|
||||
// logger.trace("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
||||
return aggregationStartCalendar;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
/**
|
||||
*
|
||||
*/
|
||||
package org.gcube.dataharvest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.utils.manager.SecretManager;
|
||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
||||
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
||||
import org.gcube.common.authorization.utils.secret.Secret;
|
||||
import org.gcube.common.authorization.utils.secret.SecretUtility;
|
||||
import org.gcube.common.keycloak.KeycloakClientFactory;
|
||||
import org.gcube.common.keycloak.KeycloakClientHelper;
|
||||
import org.gcube.common.keycloak.model.TokenResponse;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public class ContextTest {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
|
||||
|
||||
protected static final String CONFIG_INI_FILENAME = "config.ini";
|
||||
|
||||
public static final String DEFAULT_TEST_SCOPE;
|
||||
|
||||
public static final String GCUBE;
|
||||
public static final String DEVNEXT;
|
||||
public static final String NEXTNEXT;
|
||||
public static final String DEVSEC;
|
||||
public static final String DEVVRE;
|
||||
|
||||
public static final String ROOT_PROD;
|
||||
|
||||
protected static final Properties properties;
|
||||
|
||||
public static final String TYPE_PROPERTY_KEY = "type";
|
||||
public static final String USERNAME_PROPERTY_KEY = "username";
|
||||
public static final String PASSWORD_PROPERTY_KEY = "password";
|
||||
public static final String CLIENT_ID_PROPERTY_KEY = "clientId";
|
||||
|
||||
static {
|
||||
GCUBE = "/gcube";
|
||||
DEVNEXT = GCUBE + "/devNext";
|
||||
NEXTNEXT = DEVNEXT + "/NextNext";
|
||||
DEVSEC = GCUBE + "/devsec";
|
||||
DEVVRE = DEVSEC + "/devVRE";
|
||||
|
||||
ROOT_PROD = "/d4science.research-infrastructures.eu";
|
||||
|
||||
DEFAULT_TEST_SCOPE = GCUBE;
|
||||
|
||||
|
||||
properties = new Properties();
|
||||
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
|
||||
try {
|
||||
// load the properties file
|
||||
properties.load(input);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private enum Type{
|
||||
USER, CLIENT_ID
|
||||
};
|
||||
|
||||
public static void set(Secret secret) throws Exception {
|
||||
SecretManagerProvider.instance.reset();
|
||||
SecretManager secretManager = new SecretManager();
|
||||
secretManager.addSecret(secret);
|
||||
SecretManagerProvider.instance.set(secretManager);
|
||||
SecretManagerProvider.instance.get().set();
|
||||
}
|
||||
|
||||
public static void setContextByName(String fullContextName) throws Exception {
|
||||
logger.debug("Going to set credentials for context {}", fullContextName);
|
||||
Secret secret = getSecretByContextName(fullContextName);
|
||||
set(secret);
|
||||
}
|
||||
|
||||
|
||||
private static TokenResponse getJWTAccessToken(String context) throws Exception {
|
||||
Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
|
||||
|
||||
TokenResponse tr = null;
|
||||
|
||||
int index = context.indexOf('/', 1);
|
||||
String root = context.substring(0, index == -1 ? context.length() : index);
|
||||
|
||||
switch (type) {
|
||||
case CLIENT_ID:
|
||||
String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
|
||||
String clientSecret = properties.getProperty(root);
|
||||
|
||||
tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
|
||||
break;
|
||||
|
||||
case USER:
|
||||
default:
|
||||
String username = properties.getProperty(USERNAME_PROPERTY_KEY);
|
||||
String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
|
||||
|
||||
switch (root) {
|
||||
case "/gcube":
|
||||
default:
|
||||
clientId = "next.d4science.org";
|
||||
break;
|
||||
|
||||
case "/pred4s":
|
||||
clientId = "pre.d4science.org";
|
||||
break;
|
||||
|
||||
case "/d4science.research-infrastructures.eu":
|
||||
clientId = "services.d4science.org";
|
||||
break;
|
||||
}
|
||||
clientSecret = null;
|
||||
|
||||
tr = KeycloakClientHelper.getTokenForUser(context, username, password);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
return tr;
|
||||
|
||||
}
|
||||
|
||||
public static Secret getSecretByContextName(String context) throws Exception {
|
||||
TokenResponse tr = getJWTAccessToken(context);
|
||||
Secret secret = new JWTSecret(tr.getAccessToken());
|
||||
return secret;
|
||||
}
|
||||
|
||||
public static void setContext(String token) throws Exception {
|
||||
Secret secret = getSecret(token);
|
||||
set(secret);
|
||||
}
|
||||
|
||||
private static Secret getSecret(String token) throws Exception {
|
||||
Secret secret = SecretUtility.getSecretByTokenString(token);
|
||||
return secret;
|
||||
}
|
||||
|
||||
public static String getUser() {
|
||||
String user = "UNKNOWN";
|
||||
try {
|
||||
user = SecretManagerProvider.instance.get().getUser().getUsername();
|
||||
} catch(Exception e) {
|
||||
logger.error("Unable to retrieve user. {} will be used", user);
|
||||
}
|
||||
return user;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
setContextByName(ROOT_PROD);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() throws Exception {
|
||||
SecretManagerProvider.instance.reset();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,23 +1,17 @@
|
|||
package org.gcube.dataharvest;
|
||||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
|
||||
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
||||
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
|
||||
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
|
||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
|
||||
import org.gcube.dataharvest.ContextTest;
|
||||
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
|
||||
import org.gcube.dataharvest.utils.AggregationType;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
|
@ -38,7 +32,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
|||
public void testJupyterAccessesHarvester() throws Exception {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
List<Date> starts = new ArrayList<>();
|
||||
|
|
@ -50,6 +44,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
|||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
/*
|
||||
SortedSet<String> contexts = new TreeSet<>();
|
||||
|
|
@ -63,11 +58,11 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
|||
for (Date start : starts) {
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
|
||||
|
||||
for(String context : contexts) {
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
|
||||
accountingRecords.addAll(harvested);
|
||||
|
|
@ -79,7 +74,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
|||
|
||||
logger.debug("Going to insert {}", accountingRecords);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.dataharvest;
|
||||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
|
|
@ -11,9 +11,10 @@ import java.util.stream.Stream;
|
|||
|
||||
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
|
||||
import org.gcube.dataharvest.ContextTest;
|
||||
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
|
||||
import org.gcube.dataharvest.utils.AggregationType;
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
|
@ -35,7 +36,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
|||
public void testJupyterAccessesHarvester() throws Exception {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
List<Date> starts = new ArrayList<>();
|
||||
|
|
@ -48,6 +49,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
|||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
|
||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||
|
|
@ -56,11 +58,11 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
|||
for (Date start : starts) {
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
|
||||
|
||||
for(String context : contexts) {
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
|
||||
accountingRecords.addAll(harvested);
|
||||
|
|
@ -72,7 +74,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
|||
|
||||
logger.debug("Going to insert {}", accountingRecords);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||
|
||||
} catch (Throwable e) {
|
||||
|
|
@ -2,7 +2,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
|
|||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.ContextTest;
|
||||
import org.junit.Ignore;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.dataharvest;
|
||||
package org.gcube.dataharvest.plugin;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -7,7 +7,6 @@ import java.util.Date;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
|
@ -18,14 +17,15 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
|
|||
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
|
||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.utils.secret.Secret;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
||||
import org.gcube.dataharvest.ContextTest;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
||||
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||
|
|
@ -33,7 +33,6 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
|||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.utils.AggregationType;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.ContextTest;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
|
||||
import org.gcube.vremanagement.executor.api.types.Scheduling;
|
||||
|
|
@ -73,7 +72,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
dimensionMap.put(dimension.getId(), dimension);
|
||||
}
|
||||
|
||||
AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap);
|
||||
|
||||
return dao;
|
||||
}
|
||||
|
|
@ -82,7 +80,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
@Test
|
||||
public void getDimensions() {
|
||||
try {
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDao dao = AccountingDao.get();
|
||||
|
||||
|
|
@ -103,7 +101,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void launch() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||
|
||||
|
|
@ -136,7 +134,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void launchPluginOnSmartExecutor() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
String pluginName = new AccountingDashboardHarvesterPlugin().getName();
|
||||
|
||||
|
|
@ -189,7 +187,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void launchOldData() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||
|
||||
|
|
@ -226,10 +224,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void launchOldDataVREAccessesHarvester() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
|
||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
|
|
@ -252,7 +251,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
//CoreServicesAccessesHarvester vreAccessesHarvester = null;
|
||||
|
||||
for (String context : contexts) {
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
ScopeBean scopeBean = new ScopeBean(context);
|
||||
|
||||
|
|
@ -270,12 +269,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
|
||||
// Setting back token for the context
|
||||
ContextTest.set(parent.toString());
|
||||
ContextTest.setContextByName(parent.toString());
|
||||
|
||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||
|
||||
// Setting back token for the context
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -299,7 +298,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
||||
accountingRecords);
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
// dao.insertRecords(accountingRecords.toArray(new
|
||||
// AccountingRecord[1]));
|
||||
|
|
@ -310,7 +309,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
}
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
|
|
@ -324,7 +323,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
// @Test
|
||||
public void testVREAccessesHarvester() throws Exception {
|
||||
try {
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
|
|
@ -347,12 +346,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
for (Date start : starts) {
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||
|
||||
for (String contextFullname : contextFullNames) {
|
||||
|
||||
ContextTest.set(contextFullname);
|
||||
ContextTest.setContextByName(contextFullname);
|
||||
|
||||
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
|
||||
accountingRecords.addAll(harvested);
|
||||
|
|
@ -364,7 +363,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
logger.debug("{}", accountingRecords);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
// dao.insertRecords(accountingRecords.toArray(new
|
||||
// AccountingRecord[1]));
|
||||
|
||||
|
|
@ -379,7 +378,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testVREAccessesHarvesterAll() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
|
|
@ -394,6 +393,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||
accountingDataHarvesterPlugin.getConfigParameters();
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
|
||||
VREAccessesHarvester vreAccessesHarvester = null;
|
||||
|
|
@ -402,7 +402,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
for (String context : contexts) {
|
||||
// Setting the token for the context
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
ScopeBean scopeBean = new ScopeBean(context);
|
||||
|
||||
|
|
@ -419,12 +419,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
parent = scopeBean.enclosingScope();
|
||||
}
|
||||
|
||||
ContextTest.set(parent.toString());
|
||||
ContextTest.setContextByName(parent.toString());
|
||||
|
||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||
|
||||
// Setting back token for the context
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -456,7 +456,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testSocialInteraction() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
|
|
@ -480,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
for (String context : contexts) {
|
||||
// Setting the token for the context
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
logger.info("Going to harvest Social Interactions for {}", context);
|
||||
|
|
@ -494,7 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
||||
accountingRecords);
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
// dao.insertRecords(accountingRecords.toArray(new
|
||||
// AccountingRecord[1]));
|
||||
|
||||
|
|
@ -506,37 +506,37 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
}
|
||||
|
||||
@Ignore
|
||||
// @Test
|
||||
public void testMethodInvocation() {
|
||||
try {
|
||||
ContextTest.set(STOCK_ASSESMENT_VRE);
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
|
||||
|
||||
logger.debug("{}", accountingRecords);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
}
|
||||
}
|
||||
// @Ignore
|
||||
// // @Test
|
||||
// public void testMethodInvocation() {
|
||||
// try {
|
||||
// ContextTest.setContextByName(STOCK_ASSESMENT_VRE);
|
||||
//
|
||||
// AggregationType measureType = AggregationType.MONTHLY;
|
||||
//
|
||||
// Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
//
|
||||
// MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
// List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
|
||||
//
|
||||
// logger.debug("{}", accountingRecords);
|
||||
//
|
||||
// } catch (Exception e) {
|
||||
// logger.error("", e);
|
||||
// }
|
||||
// }
|
||||
|
||||
@Ignore
|
||||
//@Test
|
||||
//@Ignore
|
||||
@Test
|
||||
public void testMethodInvocationOldData() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
List<Date> starts = new ArrayList<>();
|
||||
starts.add(DateUtils.getStartCalendar(2021, Calendar.JULY, 1).getTime());
|
||||
Date start = DateUtils.getStartCalendar(2023, Calendar.JANUARY, 1).getTime();
|
||||
Date last = DateUtils.getStartCalendar(2024, Calendar.FEBRUARY, 1).getTime();
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
|
|
@ -548,32 +548,61 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
|
||||
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
|
||||
}
|
||||
|
||||
AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
|
||||
|
||||
|
||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||
|
||||
for (Date start : starts) {
|
||||
while(start.before(last)) {
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
|
||||
for (String context : contexts) {
|
||||
// Setting the token for the context
|
||||
ContextTest.set(context);
|
||||
Secret s = contextAuthorization.getCatalogueSecretForContext(context);
|
||||
ContextTest.set(s);
|
||||
|
||||
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
|
||||
accountingRecords.addAll(harvested);
|
||||
|
||||
if (context.startsWith(AccountingDashboardHarvesterPlugin.TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
logger.info("Going to harvest Method Invocations for {}", context);
|
||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
||||
start, end);
|
||||
|
||||
logger.debug("{} - {}", context, harvested);
|
||||
List<AccountingRecord> harvested = tagMeMethodInvocationHarvester.getAccountingRecords();
|
||||
logger.debug("{} - {}", context, harvested);
|
||||
accountingRecords.addAll(harvested);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
logger.info("Going to harvest Method Invocations for {}", context);
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
|
||||
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
|
||||
logger.debug("{} - {}", context, harvested);
|
||||
accountingRecords.addAll(harvested);
|
||||
} catch (Exception e) {
|
||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
start = end;
|
||||
}
|
||||
|
||||
logger.debug("Going to insert {}", accountingRecords);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
|
|
@ -589,10 +618,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testTagMeMethodInvocation() throws Exception {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
ContextTest.set(TAGME_VRE);
|
||||
ContextTest.setContextByName(TAGME_VRE);
|
||||
|
||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||
|
||||
|
|
@ -616,7 +645,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
logger.debug("{}", accountingRecords);
|
||||
}
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
// dao.insertRecords(accountingRecords.toArray(new
|
||||
// AccountingRecord[1]));
|
||||
|
||||
|
|
@ -630,10 +659,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
// @Test
|
||||
public void testGetVREUsersForSpecificVRE() {
|
||||
try {
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
||||
ContextTest.setContextByName("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
|
|
@ -645,7 +674,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
// dao.insertRecords(accountingRecords.toArray(new
|
||||
// AccountingRecord[1]));
|
||||
|
||||
|
|
@ -659,7 +688,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
// @Test
|
||||
public void testFilteringGenericResource() {
|
||||
try {
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
// Utils.setContext(RESOURCE_CATALOGUE);
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
|
@ -667,6 +696,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||
|
|
@ -685,7 +715,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testResourceCatalogueHarvester() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AggregationType measureType = AggregationType.MONTHLY;
|
||||
|
||||
|
|
@ -697,6 +727,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||
|
|
@ -715,8 +746,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testCoreServicesHarvester() {
|
||||
try {
|
||||
|
||||
String context = ROOT;
|
||||
ContextTest.set(context);
|
||||
String context = ROOT_PROD;
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
|
||||
|
|
@ -752,8 +783,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void testCatalogueHarvester() {
|
||||
try {
|
||||
|
||||
String context = ROOT;
|
||||
ContextTest.set(context);
|
||||
String context = ROOT_PROD;
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
|
|
@ -793,7 +824,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
public void addMissingVREAccesses() {
|
||||
try {
|
||||
|
||||
ContextTest.set(ROOT);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
AccountingDao dao = getAccountingDao();
|
||||
|
||||
|
|
@ -803,7 +834,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
String context = E_LEARNING_AREA_VRE;
|
||||
|
||||
// Setting the token for the context
|
||||
ContextTest.set(context);
|
||||
ContextTest.setContextByName(context);
|
||||
|
||||
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
|
||||
Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());
|
||||
|
|
@ -5,6 +5,7 @@ import java.io.InputStream;
|
|||
import java.util.Properties;
|
||||
|
||||
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
||||
import org.gcube.dataharvest.ContextTest;
|
||||
import org.junit.Ignore;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
/**
|
||||
*
|
||||
*/
|
||||
package org.gcube.dataharvest.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.utils.manager.SecretManager;
|
||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
||||
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
||||
import org.gcube.common.authorization.utils.secret.Secret;
|
||||
import org.gcube.common.keycloak.KeycloakClientFactory;
|
||||
import org.gcube.common.keycloak.model.TokenResponse;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public class ContextTest {
|
||||
|
||||
public static final String ROOT = "/d4science.research-infrastructures.eu";
|
||||
|
||||
protected static final String CONFIG_PROPERTIES_FILENAME = "config.properties";
|
||||
|
||||
protected static ContextAuthorization contextAuthorization;
|
||||
|
||||
private static Properties getProperties(){
|
||||
try {
|
||||
Properties properties = new Properties();
|
||||
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_PROPERTIES_FILENAME);
|
||||
|
||||
// load the properties file
|
||||
properties.load(input);
|
||||
|
||||
return properties;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static void set(Secret secret) throws Exception {
|
||||
SecretManagerProvider.instance.reset();
|
||||
SecretManager secretManager = new SecretManager();
|
||||
SecretManagerProvider.instance.set(secretManager);
|
||||
secretManager.addSecret(secret);
|
||||
secretManager.set();
|
||||
}
|
||||
|
||||
public static void set(String fullContextName) throws Exception {
|
||||
Secret secret = getSecret(fullContextName);
|
||||
set(secret);
|
||||
}
|
||||
|
||||
private static Secret getSecret(String fullContextName) throws Exception {
|
||||
return contextAuthorization.getSecretForContext(fullContextName);
|
||||
}
|
||||
|
||||
private static TokenResponse getJWTAccessToken(String context) throws Exception {
|
||||
ScopeProvider.instance.set(context);
|
||||
|
||||
int index = context.indexOf('/', 1);
|
||||
String root = context.substring(0, index == -1 ? context.length() : index);
|
||||
String clientSecret = getProperties().getProperty(root);
|
||||
|
||||
TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(ContextAuthorization.CLIENT_ID, clientSecret, context, null);
|
||||
return tr;
|
||||
}
|
||||
|
||||
public static Secret generateSecretByContextName(String context) throws Exception {
|
||||
TokenResponse tr = getJWTAccessToken(context);
|
||||
Secret secret = new JWTSecret(tr.getAccessToken());
|
||||
return secret;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
Secret secret = generateSecretByContextName(ROOT);
|
||||
set(secret);
|
||||
contextAuthorization = new ContextAuthorization();
|
||||
SecretManagerProvider.instance.reset();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() throws Exception {
|
||||
SecretManagerProvider.instance.reset();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -3,3 +3,4 @@
|
|||
/*.properties
|
||||
/howto.txt
|
||||
/scopedata 2.xml
|
||||
/config.ini
|
||||
|
|
|
|||
Loading…
Reference in New Issue