Compare commits
1 Commits
master
...
Feature/26
Author | SHA1 | Date |
---|---|---|
Ahmed Salah Tawfik Ibrahim | 67ad6eb052 |
|
@ -2,15 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for "accounting-dashboard-harvester-se-plugin"
|
# Changelog for "accounting-dashboard-harvester-se-plugin"
|
||||||
|
|
||||||
## [v2.5.0]
|
## [v2.3.1]
|
||||||
|
|
||||||
- Upgraded smart-executor-bom to 3.3.0 [#27999]
|
|
||||||
|
|
||||||
|
|
||||||
## [v2.4.0]
|
|
||||||
|
|
||||||
- Removed filter restriction on JobUsageRecord harvesting to get MethodInvocation
|
|
||||||
- Fixed bug on getting ScopeDescriptor for new scopes.
|
- Fixed bug on getting ScopeDescriptor for new scopes.
|
||||||
|
- Updated version of Social Client dependency (#26508)
|
||||||
|
|
||||||
|
|
||||||
## [v2.3.0]
|
## [v2.3.0]
|
||||||
|
|
10
pom.xml
10
pom.xml
|
@ -5,14 +5,14 @@
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>maven-parent</artifactId>
|
<artifactId>maven-parent</artifactId>
|
||||||
<groupId>org.gcube.tools</groupId>
|
<groupId>org.gcube.tools</groupId>
|
||||||
<version>1.2.0</version>
|
<version>1.1.0</version>
|
||||||
<relativePath />
|
<relativePath />
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<groupId>org.gcube.accounting</groupId>
|
<groupId>org.gcube.accounting</groupId>
|
||||||
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
|
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
|
||||||
<version>2.5.0</version>
|
<version>2.3.1</version>
|
||||||
<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
|
<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
|
||||||
<description>
|
<description>
|
||||||
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
|
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
|
||||||
|
@ -36,7 +36,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.vremanagement</groupId>
|
<groupId>org.gcube.vremanagement</groupId>
|
||||||
<artifactId>smart-executor-bom</artifactId>
|
<artifactId>smart-executor-bom</artifactId>
|
||||||
<version>3.3.0</version>
|
<version>3.1.0</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
@ -130,7 +130,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.social-networking</groupId>
|
<groupId>org.gcube.social-networking</groupId>
|
||||||
<artifactId>social-service-client</artifactId>
|
<artifactId>social-service-client</artifactId>
|
||||||
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
|
<version>[2.0.0-SNAPSHOT, 3.0.0)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.json</groupId>
|
<groupId>org.json</groupId>
|
||||||
|
@ -141,7 +141,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.common</groupId>
|
<groupId>org.gcube.common</groupId>
|
||||||
<artifactId>authorization-utils</artifactId>
|
<artifactId>authorization-utils</artifactId>
|
||||||
<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
|
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- Test Dependencies. Setting scope to provided to allow proper creation
|
<!-- Test Dependencies. Setting scope to provided to allow proper creation
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* @author Eric Perrone (ISTI - CNR)
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
*/
|
*/
|
||||||
public class MethodInvocationHarvester extends BasicHarvester {
|
public class MethodInvocationHarvester extends BasicHarvester {
|
||||||
|
@ -57,6 +58,7 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
||||||
|
|
||||||
SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
|
SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
|
||||||
List<Filter> filters = new ArrayList<>();
|
List<Filter> filters = new ArrayList<>();
|
||||||
|
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
||||||
|
|
||||||
Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
|
Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
|
||||||
|
|
||||||
|
@ -66,7 +68,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
||||||
AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
|
AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||||
} else {
|
} else {
|
||||||
// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
|
// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
|
||||||
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
|
|
||||||
result = accountingPersistenceQuery.getContextTimeSeries(
|
result = accountingPersistenceQuery.getContextTimeSeries(
|
||||||
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,7 @@ import org.gcube.accounting.accounting.summary.access.model.update.AccountingRec
|
||||||
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
||||||
import org.gcube.dataharvest.utils.Utils;
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
import org.gcube.portal.databook.shared.Feed;
|
import org.gcube.social_networking.socialnetworking.model.shared.Post;
|
||||||
import org.gcube.social_networking.social_networking_client_library.PostClient;
|
import org.gcube.social_networking.social_networking_client_library.PostClient;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -67,18 +67,18 @@ public class SocialInteractionsHarvester extends BasicHarvester {
|
||||||
|
|
||||||
private void getJson() throws Exception {
|
private void getJson() throws Exception {
|
||||||
PostClient postClient = new PostClient();
|
PostClient postClient = new PostClient();
|
||||||
List<Feed> vrePosts = postClient.getPostsVRE();
|
List<Post> vrePosts = postClient.getPostsVRE();
|
||||||
|
|
||||||
likes = replies = posts = 0;
|
likes = replies = posts = 0;
|
||||||
|
|
||||||
for(Feed feed : vrePosts) {
|
for(Post post : vrePosts) {
|
||||||
|
|
||||||
long time = feed.getTime().getTime();
|
long time = post.getTime().getTime();
|
||||||
|
|
||||||
if(start.getTime() <= time && time <= end.getTime()) {
|
if(start.getTime() <= time && time <= end.getTime()) {
|
||||||
posts++;
|
posts++;
|
||||||
replies += Integer.valueOf(feed.getCommentsNo());
|
replies += Integer.valueOf(post.getCommentsNo());
|
||||||
likes += Integer.valueOf(feed.getLikesNo());
|
likes += Integer.valueOf(post.getLikesNo());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,7 +91,9 @@ public class DateUtils {
|
||||||
aggregationStartCalendar.set(Calendar.MINUTE, 0);
|
aggregationStartCalendar.set(Calendar.MINUTE, 0);
|
||||||
aggregationStartCalendar.set(Calendar.SECOND, 0);
|
aggregationStartCalendar.set(Calendar.SECOND, 0);
|
||||||
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
|
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
|
||||||
// logger.trace("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
|
||||||
|
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
|
||||||
|
|
||||||
return aggregationStartCalendar;
|
return aggregationStartCalendar;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,17 +1,23 @@
|
||||||
package org.gcube.dataharvest.harvester;
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.SortedSet;
|
import java.util.SortedSet;
|
||||||
|
|
||||||
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
||||||
|
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
|
||||||
|
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
|
||||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||||
import org.gcube.dataharvest.ContextTest;
|
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||||
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
|
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
|
||||||
import org.gcube.dataharvest.utils.AggregationType;
|
import org.gcube.dataharvest.utils.AggregationType;
|
||||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||||
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.gcube.dataharvest.utils.DateUtils;
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -32,7 +38,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
||||||
public void testJupyterAccessesHarvester() throws Exception {
|
public void testJupyterAccessesHarvester() throws Exception {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
List<Date> starts = new ArrayList<>();
|
List<Date> starts = new ArrayList<>();
|
||||||
|
@ -44,7 +50,6 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
/*
|
/*
|
||||||
SortedSet<String> contexts = new TreeSet<>();
|
SortedSet<String> contexts = new TreeSet<>();
|
||||||
|
@ -58,11 +63,11 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
||||||
for (Date start : starts) {
|
for (Date start : starts) {
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
|
JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
|
||||||
|
|
||||||
for(String context : contexts) {
|
for(String context : contexts) {
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
|
List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
|
||||||
accountingRecords.addAll(harvested);
|
accountingRecords.addAll(harvested);
|
||||||
|
@ -74,7 +79,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
|
||||||
|
|
||||||
logger.debug("Going to insert {}", accountingRecords);
|
logger.debug("Going to insert {}", accountingRecords);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package org.gcube.dataharvest.plugin;
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -7,6 +7,7 @@ import java.util.Date;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Properties;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.SortedSet;
|
import java.util.SortedSet;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
|
@ -17,15 +18,14 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
|
||||||
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
|
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
|
||||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||||
import org.gcube.common.authorization.utils.secret.Secret;
|
|
||||||
import org.gcube.common.scope.impl.ScopeBean;
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||||
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
|
||||||
import org.gcube.dataharvest.ContextTest;
|
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
|
||||||
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
|
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
|
||||||
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
|
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
|
||||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
|
||||||
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||||
|
@ -33,6 +33,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||||
import org.gcube.dataharvest.utils.AggregationType;
|
import org.gcube.dataharvest.utils.AggregationType;
|
||||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||||
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.gcube.dataharvest.utils.DateUtils;
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
|
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
|
||||||
import org.gcube.vremanagement.executor.api.types.Scheduling;
|
import org.gcube.vremanagement.executor.api.types.Scheduling;
|
||||||
|
@ -72,6 +73,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
dimensionMap.put(dimension.getId(), dimension);
|
dimensionMap.put(dimension.getId(), dimension);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap);
|
||||||
|
|
||||||
return dao;
|
return dao;
|
||||||
}
|
}
|
||||||
|
@ -80,7 +82,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
@Test
|
@Test
|
||||||
public void getDimensions() {
|
public void getDimensions() {
|
||||||
try {
|
try {
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDao dao = AccountingDao.get();
|
AccountingDao dao = AccountingDao.get();
|
||||||
|
|
||||||
|
@ -101,7 +103,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void launch() {
|
public void launch() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||||
|
|
||||||
|
@ -134,7 +136,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void launchPluginOnSmartExecutor() {
|
public void launchPluginOnSmartExecutor() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
String pluginName = new AccountingDashboardHarvesterPlugin().getName();
|
String pluginName = new AccountingDashboardHarvesterPlugin().getName();
|
||||||
|
|
||||||
|
@ -187,7 +189,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void launchOldData() {
|
public void launchOldData() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||||
|
|
||||||
|
@ -224,11 +226,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void launchOldDataVREAccessesHarvester() {
|
public void launchOldDataVREAccessesHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
|
@ -251,7 +252,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
//CoreServicesAccessesHarvester vreAccessesHarvester = null;
|
//CoreServicesAccessesHarvester vreAccessesHarvester = null;
|
||||||
|
|
||||||
for (String context : contexts) {
|
for (String context : contexts) {
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
ScopeBean scopeBean = new ScopeBean(context);
|
ScopeBean scopeBean = new ScopeBean(context);
|
||||||
|
|
||||||
|
@ -269,12 +270,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setting back token for the context
|
// Setting back token for the context
|
||||||
ContextTest.setContextByName(parent.toString());
|
ContextTest.set(parent.toString());
|
||||||
|
|
||||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||||
|
|
||||||
// Setting back token for the context
|
// Setting back token for the context
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -298,7 +299,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
||||||
accountingRecords);
|
accountingRecords);
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
// dao.insertRecords(accountingRecords.toArray(new
|
// dao.insertRecords(accountingRecords.toArray(new
|
||||||
// AccountingRecord[1]));
|
// AccountingRecord[1]));
|
||||||
|
@ -309,7 +310,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
|
@ -323,7 +324,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
// @Test
|
// @Test
|
||||||
public void testVREAccessesHarvester() throws Exception {
|
public void testVREAccessesHarvester() throws Exception {
|
||||||
try {
|
try {
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
|
@ -346,12 +347,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
for (Date start : starts) {
|
for (Date start : starts) {
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||||
|
|
||||||
for (String contextFullname : contextFullNames) {
|
for (String contextFullname : contextFullNames) {
|
||||||
|
|
||||||
ContextTest.setContextByName(contextFullname);
|
ContextTest.set(contextFullname);
|
||||||
|
|
||||||
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
|
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
|
||||||
accountingRecords.addAll(harvested);
|
accountingRecords.addAll(harvested);
|
||||||
|
@ -363,7 +364,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("{}", accountingRecords);
|
logger.debug("{}", accountingRecords);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// dao.insertRecords(accountingRecords.toArray(new
|
// dao.insertRecords(accountingRecords.toArray(new
|
||||||
// AccountingRecord[1]));
|
// AccountingRecord[1]));
|
||||||
|
|
||||||
|
@ -378,7 +379,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testVREAccessesHarvesterAll() {
|
public void testVREAccessesHarvesterAll() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
|
@ -393,7 +394,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
|
||||||
accountingDataHarvesterPlugin.getConfigParameters();
|
accountingDataHarvesterPlugin.getConfigParameters();
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
VREAccessesHarvester vreAccessesHarvester = null;
|
VREAccessesHarvester vreAccessesHarvester = null;
|
||||||
|
@ -402,7 +402,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
for (String context : contexts) {
|
for (String context : contexts) {
|
||||||
// Setting the token for the context
|
// Setting the token for the context
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
ScopeBean scopeBean = new ScopeBean(context);
|
ScopeBean scopeBean = new ScopeBean(context);
|
||||||
|
|
||||||
|
@ -419,12 +419,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
parent = scopeBean.enclosingScope();
|
parent = scopeBean.enclosingScope();
|
||||||
}
|
}
|
||||||
|
|
||||||
ContextTest.setContextByName(parent.toString());
|
ContextTest.set(parent.toString());
|
||||||
|
|
||||||
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
vreAccessesHarvester = new VREAccessesHarvester(start, end);
|
||||||
|
|
||||||
// Setting back token for the context
|
// Setting back token for the context
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -456,7 +456,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testSocialInteraction() {
|
public void testSocialInteraction() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
|
@ -480,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
for (String context : contexts) {
|
for (String context : contexts) {
|
||||||
// Setting the token for the context
|
// Setting the token for the context
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
try {
|
try {
|
||||||
// Collecting info on social (posts, replies and likes)
|
// Collecting info on social (posts, replies and likes)
|
||||||
logger.info("Going to harvest Social Interactions for {}", context);
|
logger.info("Going to harvest Social Interactions for {}", context);
|
||||||
|
@ -494,7 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
|
||||||
accountingRecords);
|
accountingRecords);
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// dao.insertRecords(accountingRecords.toArray(new
|
// dao.insertRecords(accountingRecords.toArray(new
|
||||||
// AccountingRecord[1]));
|
// AccountingRecord[1]));
|
||||||
|
|
||||||
|
@ -506,37 +506,37 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// @Ignore
|
@Ignore
|
||||||
// // @Test
|
// @Test
|
||||||
// public void testMethodInvocation() {
|
public void testMethodInvocation() {
|
||||||
// try {
|
try {
|
||||||
// ContextTest.setContextByName(STOCK_ASSESMENT_VRE);
|
ContextTest.set(STOCK_ASSESMENT_VRE);
|
||||||
//
|
|
||||||
// AggregationType measureType = AggregationType.MONTHLY;
|
|
||||||
//
|
|
||||||
// Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
|
||||||
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
|
||||||
//
|
|
||||||
// MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
|
||||||
// List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
|
|
||||||
//
|
|
||||||
// logger.debug("{}", accountingRecords);
|
|
||||||
//
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// logger.error("", e);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// @Ignore
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
@Test
|
|
||||||
|
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||||
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
|
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
|
||||||
|
|
||||||
|
logger.debug("{}", accountingRecords);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ignore
|
||||||
|
//@Test
|
||||||
public void testMethodInvocationOldData() {
|
public void testMethodInvocationOldData() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
Date start = DateUtils.getStartCalendar(2023, Calendar.JANUARY, 1).getTime();
|
List<Date> starts = new ArrayList<>();
|
||||||
Date last = DateUtils.getStartCalendar(2024, Calendar.FEBRUARY, 1).getTime();
|
starts.add(DateUtils.getStartCalendar(2021, Calendar.JULY, 1).getTime());
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
|
@ -548,60 +548,31 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
|
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
|
||||||
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
|
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
|
||||||
}
|
}
|
||||||
|
|
||||||
AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
|
AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
|
||||||
|
|
||||||
|
|
||||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||||
|
|
||||||
while(start.before(last)) {
|
for (Date start : starts) {
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
|
|
||||||
for (String context : contexts) {
|
for (String context : contexts) {
|
||||||
// Setting the token for the context
|
// Setting the token for the context
|
||||||
Secret s = contextAuthorization.getCatalogueSecretForContext(context);
|
ContextTest.set(context);
|
||||||
ContextTest.set(s);
|
|
||||||
|
|
||||||
|
|
||||||
if (context.startsWith(AccountingDashboardHarvesterPlugin.TAGME_CONTEXT)) {
|
|
||||||
try {
|
|
||||||
// Collecting info on method invocation
|
|
||||||
logger.info("Going to harvest Method Invocations for {}", context);
|
|
||||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
|
||||||
start, end);
|
|
||||||
|
|
||||||
List<AccountingRecord> harvested = tagMeMethodInvocationHarvester.getAccountingRecords();
|
|
||||||
logger.debug("{} - {}", context, harvested);
|
|
||||||
accountingRecords.addAll(harvested);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
// Collecting info on method invocation
|
|
||||||
logger.info("Going to harvest Method Invocations for {}", context);
|
|
||||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
|
||||||
|
|
||||||
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
|
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
|
||||||
logger.debug("{} - {}", context, harvested);
|
|
||||||
accountingRecords.addAll(harvested);
|
accountingRecords.addAll(harvested);
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("Error harvesting Method Invocations for {}", context, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
logger.debug("{} - {}", context, harvested);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
start = end;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug("Going to insert {}", accountingRecords);
|
logger.debug("Going to insert {}", accountingRecords);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -618,10 +589,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testTagMeMethodInvocation() throws Exception {
|
public void testTagMeMethodInvocation() throws Exception {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
ContextTest.setContextByName(TAGME_VRE);
|
ContextTest.set(TAGME_VRE);
|
||||||
|
|
||||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||||
|
|
||||||
|
@ -645,7 +616,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
logger.debug("{}", accountingRecords);
|
logger.debug("{}", accountingRecords);
|
||||||
}
|
}
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// dao.insertRecords(accountingRecords.toArray(new
|
// dao.insertRecords(accountingRecords.toArray(new
|
||||||
// AccountingRecord[1]));
|
// AccountingRecord[1]));
|
||||||
|
|
||||||
|
@ -659,10 +630,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
// @Test
|
// @Test
|
||||||
public void testGetVREUsersForSpecificVRE() {
|
public void testGetVREUsersForSpecificVRE() {
|
||||||
try {
|
try {
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
ContextTest.setContextByName("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
|
@ -674,7 +645,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
|
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// dao.insertRecords(accountingRecords.toArray(new
|
// dao.insertRecords(accountingRecords.toArray(new
|
||||||
// AccountingRecord[1]));
|
// AccountingRecord[1]));
|
||||||
|
|
||||||
|
@ -688,7 +659,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
// @Test
|
// @Test
|
||||||
public void testFilteringGenericResource() {
|
public void testFilteringGenericResource() {
|
||||||
try {
|
try {
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
// Utils.setContext(RESOURCE_CATALOGUE);
|
// Utils.setContext(RESOURCE_CATALOGUE);
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
@ -696,7 +667,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||||
|
@ -715,7 +685,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testResourceCatalogueHarvester() {
|
public void testResourceCatalogueHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
|
@ -727,7 +697,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||||
|
@ -746,8 +715,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testCoreServicesHarvester() {
|
public void testCoreServicesHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
String context = ROOT_PROD;
|
String context = ROOT;
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
|
Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
|
||||||
|
@ -783,8 +752,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void testCatalogueHarvester() {
|
public void testCatalogueHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
String context = ROOT_PROD;
|
String context = ROOT;
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
|
@ -824,7 +793,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
public void addMissingVREAccesses() {
|
public void addMissingVREAccesses() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
|
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
|
@ -834,7 +803,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
String context = E_LEARNING_AREA_VRE;
|
String context = E_LEARNING_AREA_VRE;
|
||||||
|
|
||||||
// Setting the token for the context
|
// Setting the token for the context
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
|
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
|
||||||
Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());
|
Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());
|
|
@ -1,4 +1,4 @@
|
||||||
package org.gcube.dataharvest.harvester;
|
package org.gcube.dataharvest;
|
||||||
|
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
import java.time.ZoneId;
|
import java.time.ZoneId;
|
||||||
|
@ -11,10 +11,9 @@ import java.util.stream.Stream;
|
||||||
|
|
||||||
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
import org.gcube.accounting.accounting.summary.access.AccountingDao;
|
||||||
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
|
||||||
import org.gcube.dataharvest.ContextTest;
|
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
|
||||||
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
|
|
||||||
import org.gcube.dataharvest.utils.AggregationType;
|
import org.gcube.dataharvest.utils.AggregationType;
|
||||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.gcube.dataharvest.utils.DateUtils;
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -36,7 +35,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
||||||
public void testJupyterAccessesHarvester() throws Exception {
|
public void testJupyterAccessesHarvester() throws Exception {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
AccountingDao dao = getAccountingDao();
|
AccountingDao dao = getAccountingDao();
|
||||||
|
|
||||||
List<Date> starts = new ArrayList<>();
|
List<Date> starts = new ArrayList<>();
|
||||||
|
@ -49,7 +48,6 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
||||||
|
|
||||||
AggregationType measureType = AggregationType.MONTHLY;
|
AggregationType measureType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
|
||||||
SortedSet<String> contexts = contextAuthorization.getContexts();
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
List<AccountingRecord> accountingRecords = new ArrayList<>();
|
||||||
|
@ -58,11 +56,11 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
||||||
for (Date start : starts) {
|
for (Date start : starts) {
|
||||||
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
|
RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
|
||||||
|
|
||||||
for(String context : contexts) {
|
for(String context : contexts) {
|
||||||
ContextTest.setContextByName(context);
|
ContextTest.set(context);
|
||||||
|
|
||||||
List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
|
List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
|
||||||
accountingRecords.addAll(harvested);
|
accountingRecords.addAll(harvested);
|
||||||
|
@ -74,7 +72,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
|
||||||
|
|
||||||
logger.debug("Going to insert {}", accountingRecords);
|
logger.debug("Going to insert {}", accountingRecords);
|
||||||
|
|
||||||
ContextTest.setContextByName(ROOT_PROD);
|
ContextTest.set(ROOT);
|
||||||
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
|
@ -1,173 +0,0 @@
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
package org.gcube.dataharvest;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManager;
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
|
||||||
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
|
||||||
import org.gcube.common.authorization.utils.secret.Secret;
|
|
||||||
import org.gcube.common.authorization.utils.secret.SecretUtility;
|
|
||||||
import org.gcube.common.keycloak.KeycloakClientFactory;
|
|
||||||
import org.gcube.common.keycloak.KeycloakClientHelper;
|
|
||||||
import org.gcube.common.keycloak.model.TokenResponse;
|
|
||||||
import org.junit.AfterClass;
|
|
||||||
import org.junit.BeforeClass;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
|
||||||
*/
|
|
||||||
public class ContextTest {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
|
|
||||||
|
|
||||||
protected static final String CONFIG_INI_FILENAME = "config.ini";
|
|
||||||
|
|
||||||
public static final String DEFAULT_TEST_SCOPE;
|
|
||||||
|
|
||||||
public static final String GCUBE;
|
|
||||||
public static final String DEVNEXT;
|
|
||||||
public static final String NEXTNEXT;
|
|
||||||
public static final String DEVSEC;
|
|
||||||
public static final String DEVVRE;
|
|
||||||
|
|
||||||
public static final String ROOT_PROD;
|
|
||||||
|
|
||||||
protected static final Properties properties;
|
|
||||||
|
|
||||||
public static final String TYPE_PROPERTY_KEY = "type";
|
|
||||||
public static final String USERNAME_PROPERTY_KEY = "username";
|
|
||||||
public static final String PASSWORD_PROPERTY_KEY = "password";
|
|
||||||
public static final String CLIENT_ID_PROPERTY_KEY = "clientId";
|
|
||||||
|
|
||||||
static {
|
|
||||||
GCUBE = "/gcube";
|
|
||||||
DEVNEXT = GCUBE + "/devNext";
|
|
||||||
NEXTNEXT = DEVNEXT + "/NextNext";
|
|
||||||
DEVSEC = GCUBE + "/devsec";
|
|
||||||
DEVVRE = DEVSEC + "/devVRE";
|
|
||||||
|
|
||||||
ROOT_PROD = "/d4science.research-infrastructures.eu";
|
|
||||||
|
|
||||||
DEFAULT_TEST_SCOPE = GCUBE;
|
|
||||||
|
|
||||||
|
|
||||||
properties = new Properties();
|
|
||||||
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
|
|
||||||
try {
|
|
||||||
// load the properties file
|
|
||||||
properties.load(input);
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private enum Type{
|
|
||||||
USER, CLIENT_ID
|
|
||||||
};
|
|
||||||
|
|
||||||
public static void set(Secret secret) throws Exception {
|
|
||||||
SecretManagerProvider.instance.reset();
|
|
||||||
SecretManager secretManager = new SecretManager();
|
|
||||||
secretManager.addSecret(secret);
|
|
||||||
SecretManagerProvider.instance.set(secretManager);
|
|
||||||
SecretManagerProvider.instance.get().set();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void setContextByName(String fullContextName) throws Exception {
|
|
||||||
logger.debug("Going to set credentials for context {}", fullContextName);
|
|
||||||
Secret secret = getSecretByContextName(fullContextName);
|
|
||||||
set(secret);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static TokenResponse getJWTAccessToken(String context) throws Exception {
|
|
||||||
Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
|
|
||||||
|
|
||||||
TokenResponse tr = null;
|
|
||||||
|
|
||||||
int index = context.indexOf('/', 1);
|
|
||||||
String root = context.substring(0, index == -1 ? context.length() : index);
|
|
||||||
|
|
||||||
switch (type) {
|
|
||||||
case CLIENT_ID:
|
|
||||||
String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
|
|
||||||
String clientSecret = properties.getProperty(root);
|
|
||||||
|
|
||||||
tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
|
|
||||||
break;
|
|
||||||
|
|
||||||
case USER:
|
|
||||||
default:
|
|
||||||
String username = properties.getProperty(USERNAME_PROPERTY_KEY);
|
|
||||||
String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
|
|
||||||
|
|
||||||
switch (root) {
|
|
||||||
case "/gcube":
|
|
||||||
default:
|
|
||||||
clientId = "next.d4science.org";
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "/pred4s":
|
|
||||||
clientId = "pre.d4science.org";
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "/d4science.research-infrastructures.eu":
|
|
||||||
clientId = "services.d4science.org";
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
clientSecret = null;
|
|
||||||
|
|
||||||
tr = KeycloakClientHelper.getTokenForUser(context, username, password);
|
|
||||||
break;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return tr;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Secret getSecretByContextName(String context) throws Exception {
|
|
||||||
TokenResponse tr = getJWTAccessToken(context);
|
|
||||||
Secret secret = new JWTSecret(tr.getAccessToken());
|
|
||||||
return secret;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void setContext(String token) throws Exception {
|
|
||||||
Secret secret = getSecret(token);
|
|
||||||
set(secret);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Secret getSecret(String token) throws Exception {
|
|
||||||
Secret secret = SecretUtility.getSecretByTokenString(token);
|
|
||||||
return secret;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getUser() {
|
|
||||||
String user = "UNKNOWN";
|
|
||||||
try {
|
|
||||||
user = SecretManagerProvider.instance.get().getUser().getUsername();
|
|
||||||
} catch(Exception e) {
|
|
||||||
logger.error("Unable to retrieve user. {} will be used", user);
|
|
||||||
}
|
|
||||||
return user;
|
|
||||||
}
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void beforeClass() throws Exception {
|
|
||||||
setContextByName(ROOT_PROD);
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterClass
|
|
||||||
public static void afterClass() throws Exception {
|
|
||||||
SecretManagerProvider.instance.reset();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -2,7 +2,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.dataharvest.ContextTest;
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
|
@ -5,7 +5,6 @@ import java.io.InputStream;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
|
||||||
import org.gcube.dataharvest.ContextTest;
|
|
||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
|
@ -0,0 +1,92 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
package org.gcube.dataharvest.utils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.utils.manager.SecretManager;
|
||||||
|
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
||||||
|
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
||||||
|
import org.gcube.common.authorization.utils.secret.Secret;
|
||||||
|
import org.gcube.common.keycloak.KeycloakClientFactory;
|
||||||
|
import org.gcube.common.keycloak.model.TokenResponse;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
public class ContextTest {
|
||||||
|
|
||||||
|
public static final String ROOT = "/d4science.research-infrastructures.eu";
|
||||||
|
|
||||||
|
protected static final String CONFIG_PROPERTIES_FILENAME = "config.properties";
|
||||||
|
|
||||||
|
protected static ContextAuthorization contextAuthorization;
|
||||||
|
|
||||||
|
private static Properties getProperties(){
|
||||||
|
try {
|
||||||
|
Properties properties = new Properties();
|
||||||
|
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_PROPERTIES_FILENAME);
|
||||||
|
|
||||||
|
// load the properties file
|
||||||
|
properties.load(input);
|
||||||
|
|
||||||
|
return properties;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void set(Secret secret) throws Exception {
|
||||||
|
SecretManagerProvider.instance.reset();
|
||||||
|
SecretManager secretManager = new SecretManager();
|
||||||
|
SecretManagerProvider.instance.set(secretManager);
|
||||||
|
secretManager.addSecret(secret);
|
||||||
|
secretManager.set();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void set(String fullContextName) throws Exception {
|
||||||
|
Secret secret = getSecret(fullContextName);
|
||||||
|
set(secret);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Secret getSecret(String fullContextName) throws Exception {
|
||||||
|
return contextAuthorization.getSecretForContext(fullContextName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TokenResponse getJWTAccessToken(String context) throws Exception {
|
||||||
|
ScopeProvider.instance.set(context);
|
||||||
|
|
||||||
|
int index = context.indexOf('/', 1);
|
||||||
|
String root = context.substring(0, index == -1 ? context.length() : index);
|
||||||
|
String clientSecret = getProperties().getProperty(root);
|
||||||
|
|
||||||
|
TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(ContextAuthorization.CLIENT_ID, clientSecret, context, null);
|
||||||
|
return tr;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Secret generateSecretByContextName(String context) throws Exception {
|
||||||
|
TokenResponse tr = getJWTAccessToken(context);
|
||||||
|
Secret secret = new JWTSecret(tr.getAccessToken());
|
||||||
|
return secret;
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void beforeClass() throws Exception {
|
||||||
|
Secret secret = generateSecretByContextName(ROOT);
|
||||||
|
set(secret);
|
||||||
|
contextAuthorization = new ContextAuthorization();
|
||||||
|
SecretManagerProvider.instance.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void afterClass() throws Exception {
|
||||||
|
SecretManagerProvider.instance.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -3,4 +3,3 @@
|
||||||
/*.properties
|
/*.properties
|
||||||
/howto.txt
|
/howto.txt
|
||||||
/scopedata 2.xml
|
/scopedata 2.xml
|
||||||
/config.ini
|
|
||||||
|
|
Loading…
Reference in New Issue