Compare commits

..

1 Commits

Author SHA1 Message Date
Ahmed Salah Tawfik Ibrahim 67ad6eb052 Updated version of social client dependency 2024-01-25 15:45:23 +01:00
13 changed files with 203 additions and 311 deletions

View File

@ -2,10 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for "accounting-dashboard-harvester-se-plugin"
## [v2.4.0]
## [v2.3.1]
- Removed filter restriction on JobUsageRecord harvesting to get MethodInvocation
- Fixed bug on getting ScopeDescriptor for new scopes.
- Updated version of Social Client dependency (#26508)
## [v2.3.0]

View File

@ -5,14 +5,14 @@
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.2.0</version>
<version>1.1.0</version>
<relativePath />
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
<version>2.4.0</version>
<version>2.3.1</version>
<name>Accounting Dashboard Harvester Smart Executor Plugin</name>
<description>
Accounting Dashboard Harvester Smart Executor Plugin harvest accounting
@ -130,7 +130,7 @@
<dependency>
<groupId>org.gcube.social-networking</groupId>
<artifactId>social-service-client</artifactId>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
<version>[2.0.0-SNAPSHOT, 3.0.0)</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
@ -141,7 +141,7 @@
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>authorization-utils</artifactId>
<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
</dependency>
<!-- Test Dependencies. Setting scope to provided to allow proper creation

View File

@ -28,6 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class MethodInvocationHarvester extends BasicHarvester {
@ -57,6 +58,7 @@ public class MethodInvocationHarvester extends BasicHarvester {
SortedMap<Filter,SortedMap<Calendar,Info>> result = null;
List<Filter> filters = new ArrayList<>();
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
Date newMethodInvocationHarvesterStartDate = DateUtils.getStartCalendar(2017, Calendar.DECEMBER, 31).getTime();
@ -66,7 +68,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
AggregatedJobUsageRecord.class, temporalConstraint, filters, contexts, true);
} else {
// Before 31/12/2017 accounting Method Invocation using ServiceUsageRecord
filters.add(new Filter(ServiceUsageRecord.SERVICE_NAME, DATAMINER_SERVICE_NAME));
result = accountingPersistenceQuery.getContextTimeSeries(
AggregatedServiceUsageRecord.class, temporalConstraint, filters, contexts, true);
}

View File

@ -9,7 +9,7 @@ import org.gcube.accounting.accounting.summary.access.model.update.AccountingRec
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.portal.databook.shared.Feed;
import org.gcube.social_networking.socialnetworking.model.shared.Post;
import org.gcube.social_networking.social_networking_client_library.PostClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -67,18 +67,18 @@ public class SocialInteractionsHarvester extends BasicHarvester {
private void getJson() throws Exception {
PostClient postClient = new PostClient();
List<Feed> vrePosts = postClient.getPostsVRE();
List<Post> vrePosts = postClient.getPostsVRE();
likes = replies = posts = 0;
for(Feed feed : vrePosts) {
for(Post post : vrePosts) {
long time = feed.getTime().getTime();
long time = post.getTime().getTime();
if(start.getTime() <= time && time <= end.getTime()) {
posts++;
replies += Integer.valueOf(feed.getCommentsNo());
likes += Integer.valueOf(feed.getLikesNo());
replies += Integer.valueOf(post.getCommentsNo());
likes += Integer.valueOf(post.getLikesNo());
}
}

View File

@ -91,7 +91,9 @@ public class DateUtils {
aggregationStartCalendar.set(Calendar.MINUTE, 0);
aggregationStartCalendar.set(Calendar.SECOND, 0);
aggregationStartCalendar.set(Calendar.MILLISECOND, 0);
// logger.trace("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
logger.debug("{}", DEFAULT_DATE_FORMAT.format(aggregationStartCalendar.getTime()));
return aggregationStartCalendar;
}

View File

@ -1,17 +1,23 @@
package org.gcube.dataharvest.harvester;
package org.gcube.dataharvest;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.junit.Ignore;
import org.junit.Test;
@ -32,7 +38,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
public void testJupyterAccessesHarvester() throws Exception {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>();
@ -44,7 +50,6 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
AggregationType measureType = AggregationType.MONTHLY;
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
/*
SortedSet<String> contexts = new TreeSet<>();
@ -58,11 +63,11 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
JupyterAccessesHarvester jupyterAccessesHarvester = new JupyterAccessesHarvester(start, end);
for(String context : contexts) {
ContextTest.setContextByName(context);
ContextTest.set(context);
List<AccountingRecord> harvested = jupyterAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -74,7 +79,7 @@ public class AccountingDataHarvesterJupyterTest extends AccountingDataHarvesterP
logger.debug("Going to insert {}", accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));

View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest.plugin;
package org.gcube.dataharvest;
import java.time.Instant;
import java.util.ArrayList;
@ -7,6 +7,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@ -17,15 +18,14 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
@ -33,6 +33,7 @@ import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
import org.gcube.vremanagement.executor.api.types.Scheduling;
@ -72,6 +73,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
dimensionMap.put(dimension.getId(), dimension);
}
AccountingDashboardHarvesterPlugin.dimensions.set(dimensionMap);
return dao;
}
@ -80,7 +82,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
@Test
public void getDimensions() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = AccountingDao.get();
@ -101,7 +103,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launch() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
@ -134,7 +136,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchPluginOnSmartExecutor() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
String pluginName = new AccountingDashboardHarvesterPlugin().getName();
@ -187,7 +189,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchOldData() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
@ -224,11 +226,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void launchOldDataVREAccessesHarvester() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
AggregationType aggregationType = AggregationType.MONTHLY;
@ -251,7 +252,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
//CoreServicesAccessesHarvester vreAccessesHarvester = null;
for (String context : contexts) {
ContextTest.setContextByName(context);
ContextTest.set(context);
ScopeBean scopeBean = new ScopeBean(context);
@ -269,12 +270,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
// Setting back token for the context
ContextTest.setContextByName(parent.toString());
ContextTest.set(parent.toString());
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
ContextTest.setContextByName(context);
ContextTest.set(context);
}
}
@ -298,7 +299,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -309,7 +310,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
} catch (Exception e) {
logger.error("", e);
@ -323,7 +324,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testVREAccessesHarvester() throws Exception {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
@ -346,12 +347,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
for (String contextFullname : contextFullNames) {
ContextTest.setContextByName(contextFullname);
ContextTest.set(contextFullname);
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -363,7 +364,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -378,7 +379,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testVREAccessesHarvesterAll() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AggregationType measureType = AggregationType.MONTHLY;
@ -393,7 +394,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AccountingDashboardHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDashboardHarvesterPlugin();
accountingDataHarvesterPlugin.getConfigParameters();
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
VREAccessesHarvester vreAccessesHarvester = null;
@ -402,7 +402,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (String context : contexts) {
// Setting the token for the context
ContextTest.setContextByName(context);
ContextTest.set(context);
ScopeBean scopeBean = new ScopeBean(context);
@ -419,12 +419,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
parent = scopeBean.enclosingScope();
}
ContextTest.setContextByName(parent.toString());
ContextTest.set(parent.toString());
vreAccessesHarvester = new VREAccessesHarvester(start, end);
// Setting back token for the context
ContextTest.setContextByName(context);
ContextTest.set(context);
}
}
@ -456,7 +456,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testSocialInteraction() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
@ -480,7 +480,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (String context : contexts) {
// Setting the token for the context
ContextTest.setContextByName(context);
ContextTest.set(context);
try {
// Collecting info on social (posts, replies and likes)
logger.info("Going to harvest Social Interactions for {}", context);
@ -494,7 +494,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -506,37 +506,37 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
// @Ignore
// // @Test
// public void testMethodInvocation() {
// try {
// ContextTest.setContextByName(STOCK_ASSESMENT_VRE);
//
// AggregationType measureType = AggregationType.MONTHLY;
//
// Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
// Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
//
// MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
// List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
//
// logger.debug("{}", accountingRecords);
//
// } catch (Exception e) {
// logger.error("", e);
// }
// }
@Ignore
// @Test
public void testMethodInvocation() {
try {
ContextTest.set(STOCK_ASSESMENT_VRE);
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
logger.debug("{}", accountingRecords);
} catch (Exception e) {
logger.error("", e);
}
}
// @Ignore
@Test
@Ignore
//@Test
public void testMethodInvocationOldData() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
Date start = DateUtils.getStartCalendar(2023, Calendar.JANUARY, 1).getTime();
Date last = DateUtils.getStartCalendar(2024, Calendar.FEBRUARY, 1).getTime();
List<Date> starts = new ArrayList<>();
starts.add(DateUtils.getStartCalendar(2021, Calendar.JULY, 1).getTime());
AggregationType measureType = AggregationType.MONTHLY;
@ -548,60 +548,31 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
}
AccountingDashboardHarvesterPlugin.scopeDescriptors.set(scopeDescriptorMap);
List<AccountingRecord> accountingRecords = new ArrayList<>();
while(start.before(last)) {
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
for (String context : contexts) {
// Setting the token for the context
Secret s = contextAuthorization.getCatalogueSecretForContext(context);
ContextTest.set(s);
ContextTest.set(context);
if (context.startsWith(AccountingDashboardHarvesterPlugin.TAGME_CONTEXT)) {
try {
// Collecting info on method invocation
logger.info("Going to harvest Method Invocations for {}", context);
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
start, end);
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
List<AccountingRecord> harvested = tagMeMethodInvocationHarvester.getAccountingRecords();
logger.debug("{} - {}", context, harvested);
accountingRecords.addAll(harvested);
} catch (Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
} else {
try {
// Collecting info on method invocation
logger.info("Going to harvest Method Invocations for {}", context);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> harvested = methodInvocationHarvester.getAccountingRecords();
logger.debug("{} - {}", context, harvested);
accountingRecords.addAll(harvested);
} catch (Exception e) {
logger.error("Error harvesting Method Invocations for {}", context, e);
}
}
logger.debug("{} - {}", context, harvested);
}
start = end;
}
logger.debug("Going to insert {}", accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch (Exception e) {
@ -618,10 +589,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testTagMeMethodInvocation() throws Exception {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
ContextTest.setContextByName(TAGME_VRE);
ContextTest.set(TAGME_VRE);
List<AccountingRecord> accountingRecords = new ArrayList<>();
@ -645,7 +616,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", accountingRecords);
}
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -659,10 +630,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testGetVREUsersForSpecificVRE() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
ContextTest.setContextByName("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
ContextTest.set("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience");
AggregationType measureType = AggregationType.MONTHLY;
@ -674,7 +645,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// dao.insertRecords(accountingRecords.toArray(new
// AccountingRecord[1]));
@ -688,7 +659,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// @Test
public void testFilteringGenericResource() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
// Utils.setContext(RESOURCE_CATALOGUE);
AggregationType measureType = AggregationType.MONTHLY;
@ -696,7 +667,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@ -715,7 +685,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testResourceCatalogueHarvester() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AggregationType measureType = AggregationType.MONTHLY;
@ -727,7 +697,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
@ -746,8 +715,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testCoreServicesHarvester() {
try {
String context = ROOT_PROD;
ContextTest.setContextByName(context);
String context = ROOT;
ContextTest.set(context);
AccountingDao dao = getAccountingDao();
Calendar from = DateUtils.getStartCalendar(2023, Calendar.MAY, 1);
@ -783,8 +752,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void testCatalogueHarvester() {
try {
String context = ROOT_PROD;
ContextTest.setContextByName(context);
String context = ROOT;
ContextTest.set(context);
AccountingDao dao = getAccountingDao();
@ -824,7 +793,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public void addMissingVREAccesses() {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
@ -834,7 +803,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
String context = E_LEARNING_AREA_VRE;
// Setting the token for the context
ContextTest.setContextByName(context);
ContextTest.set(context);
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(context);
Dimension dimension = AccountingDashboardHarvesterPlugin.getDimension(HarvestedDataKey.ACCESSES.getKey());

View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest.harvester;
package org.gcube.dataharvest;
import java.time.LocalDate;
import java.time.ZoneId;
@ -11,10 +11,9 @@ import java.util.stream.Stream;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.plugin.AccountingDataHarvesterPluginTest;
import org.gcube.dataharvest.harvester.RStudioAccessesHarvester;
import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.junit.Ignore;
import org.junit.Test;
@ -36,7 +35,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
public void testJupyterAccessesHarvester() throws Exception {
try {
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>();
@ -49,7 +48,6 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
AggregationType measureType = AggregationType.MONTHLY;
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
List<AccountingRecord> accountingRecords = new ArrayList<>();
@ -58,11 +56,11 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
for (Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
RStudioAccessesHarvester rstudioAccessesHarvester = new RStudioAccessesHarvester(start, end);
for(String context : contexts) {
ContextTest.setContextByName(context);
ContextTest.set(context);
List<AccountingRecord> harvested = rstudioAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
@ -74,7 +72,7 @@ public class AccountingDataHarvesterRStudioTest extends AccountingDataHarvesterP
logger.debug("Going to insert {}", accountingRecords);
ContextTest.setContextByName(ROOT_PROD);
ContextTest.set(ROOT);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch (Throwable e) {

View File

@ -1,173 +0,0 @@
/**
*
*/
package org.gcube.dataharvest;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.JWTSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.authorization.utils.secret.SecretUtility;
import org.gcube.common.keycloak.KeycloakClientFactory;
import org.gcube.common.keycloak.KeycloakClientHelper;
import org.gcube.common.keycloak.model.TokenResponse;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class ContextTest {
private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
protected static final String CONFIG_INI_FILENAME = "config.ini";
public static final String DEFAULT_TEST_SCOPE;
public static final String GCUBE;
public static final String DEVNEXT;
public static final String NEXTNEXT;
public static final String DEVSEC;
public static final String DEVVRE;
public static final String ROOT_PROD;
protected static final Properties properties;
public static final String TYPE_PROPERTY_KEY = "type";
public static final String USERNAME_PROPERTY_KEY = "username";
public static final String PASSWORD_PROPERTY_KEY = "password";
public static final String CLIENT_ID_PROPERTY_KEY = "clientId";
static {
GCUBE = "/gcube";
DEVNEXT = GCUBE + "/devNext";
NEXTNEXT = DEVNEXT + "/NextNext";
DEVSEC = GCUBE + "/devsec";
DEVVRE = DEVSEC + "/devVRE";
ROOT_PROD = "/d4science.research-infrastructures.eu";
DEFAULT_TEST_SCOPE = GCUBE;
properties = new Properties();
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
try {
// load the properties file
properties.load(input);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private enum Type{
USER, CLIENT_ID
};
public static void set(Secret secret) throws Exception {
SecretManagerProvider.instance.reset();
SecretManager secretManager = new SecretManager();
secretManager.addSecret(secret);
SecretManagerProvider.instance.set(secretManager);
SecretManagerProvider.instance.get().set();
}
public static void setContextByName(String fullContextName) throws Exception {
logger.debug("Going to set credentials for context {}", fullContextName);
Secret secret = getSecretByContextName(fullContextName);
set(secret);
}
private static TokenResponse getJWTAccessToken(String context) throws Exception {
Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
TokenResponse tr = null;
int index = context.indexOf('/', 1);
String root = context.substring(0, index == -1 ? context.length() : index);
switch (type) {
case CLIENT_ID:
String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
String clientSecret = properties.getProperty(root);
tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
break;
case USER:
default:
String username = properties.getProperty(USERNAME_PROPERTY_KEY);
String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
switch (root) {
case "/gcube":
default:
clientId = "next.d4science.org";
break;
case "/pred4s":
clientId = "pre.d4science.org";
break;
case "/d4science.research-infrastructures.eu":
clientId = "services.d4science.org";
break;
}
clientSecret = null;
tr = KeycloakClientHelper.getTokenForUser(context, username, password);
break;
}
return tr;
}
public static Secret getSecretByContextName(String context) throws Exception {
TokenResponse tr = getJWTAccessToken(context);
Secret secret = new JWTSecret(tr.getAccessToken());
return secret;
}
public static void setContext(String token) throws Exception {
Secret secret = getSecret(token);
set(secret);
}
private static Secret getSecret(String token) throws Exception {
Secret secret = SecretUtility.getSecretByTokenString(token);
return secret;
}
public static String getUser() {
String user = "UNKNOWN";
try {
user = SecretManagerProvider.instance.get().getUser().getUsername();
} catch(Exception e) {
logger.error("Unable to retrieve user. {} will be used", user);
}
return user;
}
@BeforeClass
public static void beforeClass() throws Exception {
setContextByName(ROOT_PROD);
}
@AfterClass
public static void afterClass() throws Exception {
SecretManagerProvider.instance.reset();
}
}

View File

@ -2,7 +2,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
import java.util.List;
import org.gcube.dataharvest.ContextTest;
import org.gcube.dataharvest.utils.ContextTest;
import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -5,7 +5,6 @@ import java.io.InputStream;
import java.util.Properties;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.ContextTest;
import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -0,0 +1,92 @@
/**
*
*/
package org.gcube.dataharvest.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.JWTSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.keycloak.KeycloakClientFactory;
import org.gcube.common.keycloak.model.TokenResponse;
import org.gcube.common.scope.api.ScopeProvider;
import org.junit.AfterClass;
import org.junit.BeforeClass;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class ContextTest {
public static final String ROOT = "/d4science.research-infrastructures.eu";
protected static final String CONFIG_PROPERTIES_FILENAME = "config.properties";
protected static ContextAuthorization contextAuthorization;
private static Properties getProperties(){
try {
Properties properties = new Properties();
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_PROPERTIES_FILENAME);
// load the properties file
properties.load(input);
return properties;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void set(Secret secret) throws Exception {
SecretManagerProvider.instance.reset();
SecretManager secretManager = new SecretManager();
SecretManagerProvider.instance.set(secretManager);
secretManager.addSecret(secret);
secretManager.set();
}
public static void set(String fullContextName) throws Exception {
Secret secret = getSecret(fullContextName);
set(secret);
}
private static Secret getSecret(String fullContextName) throws Exception {
return contextAuthorization.getSecretForContext(fullContextName);
}
private static TokenResponse getJWTAccessToken(String context) throws Exception {
ScopeProvider.instance.set(context);
int index = context.indexOf('/', 1);
String root = context.substring(0, index == -1 ? context.length() : index);
String clientSecret = getProperties().getProperty(root);
TokenResponse tr = KeycloakClientFactory.newInstance().queryUMAToken(ContextAuthorization.CLIENT_ID, clientSecret, context, null);
return tr;
}
public static Secret generateSecretByContextName(String context) throws Exception {
TokenResponse tr = getJWTAccessToken(context);
Secret secret = new JWTSecret(tr.getAccessToken());
return secret;
}
@BeforeClass
public static void beforeClass() throws Exception {
Secret secret = generateSecretByContextName(ROOT);
set(secret);
contextAuthorization = new ContextAuthorization();
SecretManagerProvider.instance.reset();
}
@AfterClass
public static void afterClass() throws Exception {
SecretManagerProvider.instance.reset();
}
}

View File

@ -3,4 +3,3 @@
/*.properties
/howto.txt
/scopedata 2.xml
/config.ini