diff --git a/pom.xml b/pom.xml index 3f349bf..054ed27 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,11 @@ org.slf4j slf4j-api + + org.gcube.common + authorization-utils + [2.2.0, 3.0.0-SNAPSHOT) + junit diff --git a/src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java b/src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java index f087106..89e80ab 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java +++ b/src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java @@ -24,8 +24,8 @@ public abstract class DocumentElaboration { protected static final String ID = Record.ID; - protected static final int THRESHOLD_FOR_FIVE_PERCENT = 100000; - protected static final int THRESHOLD_FOR_ONE_PERCENT = 1000000; + protected static final int THRESHOLD_FOR_FIVE_PERCENT = 10000; + protected static final int THRESHOLD_FOR_ONE_PERCENT = 100000; public static final int MAX_RETRY = 7; @@ -34,6 +34,7 @@ public abstract class DocumentElaboration { protected final AggregationState finalAggregationState; protected final int rowToBeElaborated; + protected int currentlyElaborated; protected Calendar startTime; @@ -42,8 +43,9 @@ public abstract class DocumentElaboration { this.finalAggregationState = finalAggregationState; this.file = file; this.rowToBeElaborated = rowToBeElaborated; + this.currentlyElaborated = 0; } - + protected void readFile() throws Exception { FileInputStream fstream = null; DataInputStream in = null; @@ -66,7 +68,12 @@ public abstract class DocumentElaboration { percentOfNumberOfRows = percentOfNumberOfRows / 5; } - int elaborated = 0; + currentlyElaborated = 0; + int restartFrom = aggregationStatus.getRestartFrom(); + if(restartFrom>0) { + logger.info("The elaboration will be restarted from record number {}", aggregationStatus.getRestartFrom()); + } + String line; // Read File Line By Line while((line = br.readLine()) != null) { @@ -75,6 +82,9 @@ public abstract class DocumentElaboration { int i = 0; while(elaborate) { ++i; + if(currentlyElaborated=restartFrom) { + aggregationStatus.setRestartFrom(currentlyElaborated, true); + } + int elaboratedPercentage = currentlyElaborated * 100 / rowToBeElaborated; logger.info("{} - Elaborated {} rows of {} (about {}%)", aggregationStatus.getAggregationInfo(), - elaborated, rowToBeElaborated, elaboratedPercentage); + currentlyElaborated, rowToBeElaborated, elaboratedPercentage); } - if(elaborated > rowToBeElaborated) { - throw new Exception("Elaborated file line is number " + elaborated + " > " + rowToBeElaborated + if(currentlyElaborated > rowToBeElaborated) { + throw new Exception("Elaborated file line is number " + currentlyElaborated + " > " + rowToBeElaborated + " (total number of rows to elaborate). This is really strange and should not occur. Stopping execution"); } } - if(elaborated != rowToBeElaborated) { - throw new Exception("Elaborated file line is number " + elaborated + " != " + rowToBeElaborated + if(currentlyElaborated != rowToBeElaborated) { + throw new Exception("Elaborated file line is number " + currentlyElaborated + " != " + rowToBeElaborated + "(total number of rows to elaborate). This is really strange and should not occur. Stopping execution"); } - logger.info("{} - Elaborated {} rows of {} ({}%)", aggregationStatus.getAggregationInfo(), elaborated, + logger.info("{} - Elaborated {} rows of {} ({}%)", aggregationStatus.getAggregationInfo(), currentlyElaborated, rowToBeElaborated, 100); } catch(Exception e) { @@ -137,7 +150,11 @@ public abstract class DocumentElaboration { public void elaborate() throws Exception { startTime = Utility.getUTCCalendarInstance(); - readFile(); + try { + readFile(); + }catch (Exception e) { + + } afterElaboration(); aggregationStatus.setAggregationState(finalAggregationState, startTime, true); } diff --git a/src/main/java/org/gcube/accounting/aggregator/persist/InsertDocument.java b/src/main/java/org/gcube/accounting/aggregator/persist/InsertDocument.java index c288213..549285d 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persist/InsertDocument.java +++ b/src/main/java/org/gcube/accounting/aggregator/persist/InsertDocument.java @@ -10,16 +10,13 @@ import java.util.TreeMap; import org.gcube.accounting.aggregator.elaboration.Elaborator; import org.gcube.accounting.aggregator.persistence.AggregatorPersistenceDst; import org.gcube.accounting.aggregator.persistence.AggregatorPersistenceFactory; -import org.gcube.accounting.aggregator.persistence.PostgreSQLConnectorDst; import org.gcube.accounting.aggregator.status.AggregationState; import org.gcube.accounting.aggregator.status.AggregationStatus; import org.gcube.accounting.aggregator.utility.Utility; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; -import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.com.fasterxml.jackson.core.JsonProcessingException; import org.gcube.com.fasterxml.jackson.databind.JsonNode; -import org.gcube.documentstore.persistence.PersistencePostgreSQL; import org.gcube.documentstore.records.DSMapper; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java index ad741ec..b0a6165 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java @@ -61,9 +61,15 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL im } protected Connection getConnection() throws Exception { - if(connection==null) { - Class.forName("org.postgresql.Driver"); + if(connection==null || connection.isClosed()) { String url = configuration.getProperty(AccountingPersistenceQueryPostgreSQL.URL_PROPERTY_KEY); + + if(connection!=null && connection.isClosed()) { + logger.warn("The connection was closed. We should investigate why. Going to reconnect to {}.", url); + } + + Class.forName("org.postgresql.Driver"); + String username = configuration.getProperty(AccountingPersistenceConfiguration.USERNAME_PROPERTY_KEY); String password = configuration.getProperty(AccountingPersistenceConfiguration.PASSWORD_PROPERTY_KEY); @@ -126,6 +132,7 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL im stringBuffer.append(" (id, "); stringBuffer.append("record_type, aggregation_type, aggregation_start_date, aggregation_end_date, "); stringBuffer.append("original_records_number, aggregated_records_number, recovered_records_number, malformed_records_number, percentage, "); + stringBuffer.append("restart_from, "); stringBuffer.append("context, current_aggregation_state, last_update_time, previous)"); stringBuffer.append(" VALUES ("); stringBuffer.append(getValue(aggregationStatus.getUUID().toString())); @@ -150,6 +157,8 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL im stringBuffer.append(getValue(aggregationStatus.getMalformedRecordNumber())); stringBuffer.append(", "); stringBuffer.append(getValue(aggregationStatus.getPercentage())); + stringBuffer.append(", "); + stringBuffer.append(getValue(aggregationStatus.getRestartFrom())); stringBuffer.append(", "); stringBuffer.append(getValue(aggregationStatus.getContext())); @@ -168,8 +177,14 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL im if(upsert) { stringBuffer.append(") ON CONFLICT (id) DO UPDATE SET "); - stringBuffer.append("original_records_number=EXCLUDED.original_records_number, aggregated_records_number=EXCLUDED.aggregated_records_number, recovered_records_number=EXCLUDED.recovered_records_number, malformed_records_number=EXCLUDED.malformed_records_number, percentage=EXCLUDED.percentage, "); - stringBuffer.append("current_aggregation_state=EXCLUDED.current_aggregation_state, last_update_time=EXCLUDED.last_update_time, previous=EXCLUDED.previous;"); + stringBuffer.append("original_records_number=EXCLUDED.original_records_number, "); + stringBuffer.append("aggregated_records_number=EXCLUDED.aggregated_records_number, "); + stringBuffer.append("recovered_records_number=EXCLUDED.recovered_records_number, "); + stringBuffer.append("malformed_records_number=EXCLUDED.malformed_records_number, "); + stringBuffer.append("percentage=EXCLUDED.percentage, "); + stringBuffer.append("restart_from=EXCLUDED.restart_from, "); + stringBuffer.append("current_aggregation_state=EXCLUDED.current_aggregation_state, "); + stringBuffer.append("last_update_time=EXCLUDED.last_update_time, previous=EXCLUDED.previous;"); }else { stringBuffer.append(");"); } @@ -242,6 +257,9 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL im int malformedRecordNumber = resultSet.getInt("malformed_records_number"); aggregationStatus.setRecordNumbers(originalRecordsNumber, aggregatedRecordsNumber, malformedRecordNumber); + int restartFrom = resultSet.getInt("restart_from"); + aggregationStatus.setRestartFrom(restartFrom, false); + String context = resultSet.getString("context"); aggregationStatus.setContext(context); diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorDst.java b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorDst.java index c7f77da..1b1f3fb 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorDst.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorDst.java @@ -1,22 +1,7 @@ package org.gcube.accounting.aggregator.persistence; -import java.util.Calendar; -import java.util.List; -import java.util.SortedMap; -import java.util.SortedSet; - -import org.gcube.accounting.analytics.Filter; -import org.gcube.accounting.analytics.Info; -import org.gcube.accounting.analytics.NumberedFilter; -import org.gcube.accounting.analytics.TemporalConstraint; -import org.gcube.accounting.analytics.UsageValue; -import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; -import org.gcube.accounting.analytics.exception.KeyException; -import org.gcube.accounting.analytics.exception.ValueException; -import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.documentstore.persistence.PersistencePostgreSQL; -import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; /** @@ -27,6 +12,7 @@ public class PostgreSQLConnectorDst implements AggregatorPersistenceDst { protected PersistencePostgreSQL persistencePostgreSQL; protected PostgreSQLConnectorDst() throws Exception { + //AccountingPersistenceConfiguration accountingPersistenceConfiguration = new AccountingPersistenceConfiguration(PostgreSQLConnectorDst.class); AccountingPersistenceConfiguration accountingPersistenceConfiguration = new AccountingPersistenceConfiguration(PersistencePostgreSQL.class); persistencePostgreSQL = new PersistencePostgreSQL(); persistencePostgreSQL.prepareConnection(accountingPersistenceConfiguration); diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorSrc.java b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorSrc.java index bb4c902..077949b 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorSrc.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorSrc.java @@ -1,12 +1,14 @@ package org.gcube.accounting.aggregator.persistence; +import org.gcube.accounting.analytics.persistence.postgresql.AccountingPersistenceQueryPostgreSQL; + /** * @author Luca Frosini (ISTI-CNR) */ public class PostgreSQLConnectorSrc extends PostgreSQLConnector implements AggregatorPersistenceSrc { protected PostgreSQLConnectorSrc() throws Exception { - super(AggregatorPersistenceSrc.class); + super(AccountingPersistenceQueryPostgreSQL.class); } } diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorStatus.java b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorStatus.java index 04a2892..2f663d7 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorStatus.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorStatus.java @@ -1,12 +1,14 @@ package org.gcube.accounting.aggregator.persistence; +import org.gcube.accounting.analytics.persistence.postgresql.AccountingPersistenceQueryPostgreSQL; + /** * @author Luca Frosini (ISTI-CNR) */ public class PostgreSQLConnectorStatus extends PostgreSQLConnector implements AggregatorPersistenceSrc { protected PostgreSQLConnectorStatus() throws Exception { - super(AggregatorPersistenceStatus.class); + super(AccountingPersistenceQueryPostgreSQL.class); } } diff --git a/src/main/java/org/gcube/accounting/aggregator/status/AggregationStatus.java b/src/main/java/org/gcube/accounting/aggregator/status/AggregationStatus.java index 5924e2a..82bc9e5 100644 --- a/src/main/java/org/gcube/accounting/aggregator/status/AggregationStatus.java +++ b/src/main/java/org/gcube/accounting/aggregator/status/AggregationStatus.java @@ -46,6 +46,9 @@ public class AggregationStatus { @JsonProperty(required = false) protected String context; + @JsonProperty + protected int restartFrom; + @JsonProperty(required = false) protected AggregationStatus previous; @@ -90,6 +93,7 @@ public class AggregationStatus { this.uuid = UUID.randomUUID(); this.malformedRecordNumber = 0; this.previous = null; + this.restartFrom = 0; } public AggregationStatus(AggregationStatus aggregationStatus) throws Exception { @@ -98,6 +102,7 @@ public class AggregationStatus { this.uuid = aggregationStatus.getUUID(); this.malformedRecordNumber = 0; this.previous = aggregationStatus; + this.restartFrom = 0; } public AggregationInfo getAggregationInfo() { @@ -116,6 +121,8 @@ public class AggregationStatus { this.aggregationState = aggregationState; this.lastUpdateTime = endTime; + this.restartFrom = 0; + AggregationStateEvent aggregationStatusEvent = new AggregationStateEvent(aggregationState, startTime, endTime); aggregationStateEvents.add(aggregationStatusEvent); @@ -185,6 +192,18 @@ public class AggregationStatus { this.context = context; } + public int getRestartFrom() { + return restartFrom; + } + + public void setRestartFrom(int restartFrom, boolean sync) throws Exception { + this.restartFrom = restartFrom; + if (sync) { + AggregatorPersistenceFactory.getAggregatorPersistenceStatus().upsertAggregationStatus(this); + } + + } + public AggregationStatus getPrevious() { return previous; } diff --git a/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java b/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java index faad1d2..572894d 100644 --- a/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java +++ b/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java @@ -14,12 +14,7 @@ import java.util.TimeZone; import org.gcube.accounting.aggregator.aggregation.AggregationType; import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin; -import org.gcube.common.authorization.client.Constants; -import org.gcube.common.authorization.library.AuthorizationEntry; -import org.gcube.common.authorization.library.provider.AuthorizationProvider; -import org.gcube.common.authorization.library.provider.ClientInfo; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.authorization.library.utils.Caller; +import org.gcube.common.authorization.utils.manager.SecretManagerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,8 +26,7 @@ public class Utility { private static Logger logger = LoggerFactory.getLogger(Utility.class); public static String getCurrentContext() throws Exception { - String token = SecurityTokenProvider.instance.get(); - return Constants.authorizationService().get(token).getContext(); + return SecretManagerProvider.instance.get().getContext(); } @@ -170,29 +164,8 @@ public class Utility { return aggregationEndDate.getTime(); } - protected static ClientInfo getClientInfo() throws Exception { - Caller caller = AuthorizationProvider.instance.get(); - if(caller!=null){ - return caller.getClient(); - }else{ - String token = SecurityTokenProvider.instance.get(); - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - return authorizationEntry.getClientInfo(); - } - } - - public static String getUsername() throws Exception{ - try { - ClientInfo clientInfo = getClientInfo(); - String clientId = clientInfo.getId(); - if (clientId != null && clientId.compareTo("") != 0) { - return clientId; - } - throw new Exception("Username null or empty"); - } catch (Exception e) { - logger.error("Unable to retrieve user."); - throw new Exception("Unable to retrieve user.", e); - } + public static String getUsername() { + return SecretManagerProvider.instance.get().getUser().getUsername(); } diff --git a/src/test/java/org/gcube/accounting/aggregator/ContextTest.java b/src/test/java/org/gcube/accounting/aggregator/ContextTest.java index f8d73ba..367d1a3 100644 --- a/src/test/java/org/gcube/accounting/aggregator/ContextTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/ContextTest.java @@ -7,14 +7,14 @@ import java.io.IOException; import java.io.InputStream; import java.util.Properties; -import org.gcube.common.authorization.client.Constants; -import org.gcube.common.authorization.client.exceptions.ObjectNotFound; -import org.gcube.common.authorization.library.AuthorizationEntry; -import org.gcube.common.authorization.library.provider.AuthorizationProvider; -import org.gcube.common.authorization.library.provider.ClientInfo; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.authorization.library.utils.Caller; -import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.authorization.utils.manager.SecretManager; +import org.gcube.common.authorization.utils.manager.SecretManagerProvider; +import org.gcube.common.authorization.utils.secret.JWTSecret; +import org.gcube.common.authorization.utils.secret.Secret; +import org.gcube.common.authorization.utils.secret.SecretUtility; +import org.gcube.common.keycloak.KeycloakClientFactory; +import org.gcube.common.keycloak.KeycloakClientHelper; +import org.gcube.common.keycloak.model.TokenResponse; import org.junit.AfterClass; import org.junit.BeforeClass; import org.slf4j.Logger; @@ -22,82 +22,152 @@ import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) - * */ public class ContextTest { private static final Logger logger = LoggerFactory.getLogger(ContextTest.class); - protected static Properties properties; - protected static final String PROPERTIES_FILENAME = "token.properties"; + protected static final String CONFIG_INI_FILENAME = "config.ini"; - public static final String ROOT_DEV_SCOPE; - public static final String VO_DEFAULT_TEST_SCOPE; - public static final String VRE_DEFAULT_TEST_SCOPE; + public static final String DEFAULT_TEST_SCOPE; - public static final String VO_DEFAULT_TEST_SCOPE_ANOTHER_USER; + public static final String GCUBE; + public static final String DEVNEXT; + public static final String NEXTNEXT; + public static final String DEVSEC; + public static final String DEVVRE; + + public static final String ROOT_PROD; + + protected static final Properties properties; + + public static final String TYPE_PROPERTY_KEY = "type"; + public static final String USERNAME_PROPERTY_KEY = "username"; + public static final String PASSWORD_PROPERTY_KEY = "password"; + public static final String CLIENT_ID_PROPERTY_KEY = "clientId"; static { - properties = new Properties(); - InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); + GCUBE = "/gcube"; + DEVNEXT = GCUBE + "/devNext"; + NEXTNEXT = DEVNEXT + "/NextNext"; + DEVSEC = GCUBE + "/devsec"; + DEVVRE = DEVSEC + "/devVRE"; + ROOT_PROD = "/d4science.research-infrastructures.eu"; + + DEFAULT_TEST_SCOPE = GCUBE; + + + properties = new Properties(); + InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME); try { // load the properties file properties.load(input); - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } - // PARENT_DEFAULT_TEST_SCOPE = "/pred4s" - // DEFAULT_TEST_SCOPE_NAME = PARENT_DEFAULT_TEST_SCOPE + "/preprod"; - // ALTERNATIVE_TEST_SCOPE = DEFAULT_TEST_SCOPE_NAME + "/preVRE"; + } + + private enum Type{ + USER, CLIENT_ID + }; + + public static void set(Secret secret) throws Exception { + SecretManagerProvider.instance.reset(); + SecretManager secretManager = new SecretManager(); + secretManager.addSecret(secret); + SecretManagerProvider.instance.set(secretManager); + SecretManagerProvider.instance.get().set(); + } + + public static void setContextByName(String fullContextName) throws Exception { + logger.debug("Going to set credentials for context {}", fullContextName); + Secret secret = getSecretByContextName(fullContextName); + set(secret); + } + + + private static TokenResponse getJWTAccessToken(String context) throws Exception { + Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString()); + TokenResponse tr = null; - ROOT_DEV_SCOPE = "/gcube"; - VO_DEFAULT_TEST_SCOPE = ROOT_DEV_SCOPE + "/devNext"; - VRE_DEFAULT_TEST_SCOPE = VO_DEFAULT_TEST_SCOPE + "/NextNext"; + int index = context.indexOf('/', 1); + String root = context.substring(0, index == -1 ? context.length() : index); - VO_DEFAULT_TEST_SCOPE_ANOTHER_USER = "lucio.lelii_" + VO_DEFAULT_TEST_SCOPE; + switch (type) { + case CLIENT_ID: + String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY); + String clientSecret = properties.getProperty(root); + + tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null); + break; + case USER: + default: + String username = properties.getProperty(USERNAME_PROPERTY_KEY); + String password = properties.getProperty(PASSWORD_PROPERTY_KEY); + + switch (root) { + case "/gcube": + default: + clientId = "next.d4science.org"; + break; + + case "/pred4s": + clientId = "pre.d4science.org"; + break; + + case "/d4science.research-infrastructures.eu": + clientId = "services.d4science.org"; + break; + } + clientSecret = null; + + tr = KeycloakClientHelper.getTokenForUser(context, username, password); + break; + + } + + return tr; + + } + + public static Secret getSecretByContextName(String context) throws Exception { + TokenResponse tr = getJWTAccessToken(context); + Secret secret = new JWTSecret(tr.getAccessToken()); + return secret; + } + + public static void setContext(String token) throws Exception { + Secret secret = getSecret(token); + set(secret); + } + + private static Secret getSecret(String token) throws Exception { + Secret secret = SecretUtility.getSecretByTokenString(token); + return secret; + } + + public static String getUser() { + String user = "UNKNOWN"; try { - setContextByName(VO_DEFAULT_TEST_SCOPE); + user = SecretManagerProvider.instance.get().getUser().getUsername(); } catch(Exception e) { - throw new RuntimeException(e); + logger.error("Unable to retrieve user. {} will be used", user); } - } - - public static String getCurrentScope(String token) throws ObjectNotFound, Exception { - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - String context = authorizationEntry.getContext(); - logger.info("Context of token {} is {}", token, context); - return context; - } - - public static void setContextByName(String fullContextName) throws ObjectNotFound, Exception { - String token = ContextTest.properties.getProperty(fullContextName); - setContext(token); - } - - private static void setContext(String token) throws ObjectNotFound, Exception { - SecurityTokenProvider.instance.set(token); - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - ClientInfo clientInfo = authorizationEntry.getClientInfo(); - logger.debug("User : {} - Type : {}", clientInfo.getId(), clientInfo.getType().name()); - String qualifier = authorizationEntry.getQualifier(); - Caller caller = new Caller(clientInfo, qualifier); - AuthorizationProvider.instance.set(caller); - ScopeProvider.instance.set(getCurrentScope(token)); + return user; } @BeforeClass public static void beforeClass() throws Exception { - setContextByName(VO_DEFAULT_TEST_SCOPE); + setContextByName(DEFAULT_TEST_SCOPE); } @AfterClass public static void afterClass() throws Exception { - SecurityTokenProvider.instance.reset(); - ScopeProvider.instance.reset(); + SecretManagerProvider.instance.reset(); } } diff --git a/src/test/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorTest.java b/src/test/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorTest.java index 120bd04..b50953c 100644 --- a/src/test/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorTest.java @@ -27,6 +27,13 @@ public class PostgreSQLConnectorTest extends ContextTest { aggregatorPersistenceStatus = AggregatorPersistenceFactory.getAggregatorPersistenceStatus(); } + @Test + public void getAggregatorPersistenceDst() throws Exception { + ContextTest.setContextByName(ROOT_PROD); + AggregatorPersistenceDst dst = AggregatorPersistenceFactory.getAggregatorPersistenceDst(); + dst.commitAndClose(); + } + @Test public void getLastTest() throws Exception { AggregationStatus aggregationStatus = aggregatorPersistenceStatus.getLast(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, null, null); @@ -68,7 +75,7 @@ public class PostgreSQLConnectorTest extends ContextTest { logger.debug("{}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); } - @Test + //@Test public void aggregationStatusTest() throws Exception { int toRemove = -36; @@ -121,7 +128,7 @@ public class PostgreSQLConnectorTest extends ContextTest { } - @Test + //@Test public void createStartedElaboration() throws Exception { Calendar start = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 15); diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index eedadd4..5951c66 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -2,9 +2,7 @@ package org.gcube.accounting.aggregator.plugin; import java.util.Calendar; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.gcube.accounting.aggregator.ContextTest; @@ -27,7 +25,7 @@ public class AccountingAggregatorPluginTest extends ContextTest { public static final String ROOT_PROD = "/d4science.research-infrastructures.eu"; private void aggregate(String recordType, AggregationType aggregationType, Calendar aggregationStartCalendar, - Calendar aggregationEndCalendar) throws Exception { + Calendar aggregationEndCalendar, boolean forceRerun, boolean forceEarlyAggregation) throws Exception { Map inputs = new HashMap(); inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); @@ -38,8 +36,8 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, aggregationStartCalendar == null); - inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, false); - inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, false); + inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, forceEarlyAggregation); + inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, forceRerun); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false); if (aggregationStartCalendar != null) { @@ -80,12 +78,14 @@ public class AccountingAggregatorPluginTest extends ContextTest { @JsonIgnore @Test public void aggregateJobUsageRecord() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); String recordType = JobUsageRecord.class.newInstance().getRecordType(); boolean allAgregationTypes = true; + boolean forceRerun = false; + boolean forceEarlyAggregation = false; if (!allAgregationTypes) { AggregationType aggregationType = AggregationType.DAILY; @@ -95,10 +95,11 @@ public class AccountingAggregatorPluginTest extends ContextTest { // aggregationStartCalendar, 1); Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 11); - aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar); + + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRerun, forceEarlyAggregation); } else { for (AggregationType at : AggregationType.values()) { - aggregate(recordType, at, null, null); + aggregate(recordType, at, null, null, forceRerun, forceEarlyAggregation); } } } @@ -106,12 +107,14 @@ public class AccountingAggregatorPluginTest extends ContextTest { @JsonIgnore @Test public void aggregateStorageStatusRecord() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); String recordType = StorageStatusRecord.class.newInstance().getRecordType(); boolean allAgregationTypes = false; + boolean forceRerun = false; + boolean forceEarlyAggregation = false; if (!allAgregationTypes) { AggregationType aggregationType = AggregationType.DAILY; @@ -121,10 +124,10 @@ public class AccountingAggregatorPluginTest extends ContextTest { // aggregationStartCalendar, 1); Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 11); - aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar); + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRerun, forceEarlyAggregation); } else { for (AggregationType at : AggregationType.values()) { - aggregate(recordType, at, null, null); + aggregate(recordType, at, null, null, forceRerun, forceEarlyAggregation); } } } @@ -132,13 +135,16 @@ public class AccountingAggregatorPluginTest extends ContextTest { @JsonIgnore @Test public void aggregateStorageUsageRecord() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); String recordType = StorageUsageRecord.class.newInstance().getRecordType(); boolean allAgregationTypes = false; + boolean forceRerun = false; + boolean forceEarlyAggregation = false; + if (!allAgregationTypes) { AggregationType aggregationType = AggregationType.DAILY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 10); @@ -147,64 +153,126 @@ public class AccountingAggregatorPluginTest extends ContextTest { // aggregationStartCalendar, 1); Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 11); - aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar); + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRerun, forceEarlyAggregation); } else { for (AggregationType at : AggregationType.values()) { - aggregate(recordType, at, null, null); + aggregate(recordType, at, null, null, forceRerun, forceEarlyAggregation); } } } - @JsonIgnore + // @JsonIgnore @Test public void aggregateService() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + // ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); - boolean allAgregationTypes = true; + boolean allAgregationTypes = false; + boolean forceRerun = true; + boolean forceEarlyAggregation = false; if (!allAgregationTypes) { - AggregationType aggregationType = AggregationType.DAILY; - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 10); - // Calendar aggregationEndCalendar = - // Utility.getEndCalendarFromStartCalendar(aggregationType, - // aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 11); + AggregationType aggregationType = AggregationType.YEARLY; + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); +// Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2015, Calendar.JANUARY, 1); - aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar); + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRerun, forceEarlyAggregation); } else { for (AggregationType at : AggregationType.values()) { - aggregate(recordType, at, null, null); + aggregate(recordType, at, null, null, forceRerun, forceEarlyAggregation); } } } - @JsonIgnore @Test - public void aggregateAll() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + public void aggregateAprilService() throws Exception { + // ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); - Set allRecordTypes = new HashSet<>(); - allRecordTypes.add(ServiceUsageRecord.class.newInstance().getRecordType()); - allRecordTypes.add(JobUsageRecord.class.newInstance().getRecordType()); - allRecordTypes.add(StorageUsageRecord.class.newInstance().getRecordType()); - allRecordTypes.add(StorageStatusRecord.class.newInstance().getRecordType()); + String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); - for (AggregationType at : AggregationType.values()) { - for(String recordType : allRecordTypes) { - aggregate(recordType, at, null, null); + boolean allAgregationTypes = false; + boolean forceRerun = true; + boolean forceEarlyAggregation = false; + + if (!allAgregationTypes) { + AggregationType aggregationType = AggregationType.MONTHLY; + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.APRIL, 1); + Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); +// Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2015, Calendar.JANUARY, 1); + + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRerun, forceEarlyAggregation); + } else { + for (AggregationType at : AggregationType.values()) { + aggregate(recordType, at, null, null, forceRerun, forceEarlyAggregation); } } + } + +// @JsonIgnore + @Test + public void aggregateAllStorageUsageRecord() throws Exception { + // ContextTest.setContextByName(GCUBE); + // ContextTest.setContextByName(ROOT_PROD); + + String recordType = StorageUsageRecord.class.newInstance().getRecordType(); + + aggregateAll(recordType); } - + + public void aggregateAll(String recordType) throws Exception { + + boolean forceRestart = true; + boolean forceEarlyAggregation = false; + + + AggregationType aggregationType = AggregationType.YEARLY; + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1); + Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); +// while (aggregationStartCalendar.before(end)) { +// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); +// aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); +// aggregationStartCalendar = Calendar.getInstance(); +// aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); +// +// } + + + aggregationType = AggregationType.MONTHLY; + aggregationStartCalendar = Calendar.getInstance(); + aggregationStartCalendar.setTimeInMillis(end.getTimeInMillis()); + end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1); + while (aggregationStartCalendar.before(end)) { + Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); + aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); + aggregationStartCalendar = Calendar.getInstance(); + aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); + + } + + aggregationType = AggregationType.DAILY; + aggregationStartCalendar = Calendar.getInstance(); + aggregationStartCalendar.setTimeInMillis(end.getTimeInMillis()); + aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 6); + end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 9); +// while (aggregationStartCalendar.before(end)) { +// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); +// aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, true); +// aggregationStartCalendar = Calendar.getInstance(); +// aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); +// +// } + + } + @JsonIgnore @Test public void testRecovery() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); Map inputs = new HashMap(); diff --git a/src/test/java/org/gcube/accounting/aggregator/workspace/WorkSpaceManagementTest.java b/src/test/java/org/gcube/accounting/aggregator/workspace/WorkSpaceManagementTest.java index 1fa413e..9538291 100644 --- a/src/test/java/org/gcube/accounting/aggregator/workspace/WorkSpaceManagementTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/workspace/WorkSpaceManagementTest.java @@ -9,7 +9,7 @@ public class WorkSpaceManagementTest extends ContextTest { @Test public void testCreatedirectory() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(GCUBE); //ContextTest.setContextByName(AccountingAggregatorPluginTest.ROOT_PROD); WorkSpaceManagement workSpaceManagement = WorkSpaceManagement.getInstance(); diff --git a/src/test/resources/.gitignore b/src/test/resources/.gitignore index ea3ddef..f6f874a 100644 --- a/src/test/resources/.gitignore +++ b/src/test/resources/.gitignore @@ -5,3 +5,4 @@ /token.properties /d4science.research-infrastructures.eu.gcubekey /CalledMethods/ +/config.ini