added tests to investigate issues
This commit is contained in:
parent
752b933419
commit
bc4d09aeaf
2
pom.xml
2
pom.xml
|
|
@ -58,7 +58,7 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>storagehub-client-library</artifactId>
|
||||
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
|
||||
<version>1.2.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.publishing</groupId>
|
||||
|
|
|
|||
|
|
@ -184,13 +184,15 @@ public class Elaborator {
|
|||
* (even rare) where both check are valid because the first invocation happen
|
||||
* before midnight and the second after midnight (so in the next day).
|
||||
*/
|
||||
if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
|
||||
// if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
|
||||
// Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
|
||||
Persist persist = new Persist(aggregationStatus, srcBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
|
||||
persist.recover();
|
||||
/*
|
||||
}else{
|
||||
logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime));
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.Elabora
|
|||
import org.gcube.accounting.aggregator.utility.Utility;
|
||||
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -77,9 +78,64 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void aggregateStorageStatus() throws Exception {
|
||||
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
Map<String, Object> inputs = new HashMap<String, Object>();
|
||||
|
||||
|
||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
//type aggregation
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
|
||||
|
||||
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
|
||||
|
||||
|
||||
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10));
|
||||
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59));
|
||||
|
||||
|
||||
inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, StorageStatusRecord.class.newInstance().getRecordType());
|
||||
|
||||
inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false);
|
||||
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MARCH, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2018, Calendar.JANUARY, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
*/
|
||||
|
||||
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
|
||||
logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
|
||||
|
||||
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
|
||||
|
||||
plugin.launch(inputs);
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(20));
|
||||
plugin = new AccountingAggregatorPlugin();
|
||||
aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
|
||||
aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void aggregateStorage() throws Exception {
|
||||
public void aggregateStorageUsage() throws Exception {
|
||||
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
|
||||
|
|
@ -105,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.AUGUST, 1);
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.SEPTEMBER, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2016, Calendar.OCTOBER, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
|
|
@ -161,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.DECEMBER, 1);
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.FEBRUARY, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package org.gcube.accounting.aggregator.plugin;
|
|||
import java.text.ParseException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
|
@ -18,9 +19,16 @@ import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
|||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
|
||||
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
||||
import org.gcube.common.storagehub.client.dsl.ListResolver;
|
||||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||
import org.gcube.common.storagehub.model.items.Item;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.gcube.documentstore.records.RecordUtility;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -278,6 +286,31 @@ public class MyTest {
|
|||
logger.info("{}", now.getTime().after(startTime));
|
||||
logger.info("{}", now.getTime().before(endTime));
|
||||
|
||||
|
||||
}
|
||||
|
||||
public static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testStorageHub() throws Exception {
|
||||
ContextTest.setContextByName(ROOT_PROD);
|
||||
FolderContainer destinationFolder = null;
|
||||
StorageHubClient storageHubClient = new StorageHubClient();
|
||||
FolderContainer parent = storageHubClient.getWSRoot();
|
||||
ListResolver listResolver = parent.findByName("Accounting-Aggregator-Plugin");
|
||||
List<ItemContainer<? extends Item>> itemContainers = listResolver.getContainers();
|
||||
if(itemContainers.size()>=1){
|
||||
ItemContainer<? extends Item> itemContainer = itemContainers.get(0);
|
||||
if(itemContainer.getType().compareTo(ContainerType.FOLDER)==0) {
|
||||
destinationFolder = (FolderContainer) itemContainer;
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("{}", destinationFolder);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -19,7 +21,8 @@ public class LaunchGenerate {
|
|||
@SuppressWarnings("unused")
|
||||
private static final Logger logger = LoggerFactory.getLogger(LaunchGenerate.class);
|
||||
|
||||
//@Test
|
||||
@Ignore
|
||||
@Test
|
||||
public void run() throws Exception {
|
||||
File src = new File("src");
|
||||
File test = new File(src, "test");
|
||||
|
|
@ -40,4 +43,16 @@ public class LaunchGenerate {
|
|||
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testAggregationFromFile() throws Exception {
|
||||
File src = new File("src");
|
||||
File test = new File(src, "test");
|
||||
File resources = new File(test, "resources");
|
||||
File srcFile = new File(resources, "aux.json");
|
||||
|
||||
TestAggregationFromFile testAggregationFromFile = new TestAggregationFromFile(srcFile);
|
||||
testAggregationFromFile.elaborate();
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
package org.gcube.accounting.aggregator.recover;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.gcube.accounting.aggregator.aggregation.AggregatorBuffer;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.documentstore.exception.InvalidValueException;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.gcube.documentstore.records.RecordUtility;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR)
|
||||
*/
|
||||
public class TestAggregationFromFile {
|
||||
|
||||
protected Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
protected static final String ID = Record.ID;
|
||||
|
||||
static {
|
||||
/// One Record per package is enough
|
||||
RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage());
|
||||
RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage());
|
||||
}
|
||||
|
||||
protected final File srcFile;
|
||||
|
||||
protected final AggregatorBuffer aggregatorBuffer;
|
||||
|
||||
protected int elaborated;
|
||||
protected int aggregated;
|
||||
|
||||
public TestAggregationFromFile(File srcFile){
|
||||
this.srcFile = srcFile;
|
||||
this.elaborated = 0;
|
||||
this.aggregated = 0;
|
||||
this.aggregatorBuffer = new AggregatorBuffer();
|
||||
}
|
||||
|
||||
protected void readFile() throws Exception {
|
||||
try {
|
||||
// Open the file that is the first // command line parameter
|
||||
FileInputStream fstream = new FileInputStream(srcFile);
|
||||
// Get the object of DataInputStream
|
||||
DataInputStream in = new DataInputStream(fstream);
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(in));
|
||||
|
||||
String line;
|
||||
// Read File Line By Line
|
||||
while ((line = br.readLine()) != null) {
|
||||
elaborateLine(line);
|
||||
++elaborated;
|
||||
}
|
||||
|
||||
br.close();
|
||||
in.close();
|
||||
fstream.close();
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Error while elaborating file {}", srcFile.getAbsolutePath(), e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void elaborate() throws Exception{
|
||||
logger.info("Going to elaborate {}", srcFile.getAbsolutePath());
|
||||
readFile();
|
||||
}
|
||||
|
||||
protected void elaborateLine(String json) throws Exception {
|
||||
|
||||
Record record = RecordUtility.getRecord(json);
|
||||
try {
|
||||
record.validate();
|
||||
}catch (InvalidValueException e) {
|
||||
return;
|
||||
}
|
||||
record.setId(UUID.randomUUID().toString());
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
AggregatedRecord aggregatedRecord = AggregatorBuffer.getAggregatedRecord(record);
|
||||
aggregatorBuffer.aggregate(aggregatedRecord);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{"operationCount":1,"creationTime":1489770707728,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489770707728,"endTime":1489770707728,"id":"e9b6b523-c89e-4c91-a03f-4796d74ab5b1","operationResult":"SUCCESS"}
|
||||
{"operationCount":1,"creationTime":1489857068501,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489857068501,"endTime":1489857068501,"id":"28c5315f-9f70-4125-91a4-154d770f733a","operationResult":"SUCCESS"}
|
||||
{"operationCount":1,"creationTime":1489943468541,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489943468541,"endTime":1489943468541,"id":"2c8289d5-d016-4684-8045-afa2d1ca8507","operationResult":"SUCCESS"}
|
||||
{"operationCount":1,"creationTime":1490116270895,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490116270895,"endTime":1490116270895,"id":"961a213b-8266-4d03-9dc5-8b816c0238c6","operationResult":"SUCCESS"}
|
||||
{"operationCount":1,"creationTime":1490202675117,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490202675117,"endTime":1490202675117,"id":"0b906d52-6891-4904-98eb-7b6a4f06d9f0","operationResult":"SUCCESS"}
|
||||
Loading…
Reference in New Issue