diff --git a/distro/README b/distro/README index ae26293..a941085 100644 --- a/distro/README +++ b/distro/README @@ -20,12 +20,12 @@ Please see the file named "changelog.xml" in this directory for the release note Authors -------------------------------------------------- -* Luca Frosini (luca.frosini-AT-isti.cnr.it), Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). +* Alessandro Pieve (alessandro.pieve-AT-isti.cnr.it), Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). Maintainers ----------- -* Luca Frosini (luca.frosini-AT-isti.cnr.it), Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). +* Alessandro Pieve (alessandro.pieve-AT-isti.cnr.it), Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). Download information diff --git a/pom.xml b/pom.xml index d799cb9..828b1e3 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.accounting accounting-aggregator-se-plugin - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT Accounting Aggregator Accounting Aggregator Smart Executor Plugin @@ -42,21 +42,14 @@ [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) provided - - - org.gcube.common common-authorization - - org.gcube.common authorization-client - - org.gcube.core common-scope @@ -68,7 +61,6 @@ home-library-jcr [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - org.gcube.common home-library @@ -76,18 +68,15 @@ compile - org.gcube.common common-authorization - org.gcube.common authorization-client provided - org.slf4j slf4j-api @@ -114,15 +103,13 @@ [1.0.1-SNAPSHOT, 2.0.0-SNAPSHOT) - document-store-lib org.gcube.data.publishing + document-store-lib provided - org.gcube.accounting accounting-lib - [2.2.0-SNAPSHOT,3.0.0-SNAPSHOT) provided diff --git a/src/main/java/org/gcube/accounting/aggregator/madeaggregation/Aggregation.java b/src/main/java/org/gcube/accounting/aggregator/madeaggregation/Aggregation.java index 4c8a571..4a1e587 100644 --- a/src/main/java/org/gcube/accounting/aggregator/madeaggregation/Aggregation.java +++ b/src/main/java/org/gcube/accounting/aggregator/madeaggregation/Aggregation.java @@ -35,9 +35,6 @@ public class Aggregation { //list Aggregate record protected Map>> bufferedRecords = new HashMap>>(); - - - public Aggregation() { super(); } @@ -79,16 +76,13 @@ public class Aggregation { continue; } AggregationUtility util = new AggregationUtility(bufferedRecord); - //verify a record is aggregable - //logger.debug("record: {}",record.toString()); + //verify a record is aggregable if (util.isAggregable(record)){ try { AggregatedRecord bufferedAggregatedRecord = (AggregatedRecord) bufferedRecord; - // TODO check compatibility using getAggregable //logger.debug("if -- madeAggregation aggregate"); bufferedAggregatedRecord.aggregate((AggregatedRecord) record); //patch for not changed a creation time - //bufferedAggregatedRecord.setCreationTime(bufferedAggregatedRecord.getStartTime()); bufferedAggregatedRecord.setCreationTime(record.getCreationTime()); found = true; break; @@ -97,20 +91,15 @@ public class Aggregation { } } } - if(!found){ - //logger.debug("Aggregated Record not found with execption"); - //logger.debug("if -- madeAggregation not found with execption add"); records.add(record); totalBufferedRecords++; return; } }else{ - //logger.debug("else if record contains "+recordType); records = new ArrayList>(); try { - //logger.debug("else -- add getAggregatedRecord"); records.add(getAggregatedRecord(record)); } catch (Exception e) { logger.debug("pre Exception but records"); @@ -122,8 +111,6 @@ public class Aggregation { } } - - /** * Reset buffer records */ @@ -132,7 +119,6 @@ public class Aggregation { bufferedRecords.clear(); } - /** * * @return @@ -177,12 +163,5 @@ public class Aggregation { madeAggregation(record); } } - - - - - - - } diff --git a/src/main/java/org/gcube/accounting/aggregator/madeaggregation/AggregationUtility.java b/src/main/java/org/gcube/accounting/aggregator/madeaggregation/AggregationUtility.java index 81c495b..ca866fe 100644 --- a/src/main/java/org/gcube/accounting/aggregator/madeaggregation/AggregationUtility.java +++ b/src/main/java/org/gcube/accounting/aggregator/madeaggregation/AggregationUtility.java @@ -91,7 +91,6 @@ public class AggregationUtility> { Serializable recordValue = record.getResourceProperty(field); Serializable thisValue = t.getResourceProperty(field); - //logger.error("isAggregable-field:{} ,recordValue:{}, thisValue:{}",field,recordValue,thisValue); if(recordValue instanceof Comparable && thisValue instanceof Comparable){ @SuppressWarnings("rawtypes") @@ -103,14 +102,7 @@ public class AggregationUtility> { return false; } }else{ - /* - if (recordValue==null){ - //logger.trace("{} != {}", recordValue, thisValue); - return false; - } - */ if(recordValue.hashCode()!=this.hashCode()){ - //logger.trace("{} != {}", recordValue, thisValue); return false; } diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQuery.java b/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQuery.java index 8fcab3f..3bf6166 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQuery.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQuery.java @@ -9,7 +9,6 @@ package org.gcube.accounting.aggregator.persistence; public interface AggregatorPersistenceBackendQuery { public static final int KEY_VALUES_LIMIT = 25; - public void prepareConnection( AggregatorPersistenceBackendQueryConfiguration configuration) throws Exception; diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQueryConfiguration.java b/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQueryConfiguration.java index 40b9167..89b47df 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQueryConfiguration.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceBackendQueryConfiguration.java @@ -17,7 +17,7 @@ public class AggregatorPersistenceBackendQueryConfiguration extends AccountingPe } /** - * @param class1 The class of the persistence to instantiate + * @param class The class of the persistence to instantiate * @throws Exception if fails */ @SuppressWarnings({ "unchecked", "rawtypes" }) diff --git a/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java b/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java index 8447a58..fc413d8 100644 --- a/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java +++ b/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java @@ -1,10 +1,16 @@ package org.gcube.accounting.aggregator.plugin; +import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.PrintStream; import java.io.Serializable; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,13 +24,20 @@ import org.gcube.accounting.aggregator.madeaggregation.Aggregation; import org.gcube.accounting.aggregator.madeaggregation.AggregationType; import org.gcube.accounting.aggregator.persistence.AggregatorPersistenceBackendQueryConfiguration; import org.gcube.accounting.aggregator.recovery.RecoveryRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedPortletUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedTaskUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.PortletUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.TaskUsageRecord; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.documentstore.exception.InvalidValueException; import org.gcube.documentstore.persistence.PersistenceCouchBase; import org.gcube.documentstore.records.AggregatedRecord; -import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; @@ -105,15 +118,37 @@ public class AccountingAggregatorPlugin extends Plugin notDeleted = docs; List notInserted = aggregate.reallyFlush(); @@ -506,6 +574,7 @@ public class AccountingAggregatorPlugin extends Plugin mapper = new Gson().fromJson(new FileReader(new File(nameFile)), HashMap.class); @@ -208,27 +206,28 @@ public class RecoveryRecord { usageRecordType=""; if (recordType==null) recordType=""; - + JsonDocument response = null; if ((recordType.equals("ServiceUsageRecord")) || (usageRecordType.equals("ServiceUsageRecord"))){ JsonDocument document = JsonDocument.create(identifier, accounting); - JsonDocument response = bucketService.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + response = bucketService.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); } if ((recordType.equals("StorageUsageRecord")) || (usageRecordType.equals("StorageUsageRecord"))){ JsonDocument document = JsonDocument.create(identifier, accounting); - JsonDocument response = bucketStorage.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + response = bucketStorage.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); } if ((recordType.equals("JobUsageRecord")) || (usageRecordType.equals("JobUsageRecord"))){ JsonDocument document = JsonDocument.create(identifier, accounting); - JsonDocument response = bucketJob.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + response = bucketJob.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); } if ((recordType.equals("TaskUsageRecord")) || (usageRecordType.equals("TaskUsageRecord"))){ JsonDocument document = JsonDocument.create(identifier, accounting); - JsonDocument response = bucketTask.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + response = bucketTask.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); } if ((recordType.equals("PortletUsageRecord")) || (usageRecordType.equals("PortletUsageRecord"))){ JsonDocument document = JsonDocument.create(identifier, accounting); - JsonDocument response = bucketPortlet.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + response = bucketPortlet.upsert(document,PersistTo.MASTER,Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); } + logger.trace("Elaborate Insert fileJsondocument response:{}",response); }catch(Exception e){ logger.error("Problem with recovery file and insert record excepiton:{}",e.getLocalizedMessage()); throw e; diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/Tests.java b/src/test/java/org/gcube/accounting/aggregator/plugin/Tests.java index bff649b..dbe3d4e 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/Tests.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/Tests.java @@ -37,7 +37,10 @@ public class Tests { inputs.put("interval",1 ); /* OPTIONAL INPUT */ //change to time - inputs.put("startTime", 6); + //inputs.put("startTime", 6); + + //inputs.put("pathFile","/home/pieve/startTime"); + //inputs.put("endScriptTime","16:00"); //specify bucket inputs.put("bucket","accounting_service"); @@ -45,6 +48,9 @@ public class Tests { //current scope inputs.put("currentScope",false); //specify user for save to workspace + /*OPTIONAL INPUT for work a partial interval */ + //inputs.put("intervalStep",6); + //specify a recovery 0 default recovery and aggregate, 1 only aggregate, 2 only recovery inputs.put("recovery",0);