diff --git a/pom.xml b/pom.xml index bb5aaa2..f8ac6a4 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,11 @@ 42.2.19 + + org.gcube.accounting + accounting-lib + test + junit junit diff --git a/src/main/java/org/gcube/accounting/utility/postgresql/PostgreSQLQuery.java b/src/main/java/org/gcube/accounting/utility/postgresql/PostgreSQLQuery.java new file mode 100644 index 0000000..f409ce7 --- /dev/null +++ b/src/main/java/org/gcube/accounting/utility/postgresql/PostgreSQLQuery.java @@ -0,0 +1,108 @@ +package org.gcube.accounting.utility.postgresql; + +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.gcube.documentstore.records.Record; + +public class PostgreSQLQuery { + + public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z"; + + protected StringBuffer stringBuffer; + + protected String getQuotedString(String string) { + StringBuffer buffer = new StringBuffer(); + buffer.append("'"); + buffer.append(string); + buffer.append("'"); + return buffer.toString(); + } + + protected void appendString(String string) { + stringBuffer.append("'"); + stringBuffer.append(string); + stringBuffer.append("'"); + } + + protected void appendValue(Serializable serializable) { + stringBuffer.append(getValue(serializable)); + } + + protected String getValue(Serializable serializable) { + if(serializable instanceof Number) { + return serializable.toString(); + } + + if(serializable instanceof Calendar) { + Calendar calendar = (Calendar) serializable; + SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATETIME_PATTERN); + String date = simpleDateFormat.format(calendar.getTime()); + return getQuotedString(date); + } + + if(serializable instanceof Enum) { + Enum e = (Enum) serializable; + return getQuotedString(e.name()); + } + + // String, URI etc + return getQuotedString(serializable.toString()); + } + + protected void appendKey(String key) { + int lenght = key.length(); + boolean lastLowerCase = true; + for (int i=0; i keys = new TreeSet<>(record.getRequiredFields()); + StringBuffer values = new StringBuffer(); + for(String key : keys) { + if(first) { + stringBuffer.append(" ("); + values.append(" ("); + first = false; + }else { + stringBuffer.append(","); + values.append(","); + } + appendKey(key); + switch (key) { + case "creationTime": case "startTime": case "endTime": + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis((long) record.getResourceProperty(key)); + values.append(getValue(calendar)); + break; + + default: + values.append(getValue(record.getResourceProperty(key))); + break; + } + } + stringBuffer.append(") VALUES"); + stringBuffer.append(values); + stringBuffer.append(");"); + return stringBuffer.toString(); + } + +} diff --git a/src/test/java/org/gcube/accounting/utility/postgresql/PostgreSQLQueryTest.java b/src/test/java/org/gcube/accounting/utility/postgresql/PostgreSQLQueryTest.java new file mode 100644 index 0000000..80341b7 --- /dev/null +++ b/src/test/java/org/gcube/accounting/utility/postgresql/PostgreSQLQueryTest.java @@ -0,0 +1,37 @@ +package org.gcube.accounting.utility.postgresql; + +import org.gcube.accounting.datamodel.UsageRecord; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreSQLQueryTest { + + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLQueryTest.class); + + @Test + public void testSQLStatementString() throws Exception { + + UsageRecord usageRecord = TestUsageRecord.getTestAggregatedJobUsageRecord(); + PostgreSQLQuery postgreSQLQuery = new PostgreSQLQuery(); + String sql = postgreSQLQuery.getSQLInsertCommand(usageRecord); + logger.debug(sql); + + usageRecord = TestUsageRecord.getTestAggregatedPortletUsageRecord(); + sql = postgreSQLQuery.getSQLInsertCommand(usageRecord); + logger.debug(sql); + + usageRecord = TestUsageRecord.getTestAggregatedServiceUsageRecord(); + sql = postgreSQLQuery.getSQLInsertCommand(usageRecord); + logger.debug(sql); + + usageRecord = TestUsageRecord.getTestAggregatedStorageStatusRecord(); + sql = postgreSQLQuery.getSQLInsertCommand(usageRecord); + logger.debug(sql); + + usageRecord = TestUsageRecord.getTestAggregatedStorageUsageRecord(); + sql = postgreSQLQuery.getSQLInsertCommand(usageRecord); + logger.debug(sql); + } + +} diff --git a/src/test/java/org/gcube/accounting/utility/postgresql/TestUsageRecord.java b/src/test/java/org/gcube/accounting/utility/postgresql/TestUsageRecord.java new file mode 100644 index 0000000..c4af357 --- /dev/null +++ b/src/test/java/org/gcube/accounting/utility/postgresql/TestUsageRecord.java @@ -0,0 +1,371 @@ +/** + * + */ +package org.gcube.accounting.utility.postgresql; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Calendar; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.gcube.accounting.datamodel.UsageRecord.OperationResult; +import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedPortletUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; +import org.gcube.accounting.datamodel.basetypes.AbstractStorageStatusRecord; +import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.PortletUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord; +import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; +import org.gcube.accounting.persistence.AccountingPersistenceFactory; +import org.gcube.documentstore.exception.InvalidValueException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) + * + */ +public class TestUsageRecord { + + private static final Logger logger = LoggerFactory.getLogger(TestUsageRecord.class); + + public final static String TEST_CONSUMER_ID = "name.surname"; + public final static String TEST_SCOPE = "/infrastructure/vo"; + public final static String TEST_SCOPE_2 = "/infrastructure/vo/vre"; + public final static OperationResult TEST_OPERATION_RESULT = OperationResult.SUCCESS; + + public final static String TEST_SERVICE_CLASS = "TestServiceClass"; + public final static String TEST_SERVICE_NAME = "TestServiceName"; + public final static String TEST_CALLED_METHOD = "TestCalledMethod"; + public final static String TEST_CALLER_QUALIFIER = "TestCallerQualifier"; + + public final static String TEST_CALLER_HOST = "remotehost"; + public final static String TEST_HOST = "localhost"; + + public final static String TEST_PROPERTY_NAME = "TestPropertyName"; + public final static String TEST_PROPERTY_VALUE = "TestPropertyValue"; + + public final static String TEST_JOB_ID = UUID.randomUUID().toString(); + public final static String TEST_JOB_NAME = "TestJobName"; + + public final static String TEST_PORTLET_ID = "TestPortlet"; + public final static String TEST_PORTLET_OPERATION_ID = "TestPortletOperationID"; + public final static String TEST_PORTLET_MESSAGE = "TestPortletMessage"; + + private final static long MIN_DURATION = 60; // millisec + private final static long MAX_DURATION = 1000; // millisec + + static { + AccountingPersistenceFactory.initAccountingPackages(); + } + + /** + * Generate A Random long in a range between min and max. This function is + * internally used to set random duration. + * + * @return the generated random long + */ + public static long generateRandomLong(long min, long max) { + return min + (int) (Math.random() * ((max - min) + 1)); + } + + /** + * Create a valid #ServiceUsageRecord with scope set automatically. + * + * @return the created #ServiceUsageRecord + */ + public static ServiceUsageRecord createTestServiceUsageRecord() { + ServiceUsageRecord usageRecord = new ServiceUsageRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setOperationResult(TEST_OPERATION_RESULT); + usageRecord.setCallerHost(TEST_CALLER_HOST); + usageRecord.setHost(TEST_HOST); + usageRecord.setCallerQualifier(TEST_CALLER_QUALIFIER); + usageRecord.setServiceClass(TEST_SERVICE_CLASS); + usageRecord.setServiceName(TEST_SERVICE_NAME); + usageRecord.setCalledMethod(TEST_CALLED_METHOD); + + usageRecord.setDuration(generateRandomLong(MIN_DURATION, MAX_DURATION)); + + } catch (InvalidValueException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e); + throw new RuntimeException(e); + } + return usageRecord; + + } + + public final static String TEST_RESOUCE_OWNER = "resource.owner"; + public final static String TEST_RESOUCE_SCOPE = TEST_SCOPE; + + public final static String TEST_RESOURCE_URI = "testprotocol://objectURI"; + public final static String TEST_PROVIDER_URI = "testprotocol://providerURI"; + + private final static long MIN_DATA_VOLUME = 1024; + private final static long MAX_DATA_VOLUME = 10240; + + /** + * Create a valid #StorageUsageRecord with scope set automatically. + * + * @return the created #StorageUsageRecord + */ + public static StorageUsageRecord createTestStorageUsageRecord() { + StorageUsageRecord usageRecord = new StorageUsageRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setOperationResult(TEST_OPERATION_RESULT); + + usageRecord.setResourceOwner(TEST_RESOUCE_OWNER); + usageRecord.setResourceScope(TEST_RESOUCE_SCOPE); + + usageRecord.setResourceURI(new URI(TEST_RESOURCE_URI)); + usageRecord.setProviderURI(new URI(TEST_PROVIDER_URI)); + + usageRecord.setOperationType(AbstractStorageUsageRecord.OperationType.READ); + usageRecord.setDataType(AbstractStorageUsageRecord.DataType.STORAGE); + + usageRecord.setDataVolume(generateRandomLong(MIN_DATA_VOLUME, MAX_DATA_VOLUME)); + + usageRecord.setQualifier("image/png"); + + } catch (InvalidValueException | URISyntaxException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e); + throw new RuntimeException(e); + } + return usageRecord; + + } + + /** + * Create a valid #StorageVolumeUsageRecord with scope set automatically. + * + * @return the created #StorageVolumeUsageRecord + */ + public static StorageStatusRecord createTestStorageVolumeUsageRecord() { + StorageStatusRecord usageRecord = new StorageStatusRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setOperationResult(TEST_OPERATION_RESULT); + usageRecord.setDataVolume(generateRandomLong(MIN_DATA_VOLUME, MAX_DATA_VOLUME)); + usageRecord.setDataType(AbstractStorageStatusRecord.DataType.STORAGE); + usageRecord.setDataCount(generateRandomLong(MIN_DATA_VOLUME, MAX_DATA_VOLUME)); + usageRecord.setDataServiceClass("dataServiceClass"); + usageRecord.setDataServiceName("dataServiceName"); + usageRecord.setDataServiceId("dataServiceId"); + usageRecord.setProviderId(new URI(TEST_PROVIDER_URI)); + + } catch (InvalidValueException | URISyntaxException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e); + throw new RuntimeException(e); + } + return usageRecord; + + } + + /** + * @return + */ + public static JobUsageRecord createTestJobUsageRecord() { + + JobUsageRecord usageRecord = new JobUsageRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setOperationResult(TEST_OPERATION_RESULT); + usageRecord.setHost(TEST_HOST); + usageRecord.setCallerQualifier(TEST_CALLER_QUALIFIER); + usageRecord.setServiceClass(TEST_SERVICE_CLASS); + usageRecord.setServiceName(TEST_SERVICE_NAME); + usageRecord.setJobName(TEST_JOB_NAME); + usageRecord.setDuration(generateRandomLong(MIN_DURATION, MAX_DURATION)); + + } catch (InvalidValueException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e); + } + + return usageRecord; + } + + /** + * @return + */ + public static PortletUsageRecord createTestPortletUsageRecord() { + + PortletUsageRecord usageRecord = new PortletUsageRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setOperationResult(TEST_OPERATION_RESULT); + + usageRecord.setPortletId(TEST_PORTLET_ID); + usageRecord.setOperationId(TEST_PORTLET_OPERATION_ID); + usageRecord.setMessage(TEST_PORTLET_MESSAGE); + + } catch (InvalidValueException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e); + } + + return usageRecord; + } + + public static final String[] scopes = new String[] { "/gcube", "/gcube/devsec", "/gcube/devsec/devVRE", + "/gcube/devNext", "/gcube/devNext/NextNext" }; + + public static final String[] calledMethods = new String[] { "create", "read", "update", "delete", "purge", + "execute", "addToContext", "removeFromContext", "other" }; + public static final String[] users = new String[] { "luca.frosini", "lucio.lelii", "francesco.frangiacrapa", + "fabio.sinibaldi", "massimiliano.assante", "giancarlo.panichi", "leonardo.candela", "pasquale.pagano" }; + public static final String[] serviceClasses = new String[] { "information-system", "data-publishing", + "data-catalogue", "vre-management", "accounting", "data-access", "transfer" }; + public static final String[] serviceNames = new String[] { "resource-registry", "registry-publisher", + "catalogue-ws", "sdmx-publisher", "gcat", "grsf-publisher-ws", "smart-executor", "ghn-manager", + "accounting-service", "accounting-analytics", "storage-hub", "species-products-discovery", + "data-transfer-service", "uri-resolver" }; + + public static final int minutesInAnYear; + public static final int maxDuration = 450; + + private static final Random random; + + static { + random = new Random(); + minutesInAnYear = (int) TimeUnit.DAYS.toMinutes(365); + } + + public static void setCalledMethod(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomNumber = random.nextInt(calledMethods.length); + serviceUsageRecord.setCalledMethod(calledMethods[randomNumber]); + } + + public static void setCallerHostAndHost(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomNumber = random.nextInt(25); + serviceUsageRecord.setCallerHost("host" + randomNumber + ".d4science.org"); + serviceUsageRecord.setHost("host" + (25 - randomNumber) + ".d4science.org"); + } + + public static void setConsumerId(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomNumber = random.nextInt(users.length); + serviceUsageRecord.setConsumerId(users[randomNumber]); + } + + public static void setServiceClassAndName(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomNumber = random.nextInt(serviceClasses.length); + serviceUsageRecord.setServiceClass(serviceClasses[randomNumber]); + int randomInt = random.nextInt(2); + serviceUsageRecord.setServiceName(serviceNames[randomNumber + randomInt]); + } + + public static double randomBetween(int min, int max) { + return (Math.random() * (max - min + 1) + min); + } + + public static void setTiming(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int operationCount = random.nextInt(55); + operationCount = operationCount + 1; + serviceUsageRecord.setOperationCount(operationCount); + + int minutesToSubstract = random.nextInt(minutesInAnYear); + Calendar creationTime = Calendar.getInstance(); + creationTime.add(Calendar.MINUTE, -minutesToSubstract); + serviceUsageRecord.setCreationTime(creationTime); + + long duration = (long) randomBetween(90, maxDuration); + serviceUsageRecord.setDuration(duration); + + if (operationCount == 1) { + serviceUsageRecord.setMaxInvocationTime(duration); + serviceUsageRecord.setMinInvocationTime(duration); + serviceUsageRecord.setEndTime(creationTime); + serviceUsageRecord.setStartTime(creationTime); + } else { + // Random number between generated duration and 678; + long maxInvocationTime = (long) randomBetween((int) duration, 678); + serviceUsageRecord.setMaxInvocationTime(maxInvocationTime); + + // Random number between 30 and generated duration + long minInvocationTime = (long) randomBetween(30, (int) duration); + serviceUsageRecord.setMinInvocationTime(minInvocationTime); + + Calendar startTime = Calendar.getInstance(); + startTime.setTimeInMillis(creationTime.getTimeInMillis()); + int startTimeMinutesToSubstract = (int) randomBetween(2880, 1440); + startTime.add(Calendar.MINUTE, -startTimeMinutesToSubstract); + serviceUsageRecord.setStartTime(startTime); + + Calendar endTime = Calendar.getInstance(); + endTime.setTimeInMillis(creationTime.getTimeInMillis()); + int edTimeMinutesToSubstract = (int) randomBetween(startTimeMinutesToSubstract, 60); + endTime.add(Calendar.MINUTE, -edTimeMinutesToSubstract); + serviceUsageRecord.setEndTime(endTime); + } + + } + + public static void setOperationResult(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomInt = random.nextInt(OperationResult.values().length); + serviceUsageRecord.setOperationResult(OperationResult.values()[randomInt]); + } + + public static void setScope(AggregatedServiceUsageRecord serviceUsageRecord) throws Exception { + int randomInt = random.nextInt(scopes.length); + serviceUsageRecord.setScope(scopes[randomInt]); + } + + public static AggregatedServiceUsageRecord getTestAggregatedServiceUsageRecord() throws Exception { + AggregatedServiceUsageRecord serviceUsageRecord = new AggregatedServiceUsageRecord(); + serviceUsageRecord.setAggregated(true); + + setCalledMethod(serviceUsageRecord); + setCallerHostAndHost(serviceUsageRecord); + setConsumerId(serviceUsageRecord); + + serviceUsageRecord.setCallerQualifier("TOKEN"); + + setTiming(serviceUsageRecord); + + setOperationResult(serviceUsageRecord); + + setScope(serviceUsageRecord); + + setServiceClassAndName(serviceUsageRecord); + + return serviceUsageRecord; + } + + public static AggregatedJobUsageRecord getTestAggregatedJobUsageRecord() throws Exception { + JobUsageRecord jobUsageRecord = TestUsageRecord.createTestJobUsageRecord(); + AggregatedJobUsageRecord aggregatedJobUsageRecord = new AggregatedJobUsageRecord(jobUsageRecord); + return aggregatedJobUsageRecord; + } + + public static AggregatedPortletUsageRecord getTestAggregatedPortletUsageRecord() throws Exception { + PortletUsageRecord portletUsageRecord = TestUsageRecord.createTestPortletUsageRecord(); + AggregatedPortletUsageRecord aggregatedPortletUsageRecord = new AggregatedPortletUsageRecord(portletUsageRecord); + return aggregatedPortletUsageRecord; + } + + public static AggregatedServiceUsageRecord getTestSimpleAggregatedServiceUsageRecord() throws Exception { + ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord(); + AggregatedServiceUsageRecord aggregatedServiceUsageRecord = new AggregatedServiceUsageRecord(serviceUsageRecord); + return aggregatedServiceUsageRecord; + } + + public static AggregatedStorageStatusRecord getTestAggregatedStorageStatusRecord() throws Exception { + StorageStatusRecord storageStatusRecord = TestUsageRecord.createTestStorageVolumeUsageRecord(); + AggregatedStorageStatusRecord aggregatedStorageStatusRecord = new AggregatedStorageStatusRecord(storageStatusRecord); + return aggregatedStorageStatusRecord; + } + + public static AggregatedStorageUsageRecord getTestAggregatedStorageUsageRecord() throws Exception { + StorageUsageRecord storageUsageRecord = TestUsageRecord.createTestStorageUsageRecord(); + AggregatedStorageUsageRecord aggregatedStorageUsageRecord = new AggregatedStorageUsageRecord(storageUsageRecord); + return aggregatedStorageUsageRecord; + } + +}