Fixing lib

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/document-store-lib-accounting-service@160738 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2017-12-20 16:26:34 +00:00
parent dabcadaf5d
commit 707f9c442b
10 changed files with 265 additions and 226 deletions

View File

@ -19,16 +19,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class HTTPCall { public class HTTPCall {
private static final Logger logger = LoggerFactory private static final Logger logger = LoggerFactory.getLogger(HTTPCall.class);
.getLogger(HTTPCall.class);
public static final String APPLICATION_JSON_CHARSET_UTF_8 = "application/json;charset=UTF-8"; public static final String APPLICATION_JSON_CHARSET_UTF_8 = "application/json;charset=UTF-8";
public static final String APPLICATION_XML_CHARSET_UTF_8 = "application/xml;charset=UTF-8";
public enum HTTPMETHOD { public enum HTTPMETHOD {
HEAD, GET, POST, PUT, DELETE; HEAD, GET, POST, PUT, DELETE;
@Override @Override
public String toString() { public String toString() {
return this.name(); return this.name();
@ -49,50 +47,47 @@ public class HTTPCall {
this.userAgent = userAgent; this.userAgent = userAgent;
} }
protected String getParametersDataString( protected String getParametersDataString(Map<String,? extends Object> parameters)
Map<String, String> parameters)
throws UnsupportedEncodingException { throws UnsupportedEncodingException {
if (parameters == null) { if(parameters == null) {
return null; return null;
} }
StringBuilder result = new StringBuilder(); StringBuilder result = new StringBuilder();
boolean first = true; boolean first = true;
for (String key : parameters.keySet()) { for(String key : parameters.keySet()) {
if (first) { if(first) {
first = false; first = false;
} else { } else {
result.append(PARAM_SEPARATOR); result.append(PARAM_SEPARATOR);
} }
result.append(URLEncoder.encode(key, UTF8)); result.append(URLEncoder.encode(key, UTF8));
result.append(PARAM_EQUALS); result.append(PARAM_EQUALS);
result.append(URLEncoder.encode(parameters.get(key), UTF8)); result.append(URLEncoder.encode(String.valueOf(parameters.get(key)), UTF8));
} }
return result.toString(); return result.toString();
} }
protected URL getURL(String address, String path, String urlParameters) throws MalformedURLException { protected URL getURL(String address, String path, String urlParameters) throws MalformedURLException {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
stringWriter.append(address); stringWriter.append(address);
if(address.endsWith(PATH_SEPARATOR)){ if(address.endsWith(PATH_SEPARATOR)) {
if(path.startsWith(PATH_SEPARATOR)){ if(path.startsWith(PATH_SEPARATOR)) {
path = path.substring(1); path = path.substring(1);
} }
}else{ } else {
if(!path.startsWith(PATH_SEPARATOR)){ if(!path.startsWith(PATH_SEPARATOR)) {
stringWriter.append(PARAM_SEPARATOR); stringWriter.append(PARAM_SEPARATOR);
} }
} }
stringWriter.append(path); stringWriter.append(path);
if(urlParameters!=null){ if(urlParameters != null) {
stringWriter.append(PARAM_STARTER); stringWriter.append(PARAM_STARTER);
stringWriter.append(urlParameters); stringWriter.append(urlParameters);
} }
@ -100,81 +95,70 @@ public class HTTPCall {
return getURL(stringWriter.toString()); return getURL(stringWriter.toString());
} }
protected URL getURL(String urlString) throws MalformedURLException {
protected URL getURL(String urlString) throws MalformedURLException{
URL url = new URL(urlString); URL url = new URL(urlString);
if(url.getProtocol().compareTo("https")==0){ if(url.getProtocol().compareTo("https") == 0) {
url = new URL(url.getProtocol(), url.getHost(), url.getDefaultPort(), url.getFile()); url = new URL(url.getProtocol(), url.getHost(), url.getDefaultPort(), url.getFile());
} }
return url; return url;
} }
protected HttpURLConnection getConnection(String path, String urlParameters, HTTPMETHOD method, String body)
protected HttpURLConnection getConnection(String path, String urlParameters, HTTPMETHOD method, String body, String contentType)
throws Exception { throws Exception {
URL url = getURL(address, path, urlParameters); URL url = getURL(address, path, urlParameters);
return getConnection(url, method, body, contentType); return getConnection(url, method, body);
} }
protected HttpURLConnection getConnection(URL url, HTTPMETHOD method, String body, String contentType) throws Exception { protected HttpURLConnection getConnection(URL url, HTTPMETHOD method, String body) throws Exception {
HttpURLConnection connection = (HttpURLConnection) url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
if (SecurityTokenProvider.instance.get() == null) { if(SecurityTokenProvider.instance.get() == null) {
if (ScopeProvider.instance.get() == null) { if(ScopeProvider.instance.get() == null) {
throw new RuntimeException( throw new RuntimeException("Null Token and Scope. Please set your token first.");
"Null Token and Scope. Please set your token first.");
} }
connection.setRequestProperty("gcube-scope", connection.setRequestProperty("gcube-scope", ScopeProvider.instance.get());
ScopeProvider.instance.get());
} else { } else {
connection.setRequestProperty(org.gcube.common.authorization.client.Constants.TOKEN_HEADER_ENTRY, connection.setRequestProperty(org.gcube.common.authorization.client.Constants.TOKEN_HEADER_ENTRY,
SecurityTokenProvider.instance.get()); SecurityTokenProvider.instance.get());
} }
connection.setDoOutput(true); connection.setDoOutput(true);
connection.setRequestProperty("Content-type", contentType);
connection.setRequestProperty("User-Agent", userAgent);
connection.setRequestMethod(method.toString());
if (body != null connection.setRequestProperty("Content-type", APPLICATION_JSON_CHARSET_UTF_8);
&& (method == HTTPMETHOD.POST || method == HTTPMETHOD.PUT)) { connection.setRequestProperty("User-Agent", userAgent);
connection.setRequestMethod(method.toString());
if(body != null && (method == HTTPMETHOD.POST || method == HTTPMETHOD.PUT)) {
DataOutputStream wr = new DataOutputStream( DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
connection.getOutputStream()); wr.write(body.getBytes("UTF-8"));
wr.writeBytes(body);
wr.flush(); wr.flush();
wr.close(); wr.close();
} }
int responseCode = connection.getResponseCode(); int responseCode = connection.getResponseCode();
String responseMessage = connection.getResponseMessage(); String responseMessage = connection.getResponseMessage();
logger.trace("{} {} : {} - {}", logger.trace("{} {} : {} - {}", method, connection.getURL(), responseCode, responseMessage);
method, connection.getURL(), responseCode, responseMessage);
if (responseCode == HttpURLConnection.HTTP_MOVED_TEMP || if(responseCode == HttpURLConnection.HTTP_MOVED_TEMP || responseCode == HttpURLConnection.HTTP_MOVED_PERM
responseCode == HttpURLConnection.HTTP_MOVED_PERM || || responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
URL redirectURL = getURL(connection.getHeaderField("Location")); URL redirectURL = getURL(connection.getHeaderField("Location"));
logger.trace("{} is going to be redirect to {}", url.toString(), redirectURL.toString()); logger.trace("{} is going to be redirect to {}", url.toString(), redirectURL.toString());
connection = getConnection(redirectURL, method, body, contentType); connection = getConnection(redirectURL, method, body);
} }
return connection; return connection;
} }
protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException{ protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException {
StringBuilder result = new StringBuilder(); StringBuilder result = new StringBuilder();
try (BufferedReader reader = new BufferedReader( try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
new InputStreamReader(inputStream))) {
String line; String line;
while ((line = reader.readLine()) != null) { while((line = reader.readLine()) != null) {
result.append(line); result.append(line);
} }
} }
@ -182,60 +166,65 @@ public class HTTPCall {
return result; return result;
} }
public <C> C call(Class<C> clz, String path, HTTPMETHOD method) throws Exception {
return call(clz, path, method, null, null);
public void call(String path, HTTPMETHOD method, String contentType) throws Exception {
call(path, method, null, null, contentType);
} }
public void call(String path, HTTPMETHOD method, Map<String, String> parameters, String contentType) throws Exception { public <C> C call(Class<C> clz, String path, HTTPMETHOD method, Map<String,? extends Object> parameters)
call(path, method, parameters, null, contentType); throws Exception {
return call(clz, path, method, parameters, null);
} }
public void call(String path, HTTPMETHOD method, String body, String contentType) throws Exception { public <C> C call(Class<C> clz, String path, HTTPMETHOD method, String body) throws Exception {
call(path, method, null, body, contentType); return call(clz, path, method, null, body);
} }
protected void call(String path, HTTPMETHOD method, Map<String, String> parameters, String body, String contentType) throws Exception { @SuppressWarnings("unchecked")
protected <C> C call(Class<C> clz, String path, HTTPMETHOD method, Map<String,? extends Object> parameters,
String body) throws Exception {
String urlParameters = getParametersDataString(parameters); String urlParameters = getParametersDataString(parameters);
HttpURLConnection connection = getConnection(path, urlParameters, method, body, contentType); HttpURLConnection connection = getConnection(path, urlParameters, method, body);
int responseCode = connection.getResponseCode();
String responseMessage = connection.getResponseMessage();
logger.info("{} {} : {} - {}", try {
method, connection.getURL(), responseCode, responseMessage);
int responseCode = connection.getResponseCode();
if(method == HTTPMETHOD.HEAD){ String responseMessage = connection.getResponseMessage();
if(responseCode == HttpURLConnection.HTTP_NO_CONTENT){ logger.info("{} {} : {} - {}", method, connection.getURL(), responseCode, responseMessage);
throw new Exception(responseMessage);
if(method == HTTPMETHOD.HEAD) {
if(responseCode == HttpURLConnection.HTTP_NO_CONTENT) {
throw new Exception(responseMessage);
}
if(responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
throw new Exception(responseMessage);
}
if(responseCode == HttpURLConnection.HTTP_FORBIDDEN) {
throw new Exception(responseMessage);
}
} }
if(responseCode == HttpURLConnection.HTTP_NOT_FOUND){
throw new Exception(responseMessage); if(responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) {
InputStream inputStream = connection.getErrorStream();
StringBuilder result = getStringBuilder(inputStream);
String res = result.toString();
throw new Exception(res);
} }
if(responseCode == HttpURLConnection.HTTP_FORBIDDEN){
throw new Exception(responseMessage); StringBuilder result = getStringBuilder(connection.getInputStream());
}
}
if (responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) {
InputStream inputStream = connection.getErrorStream();
StringBuilder result = getStringBuilder(inputStream);
String res = result.toString(); String res = result.toString();
throw new Exception(res); logger.trace("Server returned content : {}", res);
return (C) res;
} finally {
connection.disconnect();
} }
StringBuilder result = getStringBuilder(connection.getInputStream());
String res = result.toString();
logger.trace("Server returned content : {}", res);
connection.disconnect();
} }
} }

View File

@ -21,13 +21,13 @@ public class PersistenceAccountingService extends PersistenceBackend {
private static final Logger logger = LoggerFactory.getLogger(PersistenceAccountingService.class); private static final Logger logger = LoggerFactory.getLogger(PersistenceAccountingService.class);
public static final String PATH_SERVICE_INSERT_ACCOUNTING = "/insert/record"; public static final String PATH_SERVICE_INSERT_ACCOUNTING = "/insert";
public static final String URL_PROPERTY_KEY = "URL"; public static final String URL_PROPERTY_KEY = "URL";
public static final String SERVICE_CLASS = "DataPublishing"; public static final String SERVICE_CLASS = "Accounting";
public static final String SERVICE_NAME = "AccountService"; public static final String SERVICE_NAME = "AccountingService";
public static final String SERVICE_ENTRY_NAME = "org.gcube.data.publishing.accounting.service.AccountingResource"; public static final String SERVICE_ENTRY_NAME = "org.gcube.accounting.service.AccountingResource";
private static final String USER_AGENT = "document-store-lib-accounting-service"; private static final String USER_AGENT = "document-store-lib-accounting-service";
@ -86,7 +86,8 @@ public class PersistenceAccountingService extends PersistenceBackend {
String body = DSMapper.marshal(list); String body = DSMapper.marshal(list);
logger.trace("Going to persist {}s {}", Record.class.getSimpleName(), body); logger.trace("Going to persist {}s {}", Record.class.getSimpleName(), body);
httpCall.call(PATH_SERVICE_INSERT_ACCOUNTING, HTTPMETHOD.POST, body, HTTPCall.APPLICATION_XML_CHARSET_UTF_8); httpCall.call(String.class, PATH_SERVICE_INSERT_ACCOUNTING, HTTPMETHOD.POST, body);
} }
/** /**

View File

@ -0,0 +1,29 @@
/**
*
*/
package org.gcube.documentstore.persistence;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class AccountingServiceDirectly extends AccountingServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AccountingServiceDirectly.class);
@Before
public void before() throws Exception {
PersistenceBackendConfiguration persitenceConfiguration = PersistenceBackendConfiguration
.getInstance(PersistenceAccountingService.class);
persistenceBackend = new PersistenceAccountingService();
persistenceBackend.prepareConnection(persitenceConfiguration);
logger.debug("Persistence Backend is {}", persistenceBackend.getClass().getSimpleName());
}
}

View File

@ -0,0 +1,30 @@
/**
*
*/
package org.gcube.documentstore.persistence;
import org.junit.Assert;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class AccountingServiceFromPersistence extends AccountingServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AccountingServiceFromPersistence.class);
@Before
public void before() {
String context = ScopedTest.getCurrentContext();
persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
Assert.assertTrue(persistenceBackend instanceof PersistenceAccountingService);
logger.debug("Persistence Backend is {}", persistenceBackend.getClass().getSimpleName());
}
}

View File

@ -0,0 +1,85 @@
/**
*
*/
package org.gcube.documentstore.persistence;
import java.util.Arrays;
import java.util.List;
import org.gcube.documentstore.records.DSMapper;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.utility.TestUsageRecord;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Alessandro Pieve (ISTI - CNR) alessandro.pieve@isti.cnr.it
* @author Luca Frosini (ISTI - CNR)
*/
public abstract class AccountingServiceTest extends ScopedTest {
private static final Logger logger = LoggerFactory.getLogger(AccountingServiceTest.class);
protected PersistenceBackend persistenceBackend;
@Test
public void testSingleInsertService() throws Exception {
Record record = TestUsageRecord.createTestServiceUsageRecord();
persistenceBackend.accountWithFallback(record);
}
@Test
public void testMultipleInsertService() throws Exception {
int count = 2;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestServiceUsageRecord();
}
List<Record> recordList = Arrays.asList(records);
String ret = DSMapper.marshal(recordList);
logger.debug(ret);
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertStorage() throws Exception {
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestStorageUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertJob() throws Exception {
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestJobUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertPortlet() throws Exception {
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestPortletUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
}

View File

@ -1,119 +0,0 @@
/**
*
*/
package org.gcube.documentstore.persistence;
import java.util.Arrays;
import java.util.List;
import org.gcube.documentstore.records.DSMapper;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.utility.TestUsageRecord;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Alessandro Pieve (ISTI - CNR) alessandro.pieve@isti.cnr.it
* @author Luca Frosini (ISTI - CNR)
*/
public class PersistenceAccountingServiceTest extends ScopedTest {
private static final Logger logger = LoggerFactory.getLogger(PersistenceAccountingServiceTest.class);
@Test
public void testSingleInsertService() throws Exception {
String context = ScopedTest.getCurrentContext();
PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
Record record = TestUsageRecord.createTestServiceUsageRecord();
persistenceBackend.accountWithFallback(record);
}
@Test
public void testMultipleInsertService() throws Exception {
String context = ScopedTest.getCurrentContext();
PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
int count = 2;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestServiceUsageRecord();
}
List<Record> recordList = Arrays.asList(records);
String ret = DSMapper.marshal(recordList);
logger.debug(ret);
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertStorage() throws Exception {
String context = ScopedTest.getCurrentContext();
PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestStorageUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertJob() throws Exception {
String context = ScopedTest.getCurrentContext();
PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestJobUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
@Test
public void testMultipleInsertPortlet() throws Exception {
String context = ScopedTest.getCurrentContext();
PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context);
persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context,
(FallbackPersistenceBackend) persistenceBackend);
int count = 10;
Record[] records = new Record[count];
for (int i = 0; i < count; i++) {
records[i] = TestUsageRecord.createTestPortletUsageRecord();
}
persistenceBackend.accountWithFallback(records);
}
@Test
public void testConfiguration() throws Exception {
PersistenceBackendConfiguration persitenceConfiguration = PersistenceBackendConfiguration
.getInstance(PersistenceAccountingService.class);
PersistenceAccountingService accountingService = new PersistenceAccountingService();
accountingService.prepareConnection(persitenceConfiguration);
}
}

View File

@ -42,6 +42,11 @@ public class ScopedTest {
public static final String DEFAULT_TEST_SCOPE; public static final String DEFAULT_TEST_SCOPE;
public static final String ALTERNATIVE_TEST_SCOPE; public static final String ALTERNATIVE_TEST_SCOPE;
protected static final String ACCOUNTING_SERVICE_FILENAME = "accounting-service.properties";
public static final String ACCOUNTING_SERVICE_URL_PROPERTY = "ACCOUNTING_SERVICE_URL";
public static final String ACCOUNTING_SERVICE_URL;
static { static {
Properties properties = new Properties(); Properties properties = new Properties();
InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME);
@ -60,7 +65,23 @@ public class ScopedTest {
GCUBE_DEVSEC_DEVVRE = properties.getProperty(GCUBE_DEVSEC_DEVVRE_VARNAME); GCUBE_DEVSEC_DEVVRE = properties.getProperty(GCUBE_DEVSEC_DEVVRE_VARNAME);
DEFAULT_TEST_SCOPE = GCUBE_DEVNEXT_NEXTNEXT; DEFAULT_TEST_SCOPE = GCUBE_DEVNEXT_NEXTNEXT;
ALTERNATIVE_TEST_SCOPE = GCUBE_DEVSEC; ALTERNATIVE_TEST_SCOPE = GCUBE_DEVNEXT;
properties = new Properties();
input = ScopedTest.class.getClassLoader().getResourceAsStream(ACCOUNTING_SERVICE_FILENAME);
try {
// load the properties file
properties.load(input);
} catch (IOException e) {
throw new RuntimeException(e);
}
ACCOUNTING_SERVICE_URL = properties.getProperty(ACCOUNTING_SERVICE_URL_PROPERTY);
if(ACCOUNTING_SERVICE_URL!=null){
PersistenceAccountingService.forceURL(ACCOUNTING_SERVICE_URL);
}
} }
public static String getContextFromToken(String token) throws ObjectNotFound, Exception{ public static String getContextFromToken(String token) throws ObjectNotFound, Exception{

View File

@ -0,0 +1 @@
ACCOUNTING_SERVICE_URL=http://pc-frosini.isti.cnr.it:8080/accounting-service

View File

@ -0,0 +1 @@
ACCOUNTING_SERVICE_URL=https://accounting-service-d.d4science.org:443/accounting-service

View File

@ -0,0 +1 @@
ACCOUNTING_SERVICE_URL=http://pc-frosini.isti.cnr.it:8080/accounting-service