From 9e079107f5dbc15bee4e0b267afebb846a92bdcb Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 23 Jun 2017 13:43:45 +0000 Subject: [PATCH] Added HTTPS support, HAProxy discovery support Fixes #8758. Restored Fallback capabilities Fixes #9016 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/document-store-lib-accounting-service@150499 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 5 + pom.xml | 2 +- .../documentstore/persistence/HTTPCall.java | 239 ++++++++++++++++ .../PersistenceAccountingService.java | 267 +++++------------- .../PersistenceAccountingServiceTest.java | 197 ++++++------- .../PersistenceBackendFactoryTest.java | 37 --- .../documentstore/persistence/ScopedTest.java | 100 +++++++ src/test/resources/logback-test.xml | 9 +- 8 files changed, 511 insertions(+), 345 deletions(-) create mode 100644 src/main/java/org/gcube/documentstore/persistence/HTTPCall.java delete mode 100644 src/test/java/org/gcube/documentstore/persistence/PersistenceBackendFactoryTest.java create mode 100644 src/test/java/org/gcube/documentstore/persistence/ScopedTest.java diff --git a/distro/changelog.xml b/distro/changelog.xml index e34a95f..9123f80 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,6 +1,11 @@ + + Added HTTPS support #8758 + Added HAProxy discovery support #8758 + Restored Fallback capabilities #9016 + First Release diff --git a/pom.xml b/pom.xml index d416fa1..0112795 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.data.publishing document-store-lib-accounting-service - 1.0.0-SNAPSHOT + 1.1.0-SNAPSHOT Document Store Accounting Service Connector Document Store Connector for Accounting Service diff --git a/src/main/java/org/gcube/documentstore/persistence/HTTPCall.java b/src/main/java/org/gcube/documentstore/persistence/HTTPCall.java new file mode 100644 index 0000000..000ab6f --- /dev/null +++ b/src/main/java/org/gcube/documentstore/persistence/HTTPCall.java @@ -0,0 +1,239 @@ +package org.gcube.documentstore.persistence; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.Map; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HTTPCall { + + private static final Logger logger = LoggerFactory + .getLogger(HTTPCall.class); + + public static final String APPLICATION_JSON_CHARSET_UTF_8 = "application/json;charset=UTF-8"; + public static final String APPLICATION_XML_CHARSET_UTF_8 = "application/xml;charset=UTF-8"; + + public enum HTTPMETHOD { + HEAD, GET, POST, PUT, DELETE; + + @Override + public String toString() { + return this.name(); + } + } + + public static final String PATH_SEPARATOR = "/"; + public static final String PARAM_STARTER = "?"; + public static final String PARAM_EQUALS = "="; + public static final String PARAM_SEPARATOR = "&"; + public static final String UTF8 = "UTF-8"; + + protected final String address; + protected final String userAgent; + + public HTTPCall(String address, String userAgent) { + this.address = address; + this.userAgent = userAgent; + } + + protected String getParametersDataString( + Map parameters) + throws UnsupportedEncodingException { + + if (parameters == null) { + return null; + } + + StringBuilder result = new StringBuilder(); + boolean first = true; + for (String key : parameters.keySet()) { + if (first) { + first = false; + } else { + result.append(PARAM_SEPARATOR); + } + + result.append(URLEncoder.encode(key, UTF8)); + result.append(PARAM_EQUALS); + result.append(URLEncoder.encode(parameters.get(key), UTF8)); + + } + + return result.toString(); + } + + protected URL getURL(String address, String path, String urlParameters) throws MalformedURLException { + + StringWriter stringWriter = new StringWriter(); + stringWriter.append(address); + + if(address.endsWith(PATH_SEPARATOR)){ + if(path.startsWith(PATH_SEPARATOR)){ + path = path.substring(1); + } + }else{ + if(!path.startsWith(PATH_SEPARATOR)){ + stringWriter.append(PARAM_SEPARATOR); + } + } + + stringWriter.append(path); + + if(urlParameters!=null){ + stringWriter.append(PARAM_STARTER); + stringWriter.append(urlParameters); + } + + return getURL(stringWriter.toString()); + } + + + protected URL getURL(String urlString) throws MalformedURLException{ + URL url = new URL(urlString); + if(url.getProtocol().compareTo("https")==0){ + url = new URL(url.getProtocol(), url.getHost(), url.getDefaultPort(), url.getFile()); + } + return url; + } + + + protected HttpURLConnection getConnection(String path, String urlParameters, HTTPMETHOD method, String body, String contentType) + throws Exception { + URL url = getURL(address, path, urlParameters); + return getConnection(url, method, body, contentType); + } + + protected HttpURLConnection getConnection(URL url, HTTPMETHOD method, String body, String contentType) throws Exception { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + if (SecurityTokenProvider.instance.get() == null) { + if (ScopeProvider.instance.get() == null) { + throw new RuntimeException( + "Null Token and Scope. Please set your token first."); + } + connection.setRequestProperty("gcube-scope", + ScopeProvider.instance.get()); + } else { + connection.setRequestProperty(org.gcube.common.authorization.client.Constants.TOKEN_HEADER_ENTRY, + SecurityTokenProvider.instance.get()); + } + + connection.setDoOutput(true); + + connection.setRequestProperty("Content-type", contentType); + connection.setRequestProperty("User-Agent", userAgent); + + connection.setRequestMethod(method.toString()); + + + if (body != null + && (method == HTTPMETHOD.POST || method == HTTPMETHOD.PUT)) { + + DataOutputStream wr = new DataOutputStream( + connection.getOutputStream()); + wr.writeBytes(body); + wr.flush(); + wr.close(); + } + + + int responseCode = connection.getResponseCode(); + String responseMessage = connection.getResponseMessage(); + logger.trace("{} {} : {} - {}", + method, connection.getURL(), responseCode, responseMessage); + + if (responseCode == HttpURLConnection.HTTP_MOVED_TEMP || + responseCode == HttpURLConnection.HTTP_MOVED_PERM || + responseCode == HttpURLConnection.HTTP_SEE_OTHER) { + + URL redirectURL = getURL(connection.getHeaderField("Location")); + + logger.trace("{} is going to be redirect to {}", url.toString(), redirectURL.toString()); + + connection = getConnection(redirectURL, method, body, contentType); + } + + return connection; + } + + protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException{ + StringBuilder result = new StringBuilder(); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(inputStream))) { + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + } + + return result; + } + + + + + public void call(String path, HTTPMETHOD method, String contentType) throws Exception { + call(path, method, null, null, contentType); + } + + public void call(String path, HTTPMETHOD method, Map parameters, String contentType) throws Exception { + call(path, method, parameters, null, contentType); + } + + public void call(String path, HTTPMETHOD method, String body, String contentType) throws Exception { + call(path, method, null, body, contentType); + } + + protected void call(String path, HTTPMETHOD method, Map parameters, String body, String contentType) throws Exception { + + String urlParameters = getParametersDataString(parameters); + + HttpURLConnection connection = getConnection(path, urlParameters, method, body, contentType); + + int responseCode = connection.getResponseCode(); + String responseMessage = connection.getResponseMessage(); + + logger.info("{} {} : {} - {}", + method, connection.getURL(), responseCode, responseMessage); + + if(method == HTTPMETHOD.HEAD){ + if(responseCode == HttpURLConnection.HTTP_NO_CONTENT){ + throw new Exception(responseMessage); + } + if(responseCode == HttpURLConnection.HTTP_NOT_FOUND){ + throw new Exception(responseMessage); + } + if(responseCode == HttpURLConnection.HTTP_FORBIDDEN){ + throw new Exception(responseMessage); + } + } + + + if (responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) { + InputStream inputStream = connection.getErrorStream(); + StringBuilder result = getStringBuilder(inputStream); + String res = result.toString(); + throw new Exception(res); + } + + StringBuilder result = getStringBuilder(connection.getInputStream()); + String res = result.toString(); + logger.trace("Server returned content : {}", res); + + } + + +} diff --git a/src/main/java/org/gcube/documentstore/persistence/PersistenceAccountingService.java b/src/main/java/org/gcube/documentstore/persistence/PersistenceAccountingService.java index 41eb56b..ef68391 100644 --- a/src/main/java/org/gcube/documentstore/persistence/PersistenceAccountingService.java +++ b/src/main/java/org/gcube/documentstore/persistence/PersistenceAccountingService.java @@ -3,245 +3,130 @@ */ package org.gcube.documentstore.persistence; -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.io.DataOutputStream; import java.io.StringWriter; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeoutException; +import java.util.Random; -import javax.net.ssl.HttpsURLConnection; import javax.xml.bind.JAXBContext; import org.gcube.common.resources.gcore.GCoreEndpoint; -import org.gcube.common.resources.gcore.GCoreEndpoint.Profile; -import org.gcube.common.resources.gcore.GCoreEndpoint.Profile.Endpoint; -import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.documentstore.persistence.HTTPCall.HTTPMETHOD; import org.gcube.documentstore.records.DSMapper; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.SerializableList; -import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * @author Alessandro Pieve (ISTI - CNR) alessandro.pieve@isti.cnr.it - * + * @author Luca Frosini (ISTI - CNR) luca.frosini@isti.cnr.it */ public class PersistenceAccountingService extends PersistenceBackend { private static final Logger logger = LoggerFactory.getLogger(PersistenceAccountingService.class); - public static final String URL_SERVICE_ACCOUNTING_KEY = "UrlAccountingService"; - - public static final String PATH_SERVICE_ACCOUNTING = "/accounting-service/gcube/service"; - public static final String PATH_SERVICE_INSERT_ACCOUNTING = "/insert/record"; public static final String PATH_SERVICE_INSERTS_ACCOUNTING = "/insert/records"; - public static final String PATH_SERVICE_STATUS_ACCOUNTING = "insert/getStatus"; - - public static final String GCORE_END_POINT_NAME="AccountService webapp"; - public static final String GCORE_END_RUNNING="RunningInstance"; - - public static final String RESOURCE_ACCOUNTING="org.gcube.data.publishing.accounting.service.AccountingResource"; - private static final String USER_AGENT = "Mozilla/5.0"; - private static final String CONTENT_TYPE_JSON = "application/json"; - private static final String CONTENT_TYPE_XML = "application/xml"; - - protected String urlService; - protected String context; - - + + public static final String URL_PROPERTY_KEY = "URL"; + + public static final String SERVICE_CLASS = "DataPublishing"; + public static final String SERVICE_NAME = "AccountService"; + public static final String SERVICE_ENTRY_NAME = "org.gcube.data.publishing.accounting.service.AccountingResource"; + + private static final String USER_AGENT = "document-store-lib-accounting-service"; + + private HTTPCall httpCall; + + private static String classFormat = "$resource/Profile/ServiceClass/text() eq '%1s'"; + private static String nameFormat = "$resource/Profile/ServiceName/text() eq '%1s'"; + private static String statusFormat = "$resource/Profile/DeploymentData/Status/text() eq 'ready'"; + private static String containsFormat = "$entry/@EntryName eq '%1s'"; + + private static SimpleQuery queryForService() { + return ICFactory.queryFor(GCoreEndpoint.class).addCondition(String.format(classFormat, SERVICE_CLASS)) + .addCondition(String.format(nameFormat, SERVICE_NAME)).addCondition(String.format(statusFormat)) + .addVariable("$entry", "$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint") + .addCondition(String.format(containsFormat, SERVICE_ENTRY_NAME)).setResult("$entry/text()"); + } + /** * {@inheritDoc} */ @Override - protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception { + protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception { - try{ - context = ScopeProvider.instance.get(); - logger.debug("prepareConnection context:{}",context); - ScopeProvider.instance.set(context); - SimpleQuery query = queryFor(GCoreEndpoint.class); - query.addCondition("$resource/Profile/Description/text() eq '"+ GCORE_END_POINT_NAME +"'"); - query.addCondition("$resource/Type/text() eq '"+ GCORE_END_RUNNING +"'"); - - DiscoveryClient client = clientFor(GCoreEndpoint.class); - List toReturn = client.submit(query); - - GCoreEndpoint endpoint=toReturn.get(0); - Profile profile =endpoint.profile(); - Endpoint url =profile.endpointMap().get(RESOURCE_ACCOUNTING); - - urlService=url.uri().toString(); - logger.debug("urlService from GcoreEndPoint:{}",urlService); - - - }catch(Exception e){ - try{ - urlService= configuration.getProperty(URL_SERVICE_ACCOUNTING_KEY)+PATH_SERVICE_ACCOUNTING+"/"; - logger.debug("urlService from Service End Point:{}",urlService); - }catch(Exception ex){ - logger.error("Url service not found into configuration from service point"); - throw new IllegalStateException("Url service has a null property", ex); - + String url = configuration.getProperty(URL_PROPERTY_KEY); + if (url == null || url.compareTo("") == 0) { + logger.debug("Invalid URL provided from Configuration. Looking for RunningInstance."); + SimpleQuery serviceQuery = queryForService(); + List addresses = ICFactory.client().submit(serviceQuery); + if (addresses == null || addresses.isEmpty()) { + String error = String.format("No Running Instance %s:%s found in the current context", SERVICE_CLASS, + SERVICE_NAME); + throw new Exception(error); } - + Random random = new Random(); + int index = random.nextInt(addresses.size()); + + url = addresses.get(index); } - + + logger.debug("Accounting Service URL to be contacted is {}", url); + httpCall = new HTTPCall(url, USER_AGENT); } - + /** * {@inheritDoc} */ @Override protected void reallyAccount(Record record) throws Exception { + + String marshalledRecord = DSMapper.marshal(record); + logger.trace("Goign to persist {} {}", Record.class.getSimpleName(), marshalledRecord); - logger.trace("init reallyAccount"); - String path=urlService+PATH_SERVICE_INSERT_ACCOUNTING; + httpCall.call(PATH_SERVICE_INSERT_ACCOUNTING, HTTPMETHOD.POST, marshalledRecord, HTTPCall.APPLICATION_JSON_CHARSET_UTF_8); - URL obj = new URL(path); - logger.trace("reallyAccount path:{}",path); - String recordMarshal=DSMapper.marshal(record); - int responseCode; - if (path.indexOf("http") != -1){ - logger.trace("accountWithFallback http path:{}",path); - HttpURLConnection con = (HttpURLConnection) obj.openConnection(); - con.setRequestMethod("POST"); - con.setRequestProperty("User-Agent", USER_AGENT); - con.setRequestProperty("Content-type", CONTENT_TYPE_JSON); - con.setRequestProperty("gcube-scope", context); - con.setDoOutput(true); - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - wr.writeBytes(recordMarshal); - wr.flush(); - wr.close(); - responseCode = con.getResponseCode(); - } - else{ - logger.trace("accountWithFallback https path:{}",path); - HttpsURLConnection con = (HttpsURLConnection) obj.openConnection(); - con.setRequestMethod("POST"); - con.setRequestProperty("User-Agent", USER_AGENT); - con.setRequestProperty("Content-type", CONTENT_TYPE_JSON); - con.setRequestProperty("gcube-scope", context); - con.setDoOutput(true); - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - wr.writeBytes(recordMarshal); - wr.flush(); - wr.close(); - responseCode = con.getResponseCode(); - } - //logger.debug("reallyAccount Post parameters : " + recordMarshal); - logger.debug("reallyAccount Response Code : " + responseCode); - switch (responseCode) { - case HttpURLConnection.HTTP_OK: - logger.trace("accountWithFallback - Send records to service:{}"); - break; - case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: - throw new TimeoutException("Time out for call service accounting"); - - default: - throw new Exception("Generic error for service accounting"); - - } - } - @Override protected void accountWithFallback(Record... records) throws Exception { - - logger.trace("init accountWithFallback"); - String path=urlService+PATH_SERVICE_INSERTS_ACCOUNTING; - List valuesList=new ArrayList(); - for(Record record:records){ - //logger.trace("add record:{}",record); - valuesList.add(DSMapper.marshal(record)); + try { + List valuesList = new ArrayList(); + for (Record record : records) { + valuesList.add(DSMapper.marshal(record)); + } + SerializableList list = new SerializableList(valuesList); + + JAXBContext contextRecord = JAXBContext.newInstance(SerializableList.class); + StringWriter writer = new StringWriter(); + contextRecord.createMarshaller().marshal(list, writer); + + String body = writer.toString(); + + logger.trace("Goign to persist {}s {}", Record.class.getSimpleName(), body); + + httpCall.call(PATH_SERVICE_INSERTS_ACCOUNTING, HTTPMETHOD.POST, body, HTTPCall.APPLICATION_XML_CHARSET_UTF_8); + + } catch (Exception e) { + super.accountWithFallback(records); } - SerializableList list=new SerializableList(valuesList); - - JAXBContext contextRecord = JAXBContext.newInstance(SerializableList.class); - StringWriter writer =new StringWriter(); - contextRecord.createMarshaller().marshal(list, writer); - URL obj = new URL(path); - //check if http or https - int responseCode; - if (path.indexOf("http") != -1){ - logger.trace("accountWithFallback http path:{}",path); - HttpURLConnection con = (HttpURLConnection) obj.openConnection();//add request header - con.setRequestMethod("POST"); - con.setRequestProperty("User-Agent", USER_AGENT); - con.setRequestProperty("Content-type", CONTENT_TYPE_XML); - con.setRequestProperty("gcube-scope", context); - con.setDoOutput(true); - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - wr.writeBytes(writer.toString()); - wr.flush(); - wr.close(); - responseCode = con.getResponseCode(); - } - else{ - logger.trace("accountWithFallback https path:{}",path); - HttpsURLConnection con = (HttpsURLConnection) obj.openConnection(); - con.setRequestMethod("POST"); - con.setRequestProperty("User-Agent", USER_AGENT); - con.setRequestProperty("Content-type", CONTENT_TYPE_XML); - con.setRequestProperty("gcube-scope", context); - con.setDoOutput(true); - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - wr.writeBytes(writer.toString()); - wr.flush(); - wr.close(); - responseCode = con.getResponseCode(); - } - logger.debug("accountWithFallback gcube-scope : " +context); - //logger.debug("accountWithFallback Post parameters : " + writer.toString()); - logger.debug("accountWithFallback Response Code : " + responseCode); - logger.trace("accountWithFallback - Send records to service"); - switch (responseCode) { - case HttpURLConnection.HTTP_OK: - logger.trace("accountWithFallback - Service respond ok :{}"); - break; - case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: - throw new TimeoutException("Time out for call service accounting"); - - default: - throw new Exception("Generic error for service accounting"); - - } - } - - - /** - * {@inheritDoc} - */ - @Override - public void close() throws Exception { - //TODO - } - - @Override - protected void openConnection() throws Exception { - // TODO Auto-generated method stub } @Override - protected void closeConnection() throws Exception { - // TODO Auto-generated method stub - - } + public void close() throws Exception {} @Override - protected void closeAndClean() throws Exception { - // TODO Auto-generated method stub - } + protected void openConnection() throws Exception {} + + @Override + protected void closeConnection() throws Exception {} + + @Override + protected void closeAndClean() throws Exception {} } diff --git a/src/test/java/org/gcube/documentstore/persistence/PersistenceAccountingServiceTest.java b/src/test/java/org/gcube/documentstore/persistence/PersistenceAccountingServiceTest.java index 7a524be..d1f57bc 100644 --- a/src/test/java/org/gcube/documentstore/persistence/PersistenceAccountingServiceTest.java +++ b/src/test/java/org/gcube/documentstore/persistence/PersistenceAccountingServiceTest.java @@ -5,146 +5,117 @@ package org.gcube.documentstore.persistence; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.utility.TestUsageRecord; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @author Alessandro Pieve (ISTI - CNR) alessandro.pieve@isti.cnr.it - * + * @author Luca Frosini (ISTI - CNR) */ -public class PersistenceAccountingServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(PersistenceAccountingServiceTest.class); - +public class PersistenceAccountingServiceTest extends ScopedTest { - - @Before - public void before() throws Exception{ - //SecurityTokenProvider.instance.set("36501a0d-a205-4bf1-87ad-4c7185faa0d6-98187548"); - //SecurityTokenProvider.instance.set("3acdde42-6883-4564-b3ba-69f6486f6fe0-98187548"); - - // SecurityTokenProvider.instance.set("2580f89f-d7a8-452d-a131-b3859bd771fd-98187548"); - - //ScopeProvider.instance.set("/gcube/devNext"); - //ScopeProvider.instance.set("/gcube"); - - - // ScopeProvider.instance.set("/gcube/devNext/NextNext"); - } - - @After - public void after(){ - //SecurityTokenProvider.instance.reset(); - } - - @Test - public void testSingleInsertService() throws Exception{ - // Production-Preproduction Nodes - PersistenceBackendConfiguration persitenceConfiguration = null; - PersistenceAccountingService accountingService = new PersistenceAccountingService(); - accountingService.prepareConnection(persitenceConfiguration); - + public void testSingleInsertService() throws Exception { + String context = ScopedTest.getCurrentContext(); + + PersistenceBackend persistenceBackend = PersistenceBackendFactory.getPersistenceBackend(context); + persistenceBackend = PersistenceBackendFactory.discoverPersistenceBackend(context, + (FallbackPersistenceBackend) persistenceBackend); + Record record = TestUsageRecord.createTestServiceUsageRecord(); - accountingService.reallyAccount(record); - + persistenceBackend.accountWithFallback(record); + } - @Test - public void testMultipleInsertService() throws Exception{ - // Production-Preproduction Nodes - PersistenceBackendConfiguration persitenceConfiguration = null; - PersistenceAccountingService accountingService = new PersistenceAccountingService(); - accountingService.prepareConnection(persitenceConfiguration); - Integer count=2; - Record[] records = new Record[count]; - for(int i=0; i + @@ -7,9 +9,10 @@ - - - + + + +