From efec1206952f9ee7c345e1ab26e5f2f5b94d8a53 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 26 Feb 2019 13:58:06 +0000 Subject: [PATCH] Merged from branch of release 4.13.1 ref #12988 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/gcat@177280 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 1 + pom.xml | 1 + .../gcat/persistence/ckan/CKANPackage.java | 49 +++++++++++++++++-- .../gcat/persistence/ckan/CKANResource.java | 6 ++- .../org/gcube/gcat/utils/URIResolver.java | 40 ++++++++++++--- .../org/gcube/gcat/utils/URIResolverTest.java | 19 +++++++ 6 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/gcube/gcat/utils/URIResolverTest.java diff --git a/distro/changelog.xml b/distro/changelog.xml index f073fdb..1a3d297 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -2,6 +2,7 @@ + Added Item URL via URI Resolver in extras field #13309 The final URL is retrieved only URI resolver pointing to Storage Hub #13303 Enforched items to be searchable in 'extras' field #13306 Switched item listing to use package_search in place of package_list #13307 diff --git a/pom.xml b/pom.xml index 2de4636..deb7260 100644 --- a/pom.xml +++ b/pom.xml @@ -133,6 +133,7 @@ org.gcube.common gxHTTP [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + provided diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index 6955195..98744df 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -66,7 +66,8 @@ public class CKANPackage extends CKAN { protected static final String LICENSE_KEY = "license_id"; - protected static final String ITEM_URL_KEY = "item_url"; + protected static final String EXTRAS_ITEM_URL_KEY = "Item URL"; + protected static final String AUTHOR_KEY = "author"; protected static final String AUTHOR_EMAIL_KEY = "author_email"; protected static final String OWNER_ORG_KEY = "owner_org"; @@ -278,6 +279,40 @@ public class CKANPackage extends CKAN { return created; } + protected JsonNode addExtraField(JsonNode jsonNode, String key, String value) { + ArrayNode extras = null; + boolean found = false; + if(jsonNode.has(EXTRAS_KEY)) { + extras = (ArrayNode) jsonNode.get(EXTRAS_KEY); + for(JsonNode extra : extras) { + if(extra.has(EXTRAS_KEY_KEY) && extra.get(EXTRAS_KEY_KEY).asText().compareTo(key)==0) { + ((ObjectNode) extra).put(EXTRAS_VALUE_KEY, value); + found = true; + break; + } + } + }else { + extras = mapper.createArrayNode(); + } + + if(!found) { + ObjectNode extra = mapper.createObjectNode(); + extra.put(EXTRAS_KEY_KEY, key); + extra.put(EXTRAS_VALUE_KEY, value); + extras.add(extra); + } + + return jsonNode; + } + + protected String addItemURLViaResolver(JsonNode jsonNode) { + // Adding Item URL via Resolver + URIResolver uriResolver = new URIResolver(); + String catalogueItemURL = uriResolver.getCatalogueItemURL(name); + addExtraField(jsonNode, EXTRAS_ITEM_URL_KEY, catalogueItemURL); + return catalogueItemURL; + } + // see http://docs.ckan.org/en/latest/api/#ckan.logic.action.create.package_create @Override public String create(String json) { @@ -292,16 +327,16 @@ public class CKANPackage extends CKAN { ((ObjectNode) jsonNode).remove(RESOURCES_KEY); } + String catalogueItemURL = addItemURLViaResolver(jsonNode); + super.create(getAsString(jsonNode)); this.itemID = result.get(ID_KEY).asText(); ArrayNode created = createResources(resourcesToBeCreated); ((ObjectNode) result).replace(RESOURCES_KEY, created); - // Adding Item URL via Resolver - URIResolver uriResolver = new URIResolver(); - String catalogueItemURL = uriResolver.getCatalogueItemURL(name); - ((ObjectNode) result).put(ITEM_URL_KEY, catalogueItemURL); + // Adding Item URL via Resolver as + // ((ObjectNode) result).put(ITEM_URL_KEY, catalogueItemURL); // Actions performed after a package has been correctly created on ckan. String title = result.get(TITLE_KEY).asText(); @@ -364,6 +399,8 @@ public class CKANPackage extends CKAN { ((ObjectNode) jsonNode).replace(RESOURCES_KEY, resourcesToBeSend); } + addItemURLViaResolver(jsonNode); + sendPostRequest(ITEM_UPDATE, getAsString(jsonNode)); for(String resourceId : originalResources.keySet()) { @@ -371,10 +408,12 @@ public class CKANPackage extends CKAN { ckanResource.deleteFile(); } + /* // Adding Item URL via Resolver URIResolver uriResolver = new URIResolver(); String catalogueItemURL = uriResolver.getCatalogueItemURL(name); ((ObjectNode) result).put(ITEM_URL_KEY, catalogueItemURL); + */ return getAsString(result); } catch(WebApplicationException e) { diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java index b9ec9fb..3d06e8a 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java @@ -264,7 +264,8 @@ public class CKANResource extends CKAN { * @return the public URL of the copied resource if any. It return the original URL otherwise */ protected URL copyStorageResource(URL url) { - if(isStorageFile(url)) { + persistedURL = url; + if(isStorageFile(persistedURL)) { storageHubManagement = new CatalogueStorageHubManagement(); try { persistedURL = storageHubManagement.ensureResourcePersistence(persistedURL, itemID, resourceID); @@ -279,7 +280,8 @@ public class CKANResource extends CKAN { } protected void deleteStorageResource(URL url, String resourceID, String mimetype) { - if(isStorageFile(url)) { + persistedURL = url; + if(isStorageFile(persistedURL)) { try { GXHTTPStringRequest gxhttpStringRequest = GXHTTPStringRequest.newRequest(persistedURL.toString()); HttpURLConnection httpURLConnection = gxhttpStringRequest.from(Constants.CATALOGUE_NAME).head(); diff --git a/src/main/java/org/gcube/gcat/utils/URIResolver.java b/src/main/java/org/gcube/gcat/utils/URIResolver.java index bad211f..5dc6350 100644 --- a/src/main/java/org/gcube/gcat/utils/URIResolver.java +++ b/src/main/java/org/gcube/gcat/utils/URIResolver.java @@ -1,7 +1,12 @@ package org.gcube.gcat.utils; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.HttpURLConnection; +import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.WebApplicationException; import org.gcube.common.gxhttp.request.GXHTTPStringRequest; @@ -19,10 +24,21 @@ public class URIResolver { private static final String ENTITY_NAME = "entity_name"; private static final String CATALOGUE_PLAIN_URL = "clear_url"; - protected ObjectMapper objectMapper; + protected ObjectMapper mapper; public URIResolver() { - this.objectMapper = new ObjectMapper(); + this.mapper = new ObjectMapper(); + } + + protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException { + StringBuilder result = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + } + return result; } public String getCatalogueItemURL(String name) { @@ -30,20 +46,28 @@ public class URIResolver { DataCatalogue dataCatalogue = CKAN.getCatalogue(); String uriResolverURL = dataCatalogue.getUriResolverUrl(); - ObjectNode requestContent = objectMapper.createObjectNode(); + ObjectNode requestContent = mapper.createObjectNode(); requestContent.put(CATALOGUE_CONTEXT, ContextUtility.getCurrentContext()); requestContent.put(ENTITY_TYPE, EntityContext.PRODUCT.toString()); requestContent.put(ENTITY_NAME, name); - requestContent.put(CATALOGUE_PLAIN_URL, true); + requestContent.put(CATALOGUE_PLAIN_URL, String.valueOf(true)); GXHTTPStringRequest gxhttpStringRequest = GXHTTPStringRequest.newRequest(uriResolverURL); - gxhttpStringRequest.header("User-Agent", Constants.CATALOGUE_NAME); - HttpURLConnection httpURLConnection = gxhttpStringRequest.post(objectMapper.writeValueAsString(requestContent)); + gxhttpStringRequest.from(Constants.CATALOGUE_NAME); + // gxhttpStringRequest.header("Content-type", GXConnection.APPLICATION_JSON_CHARSET_UTF_8); + gxhttpStringRequest.isExternalCall(true); + String body = mapper.writeValueAsString(requestContent); + HttpURLConnection httpURLConnection = gxhttpStringRequest.post(body); - String url = httpURLConnection.getResponseMessage(); + if(httpURLConnection.getResponseCode()!=200) { + throw new InternalServerErrorException("Unable to get Item URL via URI Resolver"); + } + + String url = getStringBuilder(httpURLConnection.getInputStream()).toString(); return url; - + } catch(WebApplicationException e) { + throw e; } catch(Exception e) { throw new WebApplicationException(e); } diff --git a/src/test/java/org/gcube/gcat/utils/URIResolverTest.java b/src/test/java/org/gcube/gcat/utils/URIResolverTest.java new file mode 100644 index 0000000..dc4ccb8 --- /dev/null +++ b/src/test/java/org/gcube/gcat/utils/URIResolverTest.java @@ -0,0 +1,19 @@ +package org.gcube.gcat.utils; + +import org.gcube.gcat.ContextTest; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class URIResolverTest extends ContextTest { + + private static final Logger logger = LoggerFactory.getLogger(URIResolverTest.class); + + @Test + public void getURL() { + URIResolver uriResolver = new URIResolver(); + String catalogueItemURL = uriResolver.getCatalogueItemURL("my_first_restful_transaction_model"); + logger.debug("Item URL is {}", catalogueItemURL); + } + +}