524 lines
16 KiB
Java
524 lines
16 KiB
Java
package org.gcube.gcat.persistence.ckan;
|
|
|
|
import java.net.HttpURLConnection;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
import java.util.UUID;
|
|
|
|
import javax.ws.rs.BadRequestException;
|
|
import javax.ws.rs.DELETE;
|
|
import javax.ws.rs.GET;
|
|
import javax.ws.rs.HEAD;
|
|
import javax.ws.rs.InternalServerErrorException;
|
|
import javax.ws.rs.NotAllowedException;
|
|
import javax.ws.rs.OPTIONS;
|
|
import javax.ws.rs.PUT;
|
|
import javax.ws.rs.WebApplicationException;
|
|
|
|
import org.apache.commons.io.FilenameUtils;
|
|
import org.apache.tika.mime.MimeType;
|
|
import org.apache.tika.mime.MimeTypes;
|
|
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
|
|
import org.gcube.gcat.utils.Constants;
|
|
import org.gcube.gcat.utils.ContextUtility;
|
|
import org.gcube.gcat.utils.HTTPCall;
|
|
import org.gcube.gcat.workspace.CatalogueStorageHubManagement;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
|
|
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
|
|
|
/**
|
|
* @author Luca Frosini (ISTI - CNR)
|
|
*/
|
|
public class CKANResource extends CKAN {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(CKANResource.class);
|
|
|
|
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.create.resource_create
|
|
public static final String RESOURCE_CREATE = CKAN.CKAN_API_PATH + "resource_create";
|
|
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.get.resource_show
|
|
public static final String RESOURCE_SHOW = CKAN.CKAN_API_PATH + "resource_show";
|
|
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.update.resource_update
|
|
public static final String RESOURCE_UPDATE = CKAN.CKAN_API_PATH + "resource_update";
|
|
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.patch.resource_patch
|
|
public static final String RESOURCE_PATCH = CKAN.CKAN_API_PATH + "resource_patch";
|
|
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.resource_delete
|
|
public static final String RESOURCE_DELETE = CKAN.CKAN_API_PATH + "resource_delete";
|
|
|
|
protected static final String URL_KEY = "url";
|
|
|
|
private static final String RESOURCES_KEY = "resources";
|
|
private static final String PACKAGE_ID_KEY = "package_id";
|
|
private static final String FORMAT_KEY = "format";
|
|
private static final String MIME_TYPE_KEY = "mimetype";
|
|
private static final String REVISION_ID_KEY = "revision_id";
|
|
|
|
private static final String TEMP = "TEMP_";
|
|
|
|
public final static String RESOURCE_NAME_REGEX = "^[\\s\\S]*$";
|
|
|
|
public static final MimeTypes ALL_MIME_TYPES;
|
|
|
|
/* TODO Remove this code ASAP. It requires a function from Storage HUB */
|
|
private static final String URI_RESOLVER_STORAGE_HUB_HOST_PROD = "data.d4science.org";
|
|
private static final String URI_RESOLVER_STORAGE_HUB_HOST_PRE = "data-pre.d4science.org";
|
|
private static final String URI_RESOLVER_STORAGE_HUB_HOST_DEV = "data.dev.d4science.org";
|
|
|
|
public static final String URI_RESOLVER_STORAGE_HUB_HOST;
|
|
public static final String URI_RESOLVER_STORAGE_HUB_PATH = "/shub/";
|
|
|
|
static {
|
|
String context = ContextUtility.getCurrentContext();
|
|
if(context.startsWith("/gcube")) {
|
|
URI_RESOLVER_STORAGE_HUB_HOST = URI_RESOLVER_STORAGE_HUB_HOST_DEV;
|
|
} else if(context.startsWith("/pred4s")){
|
|
URI_RESOLVER_STORAGE_HUB_HOST = URI_RESOLVER_STORAGE_HUB_HOST_PRE;
|
|
} else {
|
|
URI_RESOLVER_STORAGE_HUB_HOST = URI_RESOLVER_STORAGE_HUB_HOST_PROD;
|
|
}
|
|
// If you might be dealing with custom mimetypes too, then Tika supports those, and change line one to be:
|
|
// TikaConfig config = TikaConfig.getDefaultConfig();
|
|
// MimeTypes ALL_MIME_TYPES = config.getMimeRepository();
|
|
ALL_MIME_TYPES = MimeTypes.getDefaultMimeTypes();
|
|
}
|
|
/* TODO END Code to be Removed */
|
|
|
|
protected String itemID;
|
|
|
|
public String getItemID() {
|
|
return itemID;
|
|
}
|
|
|
|
protected String resourceID;
|
|
|
|
protected boolean persisted;
|
|
protected URL persistedURL;
|
|
|
|
protected String mimeType;
|
|
protected String originalFileExtension;
|
|
|
|
protected JsonNode previousRepresentation;
|
|
|
|
protected CatalogueStorageHubManagement storageHubManagement;
|
|
|
|
public URL getPersistedURL() {
|
|
return persistedURL;
|
|
}
|
|
|
|
public static String extractResourceID(JsonNode jsonNode) {
|
|
String resourceID = null;
|
|
if(jsonNode.has(ID_KEY)) {
|
|
resourceID = jsonNode.get(ID_KEY).asText();
|
|
}
|
|
return resourceID;
|
|
}
|
|
|
|
public String getResourceID() {
|
|
if(resourceID == null && previousRepresentation != null) {
|
|
resourceID = CKANResource.extractResourceID(previousRepresentation);
|
|
}
|
|
return resourceID;
|
|
}
|
|
|
|
public void setResourceID(String resourceID) {
|
|
this.resourceID = resourceID;
|
|
}
|
|
|
|
public void setPreviousRepresentation(JsonNode jsonNode) {
|
|
validate(jsonNode);
|
|
previousRepresentation = jsonNode;
|
|
}
|
|
|
|
public JsonNode getPreviousRepresentation() {
|
|
if(previousRepresentation == null && resourceID != null) {
|
|
sendGetRequest(READ, getMapWithID(resourceID));
|
|
validate(result);
|
|
previousRepresentation = result;
|
|
}
|
|
return previousRepresentation;
|
|
}
|
|
|
|
public CKANResource(String itemID) {
|
|
super();
|
|
this.nameRegex = RESOURCE_NAME_REGEX;
|
|
this.itemID = itemID;
|
|
CREATE = RESOURCE_CREATE;
|
|
READ = RESOURCE_SHOW;
|
|
UPDATE = RESOURCE_UPDATE;
|
|
PATCH = RESOURCE_PATCH;
|
|
DELETE = RESOURCE_DELETE;
|
|
PURGE = null;
|
|
persisted = false;
|
|
previousRepresentation = null;
|
|
}
|
|
|
|
@Override
|
|
public String list(int limit, int offeset) {
|
|
return list();
|
|
}
|
|
|
|
public String list() {
|
|
CKANPackage ckanPackage = new CKANPackage();
|
|
ckanPackage.setName(itemID);
|
|
String itemJson = ckanPackage.read();
|
|
JsonNode item = getAsJsonNode(itemJson);
|
|
JsonNode resources = item.get(RESOURCES_KEY);
|
|
return getAsString(resources);
|
|
}
|
|
|
|
protected String getFormat() {
|
|
String format = null;
|
|
if(originalFileExtension != null) {
|
|
format = originalFileExtension;
|
|
} else {
|
|
try {
|
|
MimeType mimeTypeClzInstance = ALL_MIME_TYPES.forName(mimeType);
|
|
format = mimeTypeClzInstance.getExtension();
|
|
// List<String> extensions = mimeTypeClzInstance.getExtensions();
|
|
if(format == null || format.compareTo("") == 0) {
|
|
format = mimeType.split("/")[1].split(";")[0];
|
|
}
|
|
} catch(Exception e) {
|
|
try {
|
|
format = mimeType.split("/")[1].split(";")[0];
|
|
} catch(Exception ex) {
|
|
format = null;
|
|
}
|
|
}
|
|
}
|
|
if(format != null && format.startsWith(".")) {
|
|
format = format.substring(1);
|
|
}
|
|
return format;
|
|
}
|
|
|
|
protected ObjectNode persistStorageFile(ObjectNode objectNode) {
|
|
|
|
if(objectNode.has(URL_KEY)) {
|
|
String urlString = objectNode.get(URL_KEY).asText();
|
|
|
|
URL url;
|
|
try {
|
|
url = new URL(urlString);
|
|
} catch(MalformedURLException e) {
|
|
throw new BadRequestException(e);
|
|
}
|
|
|
|
url = copyStorageResource(url);
|
|
|
|
if(name != null) {
|
|
objectNode.put(NAME_KEY, name);
|
|
}
|
|
|
|
if(mimeType != null) {
|
|
objectNode.put(MIME_TYPE_KEY, mimeType);
|
|
|
|
if(!objectNode.has(FORMAT_KEY)) {
|
|
String format = getFormat();
|
|
if(format != null) {
|
|
objectNode.put(FORMAT_KEY, format);
|
|
}
|
|
}
|
|
}
|
|
|
|
objectNode.put(URL_KEY, url.toString());
|
|
return objectNode;
|
|
}
|
|
|
|
String error = String.format("The content must contains the %s property", URL_KEY);
|
|
throw new BadRequestException(error);
|
|
|
|
}
|
|
|
|
protected ObjectNode validate(String json) throws MalformedURLException {
|
|
JsonNode jsonNode = getAsJsonNode(json);
|
|
return validate(jsonNode);
|
|
}
|
|
|
|
protected ObjectNode validate(JsonNode jsonNode) {
|
|
|
|
ObjectNode objectNode = (ObjectNode) jsonNode;
|
|
|
|
if(objectNode.has(PACKAGE_ID_KEY)) {
|
|
String packageId = objectNode.get(PACKAGE_ID_KEY).asText();
|
|
if(packageId.compareTo(itemID) != 0) {
|
|
String error = String.format(
|
|
"Item ID %s does not match %s which is the value of %s contained in the representation.",
|
|
itemID, packageId, PACKAGE_ID_KEY);
|
|
throw new BadRequestException(error);
|
|
}
|
|
} else {
|
|
objectNode.put(PACKAGE_ID_KEY, itemID);
|
|
}
|
|
|
|
if(objectNode.has(ID_KEY)) {
|
|
String gotId = objectNode.get(ID_KEY).asText();
|
|
if(resourceID == null) {
|
|
resourceID = gotId;
|
|
} else {
|
|
if(resourceID.compareTo(gotId) != 0) {
|
|
String error = String.format(
|
|
"Resource ID %s does not match %s which is the value of %s contained in the representation.",
|
|
resourceID, gotId, ID_KEY);
|
|
throw new BadRequestException(error);
|
|
}
|
|
}
|
|
} else {
|
|
resourceID = TEMP + UUID.randomUUID().toString();
|
|
logger.trace(
|
|
"The id of the resource with name {} for package {} has not been provided. It has been generated : {}",
|
|
name, itemID, resourceID);
|
|
}
|
|
|
|
return objectNode;
|
|
}
|
|
|
|
protected URL getFinalURL(String url) {
|
|
try {
|
|
URL urlURL = new URL(url);
|
|
return CKANResource.getFinalURL(urlURL);
|
|
} catch(MalformedURLException e) {
|
|
throw new BadRequestException(e);
|
|
}
|
|
}
|
|
|
|
public static URL getFinalURL(URL url) {
|
|
HTTPCall httpCall = new HTTPCall(url.toString());
|
|
httpCall.setgCubeTargetService(false);
|
|
URL finalURL = httpCall.getFinalURL(url);
|
|
return finalURL;
|
|
}
|
|
|
|
protected boolean isStorageFile(URL url) {
|
|
URL urlToCheck = url;
|
|
try {
|
|
urlToCheck = getFinalURL(url);
|
|
} catch(Exception e) {
|
|
// TODO Evaluate if we want to validate the URL. If the URL does not exists the service
|
|
// could decide to refuse the Resource Creation
|
|
}
|
|
if(urlToCheck.getHost().compareTo(URI_RESOLVER_STORAGE_HUB_HOST) == 0) {
|
|
if(urlToCheck.getPath().startsWith(URI_RESOLVER_STORAGE_HUB_PATH)) {
|
|
persistedURL = urlToCheck;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Check if the URl is a workspace URL so that is has to copy the resource to guarantee
|
|
* the resource remain persistent
|
|
* @param url the URL to check
|
|
* @return the public URL of the copied resource if any. It return the original URL otherwise
|
|
*/
|
|
protected URL copyStorageResource(URL url) {
|
|
persistedURL = url;
|
|
if(isStorageFile(persistedURL)) {
|
|
storageHubManagement = new CatalogueStorageHubManagement();
|
|
try {
|
|
persistedURL = storageHubManagement.ensureResourcePersistence(persistedURL, itemID, resourceID);
|
|
String originalFilename = storageHubManagement.getOriginalFilename();
|
|
name = FilenameUtils.removeExtension(originalFilename);
|
|
originalFileExtension = FilenameUtils.getExtension(originalFilename);
|
|
mimeType = storageHubManagement.getMimeType();
|
|
persisted = true;
|
|
} catch(Exception e) {
|
|
throw new InternalServerErrorException(e);
|
|
}
|
|
}
|
|
return persistedURL;
|
|
}
|
|
|
|
protected void deleteStorageResource(URL url, String resourceID, String mimetype) {
|
|
persistedURL = url;
|
|
if(isStorageFile(persistedURL)) {
|
|
try {
|
|
GXHTTPStringRequest gxhttpStringRequest = GXHTTPStringRequest.newRequest(persistedURL.toString());
|
|
HttpURLConnection httpURLConnection = gxhttpStringRequest.from(Constants.CATALOGUE_NAME).head();
|
|
String storageHubContentType = httpURLConnection.getContentType().split(";")[0];
|
|
if(mimetype.compareTo(storageHubContentType) != 0) {
|
|
mimetype = storageHubContentType;
|
|
// Using storage hub mimetype
|
|
}
|
|
} catch(Exception e) {
|
|
// using provided mimetype
|
|
}
|
|
storageHubManagement = new CatalogueStorageHubManagement();
|
|
try {
|
|
storageHubManagement.deleteResourcePersistence(itemID, resourceID, mimetype);
|
|
} catch(Exception e) {
|
|
throw new InternalServerErrorException(e);
|
|
}
|
|
}
|
|
}
|
|
|
|
protected String create(JsonNode jsonNode) {
|
|
try {
|
|
ObjectNode objectNode = validate(jsonNode);
|
|
objectNode = persistStorageFile(objectNode);
|
|
String ret = super.create(getAsString(objectNode));
|
|
if(persisted) {
|
|
String gotResourceID = result.get(ID_KEY).asText();
|
|
if(gotResourceID != null && gotResourceID.compareTo(resourceID) != 0) {
|
|
resourceID = gotResourceID;
|
|
String revisionID = result.get(REVISION_ID_KEY).asText();
|
|
storageHubManagement.renameFile(resourceID, revisionID);
|
|
}
|
|
}
|
|
return ret;
|
|
} catch(WebApplicationException e) {
|
|
// TODO Remove created file if any
|
|
throw e;
|
|
} catch(Exception e) {
|
|
// TODO Remove created file if any
|
|
throw new InternalServerErrorException(e);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public String create(String json) {
|
|
JsonNode jsonNode = getAsJsonNode(json);
|
|
return create(jsonNode);
|
|
}
|
|
|
|
@Override
|
|
public String read() {
|
|
return sendGetRequest(READ, getMapWithID(resourceID));
|
|
}
|
|
|
|
protected String update(JsonNode jsonNode) throws Exception {
|
|
ObjectNode resourceNode = (ObjectNode) jsonNode;
|
|
// This cannot be moved outside otherwise we don't
|
|
resourceNode = validate(resourceNode);
|
|
|
|
getPreviousRepresentation();
|
|
|
|
String oldURL = previousRepresentation.get(CKANResource.URL_KEY).asText();
|
|
String newURL = resourceNode.get(CKANResource.URL_KEY).asText();
|
|
|
|
if(!previousRepresentation.equals(resourceNode)) {
|
|
if(oldURL.compareTo(newURL) != 0) {
|
|
logger.trace("The URL of the resource with id {} was not changed", resourceID);
|
|
storageHubManagement = new CatalogueStorageHubManagement();
|
|
this.mimeType = previousRepresentation.get(CKANResource.MIME_TYPE_KEY).asText();
|
|
|
|
try {
|
|
storageHubManagement.retrievePersistedFile(resourceID, mimeType);
|
|
}catch (Exception e) {
|
|
// If the file was not persisted by gCat (e.g. created with the portlet) some errors can occurs
|
|
}
|
|
|
|
} else {
|
|
logger.trace("The URL of resource with id {} has been changed the old URL was {}, the new URL is {}",
|
|
resourceID, oldURL, newURL);
|
|
resourceNode = persistStorageFile(resourceNode);
|
|
/*
|
|
try {
|
|
URL urlOLD = new URL(oldURL);
|
|
deleteStorageResource(urlOLD);
|
|
}catch (Exception e) {
|
|
logger.error("Unable to remove old file at URL {}", oldURL);
|
|
}
|
|
*/
|
|
}
|
|
String ret = super.update(getAsString(resourceNode));
|
|
if(storageHubManagement.getPersistedFile()!= null) {
|
|
String revisionID = result.get(REVISION_ID_KEY).asText();
|
|
storageHubManagement.addRevisionID(resourceID, revisionID);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
return previousRepresentation.asText();
|
|
}
|
|
|
|
@Override
|
|
public String update(String json) {
|
|
try {
|
|
JsonNode jsonNode = getAsJsonNode(json);
|
|
return update(jsonNode);
|
|
} catch(WebApplicationException e) {
|
|
throw e;
|
|
} catch(Exception e) {
|
|
throw new WebApplicationException(e);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public String patch(String json) {
|
|
String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(),
|
|
PUT.class.getSimpleName(), DELETE.class.getSimpleName()};
|
|
throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed);
|
|
}
|
|
|
|
@Override
|
|
public void delete(boolean purge) {
|
|
delete();
|
|
}
|
|
|
|
@Override
|
|
public void delete() {
|
|
try {
|
|
deleteFile();
|
|
sendPostRequest(DELETE, createJsonNodeWithID(resourceID));
|
|
} catch(WebApplicationException e) {
|
|
throw e;
|
|
} catch(Exception e) {
|
|
throw new WebApplicationException(e);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
protected void purge() {
|
|
String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(),
|
|
PUT.class.getSimpleName(), DELETE.class.getSimpleName()};
|
|
throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed);
|
|
}
|
|
|
|
public JsonNode createOrUpdate(JsonNode jsonNode) {
|
|
ObjectNode resourceNode = (ObjectNode) jsonNode;
|
|
if(resourceNode.has(ID_KEY)) {
|
|
try {
|
|
update(resourceNode);
|
|
} catch(WebApplicationException e) {
|
|
throw e;
|
|
} catch(Exception e) {
|
|
throw new WebApplicationException(e);
|
|
}
|
|
} else {
|
|
create(resourceNode);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
public void deleteFile() {
|
|
try {
|
|
getPreviousRepresentation();
|
|
URL url = new URL(previousRepresentation.get(URL_KEY).asText());
|
|
mimeType = previousRepresentation.get(MIME_TYPE_KEY).asText();
|
|
deleteStorageResource(url, resourceID, mimeType);
|
|
} catch(Exception e) {
|
|
logger.error("Unable to delete resource {}",
|
|
previousRepresentation != null ? getAsString(previousRepresentation) : "");
|
|
}
|
|
}
|
|
|
|
public void rollback() {
|
|
if(previousRepresentation != null) {
|
|
try {
|
|
update(previousRepresentation);
|
|
} catch(WebApplicationException e) {
|
|
throw e;
|
|
} catch(Exception e) {
|
|
throw new WebApplicationException(e);
|
|
}
|
|
} else {
|
|
delete();
|
|
}
|
|
}
|
|
|
|
}
|