gCubeSystem
/
gcat
15
0
Fork 0
Code Issues Pull Requests Releases 13 Wiki Activity

Reorganized service

This commit is contained in:
luca.frosini 2023-07-18 16:04:16 +02:00
parent ded9f23deb
commit 04bdf3bff5
6 changed files with 147 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java
View File

@ -24,7 +24,7 @@ import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.gcat.utils.Constants;
import org.gcube.gcat.utils.HTTPCall;
import org.gcube.gcat.workspace.CatalogueStorageHubManagement;
import org.gcube.gcat.workspace.GcatStorageHubManagement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -100,7 +100,7 @@ public class CKANResource extends CKAN {
protected JsonNode previousRepresentation;
protected CatalogueStorageHubManagement storageHubManagement;
protected GcatStorageHubManagement storageHubManagement;
public URL getPersistedURL() {
return persistedURL;
@ -151,6 +151,7 @@ public class CKANResource extends CKAN {
PURGE = null;
persisted = false;
previousRepresentation = null;
storageHubManagement = new GcatStorageHubManagement();
}
@Override
@ -316,7 +317,6 @@ public class CKANResource extends CKAN {
protected URL copyStorageResource(URL url) {
persistedURL = url;
if(isStorageFile(persistedURL)) {
storageHubManagement = new CatalogueStorageHubManagement();
try {
persistedURL = storageHubManagement.ensureResourcePersistence(persistedURL, itemID, resourceID);
String originalFilename = storageHubManagement.getOriginalFilename();
@ -345,7 +345,6 @@ public class CKANResource extends CKAN {
} catch(Exception e) {
// using provided mimetype
}
storageHubManagement = new CatalogueStorageHubManagement();
try {
storageHubManagement.deleteResourcePersistence(itemID, filename, mimetype);
} catch(Exception e) {
@ -363,8 +362,9 @@ public class CKANResource extends CKAN {
String gotResourceID = result.get(ID_KEY).asText();
if(gotResourceID != null && gotResourceID.compareTo(resourceID) != 0) {
resourceID = gotResourceID;
String revisionID = result.get(REVISION_ID_KEY).asText();
storageHubManagement.renameFile(resourceID, revisionID);
// String revisionID = result.get(REVISION_ID_KEY).asText();
// storageHubManagement.renameFile(resourceID, revisionID);
storageHubManagement.renameFile(resourceID);
}
}
return ret;
@ -401,7 +401,6 @@ public class CKANResource extends CKAN {
if(!previousRepresentation.equals(resourceNode)) {
if(oldURL.compareTo(newURL) != 0) {
logger.trace("The URL of the resource with id {} was not changed", resourceID);
storageHubManagement = new CatalogueStorageHubManagement();
this.mimeType = previousRepresentation.get(CKANResource.MIME_TYPE_KEY).asText();
try {
@ -424,10 +423,10 @@ public class CKANResource extends CKAN {
*/
}
String ret = super.update(getAsString(resourceNode));
if(storageHubManagement.getPersistedFile()!= null) {
String revisionID = result.get(REVISION_ID_KEY).asText();
storageHubManagement.addRevisionID(resourceID, revisionID);
}
// if(storageHubManagement.getPersistedFile()!= null) {
// String revisionID = result.get(REVISION_ID_KEY).asText();
// storageHubManagement.addRevisionID(resourceID, revisionID);
// }
return ret;
}

48
src/main/java/org/gcube/gcat/workspace/CatalogueMetadata.java
View File

@ -1,48 +0,0 @@
package org.gcube.gcat.workspace;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.storagehub.MetadataMatcher;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class CatalogueMetadata implements MetadataMatcher {
public static final String ORIGINAL_URL = "OriginalURL";
public static final String ORIGINAL_NAME = "OriginalName";
public static final String CATALOGUE_ITEM_ID = "CatalogueItemID";
public static final String CATALOGUE_RESOURCE_ID = "CatalogueResourceID";
public static final String CATALOGUE_RESOURCE_REVISION_ID = "CatalogueResourceRevisionID";
protected String itemID;
public CatalogueMetadata(String itemID) {
this.itemID = itemID;
}
@Override
public boolean check(Metadata metadata) {
Map<String,Object> map = metadata.getMap();
Object obj = map.get(CATALOGUE_ITEM_ID);
if(obj!=null && obj.toString().compareTo(itemID) == 0) {
return true;
}
return false;
}
public Metadata getMetadata(URL url, String originalName, String resourceID) {
Map<String,Object> map = new HashMap<>();
map.put(ORIGINAL_URL, url.toString());
map.put(ORIGINAL_NAME, originalName);
map.put(CATALOGUE_ITEM_ID, itemID);
map.put(CATALOGUE_RESOURCE_ID, resourceID);
Metadata metadata = new Metadata(map);
return metadata;
}
}

40
src/main/java/org/gcube/gcat/workspace/GcatMetadataMatcher.java Normal file
View File

@ -0,0 +1,40 @@
package org.gcube.gcat.workspace;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.gcat.utils.Constants;
import org.gcube.storagehub.MetadataMatcher;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class GcatMetadataMatcher extends MetadataMatcher {
public static final String GCAT_METADATA_VERSION = "2.0.0";
public static final String CATALOGUE_ITEM_ID = "CatalogueItemID";
public GcatMetadataMatcher(String id) {
super(Constants.CATALOGUE_NAME, GCAT_METADATA_VERSION, id);
}
@Override
public boolean check(Metadata metadata) {
Map<String,Object> map = metadata.getMap();
Object obj = map.get(CATALOGUE_ITEM_ID);
if(obj!=null && obj.toString().compareTo(id) == 0) {
return true;
}
return false;
}
@Override
protected Map<String, Object> getSpecificMetadataMap() {
Map<String,Object> map = new HashMap<>();
map.put(CATALOGUE_ITEM_ID, id);
return map;
}
}

115
src/main/java/org/gcube/gcat/workspace/CatalogueStorageHubManagement.java → src/main/java/org/gcube/gcat/workspace/GcatStorageHubManagement.java
View File

@ -2,16 +2,14 @@ package org.gcube.gcat.workspace;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Map;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.gcat.utils.Constants;
import org.gcube.storagehub.MetadataMatcher;
import org.gcube.storagehub.StorageHubManagement;
import org.glassfish.jersey.media.multipart.ContentDisposition;
import org.slf4j.Logger;
@ -20,12 +18,13 @@ import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class CatalogueStorageHubManagement {
public class GcatStorageHubManagement {
private static final Logger logger = LoggerFactory.getLogger(CatalogueStorageHubManagement.class);
private static final Logger logger = LoggerFactory.getLogger(GcatStorageHubManagement.class);
protected StorageHubManagement storageHubManagement;
protected String itemId;
protected String originalFilename;
protected String mimeType;
@ -37,10 +36,15 @@ public class CatalogueStorageHubManagement {
return mimeType;
}
public CatalogueStorageHubManagement() {
public GcatStorageHubManagement() {
this.storageHubManagement = new StorageHubManagement();
}
protected MetadataMatcher getMetadataMatcher() {
MetadataMatcher metadataMatcher = new GcatMetadataMatcher(itemId);
return metadataMatcher;
}
protected String getOriginalFileName(HttpURLConnection httpURLConnection) throws Exception {
String contentDisposition = httpURLConnection.getHeaderField("Content-Disposition");
contentDisposition = contentDisposition.replaceAll("= ", "=").replaceAll(" =", "=");
@ -59,12 +63,11 @@ public class CatalogueStorageHubManagement {
HttpURLConnection httpURLConnection = gxhttpStringRequest.get();
mimeType = httpURLConnection.getContentType().split(";")[0];
originalFilename = getOriginalFileName(httpURLConnection);
CatalogueMetadata catalogueMetadata = new CatalogueMetadata(itemID);
storageHubManagement.setCheckMetadata(catalogueMetadata);
Metadata metadata = catalogueMetadata.getMetadata(persistedURL, originalFilename, resourceID);
persistedURL = storageHubManagement.persistFile(httpURLConnection.getInputStream(), resourceID, mimeType,
metadata);
// mimeType = storageHubManagement.getMimeType();
this.itemId = itemID;
storageHubManagement.setMetadataMatcher(getMetadataMatcher());
persistedURL = storageHubManagement.persistFile(httpURLConnection.getInputStream(), resourceID, mimeType);
return persistedURL;
} catch (Exception e) {
logger.error("Error while trying to persists the resource", e);
@ -80,68 +83,68 @@ public class CatalogueStorageHubManagement {
try {
secretManager.startSession(secret);
storageHubManagement = new StorageHubManagement();
CatalogueMetadata catalogueMetadata = new CatalogueMetadata(itemID);
storageHubManagement.setCheckMetadata(catalogueMetadata);
this.itemId = itemID;
storageHubManagement.setMetadataMatcher(getMetadataMatcher());
storageHubManagement.removePersistedFile(resourceID, mimeType);
} finally {
secretManager.endSession();
}
}
protected void internalAddRevisionID(String resourceID, String revisionID) throws Exception {
try {
FileContainer fileContainer = null;
AbstractFileItem fileItem = null;
try {
fileContainer = storageHubManagement.getPersistedFile();
fileItem = fileContainer.get();
}catch (Exception e) {
// This is a workaround because storage-hub invalidate the item
// when I rename it (just before this operation)
// then I get java.lang.RuntimeException: javax.ws.rs.ProcessingException: Error reading entity from input stream.
// invoking fileContainer.get()
// see issue #25373
fileContainer = storageHubManagement.getPersistedFile(resourceID, mimeType);
fileItem = fileContainer.get();
}
Metadata metadata = fileItem.getMetadata();
Map<String,Object> map = metadata.getMap();
map.put(CatalogueMetadata.CATALOGUE_RESOURCE_ID, resourceID);
map.put(CatalogueMetadata.CATALOGUE_RESOURCE_REVISION_ID, revisionID);
metadata.setMap(map);
fileContainer.setMetadata(metadata);
} catch (Exception e) {
logger.warn(
"Unable to set revision id {} to the file of resource with id {} because the file was NOT found on storage-hub.",
revisionID, resourceID);
throw e;
}
}
public void renameFile(String resourceID, String revisionID) throws Exception {
// protected void internalAddRevisionID(String resourceID, String revisionID) throws Exception {
// try {
// FileContainer fileContainer = null;
// AbstractFileItem fileItem = null;
// try {
// fileContainer = storageHubManagement.getPersistedFile();
// fileItem = fileContainer.get();
// }catch (Exception e) {
// // This is a workaround because storage-hub invalidate the item
// // when I rename it (just before this operation)
// // then I get java.lang.RuntimeException: javax.ws.rs.ProcessingException: Error reading entity from input stream.
// // invoking fileContainer.get()
// // see issue #25373
// fileContainer = storageHubManagement.getPersistedFile(resourceID, mimeType);
// fileItem = fileContainer.get();
// }
// Metadata metadata = fileItem.getMetadata();
// Map<String,Object> map = metadata.getMap();
// map.put(CatalogueMetadata.CATALOGUE_RESOURCE_ID, resourceID);
// map.put(CatalogueMetadata.CATALOGUE_RESOURCE_REVISION_ID, revisionID);
// metadata.setMap(map);
// fileContainer.setMetadata(metadata);
// } catch (Exception e) {
// logger.warn(
// "Unable to set revision id {} to the file of resource with id {} because the file was NOT found on storage-hub.",
// revisionID, resourceID);
// throw e;
// }
// }
// public void renameFile(String resourceID, String revisionID) throws Exception {
public void renameFile(String resourceID) throws Exception {
SecretManager secretManager = SecretManagerProvider.instance.get();
Secret secret = Constants.getCatalogueSecret();
try {
secretManager.startSession(secret);
FileContainer createdfile = storageHubManagement.getPersistedFile();
createdfile.rename(resourceID);
internalAddRevisionID(resourceID, revisionID);
// internalAddRevisionID(resourceID, revisionID);
} finally {
secretManager.endSession();
}
}
public void addRevisionID(String resourceID, String revisionID) throws Exception {
SecretManager secretManager = SecretManagerProvider.instance.get();
Secret secret = Constants.getCatalogueSecret();
try {
secretManager.startSession(secret);
internalAddRevisionID(resourceID, revisionID);
} finally {
secretManager.endSession();
}
}
// public void addRevisionID(String resourceID, String revisionID) throws Exception {
// SecretManager secretManager = SecretManagerProvider.instance.get();
// Secret secret = Constants.getCatalogueSecret();
// try {
// secretManager.startSession(secret);
// internalAddRevisionID(resourceID, revisionID);
// } finally {
// secretManager.endSession();
// }
// }
public FileContainer retrievePersistedFile(String filename, String mimeType) throws Exception {
SecretManager secretManager = SecretManagerProvider.instance.get();

33
src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java
View File

@ -330,8 +330,8 @@ public class CKANPackageTest extends ContextTest {
ArrayNode resourceArrayNode = itemObjectNode.putArray(CKANPackage.RESOURCES_KEY);
ObjectNode resourceNode = mapper.createObjectNode();
resourceNode.put(CKANResource.NAME_KEY, "RESTful Transaction Model");
// Workspace(luca.frosini) > RESTful Transaction Model v 1.0.pdf
resourceNode.put(CKANResource.URL_KEY, "https://goo.gl/J8AwQW");
// Workspace(luca.frosini) > RESTful Transaction Model v 1.1.pdf
resourceNode.put(CKANResource.URL_KEY, "https://data-dev.d4science.net/Qpw2");
resourceArrayNode.add(resourceNode);
ArrayNode extraArrayNode = itemObjectNode.putArray(CKANPackage.EXTRAS_KEY);
@ -400,13 +400,13 @@ public class CKANPackageTest extends ContextTest {
String updatedNotes = "A research of Luca Frosini made during the PhD";
((ObjectNode) readItemObjectNode).put(NOTES_KEY, updatedNotes);
ArrayNode resources = (ArrayNode) readItemObjectNode.get(CKANPackage.RESOURCES_KEY);
ObjectNode objectNode = (ObjectNode) resources.get(0);
// Workspace(luca.frosini) > RESTful Transaction Model v 1.1.pdf
objectNode.put(CKANResource.URL_KEY, "https://goo.gl/78ViuR");
resources.set(0, objectNode);
((ObjectNode) readItemObjectNode).replace(CKANPackage.RESOURCES_KEY, resources);
// ArrayNode resources = (ArrayNode) readItemObjectNode.get(CKANPackage.RESOURCES_KEY);
// ObjectNode objectNode = (ObjectNode) resources.get(0);
// // Workspace(luca.frosini) > RESTful Transaction Model v 1.1.pdf
// objectNode.put(CKANResource.URL_KEY, "https://data-dev.d4science.net/Qpw2");
// resources.set(0, objectNode);
//
// ((ObjectNode) readItemObjectNode).replace(CKANPackage.RESOURCES_KEY, resources);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
@ -416,11 +416,13 @@ public class CKANPackageTest extends ContextTest {
String gotUpdatedNotes = updatedItemObjectNode.get(NOTES_KEY).asText();
Assert.assertTrue(gotUpdatedNotes.compareTo(updatedNotes) == 0);
/*
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
((ObjectNode) updatedItemObjectNode).remove(CKANPackage.RESOURCES_KEY);
String secondUpdateItem = ckanPackage.update(mapper.writeValueAsString(updatedItemObjectNode));
logger.trace(secondUpdateItem);
*/
/*
ObjectNode patchObjectNode = mapper.createObjectNode();
@ -437,11 +439,12 @@ public class CKANPackageTest extends ContextTest {
Assert.assertTrue(gotPatchedNotes.compareTo(patchedNotes)==0);
*/
/*
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.purge();
logger.debug("Item {} purge successfully", ITEM_NAME_VALUE);
*/
}
@Test
@ -453,7 +456,7 @@ public class CKANPackageTest extends ContextTest {
logger.debug(ret);
}
@Ignore
// @Ignore
@Test
//(expected = NotFoundException.class)
public void delete() throws Exception {
@ -565,16 +568,16 @@ public class CKANPackageTest extends ContextTest {
// logger.debug("{}", res);
}
@Ignore
// @Test
// @Ignore
@Test
public void deleteAllItemsInAllOrganizations() {
CKANPackage ckanPackage = new CKANPackage();
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false");
UriInfo uriInfo = getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
// String res = ckanPackage.deleteAll(true);
// logger.debug("{}", res);
String res = ckanPackage.deleteAll(true);
logger.debug("{}", res);
}
@Test

39
src/test/java/org/gcube/gcat/workspace/CatalogueStorageHubManagementTest.java → src/test/java/org/gcube/gcat/workspace/GCatStorageHubManagementTest.java
View File

@ -7,15 +7,16 @@ import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.gcat.ContextTest;
import org.gcube.gcat.persistence.ckan.CKANResource;
import org.gcube.storagehub.MetadataMatcher;
import org.gcube.storagehub.StorageHubManagement;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CatalogueStorageHubManagementTest extends ContextTest {
public class GCatStorageHubManagementTest extends ContextTest {
private static final Logger logger = LoggerFactory.getLogger(CatalogueStorageHubManagementTest.class);
private static final Logger logger = LoggerFactory.getLogger(GCatStorageHubManagementTest.class);
/*
* PRE
@ -56,14 +57,14 @@ public class CatalogueStorageHubManagementTest extends ContextTest {
public static final String ITEM_ID = "MyItem";
public static final String RESOURCE_ID = "1234";
protected CatalogueStorageHubManagement catalogueStorageHubManagement;
protected GcatStorageHubManagement catalogueStorageHubManagement;
protected StorageHubManagement storageHubManagement;
protected CatalogueMetadata catalogueMetadata;
protected GcatMetadataMatcher catalogueMetadata;
public CatalogueStorageHubManagementTest() {
catalogueStorageHubManagement = new CatalogueStorageHubManagement();
public GCatStorageHubManagementTest() {
catalogueStorageHubManagement = new GcatStorageHubManagement();
storageHubManagement = catalogueStorageHubManagement.storageHubManagement;
catalogueMetadata = new CatalogueMetadata(ITEM_ID);
catalogueMetadata = new GcatMetadataMatcher(ITEM_ID);
}
@Test
@ -72,26 +73,21 @@ public class CatalogueStorageHubManagementTest extends ContextTest {
Assert.assertTrue(finalURL.toString().compareTo(ORIGINAL_STORAGE_URL_STRING) == 0);
}
protected void checkMetadata(FileContainer fileContainer, String version) {
protected void checkMetadata(FileContainer fileContainer) {
Metadata gotMetadata = fileContainer.get().getMetadata();
Map<String,Object> gotMap = gotMetadata.getMap();
CatalogueMetadata catalogueMetadata = new CatalogueMetadata(ITEM_ID);
MetadataMatcher catalogueMetadata = new GcatMetadataMatcher(ITEM_ID);
Metadata expectedMetadata = catalogueMetadata.getMetadata(ORIGINAL_STORAGE_URL, fileContainer.get().getName(),
RESOURCE_ID);
Metadata expectedMetadata = catalogueMetadata.getMetadata();
Map<String,Object> expectedMap = expectedMetadata.getMap();
for(String key : gotMap.keySet()) {
String value = (String) gotMap.get(key);
if(key.compareTo(CatalogueMetadata.CATALOGUE_RESOURCE_REVISION_ID) == 0) {
Assert.assertTrue(value.compareTo(version) == 0);
} else {
String expectedValue = (String) expectedMap.get(key);
Assert.assertTrue(value.compareTo(expectedValue) == 0);
}
}
}
@Test
public void testPersistence() throws Exception {
@ -104,13 +100,14 @@ public class CatalogueStorageHubManagementTest extends ContextTest {
FileContainer createdFileContainer = storageHubManagement.getPersistedFile();
String version = "2";
catalogueStorageHubManagement.renameFile(RESOURCE_ID, version);
checkMetadata(createdFileContainer, version);
// String version = "2";
// catalogueStorageHubManagement.renameFile(RESOURCE_ID, version);
catalogueStorageHubManagement.renameFile(RESOURCE_ID);
checkMetadata(createdFileContainer);
version = "3";
catalogueStorageHubManagement.addRevisionID(RESOURCE_ID, version);
checkMetadata(createdFileContainer, version);
// version = "3";
// catalogueStorageHubManagement.addRevisionID(RESOURCE_ID, version);
// checkMetadata(createdFileContainer);
catalogueStorageHubManagement.deleteResourcePersistence(ITEM_ID, RESOURCE_ID, MIME_TYPE);