From 405129f37f4e23175eecc22e41156c265e517dbe Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Mon, 31 Jan 2022 13:09:54 +0100 Subject: [PATCH] Registered File deletion routine --- geoportal-common/pom.xml | 1 + .../model/document/Materialization.java | 12 ++++++ .../model/document/ProfiledDocument.java | 1 - .../model/document/RegisteredFileSet.java | 1 + .../geoportal/common/model/profile/Field.java | 8 ++++ .../common/model/profile/Profile.java | 1 - .../service/engine/WorkspaceManager.java | 6 ++- .../service/engine/mongo/MongoManagerI.java | 2 +- .../engine/mongo/ProfiledMongoManager.java | 39 +++++++++++++------ .../geoportal/service/rest/GuardedMethod.java | 1 + .../service/ProfiledDocumentsTests.java | 12 +++++- 11 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/Materialization.java diff --git a/geoportal-common/pom.xml b/geoportal-common/pom.xml index fe2deb6..d7600e4 100644 --- a/geoportal-common/pom.xml +++ b/geoportal-common/pom.xml @@ -51,6 +51,7 @@ com.vdurmont semver4j 3.1.0 + compile diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/Materialization.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/Materialization.java new file mode 100644 index 0000000..7208a55 --- /dev/null +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/Materialization.java @@ -0,0 +1,12 @@ +package org.gcube.application.geoportal.common.model.document; + +import lombok.*; +import org.bson.Document; + + +public class Materialization extends Document { + + public static final String TYPE ="_type"; + + public String getType(){return super.getString(TYPE);} +} diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/ProfiledDocument.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/ProfiledDocument.java index 04ee294..0b49434 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/ProfiledDocument.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/ProfiledDocument.java @@ -4,7 +4,6 @@ import com.mongodb.client.model.geojson.GeoJsonObjectType; import com.vdurmont.semver4j.Semver; import lombok.*; import org.bson.Document; -import org.bson.types.ObjectId; @NoArgsConstructor diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java index 376999d..2d42f2c 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java @@ -22,4 +22,5 @@ public class RegisteredFileSet { private String folderID; private List payloads; + private List materializations; } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Field.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Field.java index daa780f..ed53b6c 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Field.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Field.java @@ -1,11 +1,19 @@ package org.gcube.application.geoportal.common.model.profile; +import lombok.*; import org.bson.Document; +import org.bson.types.MaxKey; +import javax.xml.bind.annotation.XmlRootElement; import java.util.HashMap; import java.util.List; import java.util.Map; +@XmlRootElement +@AllArgsConstructor +@Getter +@Setter +@ToString(callSuper = true) public class Field extends Document { public static final String TYPE="_type"; diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Profile.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Profile.java index 649e9de..eb10306 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Profile.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/profile/Profile.java @@ -49,5 +49,4 @@ public class Profile{ } - } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java index 146553b..9f3fd69 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java @@ -126,7 +126,11 @@ public class WorkspaceManager { public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException { sgClient.open(toDelete.getStorageID()).asFile().forceDelete(); } - + + public void deleteItem(String itemId)throws StorageHubException{ + sgClient.open(itemId).asItem().forceDelete(); + } + // STATIC SYNCH METHODS @Synchronized diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java index 4fd5a10..a8daf55 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java @@ -39,6 +39,6 @@ public interface MongoManagerI { public T performStep(String id, String step, Document options) throws IOException, StepException; - public T registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException; + public T registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException; public T deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException; } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index 0f40bf9..18fa72b 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -254,7 +254,7 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< * */ @Override - public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException { + public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException { List files=request.getStreams(); Document attributes =request.getAttributes(); @@ -273,14 +273,14 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+request.getFieldPath(), Response.Status.BAD_REQUEST); if(fieldDefinitions.size()>1) throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+request.getFieldPath(),Response.Status.BAD_REQUEST); - Field fieldDefinition=fieldDefinitions.get(0); + Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class); log.debug("Field definition is {}",fieldDefinition); JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson()); List found=docWrapper.getByPath(request.getDestinationPath(),RegisteredFileSet.class); - if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){ - throw new WebApplicationException("Cannot add registered fileset at "+request.getFieldPath()+" : field is not collection.",Response.Status.BAD_REQUEST); - } +// if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){ +// throw new WebApplicationException("Cannot add registered fileset at "+request.getFieldPath()+" : field is not collection.",Response.Status.BAD_REQUEST); +// } Object toSet=null; @@ -296,14 +296,14 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< case REPLACE_EXISTING: { if(found.size()>1) throw new WebApplicationException("Cannot replace multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST); - deleteFileSet(doc.get_id(),request.getDestinationPath(),false); + deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws); toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws); break; }case MERGE_EXISTING: { if(found.size()>1) throw new WebApplicationException("Cannot merge multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST); attributes.putAll(Serialization.asDocument(found.get(0))); - deleteFileSet(doc.get_id(),request.getDestinationPath(),false); + deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws); toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws); break; }case APPEND: { @@ -343,11 +343,10 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< throw new RuntimeException("Implement this"); } + + + -// @Override -// public ProfiledDocument deleteRegisteredFileSet(String id, String destination, List files) { -// throw new RuntimeException("TO IMPLEMENT"); -// } private ExecutionReport step(ProfiledDocument theDocument,String step,Document callParameters) throws StepException { log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor()); @@ -402,4 +401,22 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< attributes.putIfAbsent(RegisteredFileSet.PAYLOADS,registeredFiles); return Serialization.convert(attributes,RegisteredFileSet.class); } + + + private static ProfiledDocument deleteFileSetRoutine(ProfiledDocument doc,String fileSetPath, Boolean force, WorkspaceManager ws) throws DeletionException, StorageHubException { + log.info("Document ID {} : deleting fileset at {} [force : {}]",doc.get_id(),fileSetPath,force); + JSONPathWrapper wrapper =new JSONPathWrapper(doc.getTheDocument().toJson()); + RegisteredFileSet toDelete = Serialization.convert(wrapper.getByPath(fileSetPath).get(0),RegisteredFileSet.class); + if(toDelete.getMaterializations()!=null && !toDelete.getMaterializations().isEmpty()){ + if(!force) throw new DeletionException("Fileset (Document ID "+doc.get_id()+", path "+fileSetPath+") already materialized. Use force = true"); + else throw new RuntimeException("Implement this"); + // TODO manager force deletion + // NB handlers for materialization types + } + log.debug("Document ID {} : deleting ws folder {}",doc.get_id(),toDelete.getFolderID()); + if(toDelete.getPayloads()!=null) + ws.deleteItem(toDelete.getFolderID()); + doc.setTheDocument(Document.parse(wrapper.set(fileSetPath,null).getCtx().jsonString())); + return doc; + } } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/GuardedMethod.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/GuardedMethod.java index e7537af..423a90a 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/GuardedMethod.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/GuardedMethod.java @@ -32,6 +32,7 @@ public abstract class GuardedMethod { result=run(); return this; }catch(WebApplicationException e) { + log.error("Throwing Web Application Exception ",e); throw e; }catch(Throwable t) { log.error("Unexpected error ",t); diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java index d737a2e..910b052 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java @@ -9,6 +9,7 @@ import org.gcube.application.geoportal.common.model.document.ProfiledDocument; import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.rest.Configuration; import org.gcube.application.geoportal.common.model.rest.QueryRequest; +import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest; import org.gcube.application.geoportal.common.rest.InterfaceConstants; import org.gcube.application.geoportal.common.utils.FileSets; import org.gcube.application.geoportal.common.utils.Files; @@ -112,6 +113,7 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{ "relazioneScavo", "relazioneScavo", Document.parse("{\"titolo\" : \"mio titolo\"}"), + RegisterFileSetRequest.ClashOptions.MERGE_EXISTING, "relazione.pdf"); System.out.println(Serialization.write(doc)); } @@ -126,9 +128,17 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{ post(Entity.entity(document, MediaType.APPLICATION_JSON)),ProfiledDocument.class); } - private ProfiledDocument upload(StorageUtils storage, String id, String path, String fieldPath, Document attributes, String ...files) throws Exception { + private ProfiledDocument upload(StorageUtils storage, + String id, + String path, + String fieldDefinitionPath, + Document attributes, + RegisterFileSetRequest.ClashOptions clashPolicy, + String ...files) throws Exception { FileSets.RequestBuilder builder = FileSets.build(path); + builder.setFieldDescriptionPath(fieldDefinitionPath).setClashPolicy(clashPolicy).setAttributes(attributes); + for(String file:files) builder.add(storage.putOntoStorage(new File(TestConcessioniModel.getBaseFolder(),file),file));