Registered File deletion routine

This commit is contained in:
Fabio Sinibaldi 2022-01-31 13:09:54 +01:00
parent 42c986d3f2
commit 405129f37f
11 changed files with 68 additions and 16 deletions

View File

@ -51,6 +51,7 @@
<groupId>com.vdurmont</groupId> <groupId>com.vdurmont</groupId>
<artifactId>semver4j</artifactId> <artifactId>semver4j</artifactId>
<version>3.1.0</version> <version>3.1.0</version>
<scope>compile</scope>
</dependency> </dependency>
<!-- TEST --> <!-- TEST -->

View File

@ -0,0 +1,12 @@
package org.gcube.application.geoportal.common.model.document;
import lombok.*;
import org.bson.Document;
public class Materialization extends Document {
public static final String TYPE ="_type";
public String getType(){return super.getString(TYPE);}
}

View File

@ -4,7 +4,6 @@ import com.mongodb.client.model.geojson.GeoJsonObjectType;
import com.vdurmont.semver4j.Semver; import com.vdurmont.semver4j.Semver;
import lombok.*; import lombok.*;
import org.bson.Document; import org.bson.Document;
import org.bson.types.ObjectId;
@NoArgsConstructor @NoArgsConstructor

View File

@ -22,4 +22,5 @@ public class RegisteredFileSet {
private String folderID; private String folderID;
private List<RegisteredFile> payloads; private List<RegisteredFile> payloads;
private List<Materialization> materializations;
} }

View File

@ -1,11 +1,19 @@
package org.gcube.application.geoportal.common.model.profile; package org.gcube.application.geoportal.common.model.profile;
import lombok.*;
import org.bson.Document; import org.bson.Document;
import org.bson.types.MaxKey;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@XmlRootElement
@AllArgsConstructor
@Getter
@Setter
@ToString(callSuper = true)
public class Field extends Document { public class Field extends Document {
public static final String TYPE="_type"; public static final String TYPE="_type";

View File

@ -126,7 +126,11 @@ public class WorkspaceManager {
public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException { public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException {
sgClient.open(toDelete.getStorageID()).asFile().forceDelete(); sgClient.open(toDelete.getStorageID()).asFile().forceDelete();
} }
public void deleteItem(String itemId)throws StorageHubException{
sgClient.open(itemId).asItem().forceDelete();
}
// STATIC SYNCH METHODS // STATIC SYNCH METHODS
@Synchronized @Synchronized

View File

@ -39,6 +39,6 @@ public interface MongoManagerI<T> {
public T performStep(String id, String step, Document options) throws IOException, StepException; public T performStep(String id, String step, Document options) throws IOException, StepException;
public T registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException; public T registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException;
public T deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException; public T deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException;
} }

View File

@ -254,7 +254,7 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
* *
*/ */
@Override @Override
public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException { public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException {
List<TempFile> files=request.getStreams(); List<TempFile> files=request.getStreams();
Document attributes =request.getAttributes(); Document attributes =request.getAttributes();
@ -273,14 +273,14 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+request.getFieldPath(), Response.Status.BAD_REQUEST); throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+request.getFieldPath(), Response.Status.BAD_REQUEST);
if(fieldDefinitions.size()>1) if(fieldDefinitions.size()>1)
throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+request.getFieldPath(),Response.Status.BAD_REQUEST); throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+request.getFieldPath(),Response.Status.BAD_REQUEST);
Field fieldDefinition=fieldDefinitions.get(0); Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
log.debug("Field definition is {}",fieldDefinition); log.debug("Field definition is {}",fieldDefinition);
JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson()); JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson());
List<RegisteredFileSet> found=docWrapper.getByPath(request.getDestinationPath(),RegisteredFileSet.class); List<RegisteredFileSet> found=docWrapper.getByPath(request.getDestinationPath(),RegisteredFileSet.class);
if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){ // if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){
throw new WebApplicationException("Cannot add registered fileset at "+request.getFieldPath()+" : field is not collection.",Response.Status.BAD_REQUEST); // throw new WebApplicationException("Cannot add registered fileset at "+request.getFieldPath()+" : field is not collection.",Response.Status.BAD_REQUEST);
} // }
Object toSet=null; Object toSet=null;
@ -296,14 +296,14 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
case REPLACE_EXISTING: { case REPLACE_EXISTING: {
if(found.size()>1) if(found.size()>1)
throw new WebApplicationException("Cannot replace multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST); throw new WebApplicationException("Cannot replace multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
deleteFileSet(doc.get_id(),request.getDestinationPath(),false); deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws); toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws);
break; break;
}case MERGE_EXISTING: { }case MERGE_EXISTING: {
if(found.size()>1) if(found.size()>1)
throw new WebApplicationException("Cannot merge multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST); throw new WebApplicationException("Cannot merge multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
attributes.putAll(Serialization.asDocument(found.get(0))); attributes.putAll(Serialization.asDocument(found.get(0)));
deleteFileSet(doc.get_id(),request.getDestinationPath(),false); deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws); toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws);
break; break;
}case APPEND: { }case APPEND: {
@ -343,11 +343,10 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
throw new RuntimeException("Implement this"); throw new RuntimeException("Implement this");
} }
// @Override
// public ProfiledDocument deleteRegisteredFileSet(String id, String destination, List<TempFile> files) {
// throw new RuntimeException("TO IMPLEMENT");
// }
private ExecutionReport step(ProfiledDocument theDocument,String step,Document callParameters) throws StepException { private ExecutionReport step(ProfiledDocument theDocument,String step,Document callParameters) throws StepException {
log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor()); log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor());
@ -402,4 +401,22 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
attributes.putIfAbsent(RegisteredFileSet.PAYLOADS,registeredFiles); attributes.putIfAbsent(RegisteredFileSet.PAYLOADS,registeredFiles);
return Serialization.convert(attributes,RegisteredFileSet.class); return Serialization.convert(attributes,RegisteredFileSet.class);
} }
private static ProfiledDocument deleteFileSetRoutine(ProfiledDocument doc,String fileSetPath, Boolean force, WorkspaceManager ws) throws DeletionException, StorageHubException {
log.info("Document ID {} : deleting fileset at {} [force : {}]",doc.get_id(),fileSetPath,force);
JSONPathWrapper wrapper =new JSONPathWrapper(doc.getTheDocument().toJson());
RegisteredFileSet toDelete = Serialization.convert(wrapper.getByPath(fileSetPath).get(0),RegisteredFileSet.class);
if(toDelete.getMaterializations()!=null && !toDelete.getMaterializations().isEmpty()){
if(!force) throw new DeletionException("Fileset (Document ID "+doc.get_id()+", path "+fileSetPath+") already materialized. Use force = true");
else throw new RuntimeException("Implement this");
// TODO manager force deletion
// NB handlers for materialization types
}
log.debug("Document ID {} : deleting ws folder {}",doc.get_id(),toDelete.getFolderID());
if(toDelete.getPayloads()!=null)
ws.deleteItem(toDelete.getFolderID());
doc.setTheDocument(Document.parse(wrapper.set(fileSetPath,null).getCtx().jsonString()));
return doc;
}
} }

View File

@ -32,6 +32,7 @@ public abstract class GuardedMethod<T> {
result=run(); result=run();
return this; return this;
}catch(WebApplicationException e) { }catch(WebApplicationException e) {
log.error("Throwing Web Application Exception ",e);
throw e; throw e;
}catch(Throwable t) { }catch(Throwable t) {
log.error("Unexpected error ",t); log.error("Unexpected error ",t);

View File

@ -9,6 +9,7 @@ import org.gcube.application.geoportal.common.model.document.ProfiledDocument;
import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.model.rest.Configuration; import org.gcube.application.geoportal.common.model.rest.Configuration;
import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
import org.gcube.application.geoportal.common.rest.InterfaceConstants; import org.gcube.application.geoportal.common.rest.InterfaceConstants;
import org.gcube.application.geoportal.common.utils.FileSets; import org.gcube.application.geoportal.common.utils.FileSets;
import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.Files;
@ -112,6 +113,7 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{
"relazioneScavo", "relazioneScavo",
"relazioneScavo", "relazioneScavo",
Document.parse("{\"titolo\" : \"mio titolo\"}"), Document.parse("{\"titolo\" : \"mio titolo\"}"),
RegisterFileSetRequest.ClashOptions.MERGE_EXISTING,
"relazione.pdf"); "relazione.pdf");
System.out.println(Serialization.write(doc)); System.out.println(Serialization.write(doc));
} }
@ -126,9 +128,17 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{
post(Entity.entity(document, MediaType.APPLICATION_JSON)),ProfiledDocument.class); post(Entity.entity(document, MediaType.APPLICATION_JSON)),ProfiledDocument.class);
} }
private ProfiledDocument upload(StorageUtils storage, String id, String path, String fieldPath, Document attributes, String ...files) throws Exception { private ProfiledDocument upload(StorageUtils storage,
String id,
String path,
String fieldDefinitionPath,
Document attributes,
RegisterFileSetRequest.ClashOptions clashPolicy,
String ...files) throws Exception {
FileSets.RequestBuilder builder = FileSets.build(path); FileSets.RequestBuilder builder = FileSets.build(path);
builder.setFieldDescriptionPath(fieldDefinitionPath).setClashPolicy(clashPolicy).setAttributes(attributes);
for(String file:files) for(String file:files)
builder.add(storage.putOntoStorage(new File(TestConcessioniModel.getBaseFolder(),file),file)); builder.add(storage.putOntoStorage(new File(TestConcessioniModel.getBaseFolder(),file),file));