From 9cd78d4c4a3e4d42e2dba2f5214ef729032cbb29 Mon Sep 17 00:00:00 2001
From: Fabio Sinibaldi <fabio.sinibaldi@isti.cnr.it>
Date: Tue, 1 Feb 2022 15:24:39 +0100
Subject: [PATCH] BugFix FileSet Registration and delete

---
 .../cms/plugins/JacksonProvider.java          | 12 ---
 .../cms/serialization/JacksonProvider.java    | 39 ++++++++++
 .../{ => serialization}/Serialization.java    |  2 +-
 .../application/cms/tests/TestProfiles.java   |  3 +-
 .../plugins/ConcessioniLifeCycleManager.java  |  5 +-
 .../concessioni/model/DefaultLogicHolder.java |  7 --
 .../model/DocumentedRelazioneScavo.java       | 11 ---
 .../{utils => model}/JSONPathWrapper.java     | 10 +--
 .../model/document/RegisteredFileSet.java     | 21 +++---
 .../geoportal/service/GeoPortalService.java   |  2 +-
 .../engine/mongo/ConcessioniMongoManager.java |  2 +-
 .../service/engine/mongo/MongoManagerI.java   |  2 +-
 .../engine/mongo/ProfiledMongoManager.java    | 70 +++++++++---------
 .../service/engine/postgis/PostgisIndex.java  |  2 +-
 .../engine/providers/ProfileMapCache.java     |  2 +-
 .../service/rest/ConcessioniOverMongo.java    |  2 +-
 .../service/rest/ProfiledDocuments.java       |  2 +-
 .../service/BasicServiceTestUnit.java         |  5 +-
 .../service/ConcessioniOverMongoTest.java     |  2 +-
 .../service/ProfiledDocumentsTests.java       | 73 ++++++++++++++++---
 .../engine/materialization/SDITests.java      |  2 +-
 21 files changed, 163 insertions(+), 113 deletions(-)
 delete mode 100644 cms-plugin-framework/src/main/java/org/gcube/application/cms/plugins/JacksonProvider.java
 create mode 100644 cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/JacksonProvider.java
 rename cms-plugin-framework/src/main/java/org/gcube/application/cms/{ => serialization}/Serialization.java (99%)
 delete mode 100644 concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DefaultLogicHolder.java
 delete mode 100644 concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DocumentedRelazioneScavo.java
 rename geoportal-common/src/main/java/org/gcube/application/geoportal/common/{utils => model}/JSONPathWrapper.java (90%)

diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/plugins/JacksonProvider.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/plugins/JacksonProvider.java
deleted file mode 100644
index 51b616a..0000000
--- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/plugins/JacksonProvider.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.gcube.application.cms.plugins;
-
-import org.gcube.application.geoportal.common.JSONSerializationProvider;
-
-public class JacksonProvider implements JSONSerializationProvider {
-
-
-    @Override
-    public void setJSONWrapperDefaults() {
-
-    }
-}
diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/JacksonProvider.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/JacksonProvider.java
new file mode 100644
index 0000000..7e74818
--- /dev/null
+++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/JacksonProvider.java
@@ -0,0 +1,39 @@
+package org.gcube.application.cms.serialization;
+
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
+import com.jayway.jsonpath.spi.json.JsonProvider;
+import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
+import com.jayway.jsonpath.spi.mapper.MappingProvider;
+import org.gcube.application.geoportal.common.JSONSerializationProvider;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+public class JacksonProvider implements JSONSerializationProvider {
+
+
+    @Override
+    public void setJSONWrapperDefaults() {
+        Configuration.setDefaults(new Configuration.Defaults() {
+            private JsonProvider jacksonProvider  = new JacksonJsonProvider(Serialization.mapper);
+
+            private final MappingProvider mappingProvider = new JacksonMappingProvider(Serialization.mapper);
+            @Override
+            public JsonProvider jsonProvider() {
+                return jacksonProvider;
+            }
+
+            @Override
+            public Set<Option> options() {
+                return EnumSet.noneOf(Option.class);
+            }
+
+            @Override
+            public MappingProvider mappingProvider() {
+                return mappingProvider;
+            }
+        });
+    }
+}
diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/Serialization.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/Serialization.java
similarity index 99%
rename from cms-plugin-framework/src/main/java/org/gcube/application/cms/Serialization.java
rename to cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/Serialization.java
index cc42dd7..1e9984a 100644
--- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/Serialization.java
+++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/serialization/Serialization.java
@@ -1,4 +1,4 @@
-package org.gcube.application.cms;
+package org.gcube.application.cms.serialization;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
diff --git a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/TestProfiles.java b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/TestProfiles.java
index 06cd1ce..e02db15 100644
--- a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/TestProfiles.java
+++ b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/TestProfiles.java
@@ -1,7 +1,6 @@
 package org.gcube.application.cms.tests;
 
-import org.gcube.application.cms.Serialization;
-import org.gcube.application.cms.tests.model.concessioni.TestConcessioniModel;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.geoportal.common.model.profile.Profile;
 import org.gcube.application.geoportal.common.utils.Files;
 
diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java
index 0754e9c..3eda95e 100644
--- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java
+++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java
@@ -5,7 +5,7 @@ import org.bson.BsonDocument;
 import org.bson.BsonString;
 import org.bson.BsonValue;
 import org.bson.Document;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
 import org.gcube.application.cms.plugins.LifecycleManager;
 import org.gcube.application.cms.plugins.faults.InitializationException;
@@ -16,11 +16,8 @@ import org.gcube.application.cms.plugins.reports.ExecutionReport;
 import org.gcube.application.cms.plugins.reports.InitializationReport;
 import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
 import org.gcube.application.geoportal.common.model.document.*;
-import org.gcube.application.geoportal.common.model.legacy.RelazioneScavo;
 import org.gcube.application.geoportal.common.model.legacy.report.ConstraintCheck;
 
-import java.util.List;
-
 @Slf4j
 public class ConcessioniLifeCycleManager implements LifecycleManager {
 
diff --git a/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DefaultLogicHolder.java b/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DefaultLogicHolder.java
deleted file mode 100644
index 2938ad0..0000000
--- a/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DefaultLogicHolder.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.gcube.application.cms.custom.gna.concessioni.model;
-
-public interface DefaultLogicHolder {
-
-    public void setDefaults();
-
-}
diff --git a/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DocumentedRelazioneScavo.java b/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DocumentedRelazioneScavo.java
deleted file mode 100644
index eeb4187..0000000
--- a/concessioni-model/src/main/java/org/gcube/application/cms/custom/gna/concessioni/model/DocumentedRelazioneScavo.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.gcube.application.cms.custom.gna.concessioni.model;
-
-import org.bson.Document;
-
-public class DocumentedRelazioneScavo extends Document implements DefaultLogicHolder{
-
-    @Override
-    public void setDefaults() {
-
-    }
-}
diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/JSONPathWrapper.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/JSONPathWrapper.java
similarity index 90%
rename from geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/JSONPathWrapper.java
rename to geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/JSONPathWrapper.java
index d7295fe..a03532b 100644
--- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/JSONPathWrapper.java
+++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/JSONPathWrapper.java
@@ -1,11 +1,7 @@
-package org.gcube.application.geoportal.common.utils;
+package org.gcube.application.geoportal.common.model;
 
 
 import com.jayway.jsonpath.*;
-import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
-import com.jayway.jsonpath.spi.json.JsonProvider;
-import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
-import com.jayway.jsonpath.spi.mapper.MappingProvider;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.gcube.application.geoportal.common.JSONSerializationProvider;
@@ -59,7 +55,9 @@ public class JSONPathWrapper {
     }
 
     public <T> List<T> getByPath(String path,Class<T> clazz){
-        return ctx.read(path, new TypeRef<List<T>>() {});
+        List<T> l= ctx.read(path, new TypeRef<List<T>>() {});
+        l.removeIf(p->p==null);
+        return l;
     }
 
     public JSONPathWrapper set(String path, Object toSet){
diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java
index 2d42f2c..05ae45e 100644
--- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java
+++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/RegisteredFileSet.java
@@ -5,22 +5,25 @@ import org.bson.Document;
 
 import java.util.List;
 
-@NoArgsConstructor
-@AllArgsConstructor
+
 @Getter
 @Setter
-@ToString
-public class RegisteredFileSet {
+@ToString (callSuper = true)
+public class RegisteredFileSet extends Document {
 
     public static final String CREATION_INFO="creationInfo";
     public static final String ACCESS="access";
     public static final String FOLDER_ID="folderID";
     public static final String PAYLOADS="payloads";
+    public static final String MATERIALIZATIONS="materializations";
 
-    private AccountingInfo creationInfo;
-    private Access access;
-    private String folderID;
 
-    private List<RegisteredFile> payloads;
-    private List<Materialization> materializations;
+    public Object getCreationInfo(){ return this.get(CREATION_INFO); }
+
+    public Object getAccess(){ return this.get(ACCESS); }
+
+    public String getFolderId(){ return super.getString(FOLDER_ID); }
+
+    public List getPayloads(){return super.get(PAYLOADS,List.class);}
+    public List getMaterializations(){return super.get(MATERIALIZATIONS,List.class);}
 }
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java
index 6fedfd5..3226990 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java
@@ -7,7 +7,7 @@ import org.gcube.application.geoportal.service.rest.ConcessioniOverMongo;
 
 import org.gcube.application.geoportal.service.rest.ProfiledDocuments;
 import org.gcube.application.geoportal.service.rest.Sections;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.glassfish.jersey.server.ResourceConfig;
 
 import javax.ws.rs.ApplicationPath;
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java
index 12fe783..dcea0e8 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java
@@ -23,7 +23,7 @@ import org.gcube.application.geoportal.service.engine.WorkspaceManager.FileOptio
 import org.gcube.application.geoportal.service.engine.WorkspaceManager.FolderOptions;
 import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex;
 import org.gcube.application.geoportal.service.model.internal.faults.*;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.geoportal.service.utils.UserUtils;
 import org.gcube.common.storagehub.client.dsl.FolderContainer;
 import org.gcube.common.storagehub.model.exceptions.StorageHubException;
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java
index a8daf55..e653f92 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/MongoManagerI.java
@@ -40,5 +40,5 @@ public interface MongoManagerI<T> {
     public T performStep(String id, String step, Document options) throws IOException, StepException;
 
     public T registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException;
-    public T deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException;
+    public T deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException;
 }
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java
index d73a3be..79c2543 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java
@@ -22,13 +22,13 @@ import org.gcube.application.geoportal.common.model.profile.Profile;
 import org.gcube.application.geoportal.common.model.rest.QueryRequest;
 import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
 import org.gcube.application.geoportal.common.model.rest.TempFile;
-import org.gcube.application.geoportal.common.utils.JSONPathWrapper;
+import org.gcube.application.geoportal.common.model.JSONPathWrapper;
 import org.gcube.application.geoportal.common.utils.StorageUtils;
 import org.gcube.application.geoportal.service.engine.ImplementationProvider;
 import org.gcube.application.geoportal.service.engine.WorkspaceManager;
 import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
 import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.geoportal.service.utils.UserUtils;
 import org.gcube.common.storagehub.client.dsl.FolderContainer;
 import org.gcube.common.storagehub.model.exceptions.StorageHubException;
@@ -45,7 +45,7 @@ import java.util.Map;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.function.Consumer;
 
-import static org.gcube.application.cms.Serialization.*;
+import static org.gcube.application.cms.serialization.Serialization.*;
 
 @Slf4j
 public class ProfiledMongoManager extends MongoManager implements MongoManagerI<ProfiledDocument>{
@@ -274,20 +274,17 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
         if(fieldDefinitions.size()>1)
             throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+request.getFieldPath(),Response.Status.BAD_REQUEST);
         Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
+        if(fieldDefinition==null)
+            throw new WebApplicationException("Found field is null ["+profile.getId()+" for "+request.getFieldPath()+"]",Response.Status.BAD_REQUEST);
 
         log.debug("Field definition is {}",fieldDefinition);
         JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson());
+
         List<RegisteredFileSet> found=docWrapper.getByPath(request.getDestinationPath(),RegisteredFileSet.class);
-//        if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){
-//            throw new WebApplicationException("Cannot add registered fileset at "+request.getFieldPath()+" : field is not collection.",Response.Status.BAD_REQUEST);
-//        }
 
 
         Object toSet=null;
-
-        if(fieldDefinition.isCollection()){
-
-        }
+        Document toSetAttributes=attributes;
 
         // Manage clash options
 
@@ -297,39 +294,35 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
                     if(found.size()>1)
                         throw new WebApplicationException("Cannot replace multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
                     deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
-                    toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws);
                     break;
                 }case MERGE_EXISTING: {
                     if(found.size()>1)
                         throw new WebApplicationException("Cannot merge multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
-                    attributes.putAll(Serialization.asDocument(found.get(0)));
+                    toSetAttributes=Serialization.asDocument(found.get(0));
+                    if(attributes!=null) toSetAttributes.putAll(attributes);
                     deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
-                    toSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws);
                     break;
                 }case APPEND: {
                     if(!fieldDefinition.isCollection())
                         throw new WebApplicationException("Cannot append to "+request.getDestinationPath()+" : field "+request.getFieldPath()+" is not collection.",
                             Response.Status.BAD_REQUEST);
-                    RegisteredFileSet registeredFileSet=prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),attributes,files,storage,ws);
-                    found.add(registeredFileSet);
-                    toSet=found;
                     break;
                 }
             }
 
 
+        // Actually register Files
+        RegisteredFileSet registeredFileSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),
+                toSetAttributes,files,storage,ws);
+        if(fieldDefinition.isCollection()&&(request.getClashOption().equals(RegisterFileSetRequest.ClashOptions.APPEND))){
+            List<RegisteredFileSet> actualList= found.isEmpty()?new ArrayList<>():(List<RegisteredFileSet>)found.get(0);
+            actualList.add(registeredFileSet);
+            toSet=actualList;
+        }else toSet = registeredFileSet;
+
+
         docWrapper.set(request.getDestinationPath(),toSet);
 
-
-//        if(fieldDefinition.getMaxCardinality()>1){
-//            // Field is collection
-//            found.add(registeredFileSet);
-//            docWrapper.set(destination,found);
-//        }
-//        else {
-//            docWrapper.set(destination,registeredFileSet);
-//        }
-
         log.debug("Setting result on profiled document");
         doc.setTheDocument(Document.parse(docWrapper.getCtx().jsonString()));
 
@@ -339,8 +332,11 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
     }
 
     @Override
-    public ProfiledDocument deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException {
-        throw new RuntimeException("Implement this");
+    public ProfiledDocument deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException {
+        ProfiledDocument doc = getByID(id);
+        doc=deleteFileSetRoutine(doc,destination,force,new WorkspaceManager());
+        doc=onUpdate(doc);
+        return convert(replace(asDocumentWithId(doc),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
     }
 
     
@@ -368,16 +364,17 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
     private static final RegisteredFileSet prepareRegisteredFileSet(ProfiledDocument doc, Profile profile,String destination,
                                                                     Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
         log.debug("Preparing Registered FileSet..");
-
-        attributes.putIfAbsent(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
-        attributes.putIfAbsent(RegisteredFileSet.ACCESS,doc.getInfo().getAccess());
+        RegisteredFileSet toReturn = new RegisteredFileSet();
+        if(attributes!=null) toReturn.putAll(attributes);
+        toReturn.put(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
+        toReturn.putIfAbsent(RegisteredFileSet.ACCESS,doc.getInfo().getAccess());
         FolderContainer base=ws.createFolder(new WorkspaceManager.FolderOptions(
                 doc.get_id(),"Base Folder for profiled document. Profile "+profile.getId(),null));
 
         FolderContainer sectionFolder=ws.createFolder(new WorkspaceManager.FolderOptions(
                 doc.get_id()+destination,"Registered Fileset at path "+destination,base));
 
-        attributes.putIfAbsent(RegisteredFileSet.FOLDER_ID,sectionFolder.getId());
+        toReturn.put(RegisteredFileSet.FOLDER_ID,sectionFolder.getId());
 
         ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();
 
@@ -399,8 +396,9 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
                     IOUtils.closeQuietly(is);
             }
         }
-        attributes.putIfAbsent(RegisteredFileSet.PAYLOADS,registeredFiles);
-        return Serialization.convert(attributes,RegisteredFileSet.class);
+        toReturn.put(RegisteredFileSet.PAYLOADS,registeredFiles);
+        toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
+        return toReturn;
     }
 
 
@@ -414,9 +412,9 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
             // TODO manager force deletion
             // NB handlers for materialization types
         }
-        log.debug("Document ID {} :  deleting ws folder {}",doc.get_id(),toDelete.getFolderID());
+        log.debug("Document ID {} :  deleting ws folder {}",doc.get_id(),toDelete.getFolderId());
         if(toDelete.getPayloads()!=null)
-            ws.deleteItem(toDelete.getFolderID());
+            ws.deleteItem(toDelete.getFolderId());
         doc.setTheDocument(Document.parse(wrapper.set(fileSetPath,null).getCtx().jsonString()));
         return doc;
     }
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java
index cb34e79..ad8534b 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java
@@ -13,7 +13,7 @@ import org.gcube.application.geoportal.service.model.internal.db.PostgisTable.Fi
 import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
 import org.gcube.application.geoportal.service.model.internal.faults.PublishException;
 import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ProfileMapCache.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ProfileMapCache.java
index 54fbdc4..17c079a 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ProfileMapCache.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ProfileMapCache.java
@@ -5,7 +5,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.gcube.application.geoportal.common.model.profile.Profile;
 import org.gcube.application.geoportal.common.utils.Files;
 import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java
index c43f164..dbe247c 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java
@@ -9,7 +9,7 @@ import org.gcube.application.geoportal.common.rest.InterfaceConstants;
 import org.gcube.application.geoportal.service.engine.mongo.ConcessioniMongoManager;
 import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex;
 import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.json.JSONArray;
 
 import javax.ws.rs.*;
diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ProfiledDocuments.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ProfiledDocuments.java
index 2e93285..b158404 100644
--- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ProfiledDocuments.java
+++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/rest/ProfiledDocuments.java
@@ -10,7 +10,7 @@ import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
 import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
 import org.gcube.application.geoportal.service.engine.mongo.ProfiledMongoManager;
 import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java
index f868b7f..0a714ed 100644
--- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java
+++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java
@@ -2,11 +2,8 @@ package org.gcube.application.geoportal.service;
 
 import lombok.extern.slf4j.Slf4j;
 import org.gcube.application.cms.tests.TokenSetter;
-import org.gcube.application.geoportal.common.utils.StorageUtils;
-import org.gcube.application.geoportal.service.engine.ImplementationProvider;
-import org.gcube.application.geoportal.service.engine.providers.StorageClientProvider;
 import org.gcube.application.geoportal.service.rest.GuardedMethod;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.glassfish.jersey.test.JerseyTest;
 import org.junit.BeforeClass;
 
diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java
index c0afb9f..ab83fc4 100644
--- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java
+++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java
@@ -15,7 +15,7 @@ import org.gcube.application.geoportal.common.rest.InterfaceConstants;
 import org.gcube.application.geoportal.common.utils.FileSets;
 import org.gcube.application.geoportal.common.utils.Files;
 import org.gcube.application.geoportal.common.utils.StorageUtils;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.json.JSONObject;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java
index 0a25937..27d500e 100644
--- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java
+++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ProfiledDocumentsTests.java
@@ -1,25 +1,24 @@
 package org.gcube.application.geoportal.service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.jayway.jsonpath.TypeRef;
 import org.bson.Document;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
 import org.gcube.application.cms.tests.TokenSetter;
 import org.gcube.application.cms.tests.model.concessioni.TestConcessioniModel;
+import org.gcube.application.geoportal.common.model.JSONPathWrapper;
 import org.gcube.application.geoportal.common.model.document.ProfiledDocument;
 import org.gcube.application.geoportal.common.model.document.RegisteredFileSet;
-import org.gcube.application.geoportal.common.model.legacy.Concessione;
 import org.gcube.application.geoportal.common.model.rest.Configuration;
 import org.gcube.application.geoportal.common.model.rest.QueryRequest;
 import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
 import org.gcube.application.geoportal.common.rest.InterfaceConstants;
 import org.gcube.application.geoportal.common.utils.FileSets;
-import org.gcube.application.geoportal.common.utils.Files;
 import org.gcube.application.geoportal.common.utils.StorageUtils;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.internal.runners.TestMethod;
 
 import javax.ws.rs.client.Entity;
 import javax.ws.rs.client.WebTarget;
@@ -33,7 +32,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class ProfiledDocumentsTests extends BasicServiceTestUnit{
-	
+
 	String testProfileId="profiledConcessioni";
 
 	@Before
@@ -43,7 +42,7 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{
 
 
 	protected WebTarget baseTarget(){
-		 return target(InterfaceConstants.Methods.PROJECTS).path(testProfileId);
+		return target(InterfaceConstants.Methods.PROJECTS).path(testProfileId);
 	}
 	// GET
 	@Test
@@ -107,7 +106,7 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{
 	}
 
 	@Test
-	public void addFileSet() throws Exception {
+	public void testAddSingleFileSet() throws Exception {
 
 		ProfiledDocument doc = createNew();
 		// Try set releazione scavo
@@ -115,22 +114,72 @@ public class ProfiledDocumentsTests extends BasicServiceTestUnit{
 		String fieldPath="relazioneScavo";
 		String filename = "relazione.pdf";
 
+		// INSERT ONE, MERGE INFO (NB default values already exist)
 		doc = upload(
 				new StorageUtils(),
 				doc.get_id(),
 				fileSetPath,
 				fieldPath,
-				Document.parse("{\"titolo\" : \"mio titolo\"}"),
+				Document.parse("{\"titolo\" : \"mio titolo\",\"some\" : \"something\" }"),
 				RegisterFileSetRequest.ClashOptions.MERGE_EXISTING,
 				filename);
 		assertTrue("Relazione exists",doc.getTheDocument().containsKey(fileSetPath));
 		RegisteredFileSet set = Serialization.convert(doc.getTheDocument().get(fileSetPath), RegisteredFileSet.class);
-		System.out.println(set);
-		ProfiledConcessione.Relazione rel= Serialization.convert(doc.getTheDocument().get(fileSetPath), ProfiledConcessione.Relazione.class);
-		System.out.println(rel);
-		System.out.println(Serialization.write(doc));
+		assertTrue("Correctly merged attributes ",set.getString("titolo").equals("mio titolo"));
+		assertTrue("Correctly merged attributes ",set.getString("some").equals("something"));
+
+		// REPLACE SAME, CHANGE TITLE
+		doc = upload(
+				new StorageUtils(),
+				doc.get_id(),
+				fileSetPath,
+				fieldPath,
+				Document.parse("{\"titolo\" : \"mio altro titolo\"}"),
+				RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING,
+				filename);
+		set = Serialization.convert(doc.getTheDocument().get(fileSetPath), RegisteredFileSet.class);
+		assertTrue("Correctly merged attributes ",set.getString("titolo").equals("mio altro titolo"));
+		assertTrue("Correctly merged attributes ",!set.containsKey("some"));
+
+
+
+
 	}
 
+	@Test
+	public void testMutlipleFileSet() throws Exception {
+
+		ProfiledDocument doc = createNew();
+		// ADD 4 IMGS
+		for (int i = 0; i < 4; i++) {
+			doc = upload(
+					new StorageUtils(),
+					doc.get_id(),
+					"immagini",
+					"imgs",
+					null,
+					RegisterFileSetRequest.ClashOptions.APPEND,
+					"relazione.pdf");
+		}
+		assertTrue(doc.getTheDocument().containsKey("immagini"));
+		assertTrue("Expected 4 imgs registered",Serialization.convert(doc.getTheDocument().get("immagini"), List.class).size()==4);
+
+
+		// Replace img [3]
+		doc = upload(
+				new StorageUtils(),
+				doc.get_id(),
+				"immagini[3]",
+				"imgs",
+				Document.parse("{\"titolo\" : \"mia immagine\"}"),
+				RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING,
+				"relazione.pdf");
+
+		String title = new JSONPathWrapper(doc.getTheDocument().toJson()).
+				getByPath("immagini[3].titolo",String.class).get(0);
+
+		assertTrue("Changed Title",title.equals("mia immagine"));
+	}
 
 
 
diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java
index 0b88e1d..e21b416 100644
--- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java
+++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java
@@ -12,7 +12,7 @@ import org.gcube.application.geoportal.common.utils.Files;
 import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex;
 import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
 import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException;
-import org.gcube.application.cms.Serialization;
+import org.gcube.application.cms.serialization.Serialization;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;