gcube-cms-suite/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java

package org.gcube.application.geoportal.service.engine.mongo;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.FindOneAndReplaceOptions;
import com.mongodb.client.model.FindOneAndUpdateOptions;
import com.mongodb.client.model.ReturnDocument;
import com.vdurmont.semver4j.Semver;
import jdk.nashorn.internal.runtime.regexp.joni.Config;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.cms.plugins.LifecycleManager;
import org.gcube.application.cms.plugins.faults.EventException;
import org.gcube.application.cms.plugins.faults.InsufficientPrivileges;
import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.cms.plugins.model.PluginDescriptor;
import org.gcube.application.cms.plugins.reports.DocumentHandlingReport;
import org.gcube.application.cms.plugins.reports.EventExecutionReport;
import org.gcube.application.cms.plugins.reports.StepExecutionReport;
import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.plugins.requests.EventExecutionRequest;
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.configuration.Archive;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.*;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.accounting.AccountingInfo;
import org.gcube.application.geoportal.common.model.document.accounting.PublicationInfo;
import org.gcube.application.geoportal.common.model.document.accounting.User;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.DataAccessPolicy;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.model.configuration.Configuration;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.utils.ContextUtils;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.cms.implementations.ImplementationProvider;
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.service.engine.providers.PluginManager;
import org.gcube.application.geoportal.service.engine.providers.ucd.ProfileMap;
import org.gcube.application.geoportal.service.model.internal.faults.*;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.service.utils.UserUtils;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;

import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.InvalidParameterException;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;

import static org.gcube.application.cms.serialization.Serialization.*;

@Slf4j
public class ProfiledMongoManager extends MongoManager implements MongoManagerI<Project>{

    @Getter
    UseCaseDescriptor useCaseDescriptor;


    @Override
    protected String mongoIDFieldName() {
        return ID;
    }

    public ProfiledMongoManager(String profileId) throws ConfigurationException, RegistrationException {
        // Check UseCaseDescriptor ID
        log.info("Loading useCaseDescriptor ID {} ",profileId);
        if(profileId==null) throw new InvalidParameterException("UseCaseDescriptor ID cannot be null");
        Map<String, UseCaseDescriptor> profiles=ImplementationProvider.get().getEngineByManagedClass(ProfileMap.class);
        useCaseDescriptor = ImplementationProvider.get().getEngineByManagedClass(UCDManagerI.class).getById(profileId);
        if(useCaseDescriptor == null )
            throw new WebApplicationException("UseCaseDescriptor " + profileId + " not registered", Response.Status.NOT_FOUND);


        // Connect to DB
        init(getToUseCollectionName());

    }

    private String getToUseCollectionName(){
        //TODO collection name in UCD
        return useCaseDescriptor.getId();
    }


    protected Project lock(String id,String op) throws ProjectNotFoundException, ProjectLockedException, JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Locking {} cause {} ",id,op);

        Lock lock = new Lock();
        lock.setId(UUID.randomUUID().toString());
        lock.setInfo(UserUtils.getCurrent().asInfo());
        lock.setOperation(op);

        // find one and update
        // filter : id, lock == null
        // update with new Lock object
        Document filter = new Document(mongoIDFieldName(),asId(id)).append("$or",Arrays.asList(
                new Document(Project.LOCK,new Document("$exists",false)),
                new Document(Project.LOCK,new Document("$type","null"))));
        log.debug("Lock filter is {} ",filter.toJson());
        Object obj = getCollection().findOneAndUpdate(
                // filter by id and missing lock
                filter,
                // update lock info
                new Document("$set",new Document(Project.LOCK, Serialization.asDocument(lock))),
                new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER)
        );

        if(obj == null){
            // unable to lock, verify cause
            Project p=getByID(id);
            if(p.getLock()!=null) throw new ProjectLockedException("Project already locked : "+p.getLock());
            else {
                log.error("Unable to lock {} ",id);
                log.debug("Existing project is {} ",p);
                throw new RuntimeException("Unable to lock unlocked project "+id);
            }
        } else return Serialization.convert(obj,Project.class);
    }

    protected Project unlockAndUpdate(Project proj) throws InvalidLockException, ProjectNotFoundException, JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Unlocking {} lock is {} ",proj.getId(),proj.getLock());
        // find one and update
        Lock oldLock = proj.getLock();
        proj.setLock(null);

        Document filter = new Document(mongoIDFieldName(),asId(proj.getId())).append(Project.LOCK+"."+Lock.ID,oldLock.getId());

        Object obj = getCollection().findOneAndReplace(
                // filter by id and missing lock
                filter,
                // update lock info
                asDocumentWithId(proj),
                new FindOneAndReplaceOptions().returnDocument(ReturnDocument.AFTER));

        if(obj== null){
            // can-t unlock, check cause
            Project p = getByID(proj.getId());
            throw new InvalidLockException("Found lock for "+p.getId()+" is "+p.getLock()+", expected is "+oldLock);
        }return Serialization.convert(obj,Project.class);

        // filter : id, lock id
        // update with project (NB without lock)
        // if none matched
        // not found if proj id non existent
        // else invalid lock
//        return null;
    }

    @Getter(lazy = true)
   private final LifecycleManager manager=getLCManager();

    private LifecycleManager getLCManager() {
        try{
            LifecycleManager toReturn=null;
            //Getting Lifecycle Manager declaration from UseCaseDescriptor
            List<HandlerDeclaration> handlerDeclarations= useCaseDescriptor.getHandlersMapByType().get(PluginDescriptor.BaseTypes.LIFECYCLE_MANAGER);
            if(handlerDeclarations==null || handlerDeclarations.isEmpty()) throw new ConfigurationException("No Lifecycle Handler defined for useCaseDescriptor ID "+ useCaseDescriptor.getId());
            if(handlerDeclarations.size()>1) throw new ConfigurationException("Too many Lifecycle Handlers defined ("+handlerDeclarations+") in useCaseDescriptor ID "+ useCaseDescriptor.getId());

            HandlerDeclaration lcHandlerDeclaration=handlerDeclarations.get(0);

            // Loading Lifecycle Manager
            log.debug("Looking for handler {} ",lcHandlerDeclaration);
            toReturn=(LifecycleManager) ImplementationProvider.get().
                    getEngineByManagedClass(PluginManager.PluginMap.class).get(lcHandlerDeclaration.getId());
            if(toReturn==null) throw new ConfigurationException("Unable to find Lifecycle Manager Plugin. ID "+lcHandlerDeclaration.getId());

            return toReturn;
        } catch(Throwable t){
            log.warn("Unable to load LC Manager ",t);
            return null;
        }
    }


    @Override
    public Project registerNew(Document toRegisterDoc) throws IOException, StepException, EventException {
        log.info("Registering new document in {} ", useCaseDescriptor.getId());
        log.debug("Going to register {}",toRegisterDoc.toJson());

        Project toRegister = new Project();
        toRegister.setTheDocument(toRegisterDoc);

        PublicationInfo pubInfo=new PublicationInfo();
        pubInfo.setCreationInfo(UserUtils.getCurrent().asInfo());

        // TODO Set Access From UseCaseDescriptor
        Access access=new Access();
        access.setLicense("");
        access.setPolicy(AccessPolicy.OPEN);
        pubInfo.setAccess(access);

        toRegister.setInfo(pubInfo);

        toRegister.setProfileID(useCaseDescriptor.getId());
        toRegister.setProfileVersion(useCaseDescriptor.getVersion());
        toRegister.setVersion(new Semver("1.0.0"));


        LifecycleInformation draftInfo=new LifecycleInformation().cleanState();
        draftInfo.setPhase(LifecycleInformation.CommonPhases.DRAFT_PHASE);
        draftInfo.setLastOperationStatus(LifecycleInformation.Status.OK);
        toRegister.setLifecycleInformation(draftInfo);


        // Apply Lifecycle
        toRegister = triggerEvent(toRegister,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null);

        log.debug("Going to register {} ",toRegister);

        // Insert object
        ObjectId id =insertDoc(asDocumentWithId(toRegister));

        log.info("Obtained id {} ",id);
        try {
            return getByID(id.toHexString());
        }catch (ProjectNotFoundException | InvalidUserRoleException | UnauthorizedAccess e){
            throw new WebApplicationException("Unexpected exception while registering project ",e);
        }
    }

    @Override
    public Project update(String id, Document toSet) throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Replacing {} ",toSet);
        Project toUpdate=lock(id,"Manual update");
        try {

            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);

            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(toUpdate,u)) throw new UnauthorizedAccess("No edit rights on project "+id);


            toUpdate.setTheDocument(toSet);
            toUpdate.getLifecycleInformation().cleanState();
            toUpdate = onUpdate(toUpdate);
        }finally{
            return unlockAndUpdate(toUpdate);
        }
    }


    private Project onUpdate(Project toUpdate) throws EventException {
        UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
        toUpdate.getInfo().setLastEditInfo(u.asInfo());
        toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
        return triggerEvent(toUpdate,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null);
    }


        @Override
    public void delete(String id,boolean force) throws DeletionException, InvalidUserRoleException, ProjectLockedException, ProjectNotFoundException, UnauthorizedAccess, JsonProcessingException, InvalidLockException {
        log.info("Deleting by ID {}, force {}",id,force);
        Project doc =lock(id,"Deletion { force : "+force+"}");
        try {
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
            log.debug("Delete project {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);
            if (policy == null)  throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            if (!policy.canWrite(doc, u)) throw new UnauthorizedAccess("No edit rights on project " + id);

            doc = triggerEvent(doc, EventExecutionRequest.Events.ON_DELETE_DOCUMENT, new Document("force", force));
            //Only continue deleting if event was ok
            if(doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
                try {
                    WorkspaceManager ws = new WorkspaceManager();
                    // Get All registered Filesets with payloads
                    JSONPathWrapper wrapper = new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
                    for (Object obj : wrapper.getByPath("..*[?(@." + RegisteredFileSet.PAYLOADS + ")]")) {
                        Document fs = Serialization.asDocument(obj);
                        log.debug("Deleting {}",obj);
                        String folderId = fs.getString(RegisteredFileSet.FOLDER_ID);
                        ws.deleteItem(folderId);
                    }
                }finally {
                    if(force) super.deleteDoc(asId(id));
                }
            }
        } catch (ConfigurationException | StorageHubException e) {
            log.error("Exception while trying to delete {} [UCID {}]",id,useCaseDescriptor.getId());
            throw new DeletionException("Unable to contact Storage ",e);
        } finally{
            if(doc!=null) unlockAndUpdate(doc);
        }
    }

    @Override
    public Project getByID(String id) throws ProjectNotFoundException, InvalidUserRoleException, UnauthorizedAccess {
        User u = UserUtils.getCurrent().asInfo().getUser();
        DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Accessing Project {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);

        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles "+u.getRoles());
        }

        Document doc=getDocById(asId(id),
                policy==null?null:policy.getEnforcer().getFilter());
        if(doc==null) throw new ProjectNotFoundException("No document with ID "+id);

        Project p = convert(doc, Project.class);
        if(!policy.canRead(p,u)) throw new UnauthorizedAccess("No access rights on "+id);
        return p;
    }

    @Override
    public Iterable<Document> query(QueryRequest queryRequest) throws InvalidUserRoleException {
        LinkedBlockingQueue queue=new LinkedBlockingQueue<Project>();
        User u = UserUtils.getCurrent().asInfo().getUser();
        final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Querying {} [{}] , policy for {} is {} ",queryRequest,useCaseDescriptor.getId(),u,policy);
        // NB cannot check ownership on returned values, must specify filter
        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
        }
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.NONE)) {
            log.info("Read is NONE : Returning empty collection");
            return queue;
        }

        // NB cannot check ownership on returned values, must specify filter
        Document finalFilter=new Document();
        if(queryRequest.getFilter()!=null)
            finalFilter.putAll(queryRequest.getFilter());
        Document enforcerFilter =policy.getEnforcer().getFilter();
        if(enforcerFilter != null)
            finalFilter.putAll(enforcerFilter);
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.OWN))
            finalFilter.put(Project.INFO+"."+PublicationInfo.CREATION_INFO+"."+AccountingInfo.USER+"."+User.USERNAME,u.getUsername());
        queryRequest.setFilter(finalFilter);

        log.debug("Final filter is {}",queryRequest.getFilter());

        queryDoc(queryRequest).forEach(
                (Consumer<? super Document>) (Document d)->{try{
                    queue.put(d);
                }catch(Throwable t){log.warn("Unable to translate "+d);}});
        log.info("Returned {} elements ",queue.size());
        return queue;
    }

    @Override
    public Iterable<Project> filter(QueryRequest queryRequest) throws InvalidUserRoleException {
        LinkedBlockingQueue queue=new LinkedBlockingQueue<Project>();
        User u = UserUtils.getCurrent().asInfo().getUser();
        final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Querying {} [{}] , policy for {} is {} ",queryRequest,useCaseDescriptor.getId(),u,policy);
        // NB cannot check ownership on returned values, must specify filter
        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
        }
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.NONE)) {
            log.info("Read is NONE : Returning empty collection");
            return queue;
        }

        // NB cannot check ownership on returned values, must specify filter
        Document finalFilter=new Document();
        if(queryRequest.getFilter()!=null)
            finalFilter.putAll(queryRequest.getFilter());
        Document enforcerFilter =policy.getEnforcer().getFilter();
        if(enforcerFilter != null)
            finalFilter.putAll(enforcerFilter);
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.OWN))
            finalFilter.put(Project.INFO+"."+PublicationInfo.CREATION_INFO+"."+AccountingInfo.USER+"."+User.USERNAME,u.getUsername());
        queryRequest.setFilter(finalFilter);

        log.debug("Final filter is {}",queryRequest.getFilter());

        queryDoc(queryRequest).forEach(
                (Consumer<? super Document>) (Document d)->{try{
                    queue.put(d);
                }catch(Throwable t){log.warn("Unable to translate "+d);}});
        log.info("Returned {} elements ",queue.size());
        return queue;
    }


    @Override
    public Project performStep(String id, String step, Document options) throws StepException, JsonProcessingException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        Project document = lock(id,"Step "+step+" execution");
        try {

            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);
            // NB cannot check ownership on returned values, must specify filter
            if (policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if (!policy.canWrite(document, u)) throw new UnauthorizedAccess("No edit rights on project " + id);


            document.getLifecycleInformation().cleanState();
            document = step(document, step, options);
        }catch (InsufficientPrivileges | ConfigurationException e){
            throw e;
        } catch(Throwable t){
            log.error("[UseCaseDescriptor {} ] ERROR Invoking Step {} on document {}" , useCaseDescriptor.getId(),step,id,t);
            LifecycleInformation info = new LifecycleInformation();
            info.setPhase(document.getLifecycleInformation().getPhase());
            info.setLastOperationStatus(LifecycleInformation.Status.ERROR);
            info.addErrorMessage(t.getMessage());
            info.setLastInvokedStep(step);
            document.setLifecycleInformation(info);
        }finally{
            log.info("Storing {} [UseCaseDescriptor {}] After Step {}, Status : {} " ,id, useCaseDescriptor.getId(),
                    step,document.getLifecycleInformation().getLastOperationStatus());
            log.debug("LifecycleInformation is {} ",document.getLifecycleInformation());
            if(log.isTraceEnabled())log.trace("Document is {} ",Serialization.write(document));
            return unlockAndUpdate(document);
        }
    }



    /**
     * NB Put at path :
     *
     *     Path Examples
     *      artifact
     *      images
     *      images[1]
     *      layers[?(@.name = 'myName')].fileset
     *
     *
     *
     */
    @Override
    public Project registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.info("Registering Fileset for {} [useCaseDescriptor ID {}], Request is {} ",id, useCaseDescriptor.getId(),request);

        List<TempFile> files=request.getStreams();
        Document attributes =request.getAttributes();
        Project doc=lock(id,"Register Fileset");
        try {

            // Checking user rights on proj
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);
            // NB cannot check ownership on returned values, must specify filter
            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(doc,u)) throw new UnauthorizedAccess("No edit rights on project "+id);



            doc.getLifecycleInformation().cleanState();
            doc.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.OK);

            WorkspaceManager ws = new WorkspaceManager();
            StorageUtils storage = ImplementationProvider.get().getEngineByManagedClass(StorageUtils.class);


            log.debug("Checking field {} definition in {}", request.getFieldDefinitionPath(), useCaseDescriptor.getId());
            Field fieldDefinition = getFieldDefinition(useCaseDescriptor, request.getFieldDefinitionPath());

            JSONPathWrapper docWrapper = new JSONPathWrapper(doc.getTheDocument().toJson());

            List<String> matchingPaths = docWrapper.getMatchingPaths(request.getParentPath());
            if (matchingPaths.size() > 1)
                throw new WebApplicationException("Multiple Destination matching parent path " + request.getParentPath(), Response.Status.BAD_REQUEST);
            if (matchingPaths.isEmpty())
                throw new WebApplicationException("PArent path not found at " + request.getParentPath(), Response.Status.BAD_REQUEST);

            String parentMatchingPath = matchingPaths.get(0);
            List<Object> foundElementsByMatchingPaths = docWrapper.getByPath(parentMatchingPath);
            if (foundElementsByMatchingPaths == null || foundElementsByMatchingPaths.isEmpty())
                throw new WebApplicationException("No element found at " + parentMatchingPath, Response.Status.BAD_REQUEST);

            Document parent = Serialization.asDocument(foundElementsByMatchingPaths.get(0));


            // PREPARE REGISTERED FS

            // MANAGE CLASH
            switch (request.getClashOption()) {
                case REPLACE_EXISTING: {
                    if (fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot replace repeatable field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    // DELETE EXISTING AND PUT
                    RegisteredFileSet toDelete = Serialization.convert(parent.get(request.getFieldName()), RegisteredFileSet.class);
                    if (!(toDelete == null) && !(toDelete.isEmpty())) {
                        String path = parentMatchingPath+"."+request.getFieldName();
                        deleteFileSetRoutine(doc, false, path);

                    }
                    RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(), doc.getId(), useCaseDescriptor.getId(), request.getAttributes(), files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());
                    docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
                    break;
                }
                case MERGE_EXISTING: {
                    if (fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot merge repeatable field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    RegisteredFileSet original = Serialization.convert(parent.get(request.getFieldName()), RegisteredFileSet.class);
                    // MERGE ATTRIBUTES AND PUT
                    Document toUseAttributes = request.getAttributes();
                    if (original != null) toUseAttributes.putAll(original);
                    RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(), doc.getId(), useCaseDescriptor.getId(), toUseAttributes, files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());
                    docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
                    break;
                }
                case APPEND: {
                    if (!fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot add to single field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(), doc.getId(), useCaseDescriptor.getId(), request.getAttributes(), files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());

                    docWrapper.addElementToArray(String.format("%1ds['%2$s']", parentMatchingPath, request.getFieldName()), fs);
                    break;
                }
                default: {
                    throw new WebApplicationException("Unexpected clash policy " + request.getClashOption(), Response.Status.BAD_REQUEST);
                }
            }


            log.debug("Setting result on profiled document");
            doc.setTheDocument(Document.parse(docWrapper.getValueCTX().jsonString()));

            doc = onUpdate(doc);
        }catch (Throwable t){
            log.warn("Unexpected Exception while trying to registering fileset on {}.",id,t);
            log.debug("Request was {}",request);
            log.debug("Complete doc was {} ",doc);
        }finally {
            return unlockAndUpdate(doc);
        }
    }


    @Override
    public Project deleteFileSet(String id, String path, Boolean force) throws ConfigurationException, StorageHubException, JsonProcessingException, DeletionException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.info("Deleting Fileset for {} [useCaseDescriptor ID {}], at {} [force {} ]",id, useCaseDescriptor.getId(),path,force);
        Project doc = lock(id,"Fileset Deletion");
        try {
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.debug("Deleting Fileset for {} [{}] , policy for {} is {} ",doc.getId(),useCaseDescriptor.getId(),u,policy);
            // NB cannot check ownership on returned values, must specify filter
            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(doc,u)) throw new UnauthorizedAccess("No edit rights on project "+doc.getId());

            doc.getLifecycleInformation().cleanState();
            doc.getLifecycleInformation().cleanState().setLastOperationStatus(LifecycleInformation.Status.OK);

            doc= deleteFileSetRoutine(doc,force,path);
            doc = onUpdate(doc);
            return doc;
        }finally {
            return unlockAndUpdate(doc);
        }
    }


    private Project deleteFileSetRoutine(Project doc,Boolean force, String path) throws ConfigurationException, StorageHubException {
        JSONPathWrapper wrapper = new JSONPathWrapper(doc.getTheDocument().toJson());
        List<String> matchingPaths = wrapper.getMatchingPaths(path);
        if (matchingPaths.isEmpty())
            throw new WebApplicationException("No Registered FileSet found at " + path, Response.Status.BAD_REQUEST);
        if (matchingPaths.size() > 1)
            throw new WebApplicationException("Multiple Fileset (" + matchingPaths.size() + ") matching " + path, Response.Status.BAD_REQUEST);

        RegisteredFileSet fs = Serialization.convert(wrapper.getByPath(path), RegisteredFileSet.class);
        log.debug("Going to delete {}", fs);

        doc = triggerEvent(doc,EventExecutionRequest.Events.ON_DELETE_FILESET,new Document("force",force).append("path",path));

        // Actually delete only if event was ok
        if(doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
            // Delete from storage
            if(fs.getFolderId()!=null) {
                log.info("Deleting Fileset Folder ID {} ",fs.getFolderId());
                new WorkspaceManager().deleteItem(fs.getFolderId());
            }
            log.debug("Removing FS from document [ID : ] by path {}", doc.getId(), path);
            // Delete from document
            wrapper.setElement(path, null);
        }
        return doc;
    }



    @Override
    public Configuration getConfiguration() throws ConfigurationException{
        log.debug("Asking configuration for {} in {} ", useCaseDescriptor.getId(), UserUtils.getCurrent().getContext());
        Configuration toReturn= new Configuration();
        List<Archive> archives = new ArrayList<>();
        toReturn.setArchives(archives);
        List<Index> indexes=new ArrayList<>();
        toReturn.setIndexes(indexes);
        // Set Basic Info
        toReturn.setProfileId(this.getUseCaseDescriptor().getId());
        toReturn.setContext(ContextUtils.getCurrentScope());
        toReturn.setLastUpdatedTime(LocalDateTime.now());

        // Add Mongo Info
        Archive mongoArchive = new Archive("DOCUMENT-STORE-COLLECTION");
        MongoCollection coll=getCollection();
        mongoArchive.put("count",coll.count());
        mongoArchive.put("collection_name",getToUseCollectionName());
        archives.add(mongoArchive);

        // TODO ADD TEXT INDEXES

        // Set WS Info
        try {
            archives.add(new WorkspaceManager().getConfiguration());
        }catch (Exception e) {
            toReturn.addErrorMessage("Unable to get WS info "+e.getMessage());
            log.error("Unable to get WS Configuration",e);
        }

        // ADD LC Infos

        AccountingInfo user = UserUtils.getCurrent().asInfo();
        try{
            Configuration lcConfig = getLCManager().getCurrentConfiguration(new BaseRequest(useCaseDescriptor,user.getUser(),user.getContext()));
        log.info("Configuration is {} ",lcConfig);
        if(lcConfig.getArchives()!=null)
            archives.addAll(lcConfig.getArchives());

        if(lcConfig.getIndexes()!=null)
            indexes.addAll(lcConfig.getIndexes());
        }catch(ConfigurationException e){
            toReturn.addErrorMessage("Unable to get Lifecycle info "+e.getMessage());
            log.error("Unable to get Lifecycle info ",e);
        }

        log.debug("Returning current configuration {}",toReturn);
        return toReturn;
    }




    private Project step(Project theDocument, String step, Document callParameters) throws InsufficientPrivileges, ConfigurationException {
        try {
            log.info("[UseCaseDescriptor {}] Invoking Step {} on {}", useCaseDescriptor.getId(), step, getManager().getDescriptor());
            AccountingInfo user = UserUtils.getCurrent().asInfo();


            StepExecutionRequest request = new StepExecutionRequest(useCaseDescriptor, user.getUser(), user.getContext(), theDocument, step);

            log.debug("Requesting Step Execution {}", request);
            StepExecutionReport report = getManager().performStep(request);
            Project toReturn = report.prepareResult();

            // EVENTS
            if (report.getToTriggerEvents() != null) {
                Iterator<EventExecutionRequest> eventIT = report.getToTriggerEvents().listIterator();
                while (!toReturn.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.ERROR)
                        && eventIT.hasNext()) {
                    EventExecutionRequest triggerRequest = eventIT.next();
                    log.info("Triggering {} ", triggerRequest);
                    toReturn = triggerEvent(toReturn, triggerRequest.getEvent(), triggerRequest.getCallParameters());
                }
            }

            // STEPS
            if (report.getCascadeSteps() != null) {
                Iterator<StepExecutionRequest> stepIT = report.getCascadeSteps().listIterator();
                while (!toReturn.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.ERROR)
                        && stepIT.hasNext()) {
                    StepExecutionRequest triggerRequest = stepIT.next();
                    log.info("Triggering {} ", triggerRequest);
                    toReturn = step(toReturn, triggerRequest.getStep(), triggerRequest.getCallParameters());
                }
            }
            return report.prepareResult();
        }catch (InsufficientPrivileges | ConfigurationException e){
            throw e;
        }catch(Throwable t){
            log.error("Unable to perform step "+step,t);
            theDocument.getLifecycleInformation().addErrorMessage("Unable to perform step "+step+" cause : "+ t.getMessage());
            theDocument.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
            return theDocument;
        }
    }

    private Project triggerEvent(Project project, String event, Document parameters) {
        try{
        log.info("[UseCaseDescriptor {}] triggering event {} on {}" , useCaseDescriptor.getId(),event,getManager().getDescriptor());
            AccountingInfo user= UserUtils.getCurrent().asInfo();
        EventExecutionRequest request= new EventExecutionRequest(useCaseDescriptor,user.getUser(),user.getContext(),project,event);
        log.debug("Triggering {}",request);
        DocumentHandlingReport report = getManager().onEvent(request);
        return report.prepareResult();
        } catch (Throwable t){
            log.error("Unable to trigger event "+event,t);
            project.getLifecycleInformation().addErrorMessage("Unable to trigger "+event+" cause : "+ t.getMessage());
            project.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
            return project;
        }
    }


    private static final RegisteredFileSet prepareRegisteredFileSet(PublicationInfo defaultPublicationInfo,String docID, String profileID,
                                                                    Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
        log.debug("Preparing Registered FileSet..");
        RegisteredFileSet toReturn = new RegisteredFileSet();
        if(attributes!=null) toReturn.putAll(attributes);
        String uuid = UUID.randomUUID().toString();
        toReturn.putIfAbsent(RegisteredFileSet.UUID, uuid);
        toReturn.putIfAbsent(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
        toReturn.putIfAbsent(RegisteredFileSet.ACCESS,defaultPublicationInfo.getAccess());

        // FOLDER
        String folderID=toReturn.getFolderId();
        log.trace("Folder ID is {} ",folderID);
        FolderContainer sectionFolder=null;
        if(folderID==null || folderID.isEmpty()) {
            FolderContainer base = ws.createFolder(new WorkspaceManager.FolderOptions(
                    docID, "Base Folder for profiled document. UseCaseDescriptor " + profileID, null));
            sectionFolder = ws.createFolder(new WorkspaceManager.FolderOptions(
                    docID + "_" + uuid, "Registered Fileset uuid " + uuid, base));
            toReturn.put(RegisteredFileSet.FOLDER_ID, sectionFolder.getId());
        }else {
            sectionFolder = ws.getFolderById(folderID);
        }

        ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();

        if(toReturn.containsKey(RegisteredFileSet.PAYLOADS))
            registeredFiles.addAll(toReturn.getPayloads());

        for (TempFile f : files) {
            InputStream is=null;
            try{
                log.debug("Opening temp file {}",f);
                String fileUrl=storage.getURL(f.getId());
                log.debug("Got URL {} from ID {}",fileUrl,f.getId());
                is=new URL(fileUrl).openStream();
                RegisteredFile registered=ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(),is,
                        "Imported via gcube CMS service ", sectionFolder));
                log.debug("Registered "+registered);
                registeredFiles.add(registered);
            }catch(StorageHubException | IOException e){
                throw new StorageException("Unable to store "+f,e);
            }finally{
                if(is!=null)
                    IOUtils.closeQuietly(is);
            }
        }
        toReturn.put(RegisteredFileSet.PAYLOADS,registeredFiles);
        // TODO MERGE
        //toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
        return toReturn;
    }



    private static Field getFieldDefinition(UseCaseDescriptor useCaseDescriptor, String fieldPath)throws WebApplicationException{
        JSONPathWrapper schemaWrapper= new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
        List<Field> fieldDefinitions=schemaWrapper.getByPath(fieldPath,Field.class);
        if(fieldDefinitions==null || fieldDefinitions.isEmpty())
            throw new WebApplicationException("No Field found in schema "+ useCaseDescriptor.getId()+" at "+fieldPath, Response.Status.BAD_REQUEST);
        if(fieldDefinitions.size()>1)
            throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+ useCaseDescriptor.getId()+" for "+fieldPath,Response.Status.BAD_REQUEST);
        Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
        if(fieldDefinition==null)
            throw new WebApplicationException("Found field is null ["+ useCaseDescriptor.getId()+" for "+fieldPath+"]",Response.Status.BAD_REQUEST);
        log.trace("Field definition is {}",fieldDefinition);
        return fieldDefinition;
    }
}