gcube-cms-suite/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java

package org.gcube.application.geoportal.service.engine.mongo;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.FindOneAndReplaceOptions;
import com.mongodb.client.model.FindOneAndUpdateOptions;
import com.mongodb.client.model.ReturnDocument;
import com.vdurmont.semver4j.Semver;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.cms.implementations.ImplementationProvider;
import org.gcube.application.cms.implementations.WorkspaceManager;
import org.gcube.application.cms.implementations.faults.*;
import org.gcube.application.cms.implementations.utils.UserUtils;
import org.gcube.application.cms.plugins.LifecycleManager;
import org.gcube.application.cms.plugins.faults.EventException;
import org.gcube.application.cms.plugins.faults.InsufficientPrivileges;
import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.cms.plugins.faults.UnrecognizedStepException;
import org.gcube.application.cms.plugins.reports.DocumentHandlingReport;
import org.gcube.application.cms.plugins.reports.StepExecutionReport;
import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.plugins.requests.EventExecutionRequest;
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.configuration.Archive;
import org.gcube.application.geoportal.common.model.configuration.Configuration;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.Lock;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.accounting.AccountingInfo;
import org.gcube.application.geoportal.common.model.document.accounting.PublicationInfo;
import org.gcube.application.geoportal.common.model.document.accounting.User;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.document.relationships.Relationship;
import org.gcube.application.geoportal.common.model.plugins.LifecycleManagerDescriptor;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.*;
import org.gcube.application.geoportal.common.utils.ContextUtils;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.providers.PluginManager;

import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;

import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.InvalidParameterException;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;
import java.util.stream.Collectors;

import static org.gcube.application.cms.serialization.Serialization.asDocumentWithId;
import static org.gcube.application.cms.serialization.Serialization.convert;

@Slf4j
public class ProfiledMongoManager extends MongoManager implements MongoManagerI<Project>{

    @Getter
    UseCaseDescriptor useCaseDescriptor;


    @Override
    protected String mongoIDFieldName() {
        return ID;
    }

    public  ProfiledMongoManager(String profileId) throws ConfigurationException, RegistrationException {
        // Check UseCaseDescriptor ID
        log.info("Loading useCaseDescriptor ID {} ",profileId);
        if(profileId==null) throw new InvalidParameterException("UseCaseDescriptor ID cannot be null");
        useCaseDescriptor = ImplementationProvider.get().getProvidedObjectByClass(UCDManagerI.class).getById(profileId);
        if(useCaseDescriptor == null )
            throw new WebApplicationException("UseCaseDescriptor " + profileId + " not registered", Response.Status.NOT_FOUND);


        // Connect to DB
        init(getToUseCollectionName());

    }

    private String getToUseCollectionName(){
        //TODO collection name in UCD
        return useCaseDescriptor.getId();
    }


    protected Project lock(String id,String op) throws ProjectNotFoundException, ProjectLockedException, JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Locking {} cause {} ",id,op);

        Lock lock = new Lock();
        lock.setId(UUID.randomUUID().toString());
        lock.setInfo(UserUtils.getCurrent().asInfo());
        lock.setOperation(op);

        // find one and update
        // filter : id, lock == null
        // update with new Lock object
        Document filter = new Document(mongoIDFieldName(),asId(id)).append("$or",Arrays.asList(
                new Document(Project.LOCK,new Document("$exists",false)),
                new Document(Project.LOCK,new Document("$type","null"))));
        log.debug("Lock filter is {} ",filter.toJson());
        Object obj = getCollection().findOneAndUpdate(
                // filter by id and missing lock
                filter,
                // update lock info
                new Document("$set",new Document(Project.LOCK, Serialization.asDocument(lock))),
                new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER)
        );

        if(obj == null){
            // unable to lock, verify cause
            Project p=getByID(id);
            if(p.getLock()!=null) throw new ProjectLockedException("Project already locked : "+p.getLock());
            else {
                log.error("Unable to lock {} ",id);
                log.debug("Existing project is {} ",p);
                throw new RuntimeException("Unable to lock unlocked project "+id);
            }
        } else return Serialization.convert(obj,Project.class);
    }

    protected Project unlockAndUpdate(Project proj) throws InvalidLockException, ProjectNotFoundException, JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Unlocking for update {} lock is {} ",proj.getId(),proj.getLock());
        // find one and update
        Lock oldLock = proj.getLock();
        proj.setLock(null);

        Document filter = new Document(mongoIDFieldName(),asId(proj.getId())).append(Project.LOCK+"."+Lock.ID,oldLock.getId());

        Object obj = getCollection().findOneAndReplace(
                // filter by id and missing lock
                filter,
                // update lock info
                asDocumentWithId(proj),
                new FindOneAndReplaceOptions().returnDocument(ReturnDocument.AFTER));

        if(obj== null){
            // can-t unlock, check cause
            Project p = getByID(proj.getId());
            throw new InvalidLockException("Found lock for "+p.getId()+" is "+p.getLock()+", expected is "+oldLock);
        }return Serialization.convert(obj,Project.class);
    }

    protected Project unlock(Project proj) throws InvalidLockException, InvalidUserRoleException, ProjectNotFoundException, UnauthorizedAccess {
        log.trace("Unlocking for update {} lock is {} ",proj.getId(),proj.getLock());
        // find one and update
        Lock oldLock = proj.getLock();
        Document filter = new Document(mongoIDFieldName(),asId(proj.getId())).append(Project.LOCK+"."+Lock.ID,oldLock.getId());

        Object obj = getCollection().findOneAndUpdate(
                // filter by id and missing lock
                filter,
                // update lock info
                new Document("$set",new Document(Project.LOCK, null)),
                new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER)
        );

        if(obj== null){
            // can-t unlock, check cause
            Project p = getByID(proj.getId());
            throw new InvalidLockException("Found lock for "+p.getId()+" is "+p.getLock()+", expected is "+oldLock);
        }return Serialization.convert(obj,Project.class);
    }



    @Getter(lazy = true)
    private final LifecycleManager manager=getLCManager();

    private LifecycleManager getLCManager() {
        try{
            LifecycleManager toReturn=null;
            //Getting Lifecycle Manager declaration from UseCaseDescriptor
            List<HandlerDeclaration> handlerDeclarations= useCaseDescriptor.getHandlersMapByType().get(LifecycleManagerDescriptor.LIFECYCLE_MANAGER_TYPE);
            if(handlerDeclarations==null || handlerDeclarations.isEmpty()) throw new ConfigurationException("No Lifecycle Handler defined for useCaseDescriptor ID "+ useCaseDescriptor.getId());
            if(handlerDeclarations.size()>1) throw new ConfigurationException("Too many Lifecycle Handlers defined ("+handlerDeclarations+") in useCaseDescriptor ID "+ useCaseDescriptor.getId());

            HandlerDeclaration lcHandlerDeclaration=handlerDeclarations.get(0);

            // Loading Lifecycle Manager
            log.debug("Looking for handler {} ",lcHandlerDeclaration);
            toReturn=(LifecycleManager) ImplementationProvider.get().
                    getProvidedObjectByClass(PluginManager.PluginMap.class).get(lcHandlerDeclaration.getId());
            if(toReturn==null) throw new ConfigurationException("Unable to find Lifecycle Manager Plugin. ID "+lcHandlerDeclaration.getId());

            return toReturn;
        } catch(Throwable t){
            log.warn("Unable to load LC Manager ",t);
            return null;
        }
    }


    @Override
    public Project registerNew(Document toRegisterDoc) throws IOException, InvalidUserRoleException {
        log.info("Registering new document in {} ", useCaseDescriptor.getId());
        log.trace("Going to register {}",toRegisterDoc.toJson());


        User u = UserUtils.getCurrent().asInfo().getUser();
        final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.trace("Access policy for user {} is {} ",u,policy);

        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
        }
        if(policy.getPolicy().getWrite().equals(DataAccessPolicy.Policy.Type.none))
            throw new InvalidUserRoleException("User doesn't have write privileges " + u.getRoles());



        Project toRegister = new Project();
        toRegister.setTheDocument(toRegisterDoc);

        PublicationInfo pubInfo=new PublicationInfo();
        pubInfo.setCreationInfo(UserUtils.getCurrent().asInfo());

        // TODO Set Access From UseCaseDescriptor
        Access access=new Access();
        access.setLicense("");
        access.setPolicy(AccessPolicy.OPEN);
        pubInfo.setAccess(access);




        toRegister.setInfo(pubInfo);

        toRegister.setProfileID(useCaseDescriptor.getId());
        toRegister.setProfileVersion(useCaseDescriptor.getVersion());
        toRegister.setVersion(new Semver("1.0.0"));


        LifecycleInformation draftInfo=new LifecycleInformation().cleanState();
        draftInfo.setPhase(LifecycleInformation.CommonPhases.DRAFT_PHASE);
        draftInfo.setLastOperationStatus(LifecycleInformation.Status.OK);
        toRegister.setLifecycleInformation(draftInfo);


        // Apply Lifecycle
        toRegister = triggerEvent(toRegister,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null);

        log.debug("Going to register {} ",toRegister);

        // Insert object
        ObjectId id =insertDoc(asDocumentWithId(toRegister));

        log.info("Obtained id {} ",id);
        try {
            return getByID(id.toHexString());
        }catch (ProjectNotFoundException | InvalidUserRoleException | UnauthorizedAccess e){
            throw new WebApplicationException("Unexpected exception while registering project ",e);
        }
    }

    @Override
    public Project update(String id, Document toSet) throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.trace("Replacing {} ",toSet);
        Project toUpdate=lock(id,"Manual update");
        try {

            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);

            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(toUpdate,u)) throw new UnauthorizedAccess("No edit rights on project "+id);


            toUpdate.setTheDocument(toSet);
            toUpdate.getLifecycleInformation().cleanState();
            toUpdate = onUpdate(toUpdate);
            return unlockAndUpdate(toUpdate);
        }catch(Throwable t){
            log.error("Unexpected exception ",t);
            unlock(toUpdate);
            throw t;
        }
    }

    @Override
    public Project setRelation(String id, String relation, String targetUCD, String targetId) throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess, RegistrationException, ConfigurationException {
        Project toUpdate=lock(id,"Set Relation "+relation+" toward "+targetUCD+":"+targetId);
        try{
            log.info(getUseCaseDescriptor().getId()+":"+id+" setting relation "+relation+" toward "+targetUCD+":"+targetId);
            // Check if relation is defined
            String toSetReverseRelation=null;
            List<RelationshipDefinition> existingDefinitions = getUseCaseDescriptor().getRelationshipDefinitions();
            for(RelationshipDefinition def : existingDefinitions)
                if(def.getId().equals(relation))
                    toSetReverseRelation =def.getReverseRelationId();

            log.debug("{} reverse relation for {} is {}",getUseCaseDescriptor().getId(),relation,toSetReverseRelation);

            // check if relation existing
            List<Relationship> relations = toUpdate.getRelationshipsByName(relation);
            if(!relations.isEmpty()){
                // check if targetUCD+targetID already present
                List matching = relations.stream().filter(r ->
                        r.getTargetID().equals(targetId)&&r.getTargetUCD().equals(targetUCD)).collect(Collectors.toList());
                if(matching.size()>0) throw new WebApplicationException("Relationship "+relation+" -> "+targetUCD+" : "+targetId+" already set", Response.Status.EXPECTATION_FAILED);
            }
            // check if target exists
            ProfiledMongoManager otherManager = (targetUCD.equals(this.useCaseDescriptor.getId()))?this:new ProfiledMongoManager(targetUCD);
            Project other = getByID(targetId);

            // add relation
            Relationship rel = new Relationship();
            rel.setRelationshipName(relation);
            rel.setTargetID(targetId);
            rel.setTargetUCD(targetUCD);
            toUpdate =onUpdate(toUpdate.addRelation(rel));

            // set reverse relation
            if(toSetReverseRelation!=null){
                Relationship reverseRel = new Relationship();
                reverseRel.setRelationshipName(toSetReverseRelation);
                reverseRel.setTargetID(id);
                reverseRel.setTargetUCD(getUseCaseDescriptor().getId());
                log.info("Setting reverse relation {} ",reverseRel);
                other = otherManager.lock(other.getId(),"Setting reverse relation "+reverseRel);
                other.addRelation(reverseRel);
                otherManager.unlockAndUpdate(other);
            }

            return unlockAndUpdate(toUpdate);
        }catch(Throwable t){
            log.error("Unexpected exception ",t);
            unlock(toUpdate);
            throw t;
        }
    }

    @Override
    public Project deleteRelation(String id, String relation, String targetUCD, String targetId) throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess, RegistrationException, ConfigurationException {
        log.info("Delete relation {}:{}--{}-->{}:{} ",getUseCaseDescriptor().getId(),id,relation,targetUCD,targetId);
        Project toUpdate=lock(id,"Delete Relation");

        try{
            // SET target UCD to present UCD as default
            final String toUseTargetUCD = (targetUCD == null || targetUCD.equals(""))? getUseCaseDescriptor().getId():targetUCD;

            String toDeleteReverseRelation=null;
            List<RelationshipDefinition> existingDefinitions = getUseCaseDescriptor().getRelationshipDefinitions();
            for(RelationshipDefinition def : existingDefinitions)
                if(def.getId().equals(relation))
                    toDeleteReverseRelation =def.getReverseRelationId();

            log.debug("{} reverse relation for {} is {}",getUseCaseDescriptor().getId(),relation,toDeleteReverseRelation);

            // check if relation existing
            List<Relationship> relations = toUpdate.getRelationships();
            if(relations!=null && !relations.isEmpty()){
                int beforeSize = relations.size();

                ArrayList<Relationship> toRemove = new ArrayList<>();
                for(Relationship r : toUpdate.getRelationships()){
                    if(r.getRelationshipName().equals(relation)&&
                            r.getTargetUCD().equals(toUseTargetUCD)&&
                            r.getTargetID().equals(targetId)){
                        // set to be remove
                        log.debug("Removing {} ",r);
                        toRemove.add(r);

                        // delete reverse relation
                        if(toDeleteReverseRelation!=null){
                            log.debug("Removing reverse of {} ",r);
                            ProfiledMongoManager otherManager = (toUseTargetUCD.equals(this.useCaseDescriptor.getId()))?this:new ProfiledMongoManager(targetUCD);
                            Project other = getByID(targetId);
                            other = otherManager.lock(other.getId(),"Remove reverse relation "+toDeleteReverseRelation + " toward "+getUseCaseDescriptor().getId()+":"+id);
                            final String finalToDeleteReverseRelation = toDeleteReverseRelation;
                            other.getRelationships().removeIf(revRel ->revRel.getRelationshipName().equals(finalToDeleteReverseRelation)&&
                                    revRel.getTargetID().equals(id)&&revRel.getTargetUCD().equals(getUseCaseDescriptor().getId()));
                            otherManager.unlockAndUpdate(other);
                        }

                    }
                }

                toUpdate.getRelationships().removeAll(toRemove);

                // update only if something changed
                if(toUpdate.getRelationships().size()!=beforeSize) {
                    log.debug("Removed {} relations from {} ",(toUpdate.getRelationships().size()!=beforeSize),id);
                    return unlockAndUpdate(toUpdate);
                }
            }
            log.debug("Relationship not found. Unlocking.. ");
            return unlock(toUpdate);
        }catch(Throwable t){
            log.error("Unexpected exception ",t);
            unlock(toUpdate);
            throw t;
        }
    }


    private Project onUpdate(Project toUpdate) throws EventException {
        UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
        toUpdate.getInfo().setLastEditInfo(u.asInfo());
        toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
        return triggerEvent(toUpdate,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null);
    }


    @Override
    public void delete(String id,boolean force) throws DeletionException, InvalidUserRoleException, ProjectLockedException, ProjectNotFoundException, UnauthorizedAccess, JsonProcessingException, InvalidLockException {
        log.info("Deleting by ID {}, force {}",id,force);
        Project doc =lock(id,"Deletion { force : "+force+"}");
        boolean deleted = false;
        try {
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
            log.debug("Delete project {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);
            if (policy == null)  throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            if (!policy.canWrite(doc, u)) throw new UnauthorizedAccess("No edit rights on project " + id);

            doc = triggerEvent(doc, EventExecutionRequest.Events.ON_DELETE_DOCUMENT, new Document("force", force));
            //Only continue deleting if event was ok
            if(doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
                try {
                    WorkspaceManager ws = new WorkspaceManager();
                    // Get All registered Filesets with payloads
                    JSONPathWrapper wrapper = new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
                    for (Object obj : wrapper.getByPath("$..[?(@." + RegisteredFileSet.PAYLOADS + ")]")) {
                        Document fs = Serialization.asDocument(obj);
                        log.debug("Deleting {}",obj);
                        String folderId = fs.getString(RegisteredFileSet.FOLDER_ID);
                        ws.deleteItem(folderId);
                    }
                }finally {
                    super.deleteDoc(asId(id));
                    deleted = true;
                }
            }
        } catch (ConfigurationException | StorageHubException e) {
            log.error("Exception while trying to delete {} [UCID {}]",id,useCaseDescriptor.getId());
            throw new DeletionException("Unable to contact Storage ",e);
        } finally{
            if(doc!=null && !deleted) unlockAndUpdate(doc);
        }
    }

    @Override
    public Project getByID(String id) throws ProjectNotFoundException, InvalidUserRoleException, UnauthorizedAccess {
        User u = UserUtils.getCurrent().asInfo().getUser();
        DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Accessing Project {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);

        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles "+u.getRoles());
        }

        Document doc=getDocById(asId(id),
                (policy==null||policy.getEnforcer()==null)?null:policy.getEnforcer().getFilterDocument());
        if(doc==null) throw new ProjectNotFoundException("No document with ID "+id);

        Project p = convert(doc, Project.class);
        if(!policy.canRead(p,u)) throw new UnauthorizedAccess("No access rights on "+id);
        return p;
    }

    @Override
    public Iterable<Document> query(QueryRequest queryRequest) throws InvalidUserRoleException {
        LinkedBlockingQueue queue=new LinkedBlockingQueue<Project>();
        User u = UserUtils.getCurrent().asInfo().getUser();
        final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Querying {} [{}] , policy for {} is {} ",queryRequest,useCaseDescriptor.getId(),u,policy);
        // NB cannot check ownership on returned values, must specify filter
        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
        }
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.none)) {
            log.info("Read is NONE : Returning empty collection");
            return queue;
        }

        // NB cannot check ownership on returned values, must specify filter
        Document finalFilter=new Document();
        if(queryRequest.getFilter()!=null)
            finalFilter.putAll(queryRequest.getFilter());
        if(policy.getEnforcer() != null)
            finalFilter.putAll(policy.getEnforcer().getFilterDocument());
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.own))
            finalFilter.put(Project.INFO+"."+PublicationInfo.CREATION_INFO+"."+AccountingInfo.USER+"."+User.USERNAME,u.getUsername());
        queryRequest.setFilter(finalFilter);

        log.debug("Final filter is {}",queryRequest.getFilter());

        queryDoc(queryRequest).forEach(
                (Consumer<? super Document>) (Document d)->{try{
                    queue.put(d);
                }catch(Throwable t){log.warn("Unable to translate "+d);}});
        log.info("Returned {} elements ",queue.size());
        return queue;
    }

    @Override
    public Iterable<Project> filter(QueryRequest queryRequest) throws InvalidUserRoleException {
        LinkedBlockingQueue queue=new LinkedBlockingQueue<Project>();
        User u = UserUtils.getCurrent().asInfo().getUser();
        final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
        log.info("Querying {} [{}] , policy for {} is {} ",queryRequest,useCaseDescriptor.getId(),u,policy);
        // NB cannot check ownership on returned values, must specify filter
        if(policy == null) {
            log.warn("No policy found for {}. Returning empty ", u);
            throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
        }
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.none)) {
            log.info("Read is NONE : Returning empty collection");
            return queue;
        }

        // NB cannot check ownership on returned values, must specify filter
        Document finalFilter=new Document();
        if(queryRequest.getFilter()!=null)
            finalFilter.putAll(queryRequest.getFilter());
        if(policy.getEnforcer() != null)
            finalFilter.putAll(policy.getEnforcer().getFilterDocument());
        if(policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.own))
            finalFilter.put(Project.INFO+"."+PublicationInfo.CREATION_INFO+"."+AccountingInfo.USER+"."+User.USERNAME,u.getUsername());
        queryRequest.setFilter(finalFilter);

        log.debug("Final filter is {}",queryRequest.getFilter());

        queryDoc(queryRequest).forEach(
                (Consumer<? super Document>) (Document d)->{try{
                    queue.put(d);
                }catch(Throwable t){log.warn("Unable to translate "+d);}});
        log.info("Returned {} elements ",queue.size());
        return queue;
    }


    @Override
    public Project performStep(String id, String step, Document options) throws StepException, JsonProcessingException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess, ConfigurationException, InsufficientPrivileges {
        Project document = lock(id,"Step "+step+" execution");
        try {

            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);
            // NB cannot check ownership on returned values, must specify filter
            if (policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if (!policy.canWrite(document, u)) throw new UnauthorizedAccess("No edit rights on project " + id);


            document.getLifecycleInformation().cleanState();
            document = step(document, step, options);
            return unlockAndUpdate(document);
        }catch (UnrecognizedStepException | ConfigurationException | InsufficientPrivileges e){
            log.debug("Unable to perform step ",e);
            unlock(document);
            throw e;
        } catch(Throwable t) {
            log.error("[UseCaseDescriptor {} ] ERROR Invoking Step {} on document {}", useCaseDescriptor.getId(), step, id, t);
            LifecycleInformation info = new LifecycleInformation();
            info.setPhase(document.getLifecycleInformation().getPhase());
            info.setLastOperationStatus(LifecycleInformation.Status.ERROR);
            info.addErrorMessage(t.getMessage());
            info.setLastInvokedStep(step);
            document.setLifecycleInformation(info);
            return unlockAndUpdate(document);
        }
    }



    /**
     * NB Put at path :
     *
     *     Path Examples
     *      artifact
     *      images
     *      images[1]
     *      layers[?(@.name = 'myName')].fileset
     *
     *
     *
     */
    @Override
    public Project registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, JsonProcessingException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.info("Registering Fileset for {} [useCaseDescriptor ID {}], Request is {} ",id, useCaseDescriptor.getId(),request);

        List<TempFile> files=request.getStreams();
        Document attributes =request.getAttributes();
        Project doc=lock(id,"Register Fileset");
        try {

            // Checking user rights on proj
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.info("Registering Fileset for {} [{}] , policy for {} is {} ",id,useCaseDescriptor.getId(),u,policy);
            // NB cannot check ownership on returned values, must specify filter
            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(doc,u)) throw new UnauthorizedAccess("No edit rights on project "+id);

            doc.getLifecycleInformation().cleanState();
            doc.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.OK);

            WorkspaceManager ws = new WorkspaceManager();
            StorageUtils storage = ImplementationProvider.get().getProvidedObjectByClass(StorageUtils.class);


            log.debug("Checking field {} definition in {}", request.getFieldDefinitionPath(), useCaseDescriptor.getId());
            Field fieldDefinition = getFieldDefinition(useCaseDescriptor, request.getFieldDefinitionPath());

            JSONPathWrapper docWrapper = new JSONPathWrapper(doc.getTheDocument().toJson());

            List<String> matchingPaths = docWrapper.getMatchingPaths(request.getParentPath());
            if (matchingPaths.size() > 1)
                throw new WebApplicationException("Multiple Destination matching parent path " + request.getParentPath(), Response.Status.BAD_REQUEST);
            if (matchingPaths.isEmpty())
                throw new WebApplicationException("PArent path not found at " + request.getParentPath(), Response.Status.BAD_REQUEST);

            String parentMatchingPath = matchingPaths.get(0);
            List<Object> foundElementsByMatchingPaths = docWrapper.getByPath(parentMatchingPath);
            if (foundElementsByMatchingPaths == null || foundElementsByMatchingPaths.isEmpty())
                throw new WebApplicationException("No element found at " + parentMatchingPath, Response.Status.BAD_REQUEST);

            Document parent = Serialization.asDocument(foundElementsByMatchingPaths.get(0));


            Access toSetAccess = doc.getInfo().getAccess();
            if(request.getToSetAccess()!=null){
                //TODO validate specified Access
                toSetAccess = request.getToSetAccess();
            }

            // PREPARE REGISTERED FS

            // MANAGE CLASH
            switch (request.getClashOption()) {
                case REPLACE_EXISTING: {
                    if (fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot replace repeatable field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    // DELETE EXISTING AND PUT
                    RegisteredFileSet toDelete = Serialization.convert(parent.get(request.getFieldName()), RegisteredFileSet.class);
                    if (!(toDelete == null) && !(toDelete.isEmpty())) {
                        String path = parentMatchingPath+"."+request.getFieldName();
                        deleteFileSetRoutine(doc, false, path);

                    }
                    RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(), request.getAttributes(), files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());
                    docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
                    break;
                }
                case MERGE_EXISTING: {
                    if (fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot merge repeatable field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    RegisteredFileSet original = Serialization.convert(parent.get(request.getFieldName()), RegisteredFileSet.class);
                    // MERGE ATTRIBUTES AND PUT
                    Document toUseAttributes = request.getAttributes();
                    if (original != null) toUseAttributes.putAll(original);
                    RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(), toUseAttributes, files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());
                    docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
                    break;
                }
                case APPEND: {
                    if (!fieldDefinition.isCollection())
                        throw new WebApplicationException("Cannot add to single field " + request.getFieldDefinitionPath() + ".", Response.Status.BAD_REQUEST);
                    RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(), request.getAttributes(), files, storage, ws);
                    log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(), doc.getProfileID());

                    docWrapper.addElementToArray(String.format("%1ds['%2$s']", parentMatchingPath, request.getFieldName()), fs);
                    break;
                }
                default: {
                    throw new WebApplicationException("Unexpected clash policy " + request.getClashOption(), Response.Status.BAD_REQUEST);
                }
            }


            log.debug("Setting result on profiled document");
            doc.setTheDocument(Document.parse(docWrapper.getValueCTX().jsonString()));

            doc = onUpdate(doc);
            return unlockAndUpdate(doc);
        }catch (Throwable t) {
            log.warn("Unexpected Exception while trying to registering fileset on {}.", id, t);
            log.debug("Request was {}", request);
            log.debug("Complete doc was {} ", doc);
            unlock(doc);
            throw t;
        }
    }


    @Override
    public Project deleteFileSet(String id, String path, Boolean force) throws ConfigurationException, StorageHubException, JsonProcessingException, DeletionException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
        log.info("Deleting Fileset for {} [useCaseDescriptor ID {}], at {} [force {} ]",id, useCaseDescriptor.getId(),path,force);
        Project doc = lock(id,"Fileset Deletion");
        try {
            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);
            log.debug("Deleting Fileset for {} [{}] , policy for {} is {} ",doc.getId(),useCaseDescriptor.getId(),u,policy);
            // NB cannot check ownership on returned values, must specify filter
            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(doc,u)) throw new UnauthorizedAccess("No edit rights on project "+doc.getId());

            doc.getLifecycleInformation().cleanState();
            doc.getLifecycleInformation().cleanState().setLastOperationStatus(LifecycleInformation.Status.OK);

            doc= deleteFileSetRoutine(doc,force,path);
            return unlockAndUpdate(doc);
        }catch (Throwable t) {
            log.warn("Unexpected Exception while trying to delete fileset on {}.", id, t);
            unlock(doc);
            throw t;
        }
    }

    @Override
    public Project forceUnlock(String id) throws InvalidUserRoleException, ProjectNotFoundException, UnauthorizedAccess, JsonProcessingException, InvalidLockException {
        Project toUnlock = null;
        try{
            toUnlock = lock(id,"Check locked for force unlock");
            throw new WebApplicationException("Project "+id+" not locked", Response.Status.EXPECTATION_FAILED);
        }catch (ProjectLockedException e){
            // expected exception
            toUnlock = getByID(id);
        }finally {
            if(toUnlock!=null)
                return unlock(toUnlock);
            else throw new WebApplicationException("Unable to get Project "+id, Response.Status.EXPECTATION_FAILED);
        }
    }

    @Override
    public Project setAccessPolicy(String id, Access access) throws InvalidUserRoleException, ProjectLockedException, ProjectNotFoundException, UnauthorizedAccess, JsonProcessingException, InvalidLockException, EventException {
        log.trace("UCD {}, Project {} : Setting Access {}",useCaseDescriptor.getId(),id,access);
        Project toUpdate=lock(id,"Set Access policy");
        try {

            User u = UserUtils.getCurrent().asInfo().getUser();
            final DataAccessPolicy policy  = useCaseDescriptor.getMatching(u);

            if(policy == null) {
                log.warn("No policy found for {}. Returning empty ", u);
                throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
            }
            if(!policy.canWrite(toUpdate,u)) throw new UnauthorizedAccess("No edit rights on project "+id);


            toUpdate.getInfo().setAccess(access);
            toUpdate.getLifecycleInformation().cleanState();
            toUpdate = onUpdate(toUpdate);
            return unlockAndUpdate(toUpdate);
        }catch(Throwable t){
            log.error("Unexpected exception ",t);
            unlock(toUpdate);
            throw t;
        }
    }


    private Project deleteFileSetRoutine(Project doc,Boolean force, String path) throws ConfigurationException, StorageHubException {
        log.info("Delete[force : {}] FS at {} for {}:{}",force,path,doc.getProfileID(),doc.getId());
        JSONPathWrapper wrapper = new JSONPathWrapper(doc.getTheDocument().toJson());
        List<String> matchingPaths = wrapper.getMatchingPaths(path);
        if (matchingPaths.isEmpty())
            throw new WebApplicationException("No Registered FileSet found at " + path, Response.Status.BAD_REQUEST);
        if (matchingPaths.size() > 1)
            throw new WebApplicationException("Multiple Fileset (" + matchingPaths.size() + ") matching " + path, Response.Status.BAD_REQUEST);


        RegisteredFileSet fs = Serialization.convert(wrapper.getByPath(path).get(0), RegisteredFileSet.class);
        log.debug("Going to delete {}", fs);

        doc = triggerEvent(doc,EventExecutionRequest.Events.ON_DELETE_FILESET,new Document("force",force).append("path",path));

        // Actually delete only if event was ok
        if(doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
            // Delete from storage
            if(fs.getFolderId()!=null) {
                log.info("Deleting Fileset Folder ID {} ",fs.getFolderId());
                new WorkspaceManager().deleteItem(fs.getFolderId());
            }
            log.debug("Removing FS from document [ID : ] by path {}", doc.getId(), path);
            // Delete from document
            wrapper.setElement(path, null);
        }
        return doc;
    }



    @Override
    public Configuration getConfiguration() throws ConfigurationException{
        String context = UserUtils.getCurrent().getContext();
        log.debug("Asking configuration for {} in {} ", useCaseDescriptor.getId(), context);
        Configuration toReturn= new Configuration();
        List<Archive> archives = new ArrayList<>();
        toReturn.setArchives(archives);
        List<Index> indexes=new ArrayList<>();
        toReturn.setIndexes(indexes);
        // Set Basic Info
        toReturn.setProfileId(this.getUseCaseDescriptor().getId());
        toReturn.setContext(context);
        toReturn.setLastUpdatedTime(LocalDateTime.now());

        // Add Mongo Info
        Archive mongoArchive = new Archive("DOCUMENT-STORE-COLLECTION");
        MongoCollection coll=getCollection();
        mongoArchive.put("count",coll.count());
        mongoArchive.put("collection_name",getToUseCollectionName());

        // Get counts by PHASE, Status
        try {
            ArrayList<Document> counts = new ArrayList<>();

            coll.aggregate(Collections.singletonList(Document.parse(
                    "{\"$group\" : " +
                            "{\"_id\":{\"phase\":\"$_lifecycleInformation._phase\",\"status\":\"$_lifecycleInformation._lastOperationStatus\"}, " +
                            "\"count\":{\"$sum\":1}}}"))).forEach((Consumer) doc -> {
                try { counts.add(Serialization.asDocument(doc)); } catch (JsonProcessingException e) { log.warn("Unable to write aggregated results ",e); } });

            mongoArchive.put("countByPhase",counts);
        }catch (Throwable t){
            toReturn.addErrorMessage("Unable to get PHASE statistics "+t.getMessage());
            log.error("Unable to get PHASE statistics",t);
        }
        archives.add(mongoArchive);

        // TODO ADD TEXT INDEXES

        // Set WS Info
        try {
            archives.add(new WorkspaceManager().getConfiguration());
        }catch (Exception e) {
            toReturn.addErrorMessage("Unable to get WS info "+e.getMessage());
            log.error("Unable to get WS Configuration",e);
        }

        // ADD LC Infos

        AccountingInfo user = UserUtils.getCurrent().asInfo();
        try{
            Configuration lcConfig = getLCManager().getCurrentConfiguration(new BaseRequest(useCaseDescriptor,user.getUser(),user.getContext()));
            log.info("Configuration is {} ",lcConfig);
            if(lcConfig.getArchives()!=null)
                archives.addAll(lcConfig.getArchives());

            if(lcConfig.getIndexes()!=null)
                indexes.addAll(lcConfig.getIndexes());
        }catch(Throwable e){
            toReturn.addErrorMessage("Unable to get Lifecycle info "+e.getMessage());
            log.error("Unable to get Lifecycle info ",e);
        }

        log.debug("Returning current configuration {}",toReturn);
        return toReturn;
    }




    private Project step(Project theDocument, String step, Document callParameters) throws InsufficientPrivileges, ConfigurationException, StepException {
        try {
            log.info("[UseCaseDescriptor {}] Invoking Step {} on {}", useCaseDescriptor.getId(), step, getManager().getDescriptor());
            AccountingInfo user = UserUtils.getCurrent().asInfo();


            StepExecutionRequest request = new StepExecutionRequest(useCaseDescriptor, user.getUser(), user.getContext(), theDocument, step);
            request.setCallParameters(callParameters);

            log.debug("Requesting Step Execution {}", request);
            StepExecutionReport report = getManager().performStep(request);
            Project toReturn = report.prepareResult();

            // EVENTS
            if (report.getToTriggerEvents() != null) {
                Iterator<EventExecutionRequest> eventIT = report.getToTriggerEvents().listIterator();
                while (!toReturn.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.ERROR)
                        && eventIT.hasNext()) {
                    EventExecutionRequest triggerRequest = eventIT.next();
                    log.info("Triggering {} ", triggerRequest);
                    toReturn = triggerEvent(toReturn, triggerRequest.getEvent(), triggerRequest.getCallParameters());
                }
            }

            // STEPS
            if (report.getCascadeSteps() != null) {
                Iterator<StepExecutionRequest> stepIT = report.getCascadeSteps().listIterator();
                while (!toReturn.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.ERROR)
                        && stepIT.hasNext()) {
                    StepExecutionRequest triggerRequest = stepIT.next();
                    log.info("Triggering {} ", triggerRequest);
                    toReturn = step(toReturn, triggerRequest.getStep(), triggerRequest.getCallParameters());
                }
            }
            return report.prepareResult();
        }catch (InsufficientPrivileges | ConfigurationException | UnrecognizedStepException e){
            throw e;
        }catch(Throwable t){
            log.error("Unable to perform step "+step,t);
            theDocument.getLifecycleInformation().addErrorMessage("Unable to perform step "+step+" cause : "+ t.getMessage());
            theDocument.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
            return theDocument;
        }
    }

    private Project triggerEvent(Project project, String event, Document parameters) {
        try{
            log.info("[UseCaseDescriptor {}] triggering event {} on {}" , useCaseDescriptor.getId(),event,getManager().getDescriptor());
            AccountingInfo user= UserUtils.getCurrent().asInfo();
            EventExecutionRequest request= new EventExecutionRequest(useCaseDescriptor,user.getUser(),user.getContext(),project,event);
            request.setCallParameters(parameters);
            log.debug("Triggering {}",request);
            DocumentHandlingReport report = getManager().onEvent(request);
            return report.prepareResult();
        } catch (Throwable t){
            log.error("Unable to trigger event "+event,t);
            project.getLifecycleInformation().addErrorMessage("Unable to trigger "+event+" cause : "+ t.getMessage());
            project.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
            return project;
        }
    }


    private static final RegisteredFileSet prepareRegisteredFileSet(Access toSetAccess,String docID, String profileID,
                                                                    Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
        log.debug("Preparing Registered FileSet {}");
        RegisteredFileSet toReturn = new RegisteredFileSet();
        if(attributes!=null) toReturn.putAll(attributes);
        String uuid = UUID.randomUUID().toString();
        toReturn.putIfAbsent(RegisteredFileSet.UUID, uuid);
        toReturn.putIfAbsent(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
        toReturn.putIfAbsent(RegisteredFileSet.ACCESS,toSetAccess);

        // FOLDER
        String folderID=toReturn.getFolderId();
        log.trace("Folder ID is {} ",folderID);
        FolderContainer sectionFolder=null;
        if(folderID==null || folderID.isEmpty()) {

            // Get BASE Folder for project
            FolderContainer base = ws.getSubFolder(ws.getAppBase(),docID,"Base Folder for profiled document. UseCaseDescriptor " + profileID);
            // Create Folder for FileSet
            sectionFolder = ws.createFolder(new WorkspaceManager.FolderOptions(docID + "_" + uuid, "Registered Fileset uuid " + uuid, base));
            toReturn.put(RegisteredFileSet.FOLDER_ID, sectionFolder.getId());
        }else {
            sectionFolder = ws.getFolderById(folderID);
        }

        ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();

        if(toReturn.containsKey(RegisteredFileSet.PAYLOADS))
            registeredFiles.addAll(toReturn.getPayloads());

        for (TempFile f : files) {
            InputStream is=null;
            try{
                log.debug("Opening temp file {}",f);
                String fileUrl=null;
                if(f.getId()==null||f.getId().isEmpty())
                    fileUrl=f.getUrl();
                else fileUrl=storage.getURL(f.getId());

                log.debug("Got URL {} from ID {}",fileUrl,f.getId());
                is=new URL(fileUrl).openStream();
                RegisteredFile registered=ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(),is,
                        "Imported via gcube CMS service ", sectionFolder));
                log.debug("Registered "+registered);
                registeredFiles.add(registered);
            }catch(StorageHubException | IOException e){
                throw new StorageException("Unable to store "+f,e);
            }finally{
                if(is!=null)
                    IOUtils.closeQuietly(is);
            }
        }
        toReturn.put(RegisteredFileSet.PAYLOADS,registeredFiles);
        // TODO MERGE
        //toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
        return toReturn;
    }



    private static Field getFieldDefinition(UseCaseDescriptor useCaseDescriptor, String fieldPath)throws WebApplicationException{
        JSONPathWrapper schemaWrapper= new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
        List<Field> fieldDefinitions=schemaWrapper.getByPath(fieldPath,Field.class);
        if(fieldDefinitions==null || fieldDefinitions.isEmpty())
            throw new WebApplicationException("No Field found in schema "+ useCaseDescriptor.getId()+" at "+fieldPath, Response.Status.BAD_REQUEST);
        if(fieldDefinitions.size()>1)
            throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+ useCaseDescriptor.getId()+" for "+fieldPath,Response.Status.BAD_REQUEST);
        Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
        if(fieldDefinition==null)
            throw new WebApplicationException("Found field is null ["+ useCaseDescriptor.getId()+" for "+fieldPath+"]",Response.Status.BAD_REQUEST);
        log.trace("Field definition is {}",fieldDefinition);
        return fieldDefinition;
    }
}