gcube-cms-suite/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java

package org.gcube.application.geoportal.service.engine.mongo;

import static org.gcube.application.cms.serialization.Serialization.asDocumentWithId;
import static org.gcube.application.cms.serialization.Serialization.convert;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.InvalidParameterException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;
import java.util.stream.Collectors;

import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;

import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.cms.implementations.ImplementationProvider;
import org.gcube.application.cms.implementations.WorkspaceManager;
import org.gcube.application.cms.implementations.faults.DeletionException;
import org.gcube.application.cms.implementations.faults.InvalidLockException;
import org.gcube.application.cms.implementations.faults.InvalidUserRoleException;
import org.gcube.application.cms.implementations.faults.ProjectLockedException;
import org.gcube.application.cms.implementations.faults.ProjectNotFoundException;
import org.gcube.application.cms.implementations.faults.RegistrationException;
import org.gcube.application.cms.implementations.faults.UnauthorizedAccess;
import org.gcube.application.cms.implementations.utils.UserUtils;
import org.gcube.application.cms.plugins.LifecycleManager;
import org.gcube.application.cms.plugins.faults.EventException;
import org.gcube.application.cms.plugins.faults.InsufficientPrivileges;
import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.cms.plugins.faults.UnrecognizedStepException;
import org.gcube.application.cms.plugins.reports.DocumentHandlingReport;
import org.gcube.application.cms.plugins.reports.StepExecutionReport;
import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.plugins.requests.EventExecutionRequest;
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.configuration.Archive;
import org.gcube.application.geoportal.common.model.configuration.Configuration;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.Lock;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.accounting.AccountingInfo;
import org.gcube.application.geoportal.common.model.document.accounting.PublicationInfo;
import org.gcube.application.geoportal.common.model.document.accounting.User;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.document.relationships.Relationship;
import org.gcube.application.geoportal.common.model.plugins.LifecycleManagerDescriptor;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.DataAccessPolicy;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.RelationshipDefinition;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.providers.PluginManager;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.FindOneAndReplaceOptions;
import com.mongodb.client.model.FindOneAndUpdateOptions;
import com.mongodb.client.model.ReturnDocument;
import com.vdurmont.semver4j.Semver;

import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

/**
 * The Class ProfiledMongoManager.
 *
 * @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
 * 
 *         Apr 26, 2023
 */
@Slf4j
public class ProfiledMongoManager extends MongoManager implements MongoManagerI<Project> {

	/**
	 * Gets the use case descriptor.
	 *
	 * @return the use case descriptor
	 */
	@Getter
	UseCaseDescriptor useCaseDescriptor;

	/**
	 * Mongo ID field name.
	 *
	 * @return the string
	 */
	@Override
	protected String mongoIDFieldName() {
		return ID;
	}

	/**
	 * Instantiates a new profiled mongo manager.
	 *
	 * @param profileId the profile id
	 * @throws ConfigurationException the configuration exception
	 * @throws RegistrationException  the registration exception
	 */
	public ProfiledMongoManager(String profileId) throws ConfigurationException, RegistrationException {
		// Check UseCaseDescriptor ID
		log.info("Loading useCaseDescriptor ID {} ", profileId);
		if (profileId == null)
			throw new InvalidParameterException("UseCaseDescriptor ID cannot be null");
		useCaseDescriptor = ImplementationProvider.get().getProvidedObjectByClass(UCDManagerI.class).getById(profileId);
		if (useCaseDescriptor == null)
			throw new WebApplicationException("UseCaseDescriptor " + profileId + " not registered",
					Response.Status.NOT_FOUND);

		// Connect to DB
		init(getToUseCollectionName());

	}

	/**
	 * Gets the to use collection name.
	 *
	 * @return the to use collection name
	 */
	private String getToUseCollectionName() {
		// TODO collection name in UCD
		return useCaseDescriptor.getId();
	}

	/**
	 * Lock.
	 *
	 * @param id the id
	 * @param op the op
	 * @return the project
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	protected Project lock(String id, String op) throws ProjectNotFoundException, ProjectLockedException,
			JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
		log.trace("Locking {} cause {} ", id, op);

		Lock lock = new Lock();
		lock.setId(UUID.randomUUID().toString());
		lock.setInfo(UserUtils.getCurrent().asInfo());
		lock.setOperation(op);

		// find one and update
		// filter : id, lock == null
		// update with new Lock object
		Document filter = new Document(mongoIDFieldName(), asId(id)).append("$or",
				Arrays.asList(new Document(Project.LOCK, new Document("$exists", false)),
						new Document(Project.LOCK, new Document("$type", "null"))));
		log.debug("Lock filter is {} ", filter.toJson());
		Object obj = getCollection().findOneAndUpdate(
				// filter by id and missing lock
				filter,
				// update lock info
				new Document("$set", new Document(Project.LOCK, Serialization.asDocument(lock))),
				new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER));

		if (obj == null) {
			// unable to lock, verify cause
			Project p = getByID(id);
			if (p.getLock() != null)
				throw new ProjectLockedException("Project already locked : " + p.getLock());
			else {
				log.error("Unable to lock {} ", id);
				log.debug("Existing project is {} ", p);
				throw new RuntimeException("Unable to lock unlocked project " + id);
			}
		} else
			return Serialization.convert(obj, Project.class);
	}

	/**
	 * Unlock and update.
	 *
	 * @param proj the proj
	 * @return the project
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	protected Project unlockAndUpdate(Project proj) throws InvalidLockException, ProjectNotFoundException,
			JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
		log.info("Unlocking for update {} lock is {} ", proj.getId(), proj.getLock());
		// find one and update
		Lock oldLock = proj.getLock();
		proj.setLock(null);

		Document filter = new Document(mongoIDFieldName(), asId(proj.getId())).append(Project.LOCK + "." + Lock.ID,
				oldLock.getId());

		log.info("Filter document is {} ", filter.toJson());

		Object obj = getCollection().findOneAndReplace(
				// filter by id and missing lock
				filter,
				// update lock info
				asDocumentWithId(proj), new FindOneAndReplaceOptions().returnDocument(ReturnDocument.AFTER));

		if (obj == null) {
			// can-t unlock, check cause
			Project p = getByID(proj.getId());
			throw new InvalidLockException(
					"Found lock for " + p.getId() + " is " + p.getLock() + ", expected is " + oldLock);
		}
		return Serialization.convert(obj, Project.class);
	}

	/**
	 * Unlock and patch.
	 *
	 * @param proj the proj
	 * @return the project
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	protected Project unlockAndPatch(Project proj) throws InvalidLockException, ProjectNotFoundException,
			JsonProcessingException, InvalidUserRoleException, UnauthorizedAccess {
		log.info("Unlocking for patching {} lock is {} ", proj.getId(), proj.getLock());

		// find one and update
		Lock oldLock = proj.getLock();
		Document filter = new Document(mongoIDFieldName(), asId(proj.getId())).append(Project.LOCK + "." + Lock.ID,
				oldLock.getId());

		String documentValueAsJson = proj.getTheDocument().toJson();
		String updatedDocumentAsJson = new Document(Project.THE_DOCUMENT, documentValueAsJson).toJson();
		Document setUpdatedDocument = new Document("$set", updatedDocumentAsJson);

		log.info("Filter document is {} ", filter.toJson());
		log.info("$set is {} ", setUpdatedDocument);

		Object obj = getCollection().findOneAndUpdate(
				// filter by id and missing lock
				filter,
				// update lock info
				setUpdatedDocument, new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER));

		if (obj == null) {
			// can-t unlock, check cause
			throw new InvalidLockException(
					"Found lock for " + proj.getId() + " is " + proj.getLock() + ", expected is " + oldLock);
		}

		// I could use Serialization.convert(obj, Project.class), but to be sure I'm
		// reading again the project by
		proj = getByID(proj.getId());

		proj = unlock(proj);
		return proj;
	}

	/**
	 * Unlock.
	 *
	 * @param proj the proj
	 * @return the project
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	protected Project unlock(Project proj)
			throws InvalidLockException, InvalidUserRoleException, ProjectNotFoundException, UnauthorizedAccess {
		log.trace("Unlocking for update {} lock is {} ", proj.getId(), proj.getLock());
		// find one and update
		Lock oldLock = proj.getLock();
		Document filter = new Document(mongoIDFieldName(), asId(proj.getId())).append(Project.LOCK + "." + Lock.ID,
				oldLock.getId());

		Object obj = getCollection().findOneAndUpdate(
				// filter by id and missing lock
				filter,
				// update lock info
				new Document("$set", new Document(Project.LOCK, null)),
				new FindOneAndUpdateOptions().returnDocument(ReturnDocument.AFTER));

		if (obj == null) {
			// can-t unlock, check cause
			Project p = getByID(proj.getId());
			throw new InvalidLockException(
					"Found lock for " + p.getId() + " is " + p.getLock() + ", expected is " + oldLock);
		}
		return Serialization.convert(obj, Project.class);
	}

	/**
	 * Gets the manager.
	 *
	 * @return the manager
	 */
	@Getter(lazy = true)
	private final LifecycleManager manager = getLCManager();

	/**
	 * Gets the LC manager.
	 *
	 * @return the LC manager
	 */
	private LifecycleManager getLCManager() {
		try {
			LifecycleManager toReturn = null;
			// Getting Lifecycle Manager declaration from UseCaseDescriptor
			List<HandlerDeclaration> handlerDeclarations = useCaseDescriptor.getHandlersMapByType()
					.get(LifecycleManagerDescriptor.LIFECYCLE_MANAGER_TYPE);
			if (handlerDeclarations == null || handlerDeclarations.isEmpty())
				throw new ConfigurationException(
						"No Lifecycle Handler defined for useCaseDescriptor ID " + useCaseDescriptor.getId());
			if (handlerDeclarations.size() > 1)
				throw new ConfigurationException("Too many Lifecycle Handlers defined (" + handlerDeclarations
						+ ") in useCaseDescriptor ID " + useCaseDescriptor.getId());

			HandlerDeclaration lcHandlerDeclaration = handlerDeclarations.get(0);

			// Loading Lifecycle Manager
			log.debug("Looking for handler {} ", lcHandlerDeclaration);
			toReturn = (LifecycleManager) ImplementationProvider.get()
					.getProvidedObjectByClass(PluginManager.PluginMap.class).get(lcHandlerDeclaration.getId());
			if (toReturn == null)
				throw new ConfigurationException(
						"Unable to find Lifecycle Manager Plugin. ID " + lcHandlerDeclaration.getId());

			return toReturn;
		} catch (Throwable t) {
			log.warn("Unable to load LC Manager ", t);
			return null;
		}
	}

	/**
	 * Register new.
	 *
	 * @param toRegisterDoc the to register doc
	 * @return the project
	 * @throws IOException              Signals that an I/O exception has occurred.
	 * @throws InvalidUserRoleException the invalid user role exception
	 */
	@Override
	public Project registerNew(Document toRegisterDoc) throws IOException, InvalidUserRoleException {
		log.info("Registering new document in {} ", useCaseDescriptor.getId());
		log.trace("Going to register {}", toRegisterDoc.toJson());

		User u = UserUtils.getCurrent().asInfo().getUser();
		final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
		log.trace("Access policy for user {} is {} ", u, policy);

		if (policy == null) {
			log.warn("No policy found for {}. Returning empty ", u);
			throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
		}
		if (policy.getPolicy().getWrite().equals(DataAccessPolicy.Policy.Type.none))
			throw new InvalidUserRoleException("User doesn't have write privileges " + u.getRoles());

		Project toRegister = new Project();
		toRegister.setTheDocument(toRegisterDoc);

		PublicationInfo pubInfo = new PublicationInfo();
		pubInfo.setCreationInfo(UserUtils.getCurrent().asInfo());

		// TODO Set Access From UseCaseDescriptor
		Access access = new Access();
		access.setLicense("CC-BY-4.0");
		access.setPolicy(AccessPolicy.OPEN);
		pubInfo.setAccess(access);

		toRegister.setInfo(pubInfo);

		toRegister.setProfileID(useCaseDescriptor.getId());
		toRegister.setProfileVersion(useCaseDescriptor.getVersion());
		toRegister.setVersion(new Semver("1.0.0"));

		LifecycleInformation draftInfo = new LifecycleInformation().cleanState();
		draftInfo.setPhase(LifecycleInformation.CommonPhases.DRAFT_PHASE);
		draftInfo.setLastOperationStatus(LifecycleInformation.Status.OK);
		toRegister.setLifecycleInformation(draftInfo);

		// Apply Lifecycle
		toRegister = triggerEvent(toRegister, EventExecutionRequest.Events.ON_INIT_DOCUMENT, null);

		log.debug("Going to register {} ", toRegister);

		// Insert object
		ObjectId id = insertDoc(asDocumentWithId(toRegister));

		log.info("Obtained id {} ", id);
		try {
			return getByID(id.toHexString());
		} catch (ProjectNotFoundException | InvalidUserRoleException | UnauthorizedAccess e) {
			throw new WebApplicationException("Unexpected exception while registering project ", e);
		}
	}

	/**
	 * Update.
	 *
	 * @param id    the id
	 * @param toSet the to set
	 * @return the project
	 * @throws IOException              Signals that an I/O exception has occurred.
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	@Override
	public Project update(String id, Document toSet) throws IOException, EventException, ProjectLockedException,
			ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
		log.trace("Updating {} ", toSet);
		Project toUpdate = lock(id, "Manual update");
		try {

			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u,
					policy);

			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(toUpdate, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			toUpdate.setTheDocument(toSet);
			toUpdate.getLifecycleInformation().cleanState();
			toUpdate = onUpdate(toUpdate);
			return unlockAndUpdate(toUpdate);
		} catch (Throwable t) {
			log.error("Unexpected exception ", t);
			unlock(toUpdate);
			throw t;
		}
	}

	/**
	 * Patch.
	 *
	 * @param id    the id
	 * @param toSet the to set
	 * @return the project
	 * @throws IOException              Signals that an I/O exception has occurred.
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	@Override
	public Project patch(String id, Document toSet) throws IOException, EventException, ProjectLockedException,
			ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
		log.trace("Patching {} ", toSet);
		Project toUpdate = lock(id, "Manual patch");
		try {

			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u,
					policy);

			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(toUpdate, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			toUpdate.setTheDocument(toSet);
			toUpdate.getLifecycleInformation().cleanState();
			toUpdate = onUpdate(toUpdate);
			return unlockAndPatch(toUpdate);
		} catch (Throwable t) {
			log.error("Unexpected exception ", t);
			unlock(toUpdate);
			throw t;
		}
	}

	/**
	 * Sets the relation.
	 *
	 * @param id        the id
	 * @param relation  the relation
	 * @param targetUCD the target UCD
	 * @param targetId  the target id
	 * @return the project
	 * @throws IOException              Signals that an I/O exception has occurred.
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws RegistrationException    the registration exception
	 * @throws ConfigurationException   the configuration exception
	 */
	@Override
	public Project setRelation(String id, String relation, String targetUCD, String targetId)
			throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException,
			InvalidUserRoleException, UnauthorizedAccess, RegistrationException, ConfigurationException {
		Project toUpdate = lock(id, "Set Relation " + relation + " toward " + targetUCD + ":" + targetId);
		try {
			log.info(getUseCaseDescriptor().getId() + ":" + id + " setting relation " + relation + " toward "
					+ targetUCD + ":" + targetId);
			// Check if relation is defined
			String toSetReverseRelation = null;
			List<RelationshipDefinition> existingDefinitions = getUseCaseDescriptor().getRelationshipDefinitions();
			for (RelationshipDefinition def : existingDefinitions)
				if (def.getId().equals(relation))
					toSetReverseRelation = def.getReverseRelationId();

			log.debug("{} reverse relation for {} is {}", getUseCaseDescriptor().getId(), relation,
					toSetReverseRelation);

			// check if relation existing
			List<Relationship> relations = toUpdate.getRelationshipsByName(relation);
			if (!relations.isEmpty()) {
				// check if targetUCD+targetID already present
				List matching = relations.stream()
						.filter(r -> r.getTargetID().equals(targetId) && r.getTargetUCD().equals(targetUCD))
						.collect(Collectors.toList());
				if (matching.size() > 0)
					throw new WebApplicationException(
							"Relationship " + relation + " -> " + targetUCD + " : " + targetId + " already set",
							Response.Status.EXPECTATION_FAILED);
			}
			// check if target exists
			ProfiledMongoManager otherManager = (targetUCD.equals(this.useCaseDescriptor.getId())) ? this
					: new ProfiledMongoManager(targetUCD);
			Project other = getByID(targetId);

			// add relation
			Relationship rel = new Relationship();
			rel.setRelationshipName(relation);
			rel.setTargetID(targetId);
			rel.setTargetUCD(targetUCD);
			toUpdate = onUpdate(toUpdate.addRelation(rel));

			// set reverse relation
			if (toSetReverseRelation != null) {
				Relationship reverseRel = new Relationship();
				reverseRel.setRelationshipName(toSetReverseRelation);
				reverseRel.setTargetID(id);
				reverseRel.setTargetUCD(getUseCaseDescriptor().getId());
				log.info("Setting reverse relation {} ", reverseRel);
				other = otherManager.lock(other.getId(), "Setting reverse relation " + reverseRel);
				other.addRelation(reverseRel);
				otherManager.unlockAndUpdate(other);
			}

			return unlockAndUpdate(toUpdate);
		} catch (Throwable t) {
			log.error("Unexpected exception ", t);
			unlock(toUpdate);
			throw t;
		}
	}

	/**
	 * Delete relation.
	 *
	 * @param id        the id
	 * @param relation  the relation
	 * @param targetUCD the target UCD
	 * @param targetId  the target id
	 * @return the project
	 * @throws IOException              Signals that an I/O exception has occurred.
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws RegistrationException    the registration exception
	 * @throws ConfigurationException   the configuration exception
	 */
	@Override
	public Project deleteRelation(String id, String relation, String targetUCD, String targetId)
			throws IOException, EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException,
			InvalidUserRoleException, UnauthorizedAccess, RegistrationException, ConfigurationException {
		log.info("Delete relation {}:{}--{}-->{}:{} ", getUseCaseDescriptor().getId(), id, relation, targetUCD,
				targetId);
		Project toUpdate = lock(id, "Delete Relation");

		try {
			// SET target UCD to present UCD as default
			final String toUseTargetUCD = (targetUCD == null || targetUCD.equals("")) ? getUseCaseDescriptor().getId()
					: targetUCD;

			String toDeleteReverseRelation = null;
			List<RelationshipDefinition> existingDefinitions = getUseCaseDescriptor().getRelationshipDefinitions();
			for (RelationshipDefinition def : existingDefinitions)
				if (def.getId().equals(relation))
					toDeleteReverseRelation = def.getReverseRelationId();

			log.debug("{} reverse relation for {} is {}", getUseCaseDescriptor().getId(), relation,
					toDeleteReverseRelation);

			// check if relation existing
			List<Relationship> relations = toUpdate.getRelationships();
			if (relations != null && !relations.isEmpty()) {
				int beforeSize = relations.size();

				ArrayList<Relationship> toRemove = new ArrayList<>();
				for (Relationship r : toUpdate.getRelationships()) {
					if (r.getRelationshipName().equals(relation) && r.getTargetUCD().equals(toUseTargetUCD)
							&& r.getTargetID().equals(targetId)) {
						// set to be remove
						log.debug("Removing {} ", r);
						toRemove.add(r);

						// delete reverse relation
						if (toDeleteReverseRelation != null) {
							log.debug("Removing reverse of {} ", r);
							ProfiledMongoManager otherManager = (toUseTargetUCD.equals(this.useCaseDescriptor.getId()))
									? this
									: new ProfiledMongoManager(targetUCD);
							Project other = getByID(targetId);
							other = otherManager.lock(other.getId(), "Remove reverse relation "
									+ toDeleteReverseRelation + " toward " + getUseCaseDescriptor().getId() + ":" + id);
							final String finalToDeleteReverseRelation = toDeleteReverseRelation;
							other.getRelationships().removeIf(
									revRel -> revRel.getRelationshipName().equals(finalToDeleteReverseRelation)
											&& revRel.getTargetID().equals(id)
											&& revRel.getTargetUCD().equals(getUseCaseDescriptor().getId()));
							otherManager.unlockAndUpdate(other);
						}

					}
				}

				toUpdate.getRelationships().removeAll(toRemove);

				// update only if something changed
				if (toUpdate.getRelationships().size() != beforeSize) {
					log.debug("Removed {} relations from {} ", (toUpdate.getRelationships().size() != beforeSize), id);
					return unlockAndUpdate(toUpdate);
				}
			}
			log.debug("Relationship not found. Unlocking.. ");
			return unlock(toUpdate);
		} catch (Throwable t) {
			log.error("Unexpected exception ", t);
			unlock(toUpdate);
			throw t;
		}
	}

	/**
	 * On update.
	 *
	 * @param toUpdate the to update
	 * @return the project
	 * @throws EventException the event exception
	 */
	private Project onUpdate(Project toUpdate) throws EventException {
		UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
		toUpdate.getInfo().setLastEditInfo(u.asInfo());
		toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
		return triggerEvent(toUpdate, EventExecutionRequest.Events.ON_INIT_DOCUMENT, null);
	}

//	private Project onPatch(Project toUpdate) throws EventException {
//		UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
//		toUpdate.getInfo().setLastEditInfo(u.asInfo());
//		toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
//		return triggerEvent(toUpdate, EventExecutionRequest.Events.ON_UPDATE_DOCUMENT, null);
//	}

	/**
	 * Delete.
	 *
	 * @param id    the id
	 * @param force the force
	 * @throws DeletionException        the deletion exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidLockException     the invalid lock exception
	 */
	@Override
	public void delete(String id, boolean force)
			throws DeletionException, InvalidUserRoleException, ProjectLockedException, ProjectNotFoundException,
			UnauthorizedAccess, JsonProcessingException, InvalidLockException {
		log.info("Deleting by ID {}, force {}", id, force);
		Project doc = lock(id, "Deletion { force : " + force + "}");
		boolean deleted = false;
		try {
			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.debug("Delete project {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);
			if (policy == null)
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			if (!policy.canWrite(doc, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			doc = triggerEvent(doc, EventExecutionRequest.Events.ON_DELETE_DOCUMENT, new Document("force", force));
			// Only continue deleting if event was ok
			if (doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
				try {
					WorkspaceManager ws = new WorkspaceManager();
					// Get All registered Filesets with payloads
					JSONPathWrapper wrapper = new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
					for (Object obj : wrapper.getByPath("$..[?(@." + RegisteredFileSet.PAYLOADS + ")]")) {
						Document fs = Serialization.asDocument(obj);
						log.debug("Deleting {}", obj);
						String folderId = fs.getString(RegisteredFileSet.FOLDER_ID);
						ws.deleteItem(folderId);
					}
				} finally {
					super.deleteDoc(asId(id));
					deleted = true;
				}
			}
		} catch (ConfigurationException | StorageHubException e) {
			log.error("Exception while trying to delete {} [UCID {}]", id, useCaseDescriptor.getId());
			throw new DeletionException("Unable to contact Storage ", e);
		} finally {
			if (doc != null && !deleted)
				unlockAndUpdate(doc);
		}
	}

	/**
	 * Gets the by ID.
	 *
	 * @param id the id
	 * @return the by ID
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	@Override
	public Project getByID(String id) throws ProjectNotFoundException, InvalidUserRoleException, UnauthorizedAccess {
		User u = UserUtils.getCurrent().asInfo().getUser();
		DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
		log.info("Accessing Project {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u, policy);

		if (policy == null) {
			log.warn("No policy found for {}. Returning empty ", u);
			throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
		}

		Document doc = getDocById(asId(id),
				(policy == null || policy.getEnforcer() == null) ? null : policy.getEnforcer().getFilterDocument());
		if (doc == null)
			throw new ProjectNotFoundException("No document with ID " + id);

		Project p = convert(doc, Project.class);
		if (!policy.canRead(p, u))
			throw new UnauthorizedAccess("No access rights on " + id);
		return p;
	}

	/**
	 * Query.
	 *
	 * @param queryRequest the query request
	 * @return the iterable
	 * @throws InvalidUserRoleException the invalid user role exception
	 */
	@Override
	public Iterable<Document> query(QueryRequest queryRequest) throws InvalidUserRoleException {
		LinkedBlockingQueue queue = new LinkedBlockingQueue<Project>();
		User u = UserUtils.getCurrent().asInfo().getUser();
		final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
		log.info("Querying {} [{}] , policy for {} is {} ", queryRequest, useCaseDescriptor.getId(), u, policy);
		// NB cannot check ownership on returned values, must specify filter
		if (policy == null) {
			log.warn("No policy found for {}. Returning empty ", u);
			throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
		}
		if (policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.none)) {
			log.info("Read is NONE : Returning empty collection");
			return queue;
		}

		// NB cannot check ownership on returned values, must specify filter
		Document finalFilter = new Document();
		if (queryRequest.getFilter() != null)
			finalFilter.putAll(queryRequest.getFilter());
		if (policy.getEnforcer() != null)
			finalFilter.putAll(policy.getEnforcer().getFilterDocument());
		if (policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.own))
			finalFilter.put(Project.INFO + "." + PublicationInfo.CREATION_INFO + "." + AccountingInfo.USER + "."
					+ User.USERNAME, u.getUsername());
		queryRequest.setFilter(finalFilter);

		log.debug("Final filter is {}", queryRequest.getFilter());

		queryDoc(queryRequest).forEach((Consumer<? super Document>) (Document d) -> {
			try {
				queue.put(d);
			} catch (Throwable t) {
				log.warn("Unable to translate " + d);
			}
		});
		log.info("Returned {} elements ", queue.size());
		return queue;
	}

	/**
	 * Filter.
	 *
	 * @param queryRequest the query request
	 * @return the iterable
	 * @throws InvalidUserRoleException the invalid user role exception
	 */
	@Override
	public Iterable<Project> filter(QueryRequest queryRequest) throws InvalidUserRoleException {
		LinkedBlockingQueue queue = new LinkedBlockingQueue<Project>();
		User u = UserUtils.getCurrent().asInfo().getUser();
		final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
		log.info("Querying {} [{}] , policy for {} is {} ", queryRequest, useCaseDescriptor.getId(), u, policy);
		// NB cannot check ownership on returned values, must specify filter
		if (policy == null) {
			log.warn("No policy found for {}. Returning empty ", u);
			throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
		}
		if (policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.none)) {
			log.info("Read is NONE : Returning empty collection");
			return queue;
		}

		// NB cannot check ownership on returned values, must specify filter
		Document finalFilter = new Document();
		if (queryRequest.getFilter() != null)
			finalFilter.putAll(queryRequest.getFilter());
		if (policy.getEnforcer() != null)
			finalFilter.putAll(policy.getEnforcer().getFilterDocument());
		if (policy.getPolicy().getRead().equals(DataAccessPolicy.Policy.Type.own))
			finalFilter.put(Project.INFO + "." + PublicationInfo.CREATION_INFO + "." + AccountingInfo.USER + "."
					+ User.USERNAME, u.getUsername());
		queryRequest.setFilter(finalFilter);

		log.debug("Final filter is {}", queryRequest.getFilter());

		queryDoc(queryRequest).forEach((Consumer<? super Document>) (Document d) -> {
			try {
				queue.put(d);
			} catch (Throwable t) {
				log.warn("Unable to translate " + d);
			}
		});
		log.info("Returned {} elements ", queue.size());
		return queue;
	}

	/**
	 * Perform step.
	 *
	 * @param id      the id
	 * @param step    the step
	 * @param options the options
	 * @return the project
	 * @throws StepException            the step exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws ConfigurationException   the configuration exception
	 * @throws InsufficientPrivileges   the insufficient privileges
	 */
	@Override
	public Project performStep(String id, String step, Document options) throws StepException, JsonProcessingException,
			ProjectLockedException, ProjectNotFoundException, InvalidLockException, InvalidUserRoleException,
			UnauthorizedAccess, ConfigurationException, InsufficientPrivileges {
		Project document = lock(id, "Step " + step + " execution");
		try {

			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u,
					policy);
			// NB cannot check ownership on returned values, must specify filter
			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(document, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			document.getLifecycleInformation().cleanState();
			document = step(document, step, options);
			return unlockAndUpdate(document);
		} catch (UnrecognizedStepException | ConfigurationException | InsufficientPrivileges e) {
			log.debug("Unable to perform step ", e);
			unlock(document);
			throw e;
		} catch (Throwable t) {
			log.error("[UseCaseDescriptor {} ] ERROR Invoking Step {} on document {}", useCaseDescriptor.getId(), step,
					id, t);
			LifecycleInformation info = new LifecycleInformation();
			info.setPhase(document.getLifecycleInformation().getPhase());
			info.setLastOperationStatus(LifecycleInformation.Status.ERROR);
			info.addErrorMessage(t.getMessage());
			info.setLastInvokedStep(step);
			document.setLifecycleInformation(info);
			return unlockAndUpdate(document);
		}
	}

	/**
	 * NB Put at path :
	 * 
	 * Path Examples artifact images images[1] layers[?(@.name = 'myName')].fileset
	 *
	 * @param id      the id
	 * @param request the request
	 * @return the project
	 * @throws ConfigurationException   the configuration exception
	 * @throws StorageHubException      the storage hub exception
	 * @throws StorageException         the storage exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	@Override
	public Project registerFileSet(String id, RegisterFileSetRequest request) throws ConfigurationException,
			StorageHubException, StorageException, JsonProcessingException, EventException, ProjectLockedException,
			ProjectNotFoundException, InvalidLockException, InvalidUserRoleException, UnauthorizedAccess {
		log.info("Registering Fileset for {} [useCaseDescriptor ID {}], Request is {} ", id, useCaseDescriptor.getId(),
				request);

		List<TempFile> files = request.getStreams();
		Document attributes = request.getAttributes();
		Project doc = lock(id, "Register Fileset");
		try {

			// Checking user rights on proj
			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.info("Registering Fileset for {} [{}] , policy for {} is {} ", id, useCaseDescriptor.getId(), u,
					policy);
			// NB cannot check ownership on returned values, must specify filter
			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(doc, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			doc.getLifecycleInformation().cleanState();
			doc.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.OK);

			WorkspaceManager ws = new WorkspaceManager();
			StorageUtils storage = ImplementationProvider.get().getProvidedObjectByClass(StorageUtils.class);

			log.debug("Checking field {} definition in {}", request.getFieldDefinitionPath(),
					useCaseDescriptor.getId());
			Field fieldDefinition = getFieldDefinition(useCaseDescriptor, request.getFieldDefinitionPath());

			JSONPathWrapper docWrapper = new JSONPathWrapper(doc.getTheDocument().toJson());

			List<String> matchingPaths = docWrapper.getMatchingPaths(request.getParentPath());
			if (matchingPaths.size() > 1)
				throw new WebApplicationException(
						"Multiple Destination matching parent path " + request.getParentPath(),
						Response.Status.BAD_REQUEST);
			if (matchingPaths.isEmpty())
				throw new WebApplicationException("PArent path not found at " + request.getParentPath(),
						Response.Status.BAD_REQUEST);

			String parentMatchingPath = matchingPaths.get(0);
			List<Object> foundElementsByMatchingPaths = docWrapper.getByPath(parentMatchingPath);
			if (foundElementsByMatchingPaths == null || foundElementsByMatchingPaths.isEmpty())
				throw new WebApplicationException("No element found at " + parentMatchingPath,
						Response.Status.BAD_REQUEST);

			Document parent = Serialization.asDocument(foundElementsByMatchingPaths.get(0));

			Access toSetAccess = new Access(doc.getInfo().getAccess().getPolicy(),
					doc.getInfo().getAccess().getLicense());
			if (request.getToSetAccess() != null) {
				// TODO validate specified Access
				String requestedLicense = request.getToSetAccess().getLicense();
				if (requestedLicense != null)
					toSetAccess.setLicense(requestedLicense);

				AccessPolicy requestedPolicy = request.getToSetAccess().getPolicy();
				if (requestedPolicy != null)
					toSetAccess.setPolicy(requestedPolicy);
			}

			// PREPARE REGISTERED FS

			// MANAGE CLASH
			switch (request.getClashOption()) {
			case REPLACE_EXISTING: {
				if (fieldDefinition.isCollection())
					throw new WebApplicationException(
							"Cannot replace repeatable field " + request.getFieldDefinitionPath() + ".",
							Response.Status.BAD_REQUEST);
				// DELETE EXISTING AND PUT
				RegisteredFileSet toDelete = Serialization.convert(parent.get(request.getFieldName()),
						RegisteredFileSet.class);
				if (!(toDelete == null) && !(toDelete.isEmpty())) {
					String path = parentMatchingPath + "." + request.getFieldName();
					deleteFileSetRoutine(doc, false, false, path);

				}
				RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(),
						request.getAttributes(), files, storage, ws);
				log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(),
						doc.getProfileID());
				docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
				break;
			}
			case MERGE_EXISTING: {
				if (fieldDefinition.isCollection())
					throw new WebApplicationException(
							"Cannot merge repeatable field " + request.getFieldDefinitionPath() + ".",
							Response.Status.BAD_REQUEST);
				RegisteredFileSet original = Serialization.convert(parent.get(request.getFieldName()),
						RegisteredFileSet.class);
				// MERGE ATTRIBUTES AND PUT
				Document toUseAttributes = request.getAttributes();
				if (original != null)
					toUseAttributes.putAll(original);
				RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(),
						toUseAttributes, files, storage, ws);
				log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(),
						doc.getProfileID());
				docWrapper.putElement(parentMatchingPath, request.getFieldName(), fs);
				break;
			}
			case APPEND: {
				if (!fieldDefinition.isCollection())
					throw new WebApplicationException(
							"Cannot add to single field " + request.getFieldDefinitionPath() + ".",
							Response.Status.BAD_REQUEST);
				RegisteredFileSet fs = prepareRegisteredFileSet(toSetAccess, doc.getId(), useCaseDescriptor.getId(),
						request.getAttributes(), files, storage, ws);
				log.debug("Registered Fileset for [ID {} useCaseDescriptor {}] is {} ", fs, doc.getId(),
						doc.getProfileID());

				docWrapper.addElementToArray(String.format("%1ds['%2$s']", parentMatchingPath, request.getFieldName()),
						fs);
				break;
			}
			default: {
				throw new WebApplicationException("Unexpected clash policy " + request.getClashOption(),
						Response.Status.BAD_REQUEST);
			}
			}

			log.debug("Setting result on profiled document");
			doc.setTheDocument(Document.parse(docWrapper.getValueCTX().jsonString()));

			doc = onUpdate(doc);
			return unlockAndUpdate(doc);
		} catch (Throwable t) {
			log.warn("Unexpected Exception while trying to registering fileset on {}.", id, t);
			log.debug("Request was {}", request);
			log.debug("Complete doc was {} ", doc);
			unlock(doc);
			throw t;
		}
	}

	/**
	 * Delete file set.
	 *
	 * @param id            the id
	 * @param path          the path
	 * @param force         the force
	 * @param ignore_errors the ignore errors
	 * @return the project
	 * @throws ConfigurationException   the configuration exception
	 * @throws StorageHubException      the storage hub exception
	 * @throws JsonProcessingException  the json processing exception
	 * @throws DeletionException        the deletion exception
	 * @throws EventException           the event exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 */
	@Override
	public Project deleteFileSet(String id, String path, Boolean force, Boolean ignore_errors)
			throws ConfigurationException, StorageHubException, JsonProcessingException, DeletionException,
			EventException, ProjectLockedException, ProjectNotFoundException, InvalidLockException,
			InvalidUserRoleException, UnauthorizedAccess {
		log.info("Deleting Fileset for P_ID {} [UCD_ID {}], at {} [force {} and  ignore_errors {}]", id,
				useCaseDescriptor.getId(), path, force, ignore_errors);
		Project doc = lock(id, "Fileset Deletion");
		try {
			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);
			log.debug("Deleting Fileset for {} [{}] , policy for {} is {} ", doc.getId(), useCaseDescriptor.getId(), u,
					policy);
			// NB cannot check ownership on returned values, must specify filter
			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(doc, u))
				throw new UnauthorizedAccess("No edit rights on project " + doc.getId());

			doc.getLifecycleInformation().cleanState();
			doc.getLifecycleInformation().cleanState().setLastOperationStatus(LifecycleInformation.Status.OK);

			if (ignore_errors == null)
				ignore_errors = false;

			doc = deleteFileSetRoutine(doc, force, ignore_errors, path);
			return unlockAndUpdate(doc);
		} catch (Throwable t) {
			log.warn("Unexpected Exception while trying to delete fileset on {}.", id, t);
			unlock(doc);
			throw t;
		}
	}

	/**
	 * Force unlock.
	 *
	 * @param id the id
	 * @return the project
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidLockException     the invalid lock exception
	 */
	@Override
	public Project forceUnlock(String id) throws InvalidUserRoleException, ProjectNotFoundException, UnauthorizedAccess,
			JsonProcessingException, InvalidLockException {
		Project toUnlock = null;
		try {
			toUnlock = lock(id, "Check locked for force unlock");
			throw new WebApplicationException("Project " + id + " not locked", Response.Status.EXPECTATION_FAILED);
		} catch (ProjectLockedException e) {
			// expected exception
			toUnlock = getByID(id);
		} finally {
			if (toUnlock != null)
				return unlock(toUnlock);
			else
				throw new WebApplicationException("Unable to get Project " + id, Response.Status.EXPECTATION_FAILED);
		}
	}

	/**
	 * Sets the access policy.
	 *
	 * @param id     the id
	 * @param access the access
	 * @return the project
	 * @throws InvalidUserRoleException the invalid user role exception
	 * @throws ProjectLockedException   the project locked exception
	 * @throws ProjectNotFoundException the project not found exception
	 * @throws UnauthorizedAccess       the unauthorized access
	 * @throws JsonProcessingException  the json processing exception
	 * @throws InvalidLockException     the invalid lock exception
	 * @throws EventException           the event exception
	 */
	@Override
	public Project setAccessPolicy(String id, Access access)
			throws InvalidUserRoleException, ProjectLockedException, ProjectNotFoundException, UnauthorizedAccess,
			JsonProcessingException, InvalidLockException, EventException {
		log.trace("UCD {}, Project {} : Setting Access {}", useCaseDescriptor.getId(), id, access);
		Project toUpdate = lock(id, "Set Access policy");
		try {

			User u = UserUtils.getCurrent().asInfo().getUser();
			final DataAccessPolicy policy = useCaseDescriptor.getMatching(u);

			if (policy == null) {
				log.warn("No policy found for {}. Returning empty ", u);
				throw new InvalidUserRoleException("No policy defined for current user roles " + u.getRoles());
			}
			if (!policy.canWrite(toUpdate, u))
				throw new UnauthorizedAccess("No edit rights on project " + id);

			toUpdate.getInfo().setAccess(access);
			toUpdate.getLifecycleInformation().cleanState();
			toUpdate = onUpdate(toUpdate);
			return unlockAndUpdate(toUpdate);
		} catch (Throwable t) {
			log.error("Unexpected exception ", t);
			unlock(toUpdate);
			throw t;
		}
	}

	/**
	 * Delete file set routine. updated by Francesco
	 * 
	 * @param doc           the doc
	 * @param force         the force
	 * @param ignore_errors the ignore errors
	 * @param path          the path
	 * @return the project
	 * @throws ConfigurationException the configuration exception
	 * @throws StorageHubException    the storage hub exception
	 */
	private Project deleteFileSetRoutine(Project doc, Boolean force, Boolean ignore_errors, String path)
			throws ConfigurationException, StorageHubException {
		log.info("Delete[force : {}, ignore_errors: {}] FS at {} for {}:{}", force, ignore_errors, path,
				doc.getProfileID(), doc.getId());
		JSONPathWrapper wrapper = new JSONPathWrapper(doc.getTheDocument().toJson());
		List<String> matchingPaths = wrapper.getMatchingPaths(path);

		log.info("matchingPaths is: " + matchingPaths);

		String error = null;
		if (matchingPaths.isEmpty()) {
			error = "No Registered FileSet found at " + path;
			if (!ignore_errors) {
				throw new WebApplicationException(error, Response.Status.BAD_REQUEST);
			}
		}
		if (matchingPaths.size() > 1 && !ignore_errors) {
			error = "Multiple Fileset (" + matchingPaths.size() + ") matching " + path;
			if (!ignore_errors)
				throw new WebApplicationException(error, Response.Status.BAD_REQUEST);
		}

		if (error != null && ignore_errors) {
			log.info("Error detected {}. Ignoring it and returning input doc", error);
			return doc;
		}

		// Changed by Francesco
		RegisteredFileSet fs = null;
		error = null;
		Exception theExc = null;
		try {
			List<Object> deleteList = wrapper.getByPath(path);
			if (deleteList == null || deleteList.isEmpty()) {
				error = "List of object going to delete is null or empty";
				log.info(error);
			}

			if (error == null) {
				Object toFileSet = deleteList.get(0); // expected one
				fs = Serialization.convert(toFileSet, RegisteredFileSet.class);
				log.info("Going to delete {}", fs);
			}
		} catch (Exception e) {
			error = "Error on getting the RegisteredFileSet for path: " + path;
			log.warn(error);
			log.debug("Error is: ", e);
			theExc = e;
		}

		if (error != null) {
			if (ignore_errors) {
				log.info("ignore_errors is {}, returning input doc {}", ignore_errors, doc.getId());
				return doc;
			}
			log.info("Thrown "+ConfigurationException.class.getSimpleName()+" with message {}", theExc.getMessage());
			throw new ConfigurationException(theExc.getMessage());

		}

		// Updated by Francesco. See #24902
		doc = triggerEvent(doc, EventExecutionRequest.Events.ON_DELETE_FILESET,
				new Document("force", force).append("path", path).append("fileSetPath", path));
		// reloading the document in the wrapper
		wrapper = new JSONPathWrapper(doc.getTheDocument().toJson());

		// Actually delete only if event was ok
		if (doc.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
			// Delete from storage
			if (fs.getFolderId() != null) {
				try {
					log.info("Deleting Fileset Folder ID {} ", fs.getFolderId());
					new WorkspaceManager().deleteItem(fs.getFolderId());
				} catch (Exception e) {
					// Updated by Francesco. See #24902
					LifecycleInformation info = doc.getLifecycleInformation();
					if (info == null) {
						info = new LifecycleInformation();
					}
					info.addErrorMessage("Unable to delete the Folder ID " + fs.getFolderId() + " in the VRE Folder");
					info.setLastOperationStatus(LifecycleInformation.Status.WARNING);
					log.warn("Error on deleting the Folder ID {} in the VRE Folder", fs.getFolderId(), e);
				}
			}
			log.debug("Removing FS from document [ID : ] by path {}", doc.getId(), path);
			// Delete from document
			wrapper.setElement(path, null);

		}
		// Updated by Francesco. See #24902
		log.debug("Setting result on profiled document");
		doc.setTheDocument(Document.parse(wrapper.getValueCTX().jsonString()));
		return doc;
	}

	/**
	 * Gets the configuration.
	 *
	 * @return the configuration
	 * @throws ConfigurationException the configuration exception
	 */
	@Override
	public Configuration getConfiguration() throws ConfigurationException {
		String context = UserUtils.getCurrent().getContext();
		log.debug("Asking configuration for {} in {} ", useCaseDescriptor.getId(), context);
		Configuration toReturn = new Configuration();
		List<Archive> archives = new ArrayList<>();
		toReturn.setArchives(archives);
		List<Index> indexes = new ArrayList<>();
		toReturn.setIndexes(indexes);
		// Set Basic Info
		toReturn.setProfileId(this.getUseCaseDescriptor().getId());
		toReturn.setContext(context);
		toReturn.setLastUpdatedTime(LocalDateTime.now());

		// Add Mongo Info
		Archive mongoArchive = new Archive("DOCUMENT-STORE-COLLECTION");
		MongoCollection coll = getCollection();
		mongoArchive.put("count", coll.count());
		mongoArchive.put("collection_name", getToUseCollectionName());

		// Get counts by PHASE, Status
		try {
			ArrayList<Document> counts = new ArrayList<>();

			coll.aggregate(Collections.singletonList(Document.parse("{\"$group\" : "
					+ "{\"_id\":{\"phase\":\"$_lifecycleInformation._phase\",\"status\":\"$_lifecycleInformation._lastOperationStatus\"}, "
					+ "\"count\":{\"$sum\":1}}}"))).forEach((Consumer) doc -> {
						try {
							counts.add(Serialization.asDocument(doc));
						} catch (JsonProcessingException e) {
							log.warn("Unable to write aggregated results ", e);
						}
					});

			mongoArchive.put("countByPhase", counts);
		} catch (Throwable t) {
			toReturn.addErrorMessage("Unable to get PHASE statistics " + t.getMessage());
			log.error("Unable to get PHASE statistics", t);
		}
		archives.add(mongoArchive);

		// TODO ADD TEXT INDEXES

		// Set WS Info
		try {
			archives.add(new WorkspaceManager().getConfiguration());
		} catch (Exception e) {
			toReturn.addErrorMessage("Unable to get WS info " + e.getMessage());
			log.error("Unable to get WS Configuration", e);
		}

		// ADD LC Infos

		AccountingInfo user = UserUtils.getCurrent().asInfo();
		try {
			Configuration lcConfig = getLCManager()
					.getCurrentConfiguration(new BaseRequest(useCaseDescriptor, user.getUser(), user.getContext()));
			log.info("Configuration is {} ", lcConfig);
			if (lcConfig.getArchives() != null)
				archives.addAll(lcConfig.getArchives());

			if (lcConfig.getIndexes() != null)
				indexes.addAll(lcConfig.getIndexes());
		} catch (Throwable e) {
			toReturn.addErrorMessage("Unable to get Lifecycle info " + e.getMessage());
			log.error("Unable to get Lifecycle info ", e);
		}

		log.debug("Returning current configuration {}", toReturn);
		return toReturn;
	}

	/**
	 * Step.
	 *
	 * @param theDocument    the the document
	 * @param step           the step
	 * @param callParameters the call parameters
	 * @return the project
	 * @throws InsufficientPrivileges the insufficient privileges
	 * @throws ConfigurationException the configuration exception
	 * @throws StepException          the step exception
	 */
	private Project step(Project theDocument, String step, Document callParameters)
			throws InsufficientPrivileges, ConfigurationException, StepException {
		try {
			log.info("[UseCaseDescriptor {}] Invoking Step {} on {}", useCaseDescriptor.getId(), step,
					getManager().getDescriptor());
			AccountingInfo user = UserUtils.getCurrent().asInfo();

			StepExecutionRequest request = new StepExecutionRequest(useCaseDescriptor, user.getUser(),
					user.getContext(), theDocument, step);
			request.setCallParameters(callParameters);

			log.debug("Requesting Step Execution {}", request);
			StepExecutionReport report = getManager().performStep(request);
			Project toReturn = report.prepareResult();

			// EVENTS
			if (report.getToTriggerEvents() != null) {
				Iterator<EventExecutionRequest> eventIT = report.getToTriggerEvents().listIterator();
				while (!toReturn.getLifecycleInformation().getLastOperationStatus()
						.equals(LifecycleInformation.Status.ERROR) && eventIT.hasNext()) {
					EventExecutionRequest triggerRequest = eventIT.next();
					log.info("Triggering {} ", triggerRequest);
					toReturn = triggerEvent(toReturn, triggerRequest.getEvent(), triggerRequest.getCallParameters());
				}
			}

			// STEPS
			if (report.getCascadeSteps() != null) {
				Iterator<StepExecutionRequest> stepIT = report.getCascadeSteps().listIterator();
				while (!toReturn.getLifecycleInformation().getLastOperationStatus()
						.equals(LifecycleInformation.Status.ERROR) && stepIT.hasNext()) {
					StepExecutionRequest triggerRequest = stepIT.next();
					log.info("Triggering {} ", triggerRequest);
					toReturn = step(toReturn, triggerRequest.getStep(), triggerRequest.getCallParameters());
				}
			}
			return report.prepareResult();
		} catch (InsufficientPrivileges | ConfigurationException | UnrecognizedStepException e) {
			throw e;
		} catch (Throwable t) {
			log.error("Unable to perform step " + step, t);
			theDocument.getLifecycleInformation()
					.addErrorMessage("Unable to perform step " + step + " cause : " + t.getMessage());
			theDocument.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
			return theDocument;
		}
	}

	/**
	 * Trigger event.
	 *
	 * @param project    the project
	 * @param event      the event
	 * @param parameters the parameters
	 * @return the project
	 */
	private Project triggerEvent(Project project, String event, Document parameters) {
		try {
			log.info("[UseCaseDescriptor {}] triggering event {} on {}", useCaseDescriptor.getId(), event,
					getManager().getDescriptor());
			AccountingInfo user = UserUtils.getCurrent().asInfo();
			EventExecutionRequest request = new EventExecutionRequest(useCaseDescriptor, user.getUser(),
					user.getContext(), project, event);
			request.setCallParameters(parameters);
			log.debug("Triggering {}", request);
			DocumentHandlingReport report = getManager().onEvent(request);
			return report.prepareResult();
		} catch (Throwable t) {
			log.error("Unable to trigger event " + event, t);
			project.getLifecycleInformation()
					.addErrorMessage("Unable to trigger " + event + " cause : " + t.getMessage());
			project.getLifecycleInformation().setLastOperationStatus(LifecycleInformation.Status.ERROR);
			return project;
		}
	}

	/**
	 * Prepare registered file set.
	 *
	 * @param toSetAccess the to set access
	 * @param docID       the doc ID
	 * @param profileID   the profile ID
	 * @param attributes  the attributes
	 * @param files       the files
	 * @param storage     the storage
	 * @param ws          the ws
	 * @return the registered file set
	 * @throws StorageHubException the storage hub exception
	 * @throws StorageException    the storage exception
	 */
	private static final RegisteredFileSet prepareRegisteredFileSet(Access toSetAccess, String docID, String profileID,
			Document attributes, List<TempFile> files, StorageUtils storage, WorkspaceManager ws)
			throws StorageHubException, StorageException {
		log.info("Preparing Registered FileSet {}");
		RegisteredFileSet toReturn = new RegisteredFileSet();
		if (attributes != null)
			toReturn.putAll(attributes);
		String uuid = UUID.randomUUID().toString();
		toReturn.putIfAbsent(RegisteredFileSet.UUID, uuid);
		toReturn.putIfAbsent(RegisteredFileSet.CREATION_INFO, UserUtils.getCurrent().asInfo());
		toReturn.putIfAbsent(RegisteredFileSet.ACCESS, toSetAccess);

		// FOLDER
		String folderID = toReturn.getFolderId();
		log.trace("Folder ID is {} ", folderID);
		FolderContainer sectionFolder = null;
		if (folderID == null || folderID.isEmpty()) {

			// Get BASE Folder for project
			FolderContainer base = ws.getSubFolder(ws.getAppBase(), docID,
					"Base Folder for profiled document. UseCaseDescriptor " + profileID);
			// Create Folder for FileSet
			sectionFolder = ws.createFolder(
					new WorkspaceManager.FolderOptions(docID + "_" + uuid, "Registered Fileset uuid " + uuid, base));
			toReturn.put(RegisteredFileSet.FOLDER_ID, sectionFolder.getId());
		} else {
			sectionFolder = ws.getFolderById(folderID);
		}

		ArrayList<RegisteredFile> registeredFiles = new ArrayList<>();

		if (toReturn.containsKey(RegisteredFileSet.PAYLOADS))
			registeredFiles.addAll(toReturn.getPayloads());

		for (TempFile f : files) {
			InputStream is = null;
			try {
				log.debug("Opening temp file {}", f);
				String fileUrl = null;
				if (f.getId() == null || f.getId().isEmpty())
					fileUrl = f.getUrl();
				else
					fileUrl = storage.getURL(f.getId());

				log.info("Got URL {} from ID {}", fileUrl, f.getId());
				is = new URL(fileUrl).openStream();
				RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is,
						"Imported via gcube CMS service ", sectionFolder));
				log.info("Registered " + registered);
				registeredFiles.add(registered);
			} catch (StorageHubException | IOException e) {
				throw new StorageException("Unable to store " + f, e);
			} finally {
				if (is != null)
					IOUtils.closeQuietly(is);
			}
		}
		toReturn.put(RegisteredFileSet.PAYLOADS, registeredFiles);
		// TODO MERGE
		// toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
		return toReturn;
	}

	/**
	 * Gets the field definition.
	 *
	 * @param useCaseDescriptor the use case descriptor
	 * @param fieldPath         the field path
	 * @return the field definition
	 * @throws WebApplicationException the web application exception
	 */
	private static Field getFieldDefinition(UseCaseDescriptor useCaseDescriptor, String fieldPath)
			throws WebApplicationException {
		JSONPathWrapper schemaWrapper = new JSONPathWrapper(useCaseDescriptor.getSchema().toJson());
		List<Field> fieldDefinitions = schemaWrapper.getByPath(fieldPath, Field.class);
		if (fieldDefinitions == null || fieldDefinitions.isEmpty())
			throw new WebApplicationException(
					"No Field found in schema " + useCaseDescriptor.getId() + " at " + fieldPath,
					Response.Status.BAD_REQUEST);
		if (fieldDefinitions.size() > 1)
			throw new WebApplicationException("Multiple field definitions (" + fieldDefinitions.size() + ") found in "
					+ useCaseDescriptor.getId() + " for " + fieldPath, Response.Status.BAD_REQUEST);
		Field fieldDefinition = Serialization.convert(fieldDefinitions.get(0), Field.class);
		if (fieldDefinition == null)
			throw new WebApplicationException(
					"Found field is null [" + useCaseDescriptor.getId() + " for " + fieldPath + "]",
					Response.Status.BAD_REQUEST);
		log.trace("Field definition is {}", fieldDefinition);
		return fieldDefinition;
	}
}