gcube-cms-suite/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java

497 lines
24 KiB
Java
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package org.gcube.application.geoportal.service.engine.mongo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.jayway.jsonpath.JsonPath;
import com.mongodb.client.MongoDatabase;
import com.vdurmont.semver4j.Semver;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.cms.plugins.LifecycleManager;
import org.gcube.application.cms.plugins.faults.EventException;
import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.cms.plugins.model.PluginDescriptor;
import org.gcube.application.cms.plugins.reports.EventExecutionReport;
import org.gcube.application.cms.plugins.reports.StepExecutionReport;
import org.gcube.application.cms.plugins.requests.EventExecutionRequest;
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.document.*;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.accounting.AccountingInfo;
import org.gcube.application.geoportal.common.model.document.accounting.PublicationInfo;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.model.profile.Field;
import org.gcube.application.geoportal.common.model.profile.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.profile.Profile;
import org.gcube.application.geoportal.common.model.rest.Configuration;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.ImplementationProvider;
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.service.utils.UserUtils;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;
import static org.gcube.application.cms.serialization.Serialization.*;
@Slf4j
public class ProfiledMongoManager extends MongoManager implements MongoManagerI<ProfiledDocument>{
@Getter
Profile profile;
MongoDatabase db=null;
public ProfiledMongoManager(String profileId) throws ConfigurationException {
// Check Profile ID
log.info("Loading profile ID {} ",profileId);
if(profileId==null) throw new InvalidParameterException("Profile ID cannot be null");
Map<String,Profile> profiles=ImplementationProvider.get().getProfiles().getObject();
if(!profiles.containsKey(profileId)) {
log.debug("Asked profile {} not found. Available ones are {} ",profileId,profiles.keySet());
throw new WebApplicationException("Profile " + profileId + " not registered", Response.Status.NOT_FOUND);
}
profile=profiles.get(profileId);
log.debug("Loaded Profile {} ",profile);
// Connect to DB
String toUseDB=super.client.getConnection().getDatabase();
log.info("Connecting to DB {} ",toUseDB);
// TODO MAP OF DATABASES?
db=client.getTheClient().getDatabase(toUseDB);
}
@Getter(lazy = true)
private final LifecycleManager manager=getLCManager();
private LifecycleManager getLCManager() {
try{
LifecycleManager toReturn=null;
//Getting Lifecycle Manager declaration from Profile
List<HandlerDeclaration> handlerDeclarations= profile.getHandlersMap().get(PluginDescriptor.BaseTypes.LIFECYCLE_MANAGER);
if(handlerDeclarations==null || handlerDeclarations.isEmpty()) throw new ConfigurationException("No Lifecycle Handler defined for profile ID "+profile.getId());
if(handlerDeclarations.size()>1) throw new ConfigurationException("Too many Lifecycle Handlers defined ("+handlerDeclarations+") in profile ID "+profile.getId());
HandlerDeclaration lcHandlerDeclaration=handlerDeclarations.get(0);
// Loading Lifecycle Manager
log.debug("Looking for handler {} ",lcHandlerDeclaration);
toReturn=(LifecycleManager) ImplementationProvider.get().getPluginManager().getObject().get(lcHandlerDeclaration.getId());
if(toReturn==null) throw new ConfigurationException("Unable to find Lifecycle Manager Plugin. ID "+lcHandlerDeclaration.getId());
return toReturn;
} catch(Throwable t){
log.warn("Unable to load LC Manager ",t);
return null;
}
}
private String getCollectionName(){
// TODO Profile can directly specify, use ID only as default
return profile.getId();
}
@Override
public MongoDatabase getDatabase(){
return db;
}
@Override
public ProfiledDocument registerNew(Document toRegisterDoc) throws IOException, StepException, EventException {
log.info("Registering new document in {} ",profile.getId());
log.debug("Going to register {}",toRegisterDoc.toJson());
ProfiledDocument toRegister = new ProfiledDocument();
toRegister.setTheDocument(toRegisterDoc);
PublicationInfo pubInfo=new PublicationInfo();
pubInfo.setCreationInfo(UserUtils.getCurrent().asInfo());
// TODO Set Access From Profile
Access access=new Access();
access.setLicense("");
access.setPolicy(AccessPolicy.OPEN);
pubInfo.setAccess(access);
toRegister.setInfo(pubInfo);
toRegister.setProfileID(profile.getId());
toRegister.setProfileVersion(profile.getVersion());
toRegister.setVersion(new Semver("1.0.0"));
LifecycleInformation draftInfo=new LifecycleInformation();
draftInfo.setPhase(LifecycleInformation.DRAFT_PHASE);
toRegister.setLifecycleInformation(draftInfo);
// Apply Lifecycle
toRegister = triggerEvent(toRegister,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null).getResult();
log.debug("Going to register {} ",toRegister);
// Insert object
ObjectId id =insert(asDocumentWithId(toRegister),getCollectionName());
log.info("Obtained id {} ",id);
return getByID(id.toHexString());
}
@Override
public ProfiledDocument update(String id, Document toSet) throws IOException, EventException {
log.trace("Replacing {} ",toSet);
ProfiledDocument toUpdate=getByID(id);
toUpdate.setTheDocument(toSet);
toUpdate=onUpdate(toUpdate);
ProfiledDocument toReturn =convert(replace(asDocumentWithId(toUpdate),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
log.debug("Updated ProfiledDocument is {}",toReturn);
return toReturn;
}
private ProfiledDocument onUpdate(ProfiledDocument toUpdate) throws EventException {
UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
toUpdate.getInfo().setLastEditInfo(u.asInfo());
toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
return triggerEvent(toUpdate,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null).getResult();
}
@Override
public void delete(String id,boolean force) throws DeletionException {
log.debug("Deleting by ID {}, force {}",id,force);
try{
ProfiledDocument doc =getByID(id);
// TODO INVOKE LIFECYCLE
//if(!force&&isPublished(id)) throw new Exception("Cannot delete published documents. Unpublish it or use force = true");
try{
// TODO CHECK PHASE AND STATUS
// DEINDEX
// DEMATERIALIZE
// DELETE CONTENT
// DELETE ENTRY
throw new DeletionException("IMPLEMENT THIS");
// delete(asId(id), getCollectionName());
}catch(DeletionException e) {
//storing updated - partially deleted
// concessione=onUpdate(concessione);
// replace(asDocumentWithId(concessione), collectionName);
throw e;
}
}catch(Throwable t){
throw new DeletionException("Unable to delete "+id,t);
}
}
@Override
public ProfiledDocument getByID(String id) throws WebApplicationException{
Document doc=super.getById(asId(id),getCollectionName());
if(doc==null) throw new WebApplicationException("No document with ID "+id);
return convert(doc,ProfiledDocument.class);
}
@Override
public Iterable<Document> query(QueryRequest queryRequest) {
log.info("Querying {} ",queryRequest);
LinkedBlockingQueue queue=new LinkedBlockingQueue<Concessione>();
query(queryRequest,getCollectionName()).forEach(
(Consumer<? super Document>) (Document d)->{try{
queue.put(d);
}catch(Throwable t){log.warn("Unable to translate "+d);}});
log.info("Returned {} elements ",queue.size());
return queue;
}
@Override
public Iterable<ProfiledDocument> filter(QueryRequest queryRequest) {
log.info("Searching concessione for filter {} ",queryRequest);
LinkedBlockingQueue queue=new LinkedBlockingQueue<Concessione>();
query(queryRequest,getCollectionName()).forEach(
(Consumer<? super Document>) (Document d)->{try{
queue.put(d);
}catch(Throwable t){log.warn("Unable to translate "+d);}});
log.info("Returned {} elements ",queue.size());
return queue;
}
@Override
public ProfiledDocument performStep(String id, String step, Document options) throws StepException, JsonProcessingException {
ProfiledDocument document = getByID(id);
try{
StepExecutionReport report = step(document, step, options);
document = report.getResult();
} catch(Throwable t){
log.error("[Profile {} ] ERROR Invoking Step {} on document {}" ,profile.getId(),step,id,t);
LifecycleInformation info = new LifecycleInformation();
info.setPhase(document.getLifecycleInformation().getPhase());
info.setLastOperationStatus(LifecycleInformation.Status.ERROR);
info.addErrorMessage(t.getMessage());
info.setLastInvokedStep(step);
document.setLifecycleInformation(info);
}finally{
return convert(replace(asDocumentWithId(document),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
}
}
/**
* NB Put at path :
*
* Path Examples
* artifact
* images
* images[1]
* layers[?(@.name = 'myName')].fileset
*
*
*
*/
@Override
public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException, EventException {
log.info("Registering Fileset for {} [profile ID {}], Request is {} ",id,profile.getId(),request);
List<TempFile> files=request.getStreams();
Document attributes =request.getAttributes();
ProfiledDocument doc=getByID(id);
WorkspaceManager ws=new WorkspaceManager();
StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject();
log.debug("Checking field {} definition in {}",request.getFieldDefinitionPath(),profile.getId());
Field fieldDefinition=getFieldDefinition(profile,request.getFieldDefinitionPath());
JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson());
List<String> matchingPaths = docWrapper.getMatchingPaths(request.getParentPath());
if(matchingPaths.size()>1) throw new WebApplicationException("Multiple Destination matching parent path "+request.getParentPath(),Response.Status.BAD_REQUEST);
if(matchingPaths.isEmpty()) throw new WebApplicationException("PArent path not found at "+request.getParentPath(),Response.Status.BAD_REQUEST);
String parentMatchingPath = matchingPaths.get(0);
Document parent = Serialization.asDocument(docWrapper.getByPath(parentMatchingPath).get(0));
// PREPARE REGISTERED FS
// MANAGE CLASH
switch (request.getClashOption()){
case REPLACE_EXISTING: {
if(fieldDefinition.isCollection())
throw new WebApplicationException("Cannot replace repeatable field "+request.getFieldDefinitionPath()+".",Response.Status.BAD_REQUEST);
// DELETE EXISTING AND PUT
RegisteredFileSet toDelete = Serialization.convert(parent.get(request.getFieldName()),RegisteredFileSet.class);
if(!(toDelete == null)&&!(toDelete.isEmpty()))
deleteFileSetRoutine(toDelete,false,ws);
RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(),doc.get_id(),profile.getId(), request.getAttributes(),files,storage,ws);
log.debug("Registered Fileset for [ID {} profile {}] is {} ",fs,doc.get_id(),doc.getProfileID());
docWrapper.putElement(parentMatchingPath,request.getFieldName(),fs);
break;}
case MERGE_EXISTING: {
if(fieldDefinition.isCollection())
throw new WebApplicationException("Cannot merge repeatable field "+request.getFieldDefinitionPath()+".",Response.Status.BAD_REQUEST);
RegisteredFileSet original = Serialization.convert(parent.get(request.getFieldName()),RegisteredFileSet.class);
// MERGE ATTRIBUTES AND PUT
Document toUseAttributes=request.getAttributes();
if(original!=null) toUseAttributes.putAll(original);
RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(),doc.get_id(),profile.getId(), toUseAttributes,files,storage,ws);
log.debug("Registered Fileset for [ID {} profile {}] is {} ",fs,doc.get_id(),doc.getProfileID());
docWrapper.putElement(parentMatchingPath,request.getFieldName(),fs);
break;}
case APPEND: {
if(!fieldDefinition.isCollection())
throw new WebApplicationException("Cannot add to single field "+request.getFieldDefinitionPath()+".",Response.Status.BAD_REQUEST);
RegisteredFileSet fs = prepareRegisteredFileSet(doc.getInfo(),doc.get_id(),profile.getId(), request.getAttributes(),files,storage,ws);
log.debug("Registered Fileset for [ID {} profile {}] is {} ",fs,doc.get_id(),doc.getProfileID());
docWrapper.addElementToArray(String.format("%1ds['%2$s']",parentMatchingPath,request.getFieldName()),fs);
break;}
default: {throw new WebApplicationException("Unexpected clash policy "+request.getClashOption(),Response.Status.BAD_REQUEST);}
}
log.debug("Setting result on profiled document");
doc.setTheDocument(Document.parse(docWrapper.getValueCTX().jsonString()));
doc=onUpdate(doc);
return convert(replace(asDocumentWithId(doc),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
}
@Override
public ProfiledDocument deleteFileSet(String id, String path, Boolean force) throws ConfigurationException, StorageHubException, JsonProcessingException, DeletionException, EventException {
log.info("Deleting Fileset for {} [profile ID {}], at {} [force {} ]",id,profile.getId(),path,force);
ProfiledDocument doc = getByID(id);
JSONPathWrapper wrapper = new JSONPathWrapper(doc.getTheDocument().toJson());
List<String> matchingPaths=wrapper.getMatchingPaths(path);
if(matchingPaths.isEmpty()) throw new WebApplicationException("No Registered FileSet found at "+path,Response.Status.BAD_REQUEST);
if(matchingPaths.size()>1) throw new WebApplicationException("Multiple Fileset ("+matchingPaths.size()+") matching "+path,Response.Status.BAD_REQUEST);
RegisteredFileSet fs = Serialization.convert(wrapper.getByPath(path),RegisteredFileSet.class);
log.debug("Going to delete {}",fs);
deleteFileSetRoutine(fs,force,new WorkspaceManager());
log.debug("Removing FS from document [ID : ] by path {}",id,path);
wrapper.setElement(path,null);
doc=onUpdate(doc);
return convert(replace(asDocumentWithId(doc),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
}
@Override
public Configuration getConfiguration() throws ConfigurationException {
log.debug("Asking configuration for {} in {} ",profile.getId(), UserUtils.getCurrent().getContext());
Configuration toReturn= getManager().getCurrentConfiguration();
log.debug("Returning current configuration {}",toReturn);
return toReturn;
}
private StepExecutionReport step(ProfiledDocument theDocument, String step, Document callParameters) throws StepException {
log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor());
StepExecutionRequest request=new StepExecutionRequest();
request.setCallParameters(callParameters);
request.setDocument(theDocument);
request.setProfile(profile);
request.setStep(step);
AccountingInfo user= UserUtils.getCurrent().asInfo();
request.setUser(user.getUser());
request.setContext(user.getContext());
log.debug("Requesting Step Execution {} ",request);
StepExecutionReport report= getManager().performStep(request);
log.debug("Report is {}",report);
if(report.getResult()==null) throw new StepException("Report result is null");
return report;
}
private EventExecutionReport triggerEvent(ProfiledDocument theDocument, String event, Document parameters) throws EventException {
log.info("[Profile {} ] triggering event {} on {}" ,profile.getId(),event,getManager().getDescriptor());
EventExecutionRequest request= new EventExecutionRequest();
request.setEvent(event);
request.setProfile(profile);
request.setCallParameters(parameters);
request.setDocument(theDocument);
log.debug("Triggering {} ",request);
return getManager().onEvent(request);
}
private static final RegisteredFileSet prepareRegisteredFileSet(PublicationInfo defaultPublicationInfo,String docID, String profileID,
Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
log.debug("Preparing Registered FileSet..");
RegisteredFileSet toReturn = new RegisteredFileSet();
if(attributes!=null) toReturn.putAll(attributes);
String uuid = UUID.randomUUID().toString();
toReturn.putIfAbsent(RegisteredFileSet.UUID, uuid);
toReturn.putIfAbsent(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
toReturn.putIfAbsent(RegisteredFileSet.ACCESS,defaultPublicationInfo.getAccess());
// FOLDER
String folderID=toReturn.getFolderId();
log.trace("Folder ID is {} ",folderID);
FolderContainer sectionFolder=null;
if(folderID==null || folderID.isEmpty()) {
FolderContainer base = ws.createFolder(new WorkspaceManager.FolderOptions(
docID, "Base Folder for profiled document. Profile " + profileID, null));
sectionFolder = ws.createFolder(new WorkspaceManager.FolderOptions(
docID + "_" + uuid, "Registered Fileset uuid " + uuid, base));
toReturn.put(RegisteredFileSet.FOLDER_ID, sectionFolder.getId());
}else {
sectionFolder = ws.getFolderById(folderID);
}
ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();
if(toReturn.containsKey(RegisteredFileSet.PAYLOADS))
registeredFiles.addAll(toReturn.getPayloads());
for (TempFile f : files) {
InputStream is=null;
try{
log.debug("Opening temp file {}",f);
String fileUrl=storage.getURL(f.getId());
log.debug("Got URL {} from ID {}",fileUrl,f.getId());
is=new URL(fileUrl).openStream();
RegisteredFile registered=ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(),is,
"Imported via gcube CMS service ", sectionFolder));
log.debug("Registered "+registered);
registeredFiles.add(registered);
}catch(StorageHubException | IOException e){
throw new StorageException("Unable to store "+f,e);
}finally{
if(is!=null)
IOUtils.closeQuietly(is);
}
}
toReturn.put(RegisteredFileSet.PAYLOADS,registeredFiles);
// TODO MERGE
//toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
return toReturn;
}
private static void deleteFileSetRoutine(RegisteredFileSet fs, Boolean force, WorkspaceManager ws) throws DeletionException, StorageHubException {
log.debug("Deleting Registered FS {}");
if(fs.getMaterializations()!=null && !fs.getMaterializations().isEmpty()){
if(!force) throw new DeletionException("Fileset (uuid "+fs.getUUID()+") already materialized. Use force = true");
else throw new RuntimeException("Implement this");
// TODO manager force deletion
// NB handlers for materialization types
}
log.trace("FileSet ID {} : deleting ws folder {}",fs.getUUID(),fs.getFolderId());
if(fs.getPayloads()!=null)
ws.deleteItem(fs.getFolderId());
}
private static Field getFieldDefinition(Profile profile,String fieldPath)throws WebApplicationException{
JSONPathWrapper schemaWrapper= new JSONPathWrapper(profile.getSchema().toJson());
List<Field> fieldDefinitions=schemaWrapper.getByPath(fieldPath,Field.class);
if(fieldDefinitions==null || fieldDefinitions.isEmpty())
throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+fieldPath, Response.Status.BAD_REQUEST);
if(fieldDefinitions.size()>1)
throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+fieldPath,Response.Status.BAD_REQUEST);
Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
if(fieldDefinition==null)
throw new WebApplicationException("Found field is null ["+profile.getId()+" for "+fieldPath+"]",Response.Status.BAD_REQUEST);
log.trace("Field definition is {}",fieldDefinition);
return fieldDefinition;
}
}