470 lines
22 KiB
Java
470 lines
22 KiB
Java
package org.gcube.application.geoportal.service.engine.mongo;
|
||
|
||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||
import com.mongodb.client.MongoDatabase;
|
||
import com.vdurmont.semver4j.Semver;
|
||
import lombok.Getter;
|
||
import lombok.extern.slf4j.Slf4j;
|
||
import org.apache.commons.io.IOUtils;
|
||
import org.bson.Document;
|
||
import org.bson.types.ObjectId;
|
||
import org.gcube.application.cms.plugins.LifecycleManager;
|
||
import org.gcube.application.cms.plugins.faults.EventException;
|
||
import org.gcube.application.cms.plugins.faults.StepException;
|
||
import org.gcube.application.cms.plugins.model.PluginDescriptor;
|
||
import org.gcube.application.cms.plugins.reports.EventExecutionReport;
|
||
import org.gcube.application.cms.plugins.reports.StepExecutionReport;
|
||
import org.gcube.application.cms.plugins.requests.EventExecutionRequest;
|
||
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
|
||
import org.gcube.application.geoportal.common.faults.StorageException;
|
||
import org.gcube.application.geoportal.common.model.document.*;
|
||
import org.gcube.application.geoportal.common.model.document.access.Access;
|
||
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
|
||
import org.gcube.application.geoportal.common.model.document.accounting.AccountingInfo;
|
||
import org.gcube.application.geoportal.common.model.document.accounting.PublicationInfo;
|
||
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
|
||
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
|
||
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
|
||
import org.gcube.application.geoportal.common.model.legacy.Concessione;
|
||
import org.gcube.application.geoportal.common.model.profile.Field;
|
||
import org.gcube.application.geoportal.common.model.profile.HandlerDeclaration;
|
||
import org.gcube.application.geoportal.common.model.profile.Profile;
|
||
import org.gcube.application.geoportal.common.model.rest.Configuration;
|
||
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
|
||
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
|
||
import org.gcube.application.geoportal.common.model.rest.TempFile;
|
||
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
|
||
import org.gcube.application.geoportal.common.utils.StorageUtils;
|
||
import org.gcube.application.geoportal.service.engine.ImplementationProvider;
|
||
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
|
||
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
|
||
import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
|
||
import org.gcube.application.cms.serialization.Serialization;
|
||
import org.gcube.application.geoportal.service.utils.UserUtils;
|
||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
||
|
||
import javax.ws.rs.WebApplicationException;
|
||
import javax.ws.rs.core.Response;
|
||
import java.io.IOException;
|
||
import java.io.InputStream;
|
||
import java.net.URL;
|
||
import java.security.InvalidParameterException;
|
||
import java.util.ArrayList;
|
||
import java.util.List;
|
||
import java.util.Map;
|
||
import java.util.UUID;
|
||
import java.util.concurrent.LinkedBlockingQueue;
|
||
import java.util.function.Consumer;
|
||
|
||
import static org.gcube.application.cms.serialization.Serialization.*;
|
||
|
||
@Slf4j
|
||
public class ProfiledMongoManager extends MongoManager implements MongoManagerI<ProfiledDocument>{
|
||
|
||
@Getter
|
||
Profile profile;
|
||
MongoDatabase db=null;
|
||
|
||
|
||
|
||
|
||
public ProfiledMongoManager(String profileId) throws ConfigurationException {
|
||
// Check Profile ID
|
||
log.info("Loading profile ID {} ",profileId);
|
||
if(profileId==null) throw new InvalidParameterException("Profile ID cannot be null");
|
||
Map<String,Profile> profiles=ImplementationProvider.get().getProfiles().getObject();
|
||
if(!profiles.containsKey(profileId)) {
|
||
log.debug("Asked profile {} not found. Available ones are {} ",profileId,profiles.keySet());
|
||
throw new WebApplicationException("Profile " + profileId + " not registered", Response.Status.NOT_FOUND);
|
||
}
|
||
profile=profiles.get(profileId);
|
||
log.debug("Loaded Profile {} ",profile);
|
||
|
||
|
||
// Connect to DB
|
||
String toUseDB=super.client.getConnection().getDatabase();
|
||
log.info("Connecting to DB {} ",toUseDB);
|
||
|
||
// TODO MAP OF DATABASES?
|
||
db=client.getTheClient().getDatabase(toUseDB);
|
||
}
|
||
|
||
@Getter(lazy = true)
|
||
private final LifecycleManager manager=getLCManager();
|
||
|
||
private LifecycleManager getLCManager() {
|
||
try{
|
||
LifecycleManager toReturn=null;
|
||
//Getting Lifecycle Manager declaration from Profile
|
||
List<HandlerDeclaration> handlerDeclarations= profile.getHandlersMap().get(PluginDescriptor.BaseTypes.LIFECYCLE_MANAGER);
|
||
if(handlerDeclarations==null || handlerDeclarations.isEmpty()) throw new ConfigurationException("No Lifecycle Handler defined for profile ID "+profile.getId());
|
||
if(handlerDeclarations.size()>1) throw new ConfigurationException("Too many Lifecycle Handlers defined ("+handlerDeclarations+") in profile ID "+profile.getId());
|
||
|
||
HandlerDeclaration lcHandlerDeclaration=handlerDeclarations.get(0);
|
||
|
||
// Loading Lifecycle Manager
|
||
log.debug("Looking for handler {} ",lcHandlerDeclaration);
|
||
toReturn=(LifecycleManager) ImplementationProvider.get().getPluginManager().getObject().get(lcHandlerDeclaration.getId());
|
||
if(toReturn==null) throw new ConfigurationException("Unable to find Lifecycle Manager Plugin. ID "+lcHandlerDeclaration.getId());
|
||
|
||
return toReturn;
|
||
} catch(Throwable t){
|
||
log.warn("Unable to load LC Manager ",t);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private String getCollectionName(){
|
||
// TODO Profile can directly specify, use ID only as default
|
||
|
||
return profile.getId();
|
||
}
|
||
|
||
@Override
|
||
public MongoDatabase getDatabase(){
|
||
return db;
|
||
}
|
||
|
||
@Override
|
||
public ProfiledDocument registerNew(Document toRegisterDoc) throws IOException, StepException, EventException {
|
||
log.info("Registering new document in {} ",profile.getId());
|
||
log.debug("Going to register {}",toRegisterDoc.toJson());
|
||
|
||
ProfiledDocument toRegister = new ProfiledDocument();
|
||
toRegister.setTheDocument(toRegisterDoc);
|
||
|
||
|
||
PublicationInfo pubInfo=new PublicationInfo();
|
||
pubInfo.setCreationInfo(UserUtils.getCurrent().asInfo());
|
||
|
||
// TODO Set Access From Profile
|
||
Access access=new Access();
|
||
access.setLicense("");
|
||
access.setPolicy(AccessPolicy.OPEN);
|
||
pubInfo.setAccess(access);
|
||
|
||
toRegister.setInfo(pubInfo);
|
||
|
||
toRegister.setProfileID(profile.getId());
|
||
toRegister.setProfileVersion(profile.getVersion());
|
||
toRegister.setVersion(new Semver("1.0.0"));
|
||
|
||
|
||
LifecycleInformation draftInfo=new LifecycleInformation();
|
||
draftInfo.setPhase("DRAFT");
|
||
toRegister.setLifecycleInformation(draftInfo);
|
||
|
||
// Apply Lifecycle
|
||
toRegister = triggerEvent(toRegister,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null).getResult();
|
||
|
||
|
||
log.debug("Going to register {} ",toRegister);
|
||
|
||
// Insert object
|
||
ObjectId id =insert(asDocumentWithId(toRegister),getCollectionName());
|
||
|
||
log.info("Obtained id {} ",id);
|
||
return getByID(id.toHexString());
|
||
}
|
||
|
||
@Override
|
||
public ProfiledDocument update(String id, Document toSet) throws IOException, EventException {
|
||
log.trace("Replacing {} ",toSet);
|
||
ProfiledDocument toUpdate=getByID(id);
|
||
toUpdate.setTheDocument(toSet);
|
||
|
||
toUpdate=onUpdate(toUpdate);
|
||
ProfiledDocument toReturn =convert(replace(asDocumentWithId(toUpdate),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
|
||
log.debug("Updated ProfiledDocument is {}",toReturn);
|
||
return toReturn;
|
||
}
|
||
|
||
|
||
private ProfiledDocument onUpdate(ProfiledDocument toUpdate) throws EventException {
|
||
UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
|
||
toUpdate.getInfo().setLastEditInfo(u.asInfo());
|
||
toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
|
||
return triggerEvent(toUpdate,EventExecutionRequest.Events.ON_INIT_DOCUMENT,null).getResult();
|
||
}
|
||
|
||
@Override
|
||
public void delete(String id,boolean force) throws DeletionException {
|
||
log.debug("Deleting by ID {}, force {}",id,force);
|
||
try{
|
||
ProfiledDocument doc =getByID(id);
|
||
|
||
// TODO INVOKE LIFECYCLE
|
||
|
||
//if(!force&&isPublished(id)) throw new Exception("Cannot delete published documents. Unpublish it or use force = true");
|
||
|
||
try{
|
||
// TODO CHECK PHASE AND STATUS
|
||
// DEINDEX
|
||
// DEMATERIALIZE
|
||
// DELETE CONTENT
|
||
// DELETE ENTRY
|
||
throw new DeletionException("IMPLEMENT THIS");
|
||
// delete(asId(id), getCollectionName());
|
||
}catch(DeletionException e) {
|
||
//storing updated - partially deleted
|
||
// concessione=onUpdate(concessione);
|
||
// replace(asDocumentWithId(concessione), collectionName);
|
||
throw e;
|
||
}
|
||
}catch(Throwable t){
|
||
throw new DeletionException("Unable to delete "+id,t);
|
||
}
|
||
}
|
||
|
||
@Override
|
||
public ProfiledDocument getByID(String id) throws WebApplicationException{
|
||
Document doc=super.getById(asId(id),getCollectionName());
|
||
if(doc==null) throw new WebApplicationException("No document with ID "+id);
|
||
return convert(doc,ProfiledDocument.class);
|
||
}
|
||
|
||
@Override
|
||
public Iterable<Document> query(QueryRequest queryRequest) {
|
||
log.info("Querying {} ",queryRequest);
|
||
LinkedBlockingQueue queue=new LinkedBlockingQueue<Concessione>();
|
||
query(queryRequest,getCollectionName()).forEach(
|
||
(Consumer<? super Document>) (Document d)->{try{
|
||
queue.put(d);
|
||
}catch(Throwable t){log.warn("Unable to translate "+d);}});
|
||
log.info("Returned {} elements ",queue.size());
|
||
return queue;
|
||
}
|
||
|
||
@Override
|
||
public Iterable<ProfiledDocument> filter(QueryRequest queryRequest) {
|
||
log.info("Searching concessione for filter {} ",queryRequest);
|
||
LinkedBlockingQueue queue=new LinkedBlockingQueue<Concessione>();
|
||
query(queryRequest,getCollectionName()).forEach(
|
||
(Consumer<? super Document>) (Document d)->{try{
|
||
queue.put(d);
|
||
}catch(Throwable t){log.warn("Unable to translate "+d);}});
|
||
log.info("Returned {} elements ",queue.size());
|
||
return queue;
|
||
}
|
||
|
||
|
||
@Override
|
||
public ProfiledDocument performStep(String id, String step, Document options) throws StepException, JsonProcessingException {
|
||
ProfiledDocument document = getByID(id);
|
||
try{
|
||
StepExecutionReport report = step(document, step, options);
|
||
document = report.getResult();
|
||
} catch(Throwable t){
|
||
log.error("[Profile {} ] ERROR Invoking Step {} on document {}" ,profile.getId(),step,id,t);
|
||
LifecycleInformation info = new LifecycleInformation();
|
||
info.setPhase(document.getLifecycleInformation().getPhase());
|
||
info.setLastOperationStatus(LifecycleInformation.Status.ERROR);
|
||
info.addErrorMessage(t.getMessage());
|
||
info.setLastInvokedStep(step);
|
||
document.setLifecycleInformation(info);
|
||
}finally{
|
||
return convert(replace(asDocumentWithId(document),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
|
||
}
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* NB Put at path :
|
||
*
|
||
* Path Examples
|
||
* artifact
|
||
* images
|
||
* images[1]
|
||
* layers[?(@.name = 'myName')]
|
||
*
|
||
*
|
||
*
|
||
*/
|
||
@Override
|
||
public ProfiledDocument registerFileSet(String id,RegisterFileSetRequest request) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException, DeletionException, EventException {
|
||
List<TempFile> files=request.getStreams();
|
||
|
||
Document attributes =request.getAttributes();
|
||
|
||
log.info("Registering Fileset for {}, Request is {} ",id,request);
|
||
|
||
ProfiledDocument doc=getByID(id);
|
||
WorkspaceManager ws=new WorkspaceManager();
|
||
StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject();
|
||
|
||
|
||
log.debug("Checking {} path against profile {}",request.getFieldPath(),profile.getId());
|
||
JSONPathWrapper schemaWrapper= new JSONPathWrapper(profile.getSchema().toJson());
|
||
List<Field> fieldDefinitions=schemaWrapper.getByPath(request.getFieldPath(),Field.class);
|
||
if(fieldDefinitions==null || fieldDefinitions.isEmpty())
|
||
throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+request.getFieldPath(), Response.Status.BAD_REQUEST);
|
||
if(fieldDefinitions.size()>1)
|
||
throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+request.getFieldPath(),Response.Status.BAD_REQUEST);
|
||
Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
|
||
if(fieldDefinition==null)
|
||
throw new WebApplicationException("Found field is null ["+profile.getId()+" for "+request.getFieldPath()+"]",Response.Status.BAD_REQUEST);
|
||
|
||
log.debug("Field definition is {}",fieldDefinition);
|
||
JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson());
|
||
|
||
List<RegisteredFileSet> found=docWrapper.getByPath(request.getDestinationPath(),RegisteredFileSet.class);
|
||
|
||
|
||
Object toSet=null;
|
||
Document toSetAttributes=attributes;
|
||
|
||
// Manage clash options
|
||
|
||
if(!found.isEmpty())
|
||
switch (request.getClashOption()){
|
||
case REPLACE_EXISTING: {
|
||
if(found.size()>1)
|
||
throw new WebApplicationException("Cannot replace multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
|
||
deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
|
||
break;
|
||
}case MERGE_EXISTING: {
|
||
if(found.size()>1)
|
||
throw new WebApplicationException("Cannot merge multiple items at "+request.getDestinationPath()+".",Response.Status.BAD_REQUEST);
|
||
toSetAttributes=Serialization.asDocument(found.get(0));
|
||
if(attributes!=null) toSetAttributes.putAll(attributes);
|
||
deleteFileSetRoutine(doc,request.getDestinationPath(),false,ws);
|
||
break;
|
||
}case APPEND: {
|
||
if(!fieldDefinition.isCollection())
|
||
throw new WebApplicationException("Cannot append to "+request.getDestinationPath()+" : field "+request.getFieldPath()+" is not collection.",
|
||
Response.Status.BAD_REQUEST);
|
||
break;
|
||
}
|
||
}
|
||
|
||
|
||
// Actually register Files
|
||
RegisteredFileSet registeredFileSet = prepareRegisteredFileSet(doc,profile,request.getDestinationPath(),
|
||
toSetAttributes,files,storage,ws);
|
||
if(fieldDefinition.isCollection()&&(request.getClashOption().equals(RegisterFileSetRequest.ClashOptions.APPEND))){
|
||
List<RegisteredFileSet> actualList= found.isEmpty()?new ArrayList<>():(List<RegisteredFileSet>)found.get(0);
|
||
actualList.add(registeredFileSet);
|
||
toSet=actualList;
|
||
}else toSet = registeredFileSet;
|
||
|
||
|
||
docWrapper.set(request.getDestinationPath(),toSet);
|
||
|
||
log.debug("Setting result on profiled document");
|
||
doc.setTheDocument(Document.parse(docWrapper.getCtx().jsonString()));
|
||
|
||
doc=onUpdate(doc);
|
||
|
||
return convert(replace(asDocumentWithId(doc),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
|
||
}
|
||
|
||
@Override
|
||
public ProfiledDocument deleteFileSet(String id, String destination, Boolean force) throws ConfigurationException, StorageHubException, JsonProcessingException, DeletionException, EventException {
|
||
ProfiledDocument doc = getByID(id);
|
||
doc=deleteFileSetRoutine(doc,destination,force,new WorkspaceManager());
|
||
doc=onUpdate(doc);
|
||
return convert(replace(asDocumentWithId(doc),new ObjectId(id),getCollectionName()),ProfiledDocument.class);
|
||
}
|
||
|
||
@Override
|
||
public Configuration getConfiguration() throws ConfigurationException {
|
||
log.debug("Asking configuration for {} in {} ",profile.getId(), UserUtils.getCurrent().getContext());
|
||
Configuration toReturn= getManager().getCurrentConfiguration();
|
||
log.debug("Returning current configuration {}",toReturn);
|
||
return toReturn;
|
||
}
|
||
|
||
|
||
private StepExecutionReport step(ProfiledDocument theDocument, String step, Document callParameters) throws StepException {
|
||
log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor());
|
||
StepExecutionRequest request=new StepExecutionRequest();
|
||
request.setCallParameters(callParameters);
|
||
request.setDocument(theDocument);
|
||
request.setProfile(profile);
|
||
request.setStep(step);
|
||
AccountingInfo user= UserUtils.getCurrent().asInfo();
|
||
request.setUser(user.getUser());
|
||
request.setContext(user.getContext());
|
||
|
||
log.debug("Requesting Step Execution {} ",request);
|
||
|
||
StepExecutionReport report= getManager().performStep(request);
|
||
log.debug("Report is {}",report);
|
||
if(report.getResult()==null) throw new StepException("Report result is null");
|
||
return report;
|
||
}
|
||
|
||
private EventExecutionReport triggerEvent(ProfiledDocument theDocument, String event, Document parameters) throws EventException {
|
||
log.info("[Profile {} ] triggering event {} on {}" ,profile.getId(),event,getManager().getDescriptor());
|
||
EventExecutionRequest request= new EventExecutionRequest();
|
||
request.setEvent(event);
|
||
request.setProfile(profile);
|
||
request.setCallParameters(parameters);
|
||
request.setDocument(theDocument);
|
||
log.debug("Triggering {} ",request);
|
||
return getManager().onEvent(request);
|
||
}
|
||
|
||
|
||
private static final RegisteredFileSet prepareRegisteredFileSet(ProfiledDocument doc, Profile profile,String destination,
|
||
Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
|
||
log.debug("Preparing Registered FileSet..");
|
||
RegisteredFileSet toReturn = new RegisteredFileSet();
|
||
if(attributes!=null) toReturn.putAll(attributes);
|
||
toReturn.put(RegisteredFileSet.UUID, UUID.randomUUID().toString());
|
||
toReturn.put(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
|
||
toReturn.putIfAbsent(RegisteredFileSet.ACCESS,doc.getInfo().getAccess());
|
||
FolderContainer base=ws.createFolder(new WorkspaceManager.FolderOptions(
|
||
doc.get_id(),"Base Folder for profiled document. Profile "+profile.getId(),null));
|
||
|
||
FolderContainer sectionFolder=ws.createFolder(new WorkspaceManager.FolderOptions(
|
||
doc.get_id()+destination,"Registered Fileset at path "+destination,base));
|
||
|
||
toReturn.put(RegisteredFileSet.FOLDER_ID,sectionFolder.getId());
|
||
|
||
ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();
|
||
|
||
for (TempFile f : files) {
|
||
InputStream is=null;
|
||
try{
|
||
log.debug("Opening temp file {}",f);
|
||
String fileUrl=storage.getURL(f.getId());
|
||
log.debug("Got URL {} from ID {}",fileUrl,f.getId());
|
||
is=new URL(fileUrl).openStream();
|
||
RegisteredFile registered=ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(),is,
|
||
"Imported via gcube CMS service ", sectionFolder));
|
||
log.debug("Registered "+registered);
|
||
registeredFiles.add(registered);
|
||
}catch(StorageHubException | IOException e){
|
||
throw new StorageException("Unable to store "+f,e);
|
||
}finally{
|
||
if(is!=null)
|
||
IOUtils.closeQuietly(is);
|
||
}
|
||
}
|
||
toReturn.put(RegisteredFileSet.PAYLOADS,registeredFiles);
|
||
toReturn.remove(RegisteredFileSet.MATERIALIZATIONS);
|
||
return toReturn;
|
||
}
|
||
|
||
|
||
private static ProfiledDocument deleteFileSetRoutine(ProfiledDocument doc,String fileSetPath, Boolean force, WorkspaceManager ws) throws DeletionException, StorageHubException {
|
||
log.info("Document ID {} : deleting fileset at {} [force : {}]",doc.get_id(),fileSetPath,force);
|
||
JSONPathWrapper wrapper =new JSONPathWrapper(doc.getTheDocument().toJson());
|
||
RegisteredFileSet toDelete = Serialization.convert(wrapper.getByPath(fileSetPath).get(0),RegisteredFileSet.class);
|
||
if(toDelete.getMaterializations()!=null && !toDelete.getMaterializations().isEmpty()){
|
||
if(!force) throw new DeletionException("Fileset (Document ID "+doc.get_id()+", path "+fileSetPath+") already materialized. Use force = true");
|
||
else throw new RuntimeException("Implement this");
|
||
// TODO manager force deletion
|
||
// NB handlers for materialization types
|
||
}
|
||
log.debug("Document ID {} : deleting ws folder {}",doc.get_id(),toDelete.getFolderId());
|
||
if(toDelete.getPayloads()!=null)
|
||
ws.deleteItem(toDelete.getFolderId());
|
||
doc.setTheDocument(Document.parse(wrapper.set(fileSetPath,null).getCtx().jsonString()));
|
||
return doc;
|
||
}
|
||
}
|