Register ProfiledDocument Filesets

This commit is contained in:
Fabio Sinibaldi 2022-01-17 18:19:40 +01:00
parent a4e8cb2f1e
commit 64b03247cc
18 changed files with 360 additions and 79 deletions

View File

@ -1,6 +1,10 @@
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
# Changelog for org.gcube.application.cms.cms-test-commons
## [v1.0.2-SNAPSHOT] - 2022-01-17
- Profiles
## [v1.0.1] - 2021-09-11
- Introduced profiled documents

View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>cms-test-commons</artifactId>
<version>1.0.1</version>
<version>1.0.2-SNAPSHOT</version>
<name>CMS Test Commons</name>

View File

@ -0,0 +1,30 @@
package org.gcube.application.cms.tests;
import org.gcube.application.cms.Serialization;
import org.gcube.application.cms.tests.model.concessioni.TestConcessioniModel;
import org.gcube.application.geoportal.common.model.profile.Profile;
import org.gcube.application.geoportal.common.utils.Files;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
public class TestProfiles {
public static File BASE_FOLDER =new File("../test-data/profiledDocuments");
public static final HashMap<String, Profile> profiles =new HashMap<>();
static{
for(File f:BASE_FOLDER.listFiles()){
try {
profiles.put(f.getName(), Serialization.read(
Files.readFileAsString(f.getAbsolutePath(), Charset.defaultCharset()),Profile.class));
} catch (IOException e) {
throw new RuntimeException("Unable to read "+f.getAbsolutePath(),e);
}
}
}
}

View File

@ -4,6 +4,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# [v1.0.9-SNAPSHOT] - 2022-01-17
- Minor fixes in model
- Schema and jsonPath support
# [v1.0.8] - 2021-11-10
- Fixes [#22369](https://support.d4science.org/issues/22369)

View File

@ -47,6 +47,12 @@
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>com.vdurmont</groupId>
<artifactId>semver4j</artifactId>
<version>3.1.0</version>
</dependency>
<!-- TEST -->
<dependency>
<groupId>junit</groupId>

View File

@ -1,6 +1,7 @@
package org.gcube.application.geoportal.common.model.document;
import com.mongodb.client.model.geojson.GeoJsonObjectType;
import com.vdurmont.semver4j.Semver;
import lombok.*;
import org.bson.Document;
import org.bson.types.ObjectId;
@ -27,14 +28,14 @@ public class ProfiledDocument {
// CORE METADATA
private String _id;
private ComparableVersion version;
private Semver version;
// Publication Info
private PublicationInfo info;
// Profile reference
private String profileID;
private ComparableVersion profileVersion;
private Semver profileVersion;
private LifecycleInformation lifecycleInformation;

View File

@ -14,10 +14,8 @@ public class PublicationInfo {
public static final String LAST_EDIT_INFO="lastEditInfo";
public static final String ACCESS = "access";
private AccountingInfo creationInfo;
private AccountingInfo lastEditInfo;
private Access access;
}

View File

@ -0,0 +1,36 @@
package org.gcube.application.geoportal.common.model.profile;
import org.bson.Document;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Field extends Document {
public static final String TYPE="_type";
public static final String CHILDREN="_children";
public static final String MAX_CARDINALITY="_max";
public static final String MIN_CARDINALITY="_min";
public String getType(){
return this.getString(TYPE);
};
public List getChildren(){
return this.get(CHILDREN,List.class);
}
public Integer getMaxCardinality(){
return (Integer) this.getOrDefault(MAX_CARDINALITY,1);
}
public Integer getMinCardinality(){
return (Integer) this.getOrDefault(MIN_CARDINALITY,0);
}
public Boolean isMandatory(){
return getMinCardinality()==0;
}
}

View File

@ -0,0 +1,13 @@
package org.gcube.application.geoportal.common.model.profile;
import java.util.ArrayList;
import java.util.HashMap;
class FieldMap extends ArrayList<FieldMap.MapElement> {
public static class MapElement extends HashMap<String,Field>{
}
}

View File

@ -7,6 +7,7 @@ import java.util.Map;
import javax.xml.bind.annotation.XmlRootElement;
import com.vdurmont.semver4j.Semver;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
@ -20,13 +21,13 @@ import org.gcube.application.geoportal.common.model.document.ComparableVersion;
public class Profile{
private String id;
private ComparableVersion version;
private Semver version;
private String name;
private String description;
private AccountingInfo creationInfo;
private Document schema;
private Field schema;
private List<HandlerDeclaration> handlers;
@ -43,4 +44,7 @@ public class Profile{
});
return toReturn;
}
}

View File

@ -1,11 +1,11 @@
package org.gcube.application.geoportal.common.utils;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.*;
import lombok.Getter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
public class JSONPathWrapper {
@ -27,8 +27,39 @@ public class JSONPathWrapper {
}
public List<Object> getByPath(String path){
throw new RuntimeException("TO IMPLEMENT");
return ctx.read(path);
}
public <T> List<T> getByPath(String path,Class<T> clazz){
return ctx.read(path, new TypeRef<List<T>>() {});
}
public JSONPathWrapper set(String path, Object toSet){
ctx.set(path,toSet);
return this;
}
// public static final DocumentContext addElement(DocumentContext ctx,String path) throws JsonPathException{
// JsonPath jPath=JsonPath.compile(path);
// if(jPath.isDefinite()) {
// String parent=path.substring(0,path.lastIndexOf("."));
// List<String> found=ctx.read(parent);
// if(found==null || found.size()==0 || found.get(0)==null) {
// //missing parent, use recursion
// addElement(ctx,parent);
// }
// // found parent, adding element
// String element=path.substring(path.lastIndexOf(".")+1);
//
// Object value=new HashMap<String,String>();
// if(element.contains("[")) {
// value=new ArrayList<Object>(Collections.singletonList(value));
//
// element=element.substring(0,element.indexOf("["));
// }
//
// ctx.put(parent, element, value);
// return ctx;
// }else throw new JsonPathException("Unable to initialize non-definite path : "+path);
// }
}

View File

@ -3,7 +3,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.application.geoportal-service
## [v1.0.9-SNAPSHOT] 2022-01-17
- Plugin Management
- Plugin Management
- Profiled Document : FileSet Registration
## [v1.0.8] 2021-09-20
- Logging

View File

@ -3,6 +3,7 @@ package org.gcube.application.geoportal.service.engine;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.common.faults.PathException;
import org.gcube.application.geoportal.common.model.document.RegisteredFile;
import org.gcube.application.geoportal.common.model.legacy.WorkspaceContent;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
@ -106,6 +107,22 @@ public class WorkspaceManager {
}
public RegisteredFile registerFile(FileOptions opts) throws StorageHubException {
FileContainer item=createFileRoutine(opts);
item=sgClient.open(item.getId()).asFile();
RegisteredFile file=new RegisteredFile();
file.setLink(item.getPublicLink().toString());
file.setMimetype(item.get().getContent().getMimeType());
file.setStorageID(item.getId());
file.setName(item.get().getName());
return file;
}
public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException {
sgClient.open(toDelete.getStorageID()).asFile().forceDelete();
}

View File

@ -3,10 +3,16 @@ package org.gcube.application.geoportal.service.engine.mongo;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.bson.Document;
import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.document.ProfiledDocument;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.rest.TempFile;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import java.io.IOException;
import java.util.List;
public interface MongoManagerI<T> {
@ -16,7 +22,7 @@ public interface MongoManagerI<T> {
public T registerNew(Document toRegister) throws IOException, StepException;
// update
public T update(String id,T toSet) throws IOException, StepException;
public T update(String id,Document toSetDocument) throws IOException, StepException;
// delete
@ -31,19 +37,7 @@ public interface MongoManagerI<T> {
public Iterable<Document> query(QueryRequest request);
public Iterable<T> filter(QueryRequest request);
// materialize
public T performStep(String id, String step, Document options) throws IOException, StepException;
public T materialize(String id);
// dematerialize
public T dematerialize(String id);
// index
public T index(String id);
// deIndex
public T deIndex(String id);
public T performStep(String id, String step, Document options);
public T registerFileSet(String id, String destination, Document attributes, List<TempFile> files) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException;
}

View File

@ -2,8 +2,10 @@ package org.gcube.application.geoportal.service.engine.mongo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.mongodb.client.MongoDatabase;
import com.vdurmont.semver4j.Semver;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.cms.plugins.LifecycleManager;
@ -11,21 +13,33 @@ import org.gcube.application.cms.plugins.faults.StepException;
import org.gcube.application.cms.plugins.model.PluginDescriptor;
import org.gcube.application.cms.plugins.reports.ExecutionReport;
import org.gcube.application.cms.plugins.requests.StepExecutionRequest;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.document.*;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.model.legacy.WorkspaceContent;
import org.gcube.application.geoportal.common.model.profile.Field;
import org.gcube.application.geoportal.common.model.profile.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.profile.Profile;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.rest.TempFile;
import org.gcube.application.geoportal.common.utils.JSONPathWrapper;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.ImplementationProvider;
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
import org.gcube.application.geoportal.service.model.internal.faults.DeletionException;
import org.gcube.application.cms.Serialization;
import org.gcube.application.geoportal.service.utils.UserUtils;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
@ -89,22 +103,6 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
}
}
private ProfiledDocument onUpdate(ProfiledDocument updatedDocument) throws StepException {
UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
updatedDocument.getInfo().setLastEditInfo(u.asInfo());
return step(updatedDocument,StepExecutionRequest.Steps.ON_UPDATE_DOCUMENT).getResult();
}
// private Document asDocument(ProfiledDocument d) throws JsonProcessingException {
// return Document.parse(Serialization.write(d));
// }
//
// private ProfiledDocument asProfiledDocument(Document d) throws IOException {
// return Serialization.read(d.toJson(),ProfiledDocument.class);
// }
private String getCollectionName(){
// TODO Profile can directly specify, use ID only as default
@ -138,11 +136,11 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
toRegister.setProfileID(profile.getId());
toRegister.setProfileVersion(profile.getVersion());
toRegister.setVersion(new ComparableVersion("1.0.0"));
toRegister.setVersion(new Semver("1.0.0"));
// Apply Lifecycle
toRegister=step(toRegister,StepExecutionRequest.Steps.ON_INIT_DOCUMENT).getResult();
toRegister=step(toRegister,StepExecutionRequest.Steps.ON_INIT_DOCUMENT,null).getResult();
log.debug("Going to register {} ",toRegister);
@ -154,10 +152,22 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
}
@Override
public ProfiledDocument update(String id, ProfiledDocument toSet) throws IOException, StepException {
public ProfiledDocument update(String id, Document toSet) throws IOException, StepException {
log.trace("Replacing {} ",toSet);
toSet=onUpdate(toSet);
return convert(replace(asDocument(toSet),getCollectionName()),ProfiledDocument.class);
ProfiledDocument toUpdate=getByID(id);
toUpdate.setTheDocument(toSet);
toUpdate=onUpdate(toUpdate);
return convert(replace(asDocument(toUpdate),getCollectionName()),ProfiledDocument.class);
}
private ProfiledDocument onUpdate(ProfiledDocument toUpdate) throws StepException {
UserUtils.AuthenticatedUser u = UserUtils.getCurrent();
toUpdate.getInfo().setLastEditInfo(u.asInfo());
toUpdate.setVersion(toUpdate.getVersion().withIncPatch());
return step(toUpdate,StepExecutionRequest.Steps.ON_UPDATE_DOCUMENT,null).getResult();
}
@Override
@ -190,8 +200,10 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
}
@Override
public ProfiledDocument getByID(String id) throws IOException {
return convert(super.getById(asId(id),getCollectionName()),ProfiledDocument.class);
public ProfiledDocument getByID(String id){
Document doc=super.getById(asId(id),getCollectionName());
if(doc==null) throw new WebApplicationException("No document with ID "+id);
return convert(doc,ProfiledDocument.class);
}
@Override
@ -218,43 +230,128 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
return queue;
}
@Override
public ProfiledDocument materialize(String id) {
throw new RuntimeException("TO IMPLEMENT");
}
@Override
public ProfiledDocument dematerialize(String id) {
throw new RuntimeException("TO IMPLEMENT");
}
@Override
public ProfiledDocument index(String id) {
throw new RuntimeException("TO IMPLEMENT");
}
@Override
public ProfiledDocument deIndex(String id) {
throw new RuntimeException("TO IMPLEMENT");
}
@Override
public ProfiledDocument performStep(String id, String step, Document options) {
throw new RuntimeException("TO IMPLEMENT");
public ProfiledDocument performStep(String id, String step, Document options) throws StepException, JsonProcessingException {
ExecutionReport report = step(getByID(id), step, options);
return convert(replace(asDocument(report.getResult()),getCollectionName()),ProfiledDocument.class);
}
private ExecutionReport step(ProfiledDocument theDocument,String step) throws StepException {
log.info("[Profile {} ] Invoking Step {} on " ,profile.getId(),step,getManager().getDescriptor());
/**
* NB Put at path :
*
*
*
*/
@Override
public ProfiledDocument registerFileSet(String id, String destination, Document attributes, List<TempFile> files) throws ConfigurationException, StorageHubException, StorageException, StepException, JsonProcessingException {
log.info("Registering fileset [size : {}] for {} at {} with options {}",files.size(),id,destination,attributes);
ProfiledDocument doc=getByID(id);
WorkspaceManager ws=new WorkspaceManager();
StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject();
log.debug("Checking {}  path against profile {}",destination,profile.getId());
JSONPathWrapper schemaWrapper= new JSONPathWrapper(profile.getSchema().toJson());
List<Object> fieldDefinitions=schemaWrapper.getByPath(destination);
if(fieldDefinitions==null || fieldDefinitions.isEmpty()) throw new WebApplicationException("No Field found in schema "+profile.getId()+" at "+destination, Response.Status.BAD_REQUEST);
if(fieldDefinitions.size()>1) throw new WebApplicationException("Multiple field definitions ("+fieldDefinitions.size()+") found in "+profile.getId()+" for "+destination,Response.Status.BAD_REQUEST);
Field fieldDefinition=Serialization.convert(fieldDefinitions.get(0),Field.class);
log.debug("Field definition is {}",fieldDefinition);
JSONPathWrapper docWrapper=new JSONPathWrapper(doc.getTheDocument().toJson());
List<RegisteredFileSet> found=docWrapper.getByPath(destination,RegisteredFileSet.class);
if(fieldDefinition.getMaxCardinality()==1 && (!found.isEmpty())){
throw new WebApplicationException("Cannot add registered fileset at "+destination+" : field is not collection.",Response.Status.BAD_REQUEST);
}
RegisteredFileSet registeredFileSet=prepareRegisteredFileSet(doc,profile,destination,attributes,files,storage,ws);
log.debug("Registered fileset is {} ",registeredFileSet);
if(fieldDefinition.getMaxCardinality()>1){
// Field is collection
found.add(registeredFileSet);
docWrapper.set(destination,found);
}
else {
docWrapper.set(destination,registeredFileSet);
}
log.debug("Setting result on profiled document");
doc.setTheDocument(Document.parse(docWrapper.getCtx().jsonString()));
doc=onUpdate(doc);
return convert(replace(asDocument(doc),getCollectionName()),ProfiledDocument.class);
}
// @Override
// public ProfiledDocument deleteRegisteredFileSet(String id, String destination, List<TempFile> files) {
// throw new RuntimeException("TO IMPLEMENT");
// }
private ExecutionReport step(ProfiledDocument theDocument,String step,Document callParameters) throws StepException {
log.info("[Profile {} ] Invoking Step {} on {}" ,profile.getId(),step,getManager().getDescriptor());
StepExecutionRequest request=new StepExecutionRequest();
request.setCallParameters(callParameters);
request.setDocument(theDocument);
request.setProfile(profile);
request.setStep(StepExecutionRequest.Steps.ON_INIT_DOCUMENT);
request.setStep(step);
log.debug("Requesting Step Execution {} ",request);
ExecutionReport report= getManager().performStep(request);
log.debug("Report is {}",report);
if(report.getResult()==null) throw new StepException("Report result is null");
return report;
}
private static final RegisteredFileSet prepareRegisteredFileSet(ProfiledDocument doc, Profile profile,String destination,
Document attributes,List<TempFile> files, StorageUtils storage,WorkspaceManager ws) throws StorageHubException, StorageException {
log.debug("Preparing Registered FileSet..");
attributes.putIfAbsent(RegisteredFileSet.CREATION_INFO,UserUtils.getCurrent().asInfo());
attributes.putIfAbsent(RegisteredFileSet.ACCESS,doc.getInfo().getAccess());
FolderContainer base=ws.createFolder(new WorkspaceManager.FolderOptions(
doc.get_id(),"Base Folder for profiled document. Profile "+profile.getId(),null));
FolderContainer sectionFolder=ws.createFolder(new WorkspaceManager.FolderOptions(
doc.get_id()+destination,"Registered Fileset at path "+destination,base));
attributes.putIfAbsent(RegisteredFileSet.FOLDER_ID,sectionFolder.getId());
ArrayList<RegisteredFile> registeredFiles=new ArrayList<>();
for (TempFile f : files) {
InputStream is=null;
try{
log.debug("Opening temp file {}",f);
String fileUrl=storage.getURL(f.getId());
log.debug("Got URL {} from ID {}",fileUrl,f.getId());
is=new URL(fileUrl).openStream();
RegisteredFile registered=ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(),is,
"Imported via gcube CMS service ", sectionFolder));
log.debug("Registered "+registered);
registeredFiles.add(registered);
}catch(StorageHubException | IOException e){
throw new StorageException("Unable to store "+f,e);
}finally{
if(is!=null)
IOUtils.closeQuietly(is);
}
}
attributes.putIfAbsent(RegisteredFileSet.PAYLOADS,registeredFiles);
return Serialization.convert(attributes,RegisteredFileSet.class);
}
}

View File

@ -18,7 +18,7 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
public class ProjectTests extends BasicServiceTestUnit{
public class ProfiledDocumentsTests extends BasicServiceTestUnit{
String testProfileId="profiledConcessioni";
@ -55,12 +55,28 @@ public class ProjectTests extends BasicServiceTestUnit{
}
@Test
public void registerNew() {
public void registerNew() throws Exception {
WebTarget target=target(InterfaceConstants.Methods.PROJECTS);
Document document =new Document(Collections.singletonMap("dumbKey","dumbValue"));
System.out.println(target.path(testProfileId).request(MediaType.APPLICATION_JSON).
post(Entity.entity(document, MediaType.APPLICATION_JSON)));
check(target.path(testProfileId).request(MediaType.APPLICATION_JSON).
post(Entity.entity(document, MediaType.APPLICATION_JSON)),ProfiledDocument.class);
}
private ProfiledDocument createNew() throws Exception {
WebTarget target=target(InterfaceConstants.Methods.PROJECTS);
Document document =new Document(Collections.singletonMap("dumbKey","dumbValue"));
return check(target.path(testProfileId).request(MediaType.APPLICATION_JSON).
post(Entity.entity(document, MediaType.APPLICATION_JSON)),ProfiledDocument.class);
}
@Test
public void addFileSet() throws Exception {
// TODO Test register fileset
}

View File

@ -0,0 +1,19 @@
package org.gcube.application.geoportal.service;
import org.gcube.application.cms.tests.TestProfiles;
import org.junit.Test;
public class Profiles {
@Test
public void getFields(){
TestProfiles.profiles.forEach((k,v)->{
System.out.println(k+"\t"+v);
v.getSchema().getChildren().forEach(
i-> System.out.println(i.getClass()+"\t"+i)
);
});
}
}

View File

@ -0,0 +1,13 @@
{"id": "myProfileStructure",
"schema": {
"_type" : "complex",
"_children" :[
{ "nome" : {"_type" : "Text", "_index" : "<textIndexID>"}},
{ "abstract" : {"_type" : "complex",
"_children" :[
{"name" : {"Type" : "Text"}},
{"files" : {"Type" : "FileSet"}}
]}}
]}
}