task_28150 #19
|
|
@ -2,7 +2,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
|||
|
||||
# Changelog for org.gcube.spatial.data.gcube-sdi-suite
|
||||
|
||||
## [v1.0.6]
|
||||
## [v1.0.6-SNAPSHOT]
|
||||
|
||||
- Integrated an EventManager centrally [#26321]
|
||||
- sdi-plugins: add the logic to apply a regex to the value that must be added to index [#26322]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
# Changelog for org.gcube.application.cms-plugin-framework
|
||||
|
||||
## [v1.0.6-SNAPSHOT] - 2024-10-01
|
||||
|
||||
- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150]
|
||||
|
||||
## [v1.0.5] - 2024-07-03
|
||||
|
||||
- Implemented Event Broker [#26321]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>cms-plugin-framework</artifactId>
|
||||
<version>1.0.5</version>
|
||||
<version>1.0.6-SNAPSHOT</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.gcube.application.cms</groupId>
|
||||
|
|
|
|||
|
|
@ -22,52 +22,52 @@ import lombok.extern.slf4j.Slf4j;
|
|||
@Slf4j
|
||||
public class WorkspaceManager {
|
||||
|
||||
private static final String APP_FOLDER=".GNA_RECORDS";
|
||||
private static final String APP_FOLDER = ".GNA_RECORDS";
|
||||
|
||||
|
||||
private StorageHubClient sgClient=null;
|
||||
private StorageHubClient sgClient = null;
|
||||
@Getter
|
||||
private FolderContainer appBase=null;
|
||||
private FolderContainer appBase = null;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@AllArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public static class FolderOptions{
|
||||
public static class FolderOptions {
|
||||
@NonNull
|
||||
private String folderName;
|
||||
private String folderDescription;
|
||||
private FolderContainer parent;
|
||||
}
|
||||
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@AllArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public static class FileOptions{
|
||||
public static class FileOptions {
|
||||
@NonNull
|
||||
private String fileName;
|
||||
@NonNull
|
||||
private InputStream is;
|
||||
|
||||
|
||||
private String fileDescription;
|
||||
private FolderContainer parent;
|
||||
|
||||
// Added by Francesco, see #28150
|
||||
private Long size;
|
||||
}
|
||||
|
||||
public Archive getConfiguration(){
|
||||
public Archive getConfiguration() {
|
||||
Archive toReturn = new Archive("W-STORAGE");
|
||||
toReturn.put("folder_id",appBase.getId());
|
||||
toReturn.put("folder_id", appBase.getId());
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
public WorkspaceManager() throws ConfigurationException, StorageHubException {
|
||||
sgClient= ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class);
|
||||
appBase=getApplicationBaseFolder(sgClient);
|
||||
sgClient = ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class);
|
||||
appBase = getApplicationBaseFolder(sgClient);
|
||||
}
|
||||
|
||||
public FolderContainer createFolder(FolderOptions opts) throws StorageHubException {
|
||||
if(opts.getParent()==null)
|
||||
if (opts.getParent() == null)
|
||||
opts.setParent(appBase);
|
||||
return createFolderRoutine(opts);
|
||||
}
|
||||
|
|
@ -84,11 +84,10 @@ public class WorkspaceManager {
|
|||
sgClient.open(id).asFolder().delete();
|
||||
}
|
||||
|
||||
public FolderContainer getSubFolder(FolderContainer parentFolder,String path) throws StorageHubException {
|
||||
return getSubFolder(parentFolder,path,"");
|
||||
public FolderContainer getSubFolder(FolderContainer parentFolder, String path) throws StorageHubException {
|
||||
return getSubFolder(parentFolder, path, "");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns sub folder. Creates it if missing
|
||||
*
|
||||
|
|
@ -97,26 +96,26 @@ public class WorkspaceManager {
|
|||
* @return
|
||||
* @throws StorageHubException
|
||||
*/
|
||||
public FolderContainer getSubFolder(FolderContainer parentFolder,String path, String description) throws StorageHubException {
|
||||
try{
|
||||
public FolderContainer getSubFolder(FolderContainer parentFolder, String path, String description)
|
||||
throws StorageHubException {
|
||||
try {
|
||||
return parentFolder.openByRelativePath(path).asFolder();
|
||||
}catch(StorageHubException e) {
|
||||
log.debug("Missing subPath "+path);
|
||||
FolderContainer targetParent=parentFolder;
|
||||
String targetName=path;
|
||||
if(path.contains("/")) {
|
||||
String parent=path.substring(0, path.lastIndexOf("/"));
|
||||
log.debug("Checking intermediate "+parent);
|
||||
targetParent=getSubFolder(parentFolder,parent);
|
||||
targetName=path.substring(path.lastIndexOf("/")+1);
|
||||
} catch (StorageHubException e) {
|
||||
log.debug("Missing subPath " + path);
|
||||
FolderContainer targetParent = parentFolder;
|
||||
String targetName = path;
|
||||
if (path.contains("/")) {
|
||||
String parent = path.substring(0, path.lastIndexOf("/"));
|
||||
log.debug("Checking intermediate " + parent);
|
||||
targetParent = getSubFolder(parentFolder, parent);
|
||||
targetName = path.substring(path.lastIndexOf("/") + 1);
|
||||
}
|
||||
FolderOptions opts = new FolderOptions(targetName,description,targetParent);
|
||||
FolderOptions opts = new FolderOptions(targetName, description, targetParent);
|
||||
log.debug("Creating FOLDER {}", opts);
|
||||
return createFolder(opts);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// public WorkspaceContent storeToWS(FileOptions opts) throws FileNotFoundException, StorageHubException {
|
||||
// FileContainer item=createFileRoutine(opts);
|
||||
// item=sgClient.open(item.getId()).asFile();
|
||||
|
|
@ -131,12 +130,11 @@ public class WorkspaceManager {
|
|||
// }
|
||||
|
||||
public RegisteredFile registerFile(FileOptions opts) throws StorageHubException {
|
||||
FileContainer item=createFileRoutine(opts);
|
||||
item=sgClient.open(item.getId()).asFile();
|
||||
FileContainer item = createFileRoutine(opts);
|
||||
item = sgClient.open(item.getId()).asFile();
|
||||
|
||||
RegisteredFile file = new RegisteredFile();
|
||||
|
||||
RegisteredFile file=new RegisteredFile();
|
||||
|
||||
|
||||
file.setLink(item.getPublicLink().toString());
|
||||
file.setMimetype(item.get().getContent().getMimeType());
|
||||
file.setStorageID(item.getId());
|
||||
|
|
@ -144,26 +142,25 @@ public class WorkspaceManager {
|
|||
return file;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
// public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException {
|
||||
// sgClient.open(toDelete.getStorageID()).asFile().forceDelete();
|
||||
// }
|
||||
|
||||
public void deleteItem(String itemId)throws StorageHubException{
|
||||
public void deleteItem(String itemId) throws StorageHubException {
|
||||
sgClient.open(itemId).asItem().forceDelete();
|
||||
}
|
||||
|
||||
// STATIC SYNCH METHODS
|
||||
|
||||
// STATIC SYNCH METHODS
|
||||
|
||||
@Synchronized
|
||||
public static FolderContainer getApplicationBaseFolder(StorageHubClient sgClient) throws StorageHubException {
|
||||
FolderContainer vre=sgClient.openVREFolder();
|
||||
FolderContainer vre = sgClient.openVREFolder();
|
||||
try {
|
||||
return vre.openByRelativePath(APP_FOLDER).asFolder();
|
||||
}catch(StorageHubException e) {
|
||||
log.debug("APP Fodler missing. Initializing..");
|
||||
FolderContainer toReturn= vre.newFolder(APP_FOLDER, "Base folder for GNA records");
|
||||
} catch (StorageHubException e) {
|
||||
log.debug("APP Folder missing. Initializing..");
|
||||
FolderContainer toReturn = vre.newFolder(APP_FOLDER, "Base folder for GNA records");
|
||||
toReturn.setHidden();
|
||||
return toReturn;
|
||||
}
|
||||
|
|
@ -172,12 +169,19 @@ public class WorkspaceManager {
|
|||
@Synchronized
|
||||
private static FolderContainer createFolderRoutine(FolderOptions opts) throws StorageHubException {
|
||||
opts.setFolderName(Files.fixFilename(opts.getFolderName()));
|
||||
return opts.getParent().newFolder(opts.getFolderName(),opts.getFolderDescription());
|
||||
return opts.getParent().newFolder(opts.getFolderName(), opts.getFolderDescription());
|
||||
}
|
||||
|
||||
|
||||
@Synchronized
|
||||
private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException {
|
||||
// Updated by Francesco, see #28150
|
||||
log.info("Uploading file name: {}, in the parent folder id: {}, filesize is: " + opts.getSize(),
|
||||
opts.getFileName(), opts.getParent().getId());
|
||||
opts.setFileName(Files.fixFilename(opts.getFileName()));
|
||||
return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription());
|
||||
if (opts.getSize() == null)
|
||||
return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription());
|
||||
else
|
||||
return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription(),
|
||||
opts.getSize());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
# Changelog for org.gcube.application.geoportal-common
|
||||
|
||||
## [v1.1.1-SNAPSHOT] - 2024-10-01
|
||||
- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150]
|
||||
|
||||
## [v1.1.0] - 2024-04-08
|
||||
- Added message to StepExecutionRequest [#27192]
|
||||
- Fixed pom. Duplicate declaration of `registry-publisher` dependency
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>geoportal-common</artifactId>
|
||||
<version>1.1.0</version>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<name>Geoportal Common</name>
|
||||
|
||||
<parent>
|
||||
|
|
|
|||
|
|
@ -15,8 +15,20 @@ public class TempFile {
|
|||
private String url;
|
||||
private String filename;
|
||||
|
||||
public void validate()throws InvalidRequestException {
|
||||
if((id==null || id.isEmpty() )&&(url==null||url.isEmpty())) throw new InvalidRequestException("Invalid temp file "+this+" : ID null or empty and no url defined");
|
||||
if(filename==null || filename.isEmpty()) throw new InvalidRequestException("Invalid temp file "+this+" : filename null or empty");
|
||||
// Added by Francesco, see #28150
|
||||
private Long size;
|
||||
|
||||
public void validate() throws InvalidRequestException {
|
||||
if ((id == null || id.isEmpty()) && (url == null || url.isEmpty()))
|
||||
throw new InvalidRequestException("Invalid temp file " + this + " : ID null or empty and no url defined");
|
||||
if (filename == null || filename.isEmpty())
|
||||
throw new InvalidRequestException("Invalid temp file " + this + " : filename null or empty");
|
||||
}
|
||||
|
||||
public TempFile(String id, String url, String filename) {
|
||||
this.id = id;
|
||||
this.url = url;
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ import org.gcube.application.geoportal.common.model.document.access.Access;
|
|||
import org.gcube.application.geoportal.common.model.document.relationships.RelationshipNavigationObject;
|
||||
import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
|
||||
import org.gcube.application.geoportal.common.model.rest.DeleteRelationshipRequest;
|
||||
import org.gcube.application.geoportal.common.model.rest.PerformStepRequest;
|
||||
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
|
||||
import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
|
||||
import org.gcube.application.geoportal.common.model.rest.PerformStepRequest;
|
||||
|
||||
public interface Projects<P extends Project> {
|
||||
|
||||
|
|
|
|||
|
|
@ -13,107 +13,121 @@ import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest;
|
|||
import org.gcube.application.geoportal.common.model.rest.TempFile;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class FileSets {
|
||||
|
||||
public static class RequestBuilder {
|
||||
RegisterFileSetRequest theRequest=new RegisterFileSetRequest();
|
||||
public static class RequestBuilder {
|
||||
RegisterFileSetRequest theRequest = new RegisterFileSetRequest();
|
||||
|
||||
public RequestBuilder addAll(Collection<? extends TempFile> toAdd){
|
||||
if(theRequest.getStreams()==null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
theRequest.getStreams().addAll(toAdd);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder add(TempFile... f){
|
||||
if(theRequest.getStreams()==null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
for(TempFile temp: f )
|
||||
theRequest.getStreams().add(temp);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder addAll(Collection<? extends TempFile> toAdd) {
|
||||
if (theRequest.getStreams() == null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
theRequest.getStreams().addAll(toAdd);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder add(TempFile f){
|
||||
if(theRequest.getStreams()==null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
theRequest.getStreams().add(f);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder add(TempFile... f) {
|
||||
if (theRequest.getStreams() == null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
for (TempFile temp : f)
|
||||
theRequest.getStreams().add(temp);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setFieldDefinitionPath(String path){
|
||||
theRequest.setFieldDefinitionPath(path);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder add(TempFile f) {
|
||||
if (theRequest.getStreams() == null)
|
||||
theRequest.setStreams(new ArrayList<TempFile>());
|
||||
theRequest.getStreams().add(f);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setParentPath(String path){
|
||||
theRequest.setParentPath(path);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder setFieldDefinitionPath(String path) {
|
||||
theRequest.setFieldDefinitionPath(path);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setFieldName(String fieldName){
|
||||
theRequest.setFieldName(fieldName);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder setParentPath(String path) {
|
||||
theRequest.setParentPath(path);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path){
|
||||
theRequest.setClashOption(path);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder setFieldName(String fieldName) {
|
||||
theRequest.setFieldName(fieldName);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setAttributes(Document attributes){
|
||||
theRequest.setAttributes(attributes);
|
||||
return this;
|
||||
}
|
||||
public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path) {
|
||||
theRequest.setClashOption(path);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RegisterFileSetRequest getTheRequest(){
|
||||
if(theRequest.getClashOption()==null)
|
||||
// default Clash Policy
|
||||
setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING);
|
||||
return theRequest;}
|
||||
public RequestBuilder setAttributes(Document attributes) {
|
||||
theRequest.setAttributes(attributes);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RequestBuilder setAccess(Access access){
|
||||
theRequest.setToSetAccess(access);
|
||||
return this;
|
||||
}
|
||||
public RegisterFileSetRequest getTheRequest() {
|
||||
if (theRequest.getClashOption() == null)
|
||||
// default Clash Policy
|
||||
setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING);
|
||||
return theRequest;
|
||||
}
|
||||
|
||||
}
|
||||
public RequestBuilder setAccess(Access access) {
|
||||
theRequest.setToSetAccess(access);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static RequestBuilder build(String parent,String fieldName, String fieldDefinition) {
|
||||
return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition).setFieldName(fieldName);
|
||||
}
|
||||
public static RequestBuilder build(String parent, String fieldName, String fieldDefinition) {
|
||||
return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition)
|
||||
.setFieldName(fieldName);
|
||||
}
|
||||
|
||||
public static RequestBuilder build(String parent,String fieldName, String fieldDefinition, TempFile...files) {
|
||||
return build(parent,fieldName,fieldDefinition).add(files);
|
||||
}
|
||||
public static RequestBuilder build(String parent, String fieldName, String fieldDefinition, TempFile... files) {
|
||||
return build(parent, fieldName, fieldDefinition).add(files);
|
||||
}
|
||||
|
||||
public static TempFile asTemp(StorageUtils storage,InputStreamDescriptor descriptor) throws RemoteBackendException, FileNotFoundException {
|
||||
return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename());
|
||||
}
|
||||
public static TempFile asTemp(StorageUtils storage, InputStreamDescriptor descriptor)
|
||||
throws RemoteBackendException, FileNotFoundException {
|
||||
return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename());
|
||||
}
|
||||
|
||||
public static TempFile[] asTemp(StorageUtils storage,InputStreamDescriptor... descriptors) throws RemoteBackendException, FileNotFoundException {
|
||||
ArrayList<TempFile> toReturn=new ArrayList<TempFile>();
|
||||
for(InputStreamDescriptor desc:descriptors)
|
||||
toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename()));
|
||||
return toReturn.toArray(new TempFile[toReturn.size()]);
|
||||
}
|
||||
public static TempFile[] asTemp(StorageUtils storage, InputStreamDescriptor... descriptors)
|
||||
throws RemoteBackendException, FileNotFoundException {
|
||||
ArrayList<TempFile> toReturn = new ArrayList<TempFile>();
|
||||
for (InputStreamDescriptor desc : descriptors)
|
||||
toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename()));
|
||||
return toReturn.toArray(new TempFile[toReturn.size()]);
|
||||
}
|
||||
|
||||
public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage,
|
||||
String parentPath,String fieldName,String fieldDefinition, File directory) throws FileNotFoundException {
|
||||
public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage, String parentPath,
|
||||
String fieldName, String fieldDefinition, File directory) throws FileNotFoundException {
|
||||
|
||||
File[] children =directory.listFiles();
|
||||
InputStreamDescriptor[] iss=new InputStreamDescriptor[children.length];
|
||||
return prepareRequest(storage,parentPath,fieldName,fieldDefinition,children);
|
||||
File[] children = directory.listFiles();
|
||||
InputStreamDescriptor[] iss = new InputStreamDescriptor[children.length];
|
||||
return prepareRequest(storage, parentPath, fieldName, fieldDefinition, children);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public static RegisterFileSetRequest prepareRequest(StorageUtils storage,
|
||||
String parentPath,String fieldName,String fieldDefinition, File... toUpload) throws FileNotFoundException {
|
||||
public static RegisterFileSetRequest prepareRequest(StorageUtils storage, String parentPath, String fieldName,
|
||||
String fieldDefinition, File... toUpload) throws FileNotFoundException {
|
||||
|
||||
FileSets.RequestBuilder builder = FileSets.build(parentPath,fieldName,fieldDefinition);
|
||||
for (File f : toUpload) {
|
||||
if(!f.isDirectory())
|
||||
builder.add(FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName())));
|
||||
}
|
||||
return builder.getTheRequest();
|
||||
}
|
||||
FileSets.RequestBuilder builder = FileSets.build(parentPath, fieldName, fieldDefinition);
|
||||
for (File f : toUpload) {
|
||||
if (!f.isDirectory()) {
|
||||
long fileSize = f.length();
|
||||
TempFile file = FileSets.asTemp(storage,
|
||||
new InputStreamDescriptor(new FileInputStream(f), f.getName()));
|
||||
// Added by Francesco, see #28150
|
||||
log.debug("PrepareRequest for tempfile name " + file.getFilename() + " size: " + file.getSize());
|
||||
file.setSize(fileSize);
|
||||
builder.add(file);
|
||||
}
|
||||
}
|
||||
return builder.getTheRequest();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
package org.gcube.application.geoportal.common;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
|
@ -11,7 +9,9 @@ import java.util.Arrays;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class FilesTests {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
package org.gcube.application.geoportal.common;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.Set;
|
||||
|
||||
import com.jayway.jsonpath.Configuration;
|
||||
import com.jayway.jsonpath.Option;
|
||||
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
|
||||
|
|
@ -7,9 +10,6 @@ import com.jayway.jsonpath.spi.json.JsonProvider;
|
|||
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
|
||||
import com.jayway.jsonpath.spi.mapper.MappingProvider;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.Set;
|
||||
|
||||
public class JacksonProvider implements JSONSerializationProvider {
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,6 @@
|
|||
package org.gcube.application.geoportal.common;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.gcube.application.geoportal.common.rest.InterfaceConstants;
|
||||
import org.gcube.application.geoportal.common.utils.ContextUtils;
|
||||
import org.gcube.application.geoportal.common.utils.StorageUtils;
|
||||
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
|
||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assume.assumeTrue;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
|
|
@ -22,7 +12,16 @@ import java.util.concurrent.Executors;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.Assume.assumeTrue;
|
||||
import org.gcube.application.geoportal.common.rest.InterfaceConstants;
|
||||
import org.gcube.application.geoportal.common.utils.ContextUtils;
|
||||
import org.gcube.application.geoportal.common.utils.StorageUtils;
|
||||
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
|
||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class StorageUtilsTest {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
package org.gcube.application.geoportal.common;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
import java.util.Properties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class TokenSetter {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
package org.gcube.application.geoportal.common.legacy;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import org.gcube.application.geoportal.common.model.legacy.AccessPolicy;
|
||||
import org.gcube.application.geoportal.common.model.legacy.Concessione;
|
||||
import org.gcube.application.geoportal.common.model.legacy.RelazioneScavo;
|
||||
import org.gcube.application.geoportal.common.model.legacy.report.ConstraintCheck;
|
||||
import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class DefaultsTests {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
package org.gcube.application.geoportal.common.model;
|
||||
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import org.gcube.application.geoportal.common.model.document.Project;
|
||||
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
|
||||
import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents;
|
||||
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
import org.junit.Test;
|
||||
import static junit.framework.TestCase.assertEquals;
|
||||
import static junit.framework.TestCase.assertFalse;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
|
@ -15,7 +11,13 @@ import java.util.Collections;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static junit.framework.TestCase.*;
|
||||
import org.gcube.application.geoportal.common.model.document.Project;
|
||||
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
|
||||
import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents;
|
||||
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
|
||||
public class JSONPathTests {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
# Changelog for org.gcube.application.geoportal-service
|
||||
|
||||
## [v1.2.1-SNAPSHOT] - 2024-10-02
|
||||
|
||||
- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150]
|
||||
|
||||
## [v1.2.0] - 2024-09-24
|
||||
|
||||
- Integrated the EventManager of the cms-plugin-framework [#26321]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.gcube.application</groupId>
|
||||
<artifactId>geoportal-service</artifactId>
|
||||
<version>1.2.0</version>
|
||||
<version>1.2.1-SNAPSHOT</version>
|
||||
<name>Geoportal Service</name>
|
||||
<packaging>war</packaging>
|
||||
|
||||
|
|
|
|||
|
|
@ -1562,10 +1562,10 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI<
|
|||
else
|
||||
fileUrl = storage.getURL(f.getId());
|
||||
|
||||
log.info("Got URL {} from ID {}", fileUrl, f.getId());
|
||||
log.info("Got URL {} from ID {}, filesize {}", fileUrl, f.getId(), f.getSize());
|
||||
is = new URL(fileUrl).openStream();
|
||||
RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is,
|
||||
"Imported via gcube CMS service ", sectionFolder));
|
||||
"Imported via gcube CMS service ", sectionFolder, f.getSize()));
|
||||
log.info("Registered " + registered);
|
||||
registeredFiles.add(registered);
|
||||
} catch (StorageHubException | IOException e) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# Changelog for org.gcube.application.cms.notifications-plugins
|
||||
|
||||
## [v1.0.5] - 2024-07-03
|
||||
## [v1.0.5-SNAPSHOT] - 2024-07-03
|
||||
- Implemented the notification-plugin [#26453]
|
||||
- Updated lower range of social-service-client to `2.1.0-SNAPSHOT`
|
||||
|
||||
## [v1.0.4] - 2023-09-06
|
||||
- Using parent version range [#25572]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>notifications-plugins</artifactId>
|
||||
<version>1.0.5</version>
|
||||
<version>1.0.5-SNAPSHOT</version>
|
||||
<name>gCube CMS - Notifications Plugins</name>
|
||||
|
||||
<parent>
|
||||
|
|
@ -58,7 +58,7 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.social-networking</groupId>
|
||||
<artifactId>social-service-client</artifactId>
|
||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
<version>[2.1.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.portlets.user</groupId>
|
||||
|
|
|
|||
12
pom.xml
12
pom.xml
|
|
@ -10,7 +10,7 @@
|
|||
<groupId>org.gcube.application.cms</groupId>
|
||||
<artifactId>gcube-cms-suite</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<version>1.0.6</version>
|
||||
<version>1.0.6-SNAPSHOT</version>
|
||||
<name>Gcube CMS Suite</name>
|
||||
|
||||
<description>gCube CMS Suite is a set of components designed to manage complex space-temporal Documents defined by metadata Profiles.</description>
|
||||
|
|
@ -22,10 +22,10 @@
|
|||
<distroDirectory>distro</distroDirectory>
|
||||
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
|
||||
|
||||
<!-- default is prod -->
|
||||
<gcube-smartgears-bom-version>2.5.1</gcube-smartgears-bom-version>
|
||||
<!-- default properties -->
|
||||
<gcube-smartgears-bom-version>2.5.1-SNAPSHOT</gcube-smartgears-bom-version>
|
||||
<storagehub-version-range>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</storagehub-version-range>
|
||||
<plugin-framework-version>1.0.5-SNAPSHOT</plugin-framework-version>
|
||||
<plugin-framework-version>1.0.6-SNAPSHOT</plugin-framework-version>
|
||||
<authorization-utils-range>[2.0.0, 3.0.0-SNAPSHOT)</authorization-utils-range>
|
||||
<cms-suite-version>1.0.6-SNAPSHOT</cms-suite-version>
|
||||
</properties>
|
||||
|
|
@ -41,7 +41,7 @@
|
|||
</activation>
|
||||
<properties>
|
||||
<gcube-smartgears-bom-version>2.5.1</gcube-smartgears-bom-version>
|
||||
<plugin-framework-version>1.0.5</plugin-framework-version>
|
||||
<plugin-framework-version>1.0.6</plugin-framework-version>
|
||||
<authorization-utils-range>[2.0.0, 3.0.0-SNAPSHOT)</authorization-utils-range>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -55,7 +55,7 @@
|
|||
</activation>
|
||||
<properties>
|
||||
<gcube-smartgears-bom-version>2.5.1-SNAPSHOT</gcube-smartgears-bom-version>
|
||||
<plugin-framework-version>1.0.5-SNAPSHOT</plugin-framework-version>
|
||||
<plugin-framework-version>1.0.6-SNAPSHOT</plugin-framework-version>
|
||||
<authorization-utils-range>[2.0.0, 3.0.0-SNAPSHOT)</authorization-utils-range>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
# Changelog for org.gcube.application.cms.sdi-plugins
|
||||
|
||||
## [v1.1.4-SNAPSHOT]
|
||||
- Improved logs
|
||||
- Added fallback on /geoserver/rest path [#28150#note-8]
|
||||
|
||||
## [v1.1.3]
|
||||
- Added apply regex business logic [#26322]
|
||||
- Added reconnection attempts to DB on DB connection failure [#26322]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>sdi-plugins</artifactId>
|
||||
<version>1.1.3</version>
|
||||
<version>1.1.4-SNAPSHOT</version>
|
||||
<name>gCube CMS - SDI Plugins</name>
|
||||
|
||||
<parent>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
package org.gcube.application.cms.sdi.engine;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
|
@ -13,6 +12,7 @@ import org.gcube.data.transfer.library.DataTransferClient;
|
|||
import org.gcube.spatial.data.gis.GISInterface;
|
||||
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
|
||||
|
||||
import it.geosolutions.geoserver.rest.HTTPUtils;
|
||||
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -20,12 +20,9 @@ import lombok.extern.slf4j.Slf4j;
|
|||
@Slf4j
|
||||
public class SDIManager {
|
||||
|
||||
|
||||
|
||||
public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*");
|
||||
public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+");
|
||||
public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)");
|
||||
|
||||
public static final Pattern HOSTNAME_PATTERN = Pattern.compile("(?<=\\:\\/\\/)[^\\:]*");
|
||||
public static final Pattern PORT_PATTERN = Pattern.compile("(?<=\\:)[\\d]+");
|
||||
public static final Pattern DB_NAME_PATTERN = Pattern.compile("(?<=\\/)[^\\/]*(?=$)");
|
||||
|
||||
private final GISInterface gis;
|
||||
@Getter
|
||||
|
|
@ -35,44 +32,52 @@ public class SDIManager {
|
|||
@Getter
|
||||
private final AbstractGeoServerDescriptor currentGeoserver;
|
||||
|
||||
private final String FALLBACK_GS_REST_INTERFACE = "/rest";
|
||||
|
||||
public SDIManager() throws SDIInteractionException {
|
||||
try{
|
||||
try {
|
||||
log.debug("Initializing GIS Interface..");
|
||||
gis=GISInterface.get();
|
||||
currentGeoserver=gis.getCurrentGeoServer();
|
||||
if(currentGeoserver==null)
|
||||
gis = GISInterface.get();
|
||||
currentGeoserver = gis.getCurrentGeoServer();
|
||||
if (currentGeoserver == null)
|
||||
throw new Exception("Unable to contact data transfer for geoserver ");
|
||||
|
||||
log.debug("Found geoserver descriptor "+currentGeoserver);
|
||||
geoserverHostName=new URL(currentGeoserver.getUrl()).getHost();
|
||||
log.debug("Found geoserver descriptor " + currentGeoserver);
|
||||
geoserverHostName = new URL(currentGeoserver.getUrl()).getHost();
|
||||
|
||||
log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName);
|
||||
dtGeoServer=DataTransferClient.getInstanceByEndpoint("https://"+geoserverHostName);
|
||||
if(!currentGeoserver.getReader().existGeoserver())
|
||||
throw new Exception("Geoserver not reachable");
|
||||
}catch(Exception e) {
|
||||
throw new SDIInteractionException("Unable to initialize SDI Manager",e);
|
||||
log.debug("Contacting Data Transfer from geoserver {} ", geoserverHostName);
|
||||
String gsEP = "https://" + geoserverHostName;
|
||||
dtGeoServer = DataTransferClient.getInstanceByEndpoint(gsEP);
|
||||
if (!currentGeoserver.getReader().existGeoserver()) {
|
||||
// not a good but necessary solution here, see #28150#note-8
|
||||
String fallbackGSRestURL = currentGeoserver.getUrl() + FALLBACK_GS_REST_INTERFACE;
|
||||
log.warn("Geoserver rest interface not reachable at /rest/. Going to check "
|
||||
+ fallbackGSRestURL);
|
||||
if (!HTTPUtils.httpPing(fallbackGSRestURL, currentGeoserver.getUser(),
|
||||
currentGeoserver.getPassword())) {
|
||||
throw new Exception("Geoserver rest interface not reachable at " + fallbackGSRestURL);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("SDIManager init failed: ", e);
|
||||
throw new SDIInteractionException("Unable to initialize SDI Manager", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String createWorkspace(String toCreate) throws SDIInteractionException {
|
||||
try {
|
||||
if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) {
|
||||
log.debug("Creating workspace : "+toCreate);
|
||||
if(!currentGeoserver.getPublisher().createWorkspace(toCreate))
|
||||
throw new SDIInteractionException("Unable to create workspace "+toCreate);
|
||||
}else log.debug("Workspace "+toCreate+" exists.");
|
||||
if (!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) {
|
||||
log.debug("Creating workspace : " + toCreate);
|
||||
if (!currentGeoserver.getPublisher().createWorkspace(toCreate))
|
||||
throw new SDIInteractionException("Unable to create workspace " + toCreate);
|
||||
} else
|
||||
log.debug("Workspace " + toCreate + " exists.");
|
||||
return toCreate;
|
||||
} catch (IllegalArgumentException | MalformedURLException e) {
|
||||
throw new SDIInteractionException("Unable to create workspace "+toCreate,e);
|
||||
throw new SDIInteractionException("Unable to create workspace " + toCreate, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){
|
||||
// // ******** IDENTIFY LAYER NAME correct layer name
|
||||
// // Must be unique under same WS
|
||||
|
|
@ -94,36 +99,38 @@ public class SDIManager {
|
|||
// }
|
||||
|
||||
protected String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException {
|
||||
String storeName=encoder.getName();
|
||||
String storeName = encoder.getName();
|
||||
try {
|
||||
log.debug("Looking for datastore "+storeName+" under "+workspace);
|
||||
log.debug("Looking for datastore " + storeName + " under " + workspace);
|
||||
|
||||
if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null)
|
||||
if (currentGeoserver.getReader().getDatastore(workspace, storeName) == null)
|
||||
|
||||
if(!currentGeoserver.getDataStoreManager().create(workspace, encoder))
|
||||
throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace);
|
||||
log.debug("Store "+storeName+" exists under "+workspace);
|
||||
if (!currentGeoserver.getDataStoreManager().create(workspace, encoder))
|
||||
throw new SDIInteractionException("Unable to create store " + storeName + " in " + workspace);
|
||||
log.debug("Store " + storeName + " exists under " + workspace);
|
||||
return storeName;
|
||||
} catch (IllegalArgumentException | MalformedURLException e) {
|
||||
throw new SDIInteractionException("Unable to create store "+storeName,e);
|
||||
throw new SDIInteractionException("Unable to create store " + storeName, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection)
|
||||
throws SDIInteractionException {
|
||||
String connectionUrl = connection.getUrl();
|
||||
|
||||
protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{
|
||||
String connectionUrl=connection.getUrl();
|
||||
|
||||
Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl);
|
||||
if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection);
|
||||
Matcher hostname = HOSTNAME_PATTERN.matcher(connectionUrl);
|
||||
if (!hostname.find())
|
||||
throw new SDIInteractionException("Unable to get Hostname from " + connection);
|
||||
|
||||
Matcher port = PORT_PATTERN.matcher(connectionUrl);
|
||||
if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection);
|
||||
if (!port.find())
|
||||
throw new SDIInteractionException("Unable to get PORT from " + connection);
|
||||
|
||||
Matcher db = DB_NAME_PATTERN.matcher(connectionUrl);
|
||||
if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection);
|
||||
if (!db.find())
|
||||
throw new SDIInteractionException("Unable to get DB from " + connection);
|
||||
|
||||
|
||||
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
|
||||
GSPostGISDatastoreEncoder encoder = new GSPostGISDatastoreEncoder(storeName);
|
||||
encoder.setHost(hostname.group());
|
||||
encoder.setPort(Integer.parseInt(port.group()));
|
||||
encoder.setDatabase(db.group());
|
||||
|
|
@ -136,19 +143,20 @@ public class SDIManager {
|
|||
encoder.setFetchSize(1000);
|
||||
encoder.setValidateConnections(true);
|
||||
|
||||
return createStore(encoder,workspace);
|
||||
return createStore(encoder, workspace);
|
||||
}
|
||||
|
||||
protected String publishStyle(File sldFile, String name) throws SDIInteractionException {
|
||||
try {
|
||||
if(!currentGeoserver.getReader().existsStyle(name)) {
|
||||
log.debug("Registering style "+name);
|
||||
if(!currentGeoserver.getPublisher().publishStyle(sldFile, name))
|
||||
throw new SDIInteractionException("Unable to register style "+name);
|
||||
}else log.debug("Style "+name+" already existing");
|
||||
if (!currentGeoserver.getReader().existsStyle(name)) {
|
||||
log.debug("Registering style " + name);
|
||||
if (!currentGeoserver.getPublisher().publishStyle(sldFile, name))
|
||||
throw new SDIInteractionException("Unable to register style " + name);
|
||||
} else
|
||||
log.debug("Style " + name + " already existing");
|
||||
return name;
|
||||
} catch (IllegalArgumentException | MalformedURLException e) {
|
||||
throw new SDIInteractionException("Unable to create style "+name,e);
|
||||
throw new SDIInteractionException("Unable to create style " + name, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue