diff --git a/CHANGELOG.md b/CHANGELOG.md index 847a49b..c6f418f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.spatial.data.gcube-sdi-suite -## [v1.0.6] +## [v1.0.6-SNAPSHOT] - Integrated an EventManager centrally [#26321] - sdi-plugins: add the logic to apply a regex to the value that must be added to index [#26322] diff --git a/cms-plugin-framework/CHANGELOG.md b/cms-plugin-framework/CHANGELOG.md index 3e55ee0..11e5800 100644 --- a/cms-plugin-framework/CHANGELOG.md +++ b/cms-plugin-framework/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.cms-plugin-framework +## [v1.0.6-SNAPSHOT] - 2024-10-01 + +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.0.5] - 2024-07-03 - Implemented Event Broker [#26321] diff --git a/cms-plugin-framework/pom.xml b/cms-plugin-framework/pom.xml index 639a338..791e98b 100644 --- a/cms-plugin-framework/pom.xml +++ b/cms-plugin-framework/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 cms-plugin-framework - 1.0.5 + 1.0.6-SNAPSHOT org.gcube.application.cms diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index 56eab21..7a6acd3 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -22,52 +22,52 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class WorkspaceManager { - private static final String APP_FOLDER=".GNA_RECORDS"; + private static final String APP_FOLDER = ".GNA_RECORDS"; - - private StorageHubClient sgClient=null; + private StorageHubClient sgClient = null; @Getter - private FolderContainer appBase=null; + private FolderContainer appBase = null; @Getter @Setter @AllArgsConstructor @RequiredArgsConstructor - public static class FolderOptions{ + public static class FolderOptions { @NonNull private String folderName; private String folderDescription; private FolderContainer parent; } - + @Getter @Setter @AllArgsConstructor @RequiredArgsConstructor - public static class FileOptions{ + public static class FileOptions { @NonNull private String fileName; @NonNull private InputStream is; - + private String fileDescription; private FolderContainer parent; - + // Added by Francesco, see #28150 + private Long size; } - public Archive getConfiguration(){ + public Archive getConfiguration() { Archive toReturn = new Archive("W-STORAGE"); - toReturn.put("folder_id",appBase.getId()); + toReturn.put("folder_id", appBase.getId()); return toReturn; } public WorkspaceManager() throws ConfigurationException, StorageHubException { - sgClient= ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class); - appBase=getApplicationBaseFolder(sgClient); + sgClient = ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class); + appBase = getApplicationBaseFolder(sgClient); } public FolderContainer createFolder(FolderOptions opts) throws StorageHubException { - if(opts.getParent()==null) + if (opts.getParent() == null) opts.setParent(appBase); return createFolderRoutine(opts); } @@ -84,11 +84,10 @@ public class WorkspaceManager { sgClient.open(id).asFolder().delete(); } - public FolderContainer getSubFolder(FolderContainer parentFolder,String path) throws StorageHubException { - return getSubFolder(parentFolder,path,""); + public FolderContainer getSubFolder(FolderContainer parentFolder, String path) throws StorageHubException { + return getSubFolder(parentFolder, path, ""); } - /** * Returns sub folder. Creates it if missing * @@ -97,26 +96,26 @@ public class WorkspaceManager { * @return * @throws StorageHubException */ - public FolderContainer getSubFolder(FolderContainer parentFolder,String path, String description) throws StorageHubException { - try{ + public FolderContainer getSubFolder(FolderContainer parentFolder, String path, String description) + throws StorageHubException { + try { return parentFolder.openByRelativePath(path).asFolder(); - }catch(StorageHubException e) { - log.debug("Missing subPath "+path); - FolderContainer targetParent=parentFolder; - String targetName=path; - if(path.contains("/")) { - String parent=path.substring(0, path.lastIndexOf("/")); - log.debug("Checking intermediate "+parent); - targetParent=getSubFolder(parentFolder,parent); - targetName=path.substring(path.lastIndexOf("/")+1); + } catch (StorageHubException e) { + log.debug("Missing subPath " + path); + FolderContainer targetParent = parentFolder; + String targetName = path; + if (path.contains("/")) { + String parent = path.substring(0, path.lastIndexOf("/")); + log.debug("Checking intermediate " + parent); + targetParent = getSubFolder(parentFolder, parent); + targetName = path.substring(path.lastIndexOf("/") + 1); } - FolderOptions opts = new FolderOptions(targetName,description,targetParent); + FolderOptions opts = new FolderOptions(targetName, description, targetParent); log.debug("Creating FOLDER {}", opts); return createFolder(opts); } } - // public WorkspaceContent storeToWS(FileOptions opts) throws FileNotFoundException, StorageHubException { // FileContainer item=createFileRoutine(opts); // item=sgClient.open(item.getId()).asFile(); @@ -131,12 +130,11 @@ public class WorkspaceManager { // } public RegisteredFile registerFile(FileOptions opts) throws StorageHubException { - FileContainer item=createFileRoutine(opts); - item=sgClient.open(item.getId()).asFile(); + FileContainer item = createFileRoutine(opts); + item = sgClient.open(item.getId()).asFile(); + + RegisteredFile file = new RegisteredFile(); - RegisteredFile file=new RegisteredFile(); - - file.setLink(item.getPublicLink().toString()); file.setMimetype(item.get().getContent().getMimeType()); file.setStorageID(item.getId()); @@ -144,26 +142,25 @@ public class WorkspaceManager { return file; } - - + // public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException { // sgClient.open(toDelete.getStorageID()).asFile().forceDelete(); // } - public void deleteItem(String itemId)throws StorageHubException{ + public void deleteItem(String itemId) throws StorageHubException { sgClient.open(itemId).asItem().forceDelete(); } - // STATIC SYNCH METHODS - + // STATIC SYNCH METHODS + @Synchronized public static FolderContainer getApplicationBaseFolder(StorageHubClient sgClient) throws StorageHubException { - FolderContainer vre=sgClient.openVREFolder(); + FolderContainer vre = sgClient.openVREFolder(); try { return vre.openByRelativePath(APP_FOLDER).asFolder(); - }catch(StorageHubException e) { - log.debug("APP Fodler missing. Initializing.."); - FolderContainer toReturn= vre.newFolder(APP_FOLDER, "Base folder for GNA records"); + } catch (StorageHubException e) { + log.debug("APP Folder missing. Initializing.."); + FolderContainer toReturn = vre.newFolder(APP_FOLDER, "Base folder for GNA records"); toReturn.setHidden(); return toReturn; } @@ -172,12 +169,19 @@ public class WorkspaceManager { @Synchronized private static FolderContainer createFolderRoutine(FolderOptions opts) throws StorageHubException { opts.setFolderName(Files.fixFilename(opts.getFolderName())); - return opts.getParent().newFolder(opts.getFolderName(),opts.getFolderDescription()); + return opts.getParent().newFolder(opts.getFolderName(), opts.getFolderDescription()); } - + @Synchronized private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { + // Updated by Francesco, see #28150 + log.info("Uploading file name: {}, in the parent folder id: {}, filesize is: " + opts.getSize(), + opts.getFileName(), opts.getParent().getId()); opts.setFileName(Files.fixFilename(opts.getFileName())); - return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + if (opts.getSize() == null) + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + else + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription(), + opts.getSize()); } } diff --git a/geoportal-common/CHANGELOG.md b/geoportal-common/CHANGELOG.md index 2ff2aaf..94facb8 100644 --- a/geoportal-common/CHANGELOG.md +++ b/geoportal-common/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog for org.gcube.application.geoportal-common +## [v1.1.1-SNAPSHOT] - 2024-10-01 +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.1.0] - 2024-04-08 - Added message to StepExecutionRequest [#27192] - Fixed pom. Duplicate declaration of `registry-publisher` dependency diff --git a/geoportal-common/pom.xml b/geoportal-common/pom.xml index 85d8d72..de0c767 100644 --- a/geoportal-common/pom.xml +++ b/geoportal-common/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 geoportal-common - 1.1.0 + 1.1.1-SNAPSHOT Geoportal Common diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java index 4584a77..f5c0dd9 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java @@ -15,8 +15,20 @@ public class TempFile { private String url; private String filename; - public void validate()throws InvalidRequestException { - if((id==null || id.isEmpty() )&&(url==null||url.isEmpty())) throw new InvalidRequestException("Invalid temp file "+this+" : ID null or empty and no url defined"); - if(filename==null || filename.isEmpty()) throw new InvalidRequestException("Invalid temp file "+this+" : filename null or empty"); + // Added by Francesco, see #28150 + private Long size; + + public void validate() throws InvalidRequestException { + if ((id == null || id.isEmpty()) && (url == null || url.isEmpty())) + throw new InvalidRequestException("Invalid temp file " + this + " : ID null or empty and no url defined"); + if (filename == null || filename.isEmpty()) + throw new InvalidRequestException("Invalid temp file " + this + " : filename null or empty"); } + + public TempFile(String id, String url, String filename) { + this.id = id; + this.url = url; + this.filename = filename; + } + } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java index 1a9e207..e408354 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java @@ -11,9 +11,9 @@ import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.relationships.RelationshipNavigationObject; import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest; import org.gcube.application.geoportal.common.model.rest.DeleteRelationshipRequest; +import org.gcube.application.geoportal.common.model.rest.PerformStepRequest; import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest; -import org.gcube.application.geoportal.common.model.rest.PerformStepRequest; public interface Projects

{ diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java index 45285d0..ddbb68a 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java @@ -13,107 +13,121 @@ import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest; import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class FileSets { - public static class RequestBuilder { - RegisterFileSetRequest theRequest=new RegisterFileSetRequest(); + public static class RequestBuilder { + RegisterFileSetRequest theRequest = new RegisterFileSetRequest(); - public RequestBuilder addAll(Collection toAdd){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - theRequest.getStreams().addAll(toAdd); - return this; - } - public RequestBuilder add(TempFile... f){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - for(TempFile temp: f ) - theRequest.getStreams().add(temp); - return this; - } + public RequestBuilder addAll(Collection toAdd) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + theRequest.getStreams().addAll(toAdd); + return this; + } - public RequestBuilder add(TempFile f){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - theRequest.getStreams().add(f); - return this; - } + public RequestBuilder add(TempFile... f) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + for (TempFile temp : f) + theRequest.getStreams().add(temp); + return this; + } - public RequestBuilder setFieldDefinitionPath(String path){ - theRequest.setFieldDefinitionPath(path); - return this; - } + public RequestBuilder add(TempFile f) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + theRequest.getStreams().add(f); + return this; + } - public RequestBuilder setParentPath(String path){ - theRequest.setParentPath(path); - return this; - } + public RequestBuilder setFieldDefinitionPath(String path) { + theRequest.setFieldDefinitionPath(path); + return this; + } - public RequestBuilder setFieldName(String fieldName){ - theRequest.setFieldName(fieldName); - return this; - } + public RequestBuilder setParentPath(String path) { + theRequest.setParentPath(path); + return this; + } - public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path){ - theRequest.setClashOption(path); - return this; - } + public RequestBuilder setFieldName(String fieldName) { + theRequest.setFieldName(fieldName); + return this; + } - public RequestBuilder setAttributes(Document attributes){ - theRequest.setAttributes(attributes); - return this; - } + public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path) { + theRequest.setClashOption(path); + return this; + } - public RegisterFileSetRequest getTheRequest(){ - if(theRequest.getClashOption()==null) - // default Clash Policy - setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING); - return theRequest;} + public RequestBuilder setAttributes(Document attributes) { + theRequest.setAttributes(attributes); + return this; + } - public RequestBuilder setAccess(Access access){ - theRequest.setToSetAccess(access); - return this; - } + public RegisterFileSetRequest getTheRequest() { + if (theRequest.getClashOption() == null) + // default Clash Policy + setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING); + return theRequest; + } - } + public RequestBuilder setAccess(Access access) { + theRequest.setToSetAccess(access); + return this; + } + } - public static RequestBuilder build(String parent,String fieldName, String fieldDefinition) { - return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition).setFieldName(fieldName); - } + public static RequestBuilder build(String parent, String fieldName, String fieldDefinition) { + return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition) + .setFieldName(fieldName); + } - public static RequestBuilder build(String parent,String fieldName, String fieldDefinition, TempFile...files) { - return build(parent,fieldName,fieldDefinition).add(files); - } + public static RequestBuilder build(String parent, String fieldName, String fieldDefinition, TempFile... files) { + return build(parent, fieldName, fieldDefinition).add(files); + } - public static TempFile asTemp(StorageUtils storage,InputStreamDescriptor descriptor) throws RemoteBackendException, FileNotFoundException { - return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename()); - } + public static TempFile asTemp(StorageUtils storage, InputStreamDescriptor descriptor) + throws RemoteBackendException, FileNotFoundException { + return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename()); + } - public static TempFile[] asTemp(StorageUtils storage,InputStreamDescriptor... descriptors) throws RemoteBackendException, FileNotFoundException { - ArrayList toReturn=new ArrayList(); - for(InputStreamDescriptor desc:descriptors) - toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename())); - return toReturn.toArray(new TempFile[toReturn.size()]); - } + public static TempFile[] asTemp(StorageUtils storage, InputStreamDescriptor... descriptors) + throws RemoteBackendException, FileNotFoundException { + ArrayList toReturn = new ArrayList(); + for (InputStreamDescriptor desc : descriptors) + toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename())); + return toReturn.toArray(new TempFile[toReturn.size()]); + } - public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage, - String parentPath,String fieldName,String fieldDefinition, File directory) throws FileNotFoundException { + public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage, String parentPath, + String fieldName, String fieldDefinition, File directory) throws FileNotFoundException { - File[] children =directory.listFiles(); - InputStreamDescriptor[] iss=new InputStreamDescriptor[children.length]; - return prepareRequest(storage,parentPath,fieldName,fieldDefinition,children); + File[] children = directory.listFiles(); + InputStreamDescriptor[] iss = new InputStreamDescriptor[children.length]; + return prepareRequest(storage, parentPath, fieldName, fieldDefinition, children); - } + } - public static RegisterFileSetRequest prepareRequest(StorageUtils storage, - String parentPath,String fieldName,String fieldDefinition, File... toUpload) throws FileNotFoundException { + public static RegisterFileSetRequest prepareRequest(StorageUtils storage, String parentPath, String fieldName, + String fieldDefinition, File... toUpload) throws FileNotFoundException { - FileSets.RequestBuilder builder = FileSets.build(parentPath,fieldName,fieldDefinition); - for (File f : toUpload) { - if(!f.isDirectory()) - builder.add(FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName()))); - } - return builder.getTheRequest(); - } + FileSets.RequestBuilder builder = FileSets.build(parentPath, fieldName, fieldDefinition); + for (File f : toUpload) { + if (!f.isDirectory()) { + long fileSize = f.length(); + TempFile file = FileSets.asTemp(storage, + new InputStreamDescriptor(new FileInputStream(f), f.getName())); + // Added by Francesco, see #28150 + log.debug("PrepareRequest for tempfile name " + file.getFilename() + " size: " + file.getSize()); + file.setSize(fileSize); + builder.add(file); + } + } + return builder.getTheRequest(); + } } diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java index f511c2f..0b7d68a 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java @@ -1,8 +1,6 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; -import org.gcube.application.geoportal.common.utils.Files; -import org.junit.Test; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -11,7 +9,9 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertTrue; +import org.gcube.application.geoportal.common.utils.Files; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class FilesTests { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java index f75d6bb..a3006b2 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java @@ -1,5 +1,8 @@ package org.gcube.application.geoportal.common; +import java.util.EnumSet; +import java.util.Set; + import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.spi.json.JacksonJsonProvider; @@ -7,9 +10,6 @@ import com.jayway.jsonpath.spi.json.JsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import com.jayway.jsonpath.spi.mapper.MappingProvider; -import java.util.EnumSet; -import java.util.Set; - public class JacksonProvider implements JSONSerializationProvider { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java index 90603e2..ecd61bb 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java @@ -1,16 +1,6 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; -import org.gcube.application.geoportal.common.rest.InterfaceConstants; -import org.gcube.application.geoportal.common.utils.ContextUtils; -import org.gcube.application.geoportal.common.utils.StorageUtils; -import org.gcube.application.geoportal.common.utils.tests.GCubeTest; -import org.gcube.contentmanagement.blobstorage.service.IClient; -import org.gcube.contentmanager.storageclient.wrapper.AccessType; -import org.gcube.contentmanager.storageclient.wrapper.MemoryType; -import org.gcube.contentmanager.storageclient.wrapper.StorageClient; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assume.assumeTrue; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -22,7 +12,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static org.junit.Assume.assumeTrue; +import org.gcube.application.geoportal.common.rest.InterfaceConstants; +import org.gcube.application.geoportal.common.utils.ContextUtils; +import org.gcube.application.geoportal.common.utils.StorageUtils; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class StorageUtilsTest { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java index b8d27bd..0c0de3b 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java @@ -1,10 +1,11 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; +import java.util.Properties; + import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; -import java.util.Properties; +import lombok.extern.slf4j.Slf4j; @Slf4j public class TokenSetter { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java index 7b81549..1bd7ab0 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java @@ -1,13 +1,12 @@ package org.gcube.application.geoportal.common.legacy; +import static org.junit.Assert.assertEquals; + import org.gcube.application.geoportal.common.model.legacy.AccessPolicy; import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.legacy.RelazioneScavo; import org.gcube.application.geoportal.common.model.legacy.report.ConstraintCheck; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; public class DefaultsTests { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java index cdbfad0..24c162e 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java @@ -1,12 +1,8 @@ package org.gcube.application.geoportal.common.model; -import com.jayway.jsonpath.JsonPath; -import org.gcube.application.geoportal.common.model.document.Project; -import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; -import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents; -import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; -import org.gcube.application.geoportal.common.utils.Files; -import org.junit.Test; +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; import java.io.File; import java.io.IOException; @@ -15,7 +11,13 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import static junit.framework.TestCase.*; +import org.gcube.application.geoportal.common.model.document.Project; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; +import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents; +import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; +import org.gcube.application.geoportal.common.utils.Files; + +import com.jayway.jsonpath.JsonPath; public class JSONPathTests { diff --git a/geoportal-service/CHANGELOG.md b/geoportal-service/CHANGELOG.md index 9cb5a83..22d35c2 100644 --- a/geoportal-service/CHANGELOG.md +++ b/geoportal-service/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.geoportal-service +## [v1.2.1-SNAPSHOT] - 2024-10-02 + +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.2.0] - 2024-09-24 - Integrated the EventManager of the cms-plugin-framework [#26321] diff --git a/geoportal-service/pom.xml b/geoportal-service/pom.xml index c8c58ef..1ad86b4 100644 --- a/geoportal-service/pom.xml +++ b/geoportal-service/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.gcube.application geoportal-service - 1.2.0 + 1.2.1-SNAPSHOT Geoportal Service war diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index 2cde1bc..ca05a38 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -1562,10 +1562,10 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< else fileUrl = storage.getURL(f.getId()); - log.info("Got URL {} from ID {}", fileUrl, f.getId()); + log.info("Got URL {} from ID {}, filesize {}", fileUrl, f.getId(), f.getSize()); is = new URL(fileUrl).openStream(); RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is, - "Imported via gcube CMS service ", sectionFolder)); + "Imported via gcube CMS service ", sectionFolder, f.getSize())); log.info("Registered " + registered); registeredFiles.add(registered); } catch (StorageHubException | IOException e) { diff --git a/notifications-plugins/CHANGELOG.md b/notifications-plugins/CHANGELOG.md index 2a465d5..5209f50 100644 --- a/notifications-plugins/CHANGELOG.md +++ b/notifications-plugins/CHANGELOG.md @@ -1,7 +1,8 @@ # Changelog for org.gcube.application.cms.notifications-plugins -## [v1.0.5] - 2024-07-03 +## [v1.0.5-SNAPSHOT] - 2024-07-03 - Implemented the notification-plugin [#26453] +- Updated lower range of social-service-client to `2.1.0-SNAPSHOT` ## [v1.0.4] - 2023-09-06 - Using parent version range [#25572] diff --git a/notifications-plugins/pom.xml b/notifications-plugins/pom.xml index fd92e15..5dadb66 100644 --- a/notifications-plugins/pom.xml +++ b/notifications-plugins/pom.xml @@ -5,7 +5,7 @@ 4.0.0 notifications-plugins - 1.0.5 + 1.0.5-SNAPSHOT gCube CMS - Notifications Plugins @@ -58,7 +58,7 @@ org.gcube.social-networking social-service-client - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + [2.1.0-SNAPSHOT, 3.0.0-SNAPSHOT) org.gcube.portlets.user diff --git a/pom.xml b/pom.xml index ba4dc89..1241f3a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.gcube.application.cms gcube-cms-suite pom - 1.0.6 + 1.0.6-SNAPSHOT Gcube CMS Suite gCube CMS Suite is a set of components designed to manage complex space-temporal Documents defined by metadata Profiles. @@ -22,10 +22,10 @@ distro https://code-repo.d4science.org/gCubeSystem - - 2.5.1 + + 2.5.1-SNAPSHOT [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - 1.0.5-SNAPSHOT + 1.0.6-SNAPSHOT [2.0.0, 3.0.0-SNAPSHOT) 1.0.6-SNAPSHOT @@ -41,7 +41,7 @@ 2.5.1 - 1.0.5 + 1.0.6 [2.0.0, 3.0.0-SNAPSHOT) @@ -55,7 +55,7 @@ 2.5.1-SNAPSHOT - 1.0.5-SNAPSHOT + 1.0.6-SNAPSHOT [2.0.0, 3.0.0-SNAPSHOT) diff --git a/sdi-plugins/CHANGELOG.md b/sdi-plugins/CHANGELOG.md index e581b54..27bdb59 100644 --- a/sdi-plugins/CHANGELOG.md +++ b/sdi-plugins/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.cms.sdi-plugins +## [v1.1.4-SNAPSHOT] +- Improved logs +- Added fallback on /geoserver/rest path [#28150#note-8] + ## [v1.1.3] - Added apply regex business logic [#26322] - Added reconnection attempts to DB on DB connection failure [#26322] diff --git a/sdi-plugins/pom.xml b/sdi-plugins/pom.xml index 3380356..8811cf9 100644 --- a/sdi-plugins/pom.xml +++ b/sdi-plugins/pom.xml @@ -5,7 +5,7 @@ 4.0.0 sdi-plugins - 1.1.3 + 1.1.4-SNAPSHOT gCube CMS - SDI Plugins diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java index 6b35b8a..7cc0778 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -1,6 +1,5 @@ package org.gcube.application.cms.sdi.engine; - import java.io.File; import java.net.MalformedURLException; import java.net.URL; @@ -13,6 +12,7 @@ import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.spatial.data.gis.GISInterface; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; +import it.geosolutions.geoserver.rest.HTTPUtils; import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -20,12 +20,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class SDIManager { - - - public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); - public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); - public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); - + public static final Pattern HOSTNAME_PATTERN = Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); + public static final Pattern PORT_PATTERN = Pattern.compile("(?<=\\:)[\\d]+"); + public static final Pattern DB_NAME_PATTERN = Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); private final GISInterface gis; @Getter @@ -35,44 +32,52 @@ public class SDIManager { @Getter private final AbstractGeoServerDescriptor currentGeoserver; + private final String FALLBACK_GS_REST_INTERFACE = "/rest"; public SDIManager() throws SDIInteractionException { - try{ + try { log.debug("Initializing GIS Interface.."); - gis=GISInterface.get(); - currentGeoserver=gis.getCurrentGeoServer(); - if(currentGeoserver==null) + gis = GISInterface.get(); + currentGeoserver = gis.getCurrentGeoServer(); + if (currentGeoserver == null) throw new Exception("Unable to contact data transfer for geoserver "); - log.debug("Found geoserver descriptor "+currentGeoserver); - geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); + log.debug("Found geoserver descriptor " + currentGeoserver); + geoserverHostName = new URL(currentGeoserver.getUrl()).getHost(); - log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); - dtGeoServer=DataTransferClient.getInstanceByEndpoint("https://"+geoserverHostName); - if(!currentGeoserver.getReader().existGeoserver()) - throw new Exception("Geoserver not reachable"); - }catch(Exception e) { - throw new SDIInteractionException("Unable to initialize SDI Manager",e); + log.debug("Contacting Data Transfer from geoserver {} ", geoserverHostName); + String gsEP = "https://" + geoserverHostName; + dtGeoServer = DataTransferClient.getInstanceByEndpoint(gsEP); + if (!currentGeoserver.getReader().existGeoserver()) { + // not a good but necessary solution here, see #28150#note-8 + String fallbackGSRestURL = currentGeoserver.getUrl() + FALLBACK_GS_REST_INTERFACE; + log.warn("Geoserver rest interface not reachable at /rest/. Going to check " + + fallbackGSRestURL); + if (!HTTPUtils.httpPing(fallbackGSRestURL, currentGeoserver.getUser(), + currentGeoserver.getPassword())) { + throw new Exception("Geoserver rest interface not reachable at " + fallbackGSRestURL); + } + } + } catch (Exception e) { + log.error("SDIManager init failed: ", e); + throw new SDIInteractionException("Unable to initialize SDI Manager", e); } } - - - public String createWorkspace(String toCreate) throws SDIInteractionException { try { - if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { - log.debug("Creating workspace : "+toCreate); - if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) - throw new SDIInteractionException("Unable to create workspace "+toCreate); - }else log.debug("Workspace "+toCreate+" exists."); + if (!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { + log.debug("Creating workspace : " + toCreate); + if (!currentGeoserver.getPublisher().createWorkspace(toCreate)) + throw new SDIInteractionException("Unable to create workspace " + toCreate); + } else + log.debug("Workspace " + toCreate + " exists."); return toCreate; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create workspace "+toCreate,e); + throw new SDIInteractionException("Unable to create workspace " + toCreate, e); } } - // protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){ // // ******** IDENTIFY LAYER NAME correct layer name // // Must be unique under same WS @@ -94,36 +99,38 @@ public class SDIManager { // } protected String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { - String storeName=encoder.getName(); + String storeName = encoder.getName(); try { - log.debug("Looking for datastore "+storeName+" under "+workspace); + log.debug("Looking for datastore " + storeName + " under " + workspace); - if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) + if (currentGeoserver.getReader().getDatastore(workspace, storeName) == null) - if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) - throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); - log.debug("Store "+storeName+" exists under "+workspace); + if (!currentGeoserver.getDataStoreManager().create(workspace, encoder)) + throw new SDIInteractionException("Unable to create store " + storeName + " in " + workspace); + log.debug("Store " + storeName + " exists under " + workspace); return storeName; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create store "+storeName,e); + throw new SDIInteractionException("Unable to create store " + storeName, e); } } + protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) + throws SDIInteractionException { + String connectionUrl = connection.getUrl(); - protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ - String connectionUrl=connection.getUrl(); - - Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); - if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); + Matcher hostname = HOSTNAME_PATTERN.matcher(connectionUrl); + if (!hostname.find()) + throw new SDIInteractionException("Unable to get Hostname from " + connection); Matcher port = PORT_PATTERN.matcher(connectionUrl); - if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); + if (!port.find()) + throw new SDIInteractionException("Unable to get PORT from " + connection); Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); - if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); + if (!db.find()) + throw new SDIInteractionException("Unable to get DB from " + connection); - - GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); + GSPostGISDatastoreEncoder encoder = new GSPostGISDatastoreEncoder(storeName); encoder.setHost(hostname.group()); encoder.setPort(Integer.parseInt(port.group())); encoder.setDatabase(db.group()); @@ -136,19 +143,20 @@ public class SDIManager { encoder.setFetchSize(1000); encoder.setValidateConnections(true); - return createStore(encoder,workspace); + return createStore(encoder, workspace); } protected String publishStyle(File sldFile, String name) throws SDIInteractionException { try { - if(!currentGeoserver.getReader().existsStyle(name)) { - log.debug("Registering style "+name); - if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) - throw new SDIInteractionException("Unable to register style "+name); - }else log.debug("Style "+name+" already existing"); + if (!currentGeoserver.getReader().existsStyle(name)) { + log.debug("Registering style " + name); + if (!currentGeoserver.getPublisher().publishStyle(sldFile, name)) + throw new SDIInteractionException("Unable to register style " + name); + } else + log.debug("Style " + name + " already existing"); return name; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create style "+name,e); + throw new SDIInteractionException("Unable to create style " + name, e); } }