From 0808d1e12c653a003b9dbfdd01e94fb88d59b15f Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Tue, 1 Oct 2024 15:32:19 +0200 Subject: [PATCH] Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] --- cms-plugin-framework/CHANGELOG.md | 4 ++++ cms-plugin-framework/pom.xml | 2 +- .../cms/implementations/WorkspaceManager.java | 11 ++++++++++- geoportal-common/CHANGELOG.md | 3 +++ geoportal-common/pom.xml | 2 +- .../geoportal/common/model/rest/TempFile.java | 18 +++++++++++++++--- .../geoportal/common/utils/FileSets.java | 8 ++++++-- .../engine/mongo/ProfiledMongoManager.java | 2 +- 8 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cms-plugin-framework/CHANGELOG.md b/cms-plugin-framework/CHANGELOG.md index 3e55ee0..11e5800 100644 --- a/cms-plugin-framework/CHANGELOG.md +++ b/cms-plugin-framework/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.cms-plugin-framework +## [v1.0.6-SNAPSHOT] - 2024-10-01 + +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.0.5] - 2024-07-03 - Implemented Event Broker [#26321] diff --git a/cms-plugin-framework/pom.xml b/cms-plugin-framework/pom.xml index 639a338..791e98b 100644 --- a/cms-plugin-framework/pom.xml +++ b/cms-plugin-framework/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 cms-plugin-framework - 1.0.5 + 1.0.6-SNAPSHOT org.gcube.application.cms diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index 56eab21..bb2246d 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -52,6 +52,8 @@ public class WorkspaceManager { private String fileDescription; private FolderContainer parent; + //Added by Francesco, see #28150 + private Long size; } @@ -177,7 +179,14 @@ public class WorkspaceManager { @Synchronized private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { + // Updated by Francesco, see #28150 + log.debug("Uploading file name: {}, in the parent folder id: {}, filesize is: {}", opts.getFileName(), + opts.getParent().getId(), opts.getSize()); opts.setFileName(Files.fixFilename(opts.getFileName())); - return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + if (opts.getSize() == null) + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + else + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription(), + opts.getSize()); } } diff --git a/geoportal-common/CHANGELOG.md b/geoportal-common/CHANGELOG.md index 2ff2aaf..94facb8 100644 --- a/geoportal-common/CHANGELOG.md +++ b/geoportal-common/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog for org.gcube.application.geoportal-common +## [v1.1.1-SNAPSHOT] - 2024-10-01 +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.1.0] - 2024-04-08 - Added message to StepExecutionRequest [#27192] - Fixed pom. Duplicate declaration of `registry-publisher` dependency diff --git a/geoportal-common/pom.xml b/geoportal-common/pom.xml index 85d8d72..de0c767 100644 --- a/geoportal-common/pom.xml +++ b/geoportal-common/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 geoportal-common - 1.1.0 + 1.1.1-SNAPSHOT Geoportal Common diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java index 4584a77..f5c0dd9 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java @@ -15,8 +15,20 @@ public class TempFile { private String url; private String filename; - public void validate()throws InvalidRequestException { - if((id==null || id.isEmpty() )&&(url==null||url.isEmpty())) throw new InvalidRequestException("Invalid temp file "+this+" : ID null or empty and no url defined"); - if(filename==null || filename.isEmpty()) throw new InvalidRequestException("Invalid temp file "+this+" : filename null or empty"); + // Added by Francesco, see #28150 + private Long size; + + public void validate() throws InvalidRequestException { + if ((id == null || id.isEmpty()) && (url == null || url.isEmpty())) + throw new InvalidRequestException("Invalid temp file " + this + " : ID null or empty and no url defined"); + if (filename == null || filename.isEmpty()) + throw new InvalidRequestException("Invalid temp file " + this + " : filename null or empty"); } + + public TempFile(String id, String url, String filename) { + this.id = id; + this.url = url; + this.filename = filename; + } + } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java index 45285d0..a2ee2f0 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java @@ -111,8 +111,12 @@ public class FileSets { FileSets.RequestBuilder builder = FileSets.build(parentPath,fieldName,fieldDefinition); for (File f : toUpload) { - if(!f.isDirectory()) - builder.add(FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName()))); + if(!f.isDirectory()) { + TempFile file = FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName())); + //Added by Francesco, see #28150 + file.setSize(f.length()); + builder.add(file); + } } return builder.getTheRequest(); } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index 2cde1bc..c3945e5 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -1562,7 +1562,7 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< else fileUrl = storage.getURL(f.getId()); - log.info("Got URL {} from ID {}", fileUrl, f.getId()); + log.info("Got URL {} from ID {}, filesize {}", fileUrl, f.getId(), f.getSize()); is = new URL(fileUrl).openStream(); RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is, "Imported via gcube CMS service ", sectionFolder));