From 0808d1e12c653a003b9dbfdd01e94fb88d59b15f Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Tue, 1 Oct 2024 15:32:19 +0200 Subject: [PATCH 01/14] Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] --- cms-plugin-framework/CHANGELOG.md | 4 ++++ cms-plugin-framework/pom.xml | 2 +- .../cms/implementations/WorkspaceManager.java | 11 ++++++++++- geoportal-common/CHANGELOG.md | 3 +++ geoportal-common/pom.xml | 2 +- .../geoportal/common/model/rest/TempFile.java | 18 +++++++++++++++--- .../geoportal/common/utils/FileSets.java | 8 ++++++-- .../engine/mongo/ProfiledMongoManager.java | 2 +- 8 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cms-plugin-framework/CHANGELOG.md b/cms-plugin-framework/CHANGELOG.md index 3e55ee0..11e5800 100644 --- a/cms-plugin-framework/CHANGELOG.md +++ b/cms-plugin-framework/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.cms-plugin-framework +## [v1.0.6-SNAPSHOT] - 2024-10-01 + +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.0.5] - 2024-07-03 - Implemented Event Broker [#26321] diff --git a/cms-plugin-framework/pom.xml b/cms-plugin-framework/pom.xml index 639a338..791e98b 100644 --- a/cms-plugin-framework/pom.xml +++ b/cms-plugin-framework/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 cms-plugin-framework - 1.0.5 + 1.0.6-SNAPSHOT org.gcube.application.cms diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index 56eab21..bb2246d 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -52,6 +52,8 @@ public class WorkspaceManager { private String fileDescription; private FolderContainer parent; + //Added by Francesco, see #28150 + private Long size; } @@ -177,7 +179,14 @@ public class WorkspaceManager { @Synchronized private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { + // Updated by Francesco, see #28150 + log.debug("Uploading file name: {}, in the parent folder id: {}, filesize is: {}", opts.getFileName(), + opts.getParent().getId(), opts.getSize()); opts.setFileName(Files.fixFilename(opts.getFileName())); - return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + if (opts.getSize() == null) + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); + else + return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription(), + opts.getSize()); } } diff --git a/geoportal-common/CHANGELOG.md b/geoportal-common/CHANGELOG.md index 2ff2aaf..94facb8 100644 --- a/geoportal-common/CHANGELOG.md +++ b/geoportal-common/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog for org.gcube.application.geoportal-common +## [v1.1.1-SNAPSHOT] - 2024-10-01 +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.1.0] - 2024-04-08 - Added message to StepExecutionRequest [#27192] - Fixed pom. Duplicate declaration of `registry-publisher` dependency diff --git a/geoportal-common/pom.xml b/geoportal-common/pom.xml index 85d8d72..de0c767 100644 --- a/geoportal-common/pom.xml +++ b/geoportal-common/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 geoportal-common - 1.1.0 + 1.1.1-SNAPSHOT Geoportal Common diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java index 4584a77..f5c0dd9 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/rest/TempFile.java @@ -15,8 +15,20 @@ public class TempFile { private String url; private String filename; - public void validate()throws InvalidRequestException { - if((id==null || id.isEmpty() )&&(url==null||url.isEmpty())) throw new InvalidRequestException("Invalid temp file "+this+" : ID null or empty and no url defined"); - if(filename==null || filename.isEmpty()) throw new InvalidRequestException("Invalid temp file "+this+" : filename null or empty"); + // Added by Francesco, see #28150 + private Long size; + + public void validate() throws InvalidRequestException { + if ((id == null || id.isEmpty()) && (url == null || url.isEmpty())) + throw new InvalidRequestException("Invalid temp file " + this + " : ID null or empty and no url defined"); + if (filename == null || filename.isEmpty()) + throw new InvalidRequestException("Invalid temp file " + this + " : filename null or empty"); } + + public TempFile(String id, String url, String filename) { + this.id = id; + this.url = url; + this.filename = filename; + } + } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java index 45285d0..a2ee2f0 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java @@ -111,8 +111,12 @@ public class FileSets { FileSets.RequestBuilder builder = FileSets.build(parentPath,fieldName,fieldDefinition); for (File f : toUpload) { - if(!f.isDirectory()) - builder.add(FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName()))); + if(!f.isDirectory()) { + TempFile file = FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName())); + //Added by Francesco, see #28150 + file.setSize(f.length()); + builder.add(file); + } } return builder.getTheRequest(); } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index 2cde1bc..c3945e5 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -1562,7 +1562,7 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< else fileUrl = storage.getURL(f.getId()); - log.info("Got URL {} from ID {}", fileUrl, f.getId()); + log.info("Got URL {} from ID {}, filesize {}", fileUrl, f.getId(), f.getSize()); is = new URL(fileUrl).openStream(); RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is, "Imported via gcube CMS service ", sectionFolder)); -- 2.17.1 From 04093cd33b47f7a9b24a15f30eaff70ea6cabbf9 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Tue, 1 Oct 2024 16:00:49 +0200 Subject: [PATCH 02/14] updated logs --- .../application/cms/implementations/WorkspaceManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index bb2246d..b40cd99 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -164,7 +164,7 @@ public class WorkspaceManager { try { return vre.openByRelativePath(APP_FOLDER).asFolder(); }catch(StorageHubException e) { - log.debug("APP Fodler missing. Initializing.."); + log.debug("APP Folder missing. Initializing.."); FolderContainer toReturn= vre.newFolder(APP_FOLDER, "Base folder for GNA records"); toReturn.setHidden(); return toReturn; @@ -180,7 +180,7 @@ public class WorkspaceManager { @Synchronized private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { // Updated by Francesco, see #28150 - log.debug("Uploading file name: {}, in the parent folder id: {}, filesize is: {}", opts.getFileName(), + log.info("Uploading file name: {}, in the parent folder id: {}, filesize is: {}", opts.getFileName(), opts.getParent().getId(), opts.getSize()); opts.setFileName(Files.fixFilename(opts.getFileName())); if (opts.getSize() == null) -- 2.17.1 From 7e992465f244c1729d84e6507e51589090b21ed7 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Tue, 1 Oct 2024 16:04:44 +0200 Subject: [PATCH 03/14] to build SNAPSHOT --- CHANGELOG.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 847a49b..c6f418f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.spatial.data.gcube-sdi-suite -## [v1.0.6] +## [v1.0.6-SNAPSHOT] - Integrated an EventManager centrally [#26321] - sdi-plugins: add the logic to apply a regex to the value that must be added to index [#26322] diff --git a/pom.xml b/pom.xml index ba4dc89..631adab 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.gcube.application.cms gcube-cms-suite pom - 1.0.6 + 1.0.6-SNAPSHOT Gcube CMS Suite gCube CMS Suite is a set of components designed to manage complex space-temporal Documents defined by metadata Profiles. -- 2.17.1 From 814753ca1895ff46871a7b27a754640e90144db9 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 11:36:13 +0200 Subject: [PATCH 04/14] added logs --- .../java/org/gcube/application/cms/sdi/engine/SDIManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java index 6b35b8a..4b9673b 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -52,6 +52,7 @@ public class SDIManager { if(!currentGeoserver.getReader().existGeoserver()) throw new Exception("Geoserver not reachable"); }catch(Exception e) { + log.error("SDIManager init failed: ",e); throw new SDIInteractionException("Unable to initialize SDI Manager",e); } } -- 2.17.1 From 391d0a2c3fd2c0f51ce4143c77686dd98c3e7db3 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 11:50:27 +0200 Subject: [PATCH 05/14] improved logs --- sdi-plugins/CHANGELOG.md | 3 +++ sdi-plugins/pom.xml | 2 +- .../org/gcube/application/cms/sdi/engine/SDIManager.java | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sdi-plugins/CHANGELOG.md b/sdi-plugins/CHANGELOG.md index e581b54..6ce4244 100644 --- a/sdi-plugins/CHANGELOG.md +++ b/sdi-plugins/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog for org.gcube.application.cms.sdi-plugins +## [v1.1.4-SNAPSHOT] +- Improved logs + ## [v1.1.3] - Added apply regex business logic [#26322] - Added reconnection attempts to DB on DB connection failure [#26322] diff --git a/sdi-plugins/pom.xml b/sdi-plugins/pom.xml index 3380356..8811cf9 100644 --- a/sdi-plugins/pom.xml +++ b/sdi-plugins/pom.xml @@ -5,7 +5,7 @@ 4.0.0 sdi-plugins - 1.1.3 + 1.1.4-SNAPSHOT gCube CMS - SDI Plugins diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java index 4b9673b..b579d94 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -48,9 +48,10 @@ public class SDIManager { geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); - dtGeoServer=DataTransferClient.getInstanceByEndpoint("https://"+geoserverHostName); + String gsEP = "https://"+geoserverHostName; + dtGeoServer=DataTransferClient.getInstanceByEndpoint(gsEP); if(!currentGeoserver.getReader().existGeoserver()) - throw new Exception("Geoserver not reachable"); + throw new Exception("Geoserver not reachable at "+gsEP); }catch(Exception e) { log.error("SDIManager init failed: ",e); throw new SDIInteractionException("Unable to initialize SDI Manager",e); -- 2.17.1 From 2f372dd6c6bfc2869d2a2edbbf9538a7bbf99939 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 12:44:37 +0200 Subject: [PATCH 06/14] updated plugin-framework-version at 1.0.6-SNAPSHOT --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 631adab..ca4df20 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 2.5.1 [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - 1.0.5-SNAPSHOT + 1.0.6 [2.0.0, 3.0.0-SNAPSHOT) 1.0.6-SNAPSHOT @@ -41,7 +41,7 @@ 2.5.1 - 1.0.5 + 1.0.6 [2.0.0, 3.0.0-SNAPSHOT) @@ -55,7 +55,7 @@ 2.5.1-SNAPSHOT - 1.0.5-SNAPSHOT + 1.0.6-SNAPSHOT [2.0.0, 3.0.0-SNAPSHOT) -- 2.17.1 From d22e5dceeeb8976f5969a75cfaf40d71d6b32c44 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 12:47:07 +0200 Subject: [PATCH 07/14] updated default properties --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ca4df20..5c825c0 100644 --- a/pom.xml +++ b/pom.xml @@ -22,10 +22,10 @@ distro https://code-repo.d4science.org/gCubeSystem - + 2.5.1 [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - 1.0.6 + 1.0.6-SNAPSHOT [2.0.0, 3.0.0-SNAPSHOT) 1.0.6-SNAPSHOT -- 2.17.1 From b9fa10a69953a0bd1ef407698904bc0548c05906 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 12:53:03 +0200 Subject: [PATCH 08/14] using gcube-smartgears-bom-version 2.5.1{-SNAPSHOT} --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5c825c0..1241f3a 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ https://code-repo.d4science.org/gCubeSystem - 2.5.1 + 2.5.1-SNAPSHOT [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) 1.0.6-SNAPSHOT [2.0.0, 3.0.0-SNAPSHOT) -- 2.17.1 From 390d9892626245df352e2d921037a3130663da7e Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 14:35:09 +0200 Subject: [PATCH 09/14] - Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] --- geoportal-service/CHANGELOG.md | 4 ++++ geoportal-service/pom.xml | 2 +- .../geoportal/service/engine/mongo/ProfiledMongoManager.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/geoportal-service/CHANGELOG.md b/geoportal-service/CHANGELOG.md index 9cb5a83..22d35c2 100644 --- a/geoportal-service/CHANGELOG.md +++ b/geoportal-service/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for org.gcube.application.geoportal-service +## [v1.2.1-SNAPSHOT] - 2024-10-02 + +- Included the file size to reduce/optimize the time to upload files to the storage hub [#28150] + ## [v1.2.0] - 2024-09-24 - Integrated the EventManager of the cms-plugin-framework [#26321] diff --git a/geoportal-service/pom.xml b/geoportal-service/pom.xml index c8c58ef..1ad86b4 100644 --- a/geoportal-service/pom.xml +++ b/geoportal-service/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.gcube.application geoportal-service - 1.2.0 + 1.2.1-SNAPSHOT Geoportal Service war diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index c3945e5..ca05a38 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -1565,7 +1565,7 @@ public class ProfiledMongoManager extends MongoManager implements MongoManagerI< log.info("Got URL {} from ID {}, filesize {}", fileUrl, f.getId(), f.getSize()); is = new URL(fileUrl).openStream(); RegisteredFile registered = ws.registerFile(new WorkspaceManager.FileOptions(f.getFilename(), is, - "Imported via gcube CMS service ", sectionFolder)); + "Imported via gcube CMS service ", sectionFolder, f.getSize())); log.info("Registered " + registered); registeredFiles.add(registered); } catch (StorageHubException | IOException e) { -- 2.17.1 From 91ec8063ebd6569f1b3bb9e94f6aa3ae97f2258c Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 14:50:15 +0200 Subject: [PATCH 10/14] just to build snapshot version --- notifications-plugins/CHANGELOG.md | 2 +- notifications-plugins/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notifications-plugins/CHANGELOG.md b/notifications-plugins/CHANGELOG.md index 2a465d5..d5ac811 100644 --- a/notifications-plugins/CHANGELOG.md +++ b/notifications-plugins/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog for org.gcube.application.cms.notifications-plugins -## [v1.0.5] - 2024-07-03 +## [v1.0.5-SNAPSHOT] - 2024-07-03 - Implemented the notification-plugin [#26453] ## [v1.0.4] - 2023-09-06 diff --git a/notifications-plugins/pom.xml b/notifications-plugins/pom.xml index fd92e15..59b281c 100644 --- a/notifications-plugins/pom.xml +++ b/notifications-plugins/pom.xml @@ -5,7 +5,7 @@ 4.0.0 notifications-plugins - 1.0.5 + 1.0.5-SNAPSHOT gCube CMS - Notifications Plugins -- 2.17.1 From e83c814fb64ff500450344ea19133134a36aa230 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 15:19:04 +0200 Subject: [PATCH 11/14] - Updated lower range of social-service-client to `2.1.0-SNAPSHOT` --- notifications-plugins/CHANGELOG.md | 1 + notifications-plugins/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/notifications-plugins/CHANGELOG.md b/notifications-plugins/CHANGELOG.md index d5ac811..5209f50 100644 --- a/notifications-plugins/CHANGELOG.md +++ b/notifications-plugins/CHANGELOG.md @@ -2,6 +2,7 @@ ## [v1.0.5-SNAPSHOT] - 2024-07-03 - Implemented the notification-plugin [#26453] +- Updated lower range of social-service-client to `2.1.0-SNAPSHOT` ## [v1.0.4] - 2023-09-06 - Using parent version range [#25572] diff --git a/notifications-plugins/pom.xml b/notifications-plugins/pom.xml index 59b281c..5dadb66 100644 --- a/notifications-plugins/pom.xml +++ b/notifications-plugins/pom.xml @@ -58,7 +58,7 @@ org.gcube.social-networking social-service-client - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + [2.1.0-SNAPSHOT, 3.0.0-SNAPSHOT) org.gcube.portlets.user -- 2.17.1 From d4189e0c982a29f9def2ce052825fb14e844d267 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 16:08:48 +0200 Subject: [PATCH 12/14] added some logs --- .../cms/implementations/WorkspaceManager.java | 91 +++++---- .../geoportal/common/rest/Projects.java | 2 +- .../geoportal/common/utils/FileSets.java | 180 +++++++++--------- .../geoportal/common/FilesTests.java | 8 +- .../geoportal/common/JacksonProvider.java | 6 +- .../geoportal/common/StorageUtilsTest.java | 23 ++- .../geoportal/common/TokenSetter.java | 5 +- .../common/legacy/DefaultsTests.java | 5 +- .../geoportal/common/model/JSONPathTests.java | 18 +- 9 files changed, 172 insertions(+), 166 deletions(-) diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index b40cd99..7a6acd3 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -22,54 +22,52 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class WorkspaceManager { - private static final String APP_FOLDER=".GNA_RECORDS"; + private static final String APP_FOLDER = ".GNA_RECORDS"; - - private StorageHubClient sgClient=null; + private StorageHubClient sgClient = null; @Getter - private FolderContainer appBase=null; + private FolderContainer appBase = null; @Getter @Setter @AllArgsConstructor @RequiredArgsConstructor - public static class FolderOptions{ + public static class FolderOptions { @NonNull private String folderName; private String folderDescription; private FolderContainer parent; } - + @Getter @Setter @AllArgsConstructor @RequiredArgsConstructor - public static class FileOptions{ + public static class FileOptions { @NonNull private String fileName; @NonNull private InputStream is; - + private String fileDescription; private FolderContainer parent; - //Added by Francesco, see #28150 + // Added by Francesco, see #28150 private Long size; - } - public Archive getConfiguration(){ + public Archive getConfiguration() { Archive toReturn = new Archive("W-STORAGE"); - toReturn.put("folder_id",appBase.getId()); + toReturn.put("folder_id", appBase.getId()); return toReturn; } public WorkspaceManager() throws ConfigurationException, StorageHubException { - sgClient= ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class); - appBase=getApplicationBaseFolder(sgClient); + sgClient = ImplementationProvider.get().getProvidedObjectByClass(StorageHubClient.class); + appBase = getApplicationBaseFolder(sgClient); } public FolderContainer createFolder(FolderOptions opts) throws StorageHubException { - if(opts.getParent()==null) + if (opts.getParent() == null) opts.setParent(appBase); return createFolderRoutine(opts); } @@ -86,11 +84,10 @@ public class WorkspaceManager { sgClient.open(id).asFolder().delete(); } - public FolderContainer getSubFolder(FolderContainer parentFolder,String path) throws StorageHubException { - return getSubFolder(parentFolder,path,""); + public FolderContainer getSubFolder(FolderContainer parentFolder, String path) throws StorageHubException { + return getSubFolder(parentFolder, path, ""); } - /** * Returns sub folder. Creates it if missing * @@ -99,26 +96,26 @@ public class WorkspaceManager { * @return * @throws StorageHubException */ - public FolderContainer getSubFolder(FolderContainer parentFolder,String path, String description) throws StorageHubException { - try{ + public FolderContainer getSubFolder(FolderContainer parentFolder, String path, String description) + throws StorageHubException { + try { return parentFolder.openByRelativePath(path).asFolder(); - }catch(StorageHubException e) { - log.debug("Missing subPath "+path); - FolderContainer targetParent=parentFolder; - String targetName=path; - if(path.contains("/")) { - String parent=path.substring(0, path.lastIndexOf("/")); - log.debug("Checking intermediate "+parent); - targetParent=getSubFolder(parentFolder,parent); - targetName=path.substring(path.lastIndexOf("/")+1); + } catch (StorageHubException e) { + log.debug("Missing subPath " + path); + FolderContainer targetParent = parentFolder; + String targetName = path; + if (path.contains("/")) { + String parent = path.substring(0, path.lastIndexOf("/")); + log.debug("Checking intermediate " + parent); + targetParent = getSubFolder(parentFolder, parent); + targetName = path.substring(path.lastIndexOf("/") + 1); } - FolderOptions opts = new FolderOptions(targetName,description,targetParent); + FolderOptions opts = new FolderOptions(targetName, description, targetParent); log.debug("Creating FOLDER {}", opts); return createFolder(opts); } } - // public WorkspaceContent storeToWS(FileOptions opts) throws FileNotFoundException, StorageHubException { // FileContainer item=createFileRoutine(opts); // item=sgClient.open(item.getId()).asFile(); @@ -133,12 +130,11 @@ public class WorkspaceManager { // } public RegisteredFile registerFile(FileOptions opts) throws StorageHubException { - FileContainer item=createFileRoutine(opts); - item=sgClient.open(item.getId()).asFile(); + FileContainer item = createFileRoutine(opts); + item = sgClient.open(item.getId()).asFile(); + + RegisteredFile file = new RegisteredFile(); - RegisteredFile file=new RegisteredFile(); - - file.setLink(item.getPublicLink().toString()); file.setMimetype(item.get().getContent().getMimeType()); file.setStorageID(item.getId()); @@ -146,26 +142,25 @@ public class WorkspaceManager { return file; } - - + // public void deleteFromWS(WorkspaceContent toDelete) throws StorageHubException { // sgClient.open(toDelete.getStorageID()).asFile().forceDelete(); // } - public void deleteItem(String itemId)throws StorageHubException{ + public void deleteItem(String itemId) throws StorageHubException { sgClient.open(itemId).asItem().forceDelete(); } - // STATIC SYNCH METHODS - + // STATIC SYNCH METHODS + @Synchronized public static FolderContainer getApplicationBaseFolder(StorageHubClient sgClient) throws StorageHubException { - FolderContainer vre=sgClient.openVREFolder(); + FolderContainer vre = sgClient.openVREFolder(); try { return vre.openByRelativePath(APP_FOLDER).asFolder(); - }catch(StorageHubException e) { + } catch (StorageHubException e) { log.debug("APP Folder missing. Initializing.."); - FolderContainer toReturn= vre.newFolder(APP_FOLDER, "Base folder for GNA records"); + FolderContainer toReturn = vre.newFolder(APP_FOLDER, "Base folder for GNA records"); toReturn.setHidden(); return toReturn; } @@ -174,14 +169,14 @@ public class WorkspaceManager { @Synchronized private static FolderContainer createFolderRoutine(FolderOptions opts) throws StorageHubException { opts.setFolderName(Files.fixFilename(opts.getFolderName())); - return opts.getParent().newFolder(opts.getFolderName(),opts.getFolderDescription()); + return opts.getParent().newFolder(opts.getFolderName(), opts.getFolderDescription()); } - + @Synchronized private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { // Updated by Francesco, see #28150 - log.info("Uploading file name: {}, in the parent folder id: {}, filesize is: {}", opts.getFileName(), - opts.getParent().getId(), opts.getSize()); + log.info("Uploading file name: {}, in the parent folder id: {}, filesize is: " + opts.getSize(), + opts.getFileName(), opts.getParent().getId()); opts.setFileName(Files.fixFilename(opts.getFileName())); if (opts.getSize() == null) return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java index 1a9e207..e408354 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/rest/Projects.java @@ -11,9 +11,9 @@ import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.relationships.RelationshipNavigationObject; import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest; import org.gcube.application.geoportal.common.model.rest.DeleteRelationshipRequest; +import org.gcube.application.geoportal.common.model.rest.PerformStepRequest; import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest; -import org.gcube.application.geoportal.common.model.rest.PerformStepRequest; public interface Projects

{ diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java index a2ee2f0..76619fd 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java @@ -13,111 +13,121 @@ import org.gcube.application.geoportal.common.model.rest.RegisterFileSetRequest; import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class FileSets { - public static class RequestBuilder { - RegisterFileSetRequest theRequest=new RegisterFileSetRequest(); + public static class RequestBuilder { + RegisterFileSetRequest theRequest = new RegisterFileSetRequest(); - public RequestBuilder addAll(Collection toAdd){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - theRequest.getStreams().addAll(toAdd); - return this; - } - public RequestBuilder add(TempFile... f){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - for(TempFile temp: f ) - theRequest.getStreams().add(temp); - return this; - } + public RequestBuilder addAll(Collection toAdd) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + theRequest.getStreams().addAll(toAdd); + return this; + } - public RequestBuilder add(TempFile f){ - if(theRequest.getStreams()==null) - theRequest.setStreams(new ArrayList()); - theRequest.getStreams().add(f); - return this; - } + public RequestBuilder add(TempFile... f) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + for (TempFile temp : f) + theRequest.getStreams().add(temp); + return this; + } - public RequestBuilder setFieldDefinitionPath(String path){ - theRequest.setFieldDefinitionPath(path); - return this; - } + public RequestBuilder add(TempFile f) { + if (theRequest.getStreams() == null) + theRequest.setStreams(new ArrayList()); + theRequest.getStreams().add(f); + return this; + } - public RequestBuilder setParentPath(String path){ - theRequest.setParentPath(path); - return this; - } + public RequestBuilder setFieldDefinitionPath(String path) { + theRequest.setFieldDefinitionPath(path); + return this; + } - public RequestBuilder setFieldName(String fieldName){ - theRequest.setFieldName(fieldName); - return this; - } + public RequestBuilder setParentPath(String path) { + theRequest.setParentPath(path); + return this; + } - public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path){ - theRequest.setClashOption(path); - return this; - } + public RequestBuilder setFieldName(String fieldName) { + theRequest.setFieldName(fieldName); + return this; + } - public RequestBuilder setAttributes(Document attributes){ - theRequest.setAttributes(attributes); - return this; - } + public RequestBuilder setClashPolicy(RegisterFileSetRequest.ClashOptions path) { + theRequest.setClashOption(path); + return this; + } - public RegisterFileSetRequest getTheRequest(){ - if(theRequest.getClashOption()==null) - // default Clash Policy - setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING); - return theRequest;} + public RequestBuilder setAttributes(Document attributes) { + theRequest.setAttributes(attributes); + return this; + } - public RequestBuilder setAccess(Access access){ - theRequest.setToSetAccess(access); - return this; - } + public RegisterFileSetRequest getTheRequest() { + if (theRequest.getClashOption() == null) + // default Clash Policy + setClashPolicy(RegisterFileSetRequest.ClashOptions.REPLACE_EXISTING); + return theRequest; + } - } + public RequestBuilder setAccess(Access access) { + theRequest.setToSetAccess(access); + return this; + } + } - public static RequestBuilder build(String parent,String fieldName, String fieldDefinition) { - return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition).setFieldName(fieldName); - } + public static RequestBuilder build(String parent, String fieldName, String fieldDefinition) { + return new RequestBuilder().setParentPath(parent).setFieldDefinitionPath(fieldDefinition) + .setFieldName(fieldName); + } - public static RequestBuilder build(String parent,String fieldName, String fieldDefinition, TempFile...files) { - return build(parent,fieldName,fieldDefinition).add(files); - } + public static RequestBuilder build(String parent, String fieldName, String fieldDefinition, TempFile... files) { + return build(parent, fieldName, fieldDefinition).add(files); + } - public static TempFile asTemp(StorageUtils storage,InputStreamDescriptor descriptor) throws RemoteBackendException, FileNotFoundException { - return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename()); - } + public static TempFile asTemp(StorageUtils storage, InputStreamDescriptor descriptor) + throws RemoteBackendException, FileNotFoundException { + return storage.putOntoStorage(descriptor.getStream(), descriptor.getFilename()); + } - public static TempFile[] asTemp(StorageUtils storage,InputStreamDescriptor... descriptors) throws RemoteBackendException, FileNotFoundException { - ArrayList toReturn=new ArrayList(); - for(InputStreamDescriptor desc:descriptors) - toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename())); - return toReturn.toArray(new TempFile[toReturn.size()]); - } + public static TempFile[] asTemp(StorageUtils storage, InputStreamDescriptor... descriptors) + throws RemoteBackendException, FileNotFoundException { + ArrayList toReturn = new ArrayList(); + for (InputStreamDescriptor desc : descriptors) + toReturn.add(storage.putOntoStorage(desc.getStream(), desc.getFilename())); + return toReturn.toArray(new TempFile[toReturn.size()]); + } - public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage, - String parentPath,String fieldName,String fieldDefinition, File directory) throws FileNotFoundException { + public static RegisterFileSetRequest prepareRequestFromFolder(StorageUtils storage, String parentPath, + String fieldName, String fieldDefinition, File directory) throws FileNotFoundException { - File[] children =directory.listFiles(); - InputStreamDescriptor[] iss=new InputStreamDescriptor[children.length]; - return prepareRequest(storage,parentPath,fieldName,fieldDefinition,children); + File[] children = directory.listFiles(); + InputStreamDescriptor[] iss = new InputStreamDescriptor[children.length]; + return prepareRequest(storage, parentPath, fieldName, fieldDefinition, children); - } + } - public static RegisterFileSetRequest prepareRequest(StorageUtils storage, - String parentPath,String fieldName,String fieldDefinition, File... toUpload) throws FileNotFoundException { + public static RegisterFileSetRequest prepareRequest(StorageUtils storage, String parentPath, String fieldName, + String fieldDefinition, File... toUpload) throws FileNotFoundException { - FileSets.RequestBuilder builder = FileSets.build(parentPath,fieldName,fieldDefinition); - for (File f : toUpload) { - if(!f.isDirectory()) { - TempFile file = FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName())); - //Added by Francesco, see #28150 - file.setSize(f.length()); - builder.add(file); - } - } - return builder.getTheRequest(); - } + FileSets.RequestBuilder builder = FileSets.build(parentPath, fieldName, fieldDefinition); + for (File f : toUpload) { + if (!f.isDirectory()) { + long fileSize = f.length(); + TempFile file = FileSets.asTemp(storage, + new InputStreamDescriptor(new FileInputStream(f), f.getName())); + // Added by Francesco, see #28150 + log.info("PrepareRequest for tempfile name " + file.getFilename() + " size: " + file.getSize()); + file.setSize(fileSize); + builder.add(file); + } + } + return builder.getTheRequest(); + } } diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java index f511c2f..0b7d68a 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java @@ -1,8 +1,6 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; -import org.gcube.application.geoportal.common.utils.Files; -import org.junit.Test; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -11,7 +9,9 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertTrue; +import org.gcube.application.geoportal.common.utils.Files; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class FilesTests { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java index f75d6bb..a3006b2 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/JacksonProvider.java @@ -1,5 +1,8 @@ package org.gcube.application.geoportal.common; +import java.util.EnumSet; +import java.util.Set; + import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.spi.json.JacksonJsonProvider; @@ -7,9 +10,6 @@ import com.jayway.jsonpath.spi.json.JsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import com.jayway.jsonpath.spi.mapper.MappingProvider; -import java.util.EnumSet; -import java.util.Set; - public class JacksonProvider implements JSONSerializationProvider { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java index 90603e2..ecd61bb 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java @@ -1,16 +1,6 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; -import org.gcube.application.geoportal.common.rest.InterfaceConstants; -import org.gcube.application.geoportal.common.utils.ContextUtils; -import org.gcube.application.geoportal.common.utils.StorageUtils; -import org.gcube.application.geoportal.common.utils.tests.GCubeTest; -import org.gcube.contentmanagement.blobstorage.service.IClient; -import org.gcube.contentmanager.storageclient.wrapper.AccessType; -import org.gcube.contentmanager.storageclient.wrapper.MemoryType; -import org.gcube.contentmanager.storageclient.wrapper.StorageClient; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assume.assumeTrue; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -22,7 +12,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static org.junit.Assume.assumeTrue; +import org.gcube.application.geoportal.common.rest.InterfaceConstants; +import org.gcube.application.geoportal.common.utils.ContextUtils; +import org.gcube.application.geoportal.common.utils.StorageUtils; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class StorageUtilsTest { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java index b8d27bd..0c0de3b 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java @@ -1,10 +1,11 @@ package org.gcube.application.geoportal.common; -import lombok.extern.slf4j.Slf4j; +import java.util.Properties; + import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; -import java.util.Properties; +import lombok.extern.slf4j.Slf4j; @Slf4j public class TokenSetter { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java index 7b81549..1bd7ab0 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/legacy/DefaultsTests.java @@ -1,13 +1,12 @@ package org.gcube.application.geoportal.common.legacy; +import static org.junit.Assert.assertEquals; + import org.gcube.application.geoportal.common.model.legacy.AccessPolicy; import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.legacy.RelazioneScavo; import org.gcube.application.geoportal.common.model.legacy.report.ConstraintCheck; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; public class DefaultsTests { diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java index cdbfad0..24c162e 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/model/JSONPathTests.java @@ -1,12 +1,8 @@ package org.gcube.application.geoportal.common.model; -import com.jayway.jsonpath.JsonPath; -import org.gcube.application.geoportal.common.model.document.Project; -import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; -import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents; -import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; -import org.gcube.application.geoportal.common.utils.Files; -import org.junit.Test; +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; import java.io.File; import java.io.IOException; @@ -15,7 +11,13 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import static junit.framework.TestCase.*; +import org.gcube.application.geoportal.common.model.document.Project; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; +import org.gcube.application.geoportal.common.model.document.lifecycle.TriggeredEvents; +import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; +import org.gcube.application.geoportal.common.utils.Files; + +import com.jayway.jsonpath.JsonPath; public class JSONPathTests { -- 2.17.1 From b7cd1f8e2096f52e70a4435781fb73bbbbe5c2ca Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Wed, 2 Oct 2024 17:11:13 +0200 Subject: [PATCH 13/14] moved log at debug --- .../org/gcube/application/geoportal/common/utils/FileSets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java index 76619fd..ddbb68a 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/FileSets.java @@ -123,7 +123,7 @@ public class FileSets { TempFile file = FileSets.asTemp(storage, new InputStreamDescriptor(new FileInputStream(f), f.getName())); // Added by Francesco, see #28150 - log.info("PrepareRequest for tempfile name " + file.getFilename() + " size: " + file.getSize()); + log.debug("PrepareRequest for tempfile name " + file.getFilename() + " size: " + file.getSize()); file.setSize(fileSize); builder.add(file); } -- 2.17.1 From 5a15af00366e64268728e875728adaa60ad759fd Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Fri, 4 Oct 2024 11:25:07 +0200 Subject: [PATCH 14/14] Added fallback on /geoserver/rest path [#28150#note-8] --- sdi-plugins/CHANGELOG.md | 1 + .../cms/sdi/engine/SDIManager.java | 114 +++++++++--------- 2 files changed, 61 insertions(+), 54 deletions(-) diff --git a/sdi-plugins/CHANGELOG.md b/sdi-plugins/CHANGELOG.md index 6ce4244..27bdb59 100644 --- a/sdi-plugins/CHANGELOG.md +++ b/sdi-plugins/CHANGELOG.md @@ -2,6 +2,7 @@ ## [v1.1.4-SNAPSHOT] - Improved logs +- Added fallback on /geoserver/rest path [#28150#note-8] ## [v1.1.3] - Added apply regex business logic [#26322] diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java index b579d94..7cc0778 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -1,6 +1,5 @@ package org.gcube.application.cms.sdi.engine; - import java.io.File; import java.net.MalformedURLException; import java.net.URL; @@ -13,6 +12,7 @@ import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.spatial.data.gis.GISInterface; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; +import it.geosolutions.geoserver.rest.HTTPUtils; import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -20,12 +20,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class SDIManager { - - - public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); - public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); - public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); - + public static final Pattern HOSTNAME_PATTERN = Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); + public static final Pattern PORT_PATTERN = Pattern.compile("(?<=\\:)[\\d]+"); + public static final Pattern DB_NAME_PATTERN = Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); private final GISInterface gis; @Getter @@ -35,46 +32,52 @@ public class SDIManager { @Getter private final AbstractGeoServerDescriptor currentGeoserver; + private final String FALLBACK_GS_REST_INTERFACE = "/rest"; public SDIManager() throws SDIInteractionException { - try{ + try { log.debug("Initializing GIS Interface.."); - gis=GISInterface.get(); - currentGeoserver=gis.getCurrentGeoServer(); - if(currentGeoserver==null) + gis = GISInterface.get(); + currentGeoserver = gis.getCurrentGeoServer(); + if (currentGeoserver == null) throw new Exception("Unable to contact data transfer for geoserver "); - log.debug("Found geoserver descriptor "+currentGeoserver); - geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); + log.debug("Found geoserver descriptor " + currentGeoserver); + geoserverHostName = new URL(currentGeoserver.getUrl()).getHost(); - log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); - String gsEP = "https://"+geoserverHostName; - dtGeoServer=DataTransferClient.getInstanceByEndpoint(gsEP); - if(!currentGeoserver.getReader().existGeoserver()) - throw new Exception("Geoserver not reachable at "+gsEP); - }catch(Exception e) { - log.error("SDIManager init failed: ",e); - throw new SDIInteractionException("Unable to initialize SDI Manager",e); + log.debug("Contacting Data Transfer from geoserver {} ", geoserverHostName); + String gsEP = "https://" + geoserverHostName; + dtGeoServer = DataTransferClient.getInstanceByEndpoint(gsEP); + if (!currentGeoserver.getReader().existGeoserver()) { + // not a good but necessary solution here, see #28150#note-8 + String fallbackGSRestURL = currentGeoserver.getUrl() + FALLBACK_GS_REST_INTERFACE; + log.warn("Geoserver rest interface not reachable at /rest/. Going to check " + + fallbackGSRestURL); + if (!HTTPUtils.httpPing(fallbackGSRestURL, currentGeoserver.getUser(), + currentGeoserver.getPassword())) { + throw new Exception("Geoserver rest interface not reachable at " + fallbackGSRestURL); + } + } + } catch (Exception e) { + log.error("SDIManager init failed: ", e); + throw new SDIInteractionException("Unable to initialize SDI Manager", e); } } - - - public String createWorkspace(String toCreate) throws SDIInteractionException { try { - if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { - log.debug("Creating workspace : "+toCreate); - if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) - throw new SDIInteractionException("Unable to create workspace "+toCreate); - }else log.debug("Workspace "+toCreate+" exists."); + if (!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { + log.debug("Creating workspace : " + toCreate); + if (!currentGeoserver.getPublisher().createWorkspace(toCreate)) + throw new SDIInteractionException("Unable to create workspace " + toCreate); + } else + log.debug("Workspace " + toCreate + " exists."); return toCreate; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create workspace "+toCreate,e); + throw new SDIInteractionException("Unable to create workspace " + toCreate, e); } } - // protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){ // // ******** IDENTIFY LAYER NAME correct layer name // // Must be unique under same WS @@ -96,36 +99,38 @@ public class SDIManager { // } protected String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { - String storeName=encoder.getName(); + String storeName = encoder.getName(); try { - log.debug("Looking for datastore "+storeName+" under "+workspace); + log.debug("Looking for datastore " + storeName + " under " + workspace); - if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) + if (currentGeoserver.getReader().getDatastore(workspace, storeName) == null) - if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) - throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); - log.debug("Store "+storeName+" exists under "+workspace); + if (!currentGeoserver.getDataStoreManager().create(workspace, encoder)) + throw new SDIInteractionException("Unable to create store " + storeName + " in " + workspace); + log.debug("Store " + storeName + " exists under " + workspace); return storeName; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create store "+storeName,e); + throw new SDIInteractionException("Unable to create store " + storeName, e); } } + protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) + throws SDIInteractionException { + String connectionUrl = connection.getUrl(); - protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ - String connectionUrl=connection.getUrl(); - - Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); - if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); + Matcher hostname = HOSTNAME_PATTERN.matcher(connectionUrl); + if (!hostname.find()) + throw new SDIInteractionException("Unable to get Hostname from " + connection); Matcher port = PORT_PATTERN.matcher(connectionUrl); - if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); + if (!port.find()) + throw new SDIInteractionException("Unable to get PORT from " + connection); Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); - if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); + if (!db.find()) + throw new SDIInteractionException("Unable to get DB from " + connection); - - GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); + GSPostGISDatastoreEncoder encoder = new GSPostGISDatastoreEncoder(storeName); encoder.setHost(hostname.group()); encoder.setPort(Integer.parseInt(port.group())); encoder.setDatabase(db.group()); @@ -138,19 +143,20 @@ public class SDIManager { encoder.setFetchSize(1000); encoder.setValidateConnections(true); - return createStore(encoder,workspace); + return createStore(encoder, workspace); } protected String publishStyle(File sldFile, String name) throws SDIInteractionException { try { - if(!currentGeoserver.getReader().existsStyle(name)) { - log.debug("Registering style "+name); - if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) - throw new SDIInteractionException("Unable to register style "+name); - }else log.debug("Style "+name+" already existing"); + if (!currentGeoserver.getReader().existsStyle(name)) { + log.debug("Registering style " + name); + if (!currentGeoserver.getPublisher().publishStyle(sldFile, name)) + throw new SDIInteractionException("Unable to register style " + name); + } else + log.debug("Style " + name + " already existing"); return name; } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create style "+name,e); + throw new SDIInteractionException("Unable to create style " + name, e); } } -- 2.17.1