diff --git a/cms-plugin-framework/pom.xml b/cms-plugin-framework/pom.xml index 4adfaaf..bf3b4e2 100644 --- a/cms-plugin-framework/pom.xml +++ b/cms-plugin-framework/pom.xml @@ -80,6 +80,12 @@ jackson-datatype-jsr310 + + + org.gcube.common + storagehub-client-library + + diff --git a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/ImplementationProvider.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/ImplementationProvider.java index 33b17be..d3fa662 100644 --- a/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/ImplementationProvider.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/ImplementationProvider.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.gcube.application.cms.caches.Cache; import org.gcube.application.cms.caches.Engine; import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; import java.util.HashMap; import java.util.HashSet; @@ -43,6 +44,7 @@ public class ImplementationProvider { private ImplementationProvider(){ //Defaults setEngine(new DefaultISProvider(),ISInterface.class); + setEngine(new StorageHubProvider(), StorageHubClient.class); } public Map getManagerList(){ HashMap toReturn=new HashMap<>(); diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/StorageHubProvider.java similarity index 87% rename from geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java rename to cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/StorageHubProvider.java index 8d21ec4..ac32917 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/StorageHubProvider.java @@ -1,4 +1,4 @@ -package org.gcube.application.geoportal.service.engine.providers; +package org.gcube.application.cms.implementations; import org.gcube.application.cms.caches.Engine; import org.gcube.application.geoportal.common.model.rest.ConfigurationException; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java similarity index 97% rename from geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java rename to cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java index 0e057e8..571494d 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java +++ b/cms-plugin-framework/src/main/java/org/gcube/application/cms/implementations/WorkspaceManager.java @@ -1,4 +1,4 @@ -package org.gcube.application.geoportal.service.engine; +package org.gcube.application.cms.implementations; import lombok.*; import lombok.extern.slf4j.Slf4j; @@ -13,7 +13,6 @@ import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; -import javax.validation.constraints.NotNull; import java.io.FileNotFoundException; import java.io.InputStream; @@ -32,7 +31,7 @@ public class WorkspaceManager { @AllArgsConstructor @RequiredArgsConstructor public static class FolderOptions{ - @NotNull + @NonNull private String folderName; private String folderDescription; private FolderContainer parent; @@ -43,7 +42,7 @@ public class WorkspaceManager { @AllArgsConstructor @RequiredArgsConstructor public static class FileOptions{ - @NotNull + @NonNull private String fileName; @NonNull private InputStream is; diff --git a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicPluginTest.java b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicPluginTest.java index 925d504..544ef46 100644 --- a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicPluginTest.java +++ b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicPluginTest.java @@ -1,10 +1,15 @@ package org.gcube.application.cms.tests.plugins; import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.caches.Engine; +import org.gcube.application.cms.implementations.ImplementationProvider; import org.gcube.application.cms.plugins.Plugin; import org.gcube.application.cms.plugins.PluginsReflections; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.cms.tests.model.BasicTests; import org.gcube.application.geoportal.common.model.document.accounting.Context; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import org.junit.Assert; import org.junit.BeforeClass; @@ -37,6 +42,23 @@ public abstract class BasicPluginTest extends BasicTests { plugins.forEach(PluginCheks.descriptor); plugins.forEach(PluginCheks.init); System.out.println("Plugin Loading OK"); + + + ImplementationProvider.get().setEngine(new Engine() { + @Override + public void init() {} + + @Override + public void shutdown() {} + + @Override + public StorageUtils getObject() throws ConfigurationException { + TokenSetter.set(GCubeTest.getContext()); + return new StorageUtils(); + } + + },StorageUtils.class); + } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GCubeSDILayer.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GCubeSDILayer.java index cfe40f2..7a192d9 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GCubeSDILayer.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GCubeSDILayer.java @@ -64,9 +64,9 @@ public class GCubeSDILayer extends Materialization { return toReturn; } - public static final BBOX WORLD=new BBOX(180d,90d,-180d,-90d); + public static final BBOX WORLD=new BBOX(180d,-180d,90d,-90d); - public static final BBOX WORLD_3D=new BBOX(180d,90d,-180d,-90d); + public static final BBOX WORLD_3D=new BBOX(180d,-180d,90d,-90d); public static final String MAX_X="_maxX"; public static final String MAX_Y="_maxY"; diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java index e0d016f..98e4fb9 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java @@ -1,11 +1,14 @@ package org.gcube.application.geoportal.common.utils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.nio.charset.Charset; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -98,4 +101,15 @@ public class Files { return toFix.toLowerCase(). replaceAll("[\\-\\*\\+\\/\\\\ \\[\\]\\(\\)\\.\\\"\\:\\;\\|\\=]","_")+extension; } + + public static final File downloadFromUrl(String name,String url) throws IOException { + Path dest =java.nio.file.Files.createTempDirectory("downloads_").resolve(name); + InputStream is = null; + try { + java.nio.file.Files.copy(new URL(url).openStream(), dest); + return dest.toFile(); + }finally { + if(is!=null) IOUtils.closeQuietly(is); + } + } } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java index bc0c203..9424ada 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java @@ -1,11 +1,11 @@ package org.gcube.application.geoportal.common.utils; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.UUID; +import org.gcube.application.geoportal.common.faults.StorageException; import org.gcube.application.geoportal.common.rest.InterfaceConstants; import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.contentmanagement.blobstorage.service.IClient; @@ -74,4 +74,13 @@ public class StorageUtils { public static final String getUniqueString(){ return UUID.randomUUID().toString(); } + + public File download(String id,String name) throws IOException, StorageException { + Path p = Files.createTempDirectory(id).resolve(name); + + client.get().LFile(p.toAbsolutePath().toString()).RFile(id); + File toReturn =p.toFile(); + if(!toReturn.exists()) throw new StorageException("Unable to download "+id); + return p.toFile(); + } } diff --git a/geoportal-service/pom.xml b/geoportal-service/pom.xml index 4e0f4f3..079004c 100644 --- a/geoportal-service/pom.xml +++ b/geoportal-service/pom.xml @@ -84,11 +84,6 @@ storage-manager-wrapper - - - org.gcube.common - storagehub-client-library - diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java index 1338ec3..9919c1a 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/GeoPortalService.java @@ -4,14 +4,13 @@ import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import lombok.extern.slf4j.Slf4j; import org.gcube.application.cms.caches.Engine; import org.gcube.application.cms.implementations.ImplementationProvider; +import org.gcube.application.cms.implementations.StorageHubProvider; import org.gcube.application.geoportal.common.rest.InterfaceConstants; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.engine.mongo.UCDManagerI; -import org.gcube.application.geoportal.service.engine.postgis.PostgisDBManager; import org.gcube.application.geoportal.service.engine.postgis.PostgisDBManagerI; import org.gcube.application.geoportal.service.engine.providers.*; import org.gcube.application.geoportal.service.engine.providers.ucd.ProfileMap; -import org.gcube.application.geoportal.service.engine.providers.ucd.LocalFolderProfileMapCache; import org.gcube.application.geoportal.service.engine.providers.ucd.SingleISResourceUCDProvider; import org.gcube.application.geoportal.service.engine.providers.ucd.UCDManager; import org.gcube.application.geoportal.service.model.internal.db.Mongo; @@ -20,7 +19,6 @@ import org.gcube.application.geoportal.service.rest.*; import org.gcube.application.cms.serialization.Serialization; import org.gcube.common.storagehub.client.dsl.StorageHubClient; -import org.geotoolkit.internal.sql.table.Entry; import org.glassfish.jersey.server.ResourceConfig; import javax.ws.rs.ApplicationPath; @@ -52,7 +50,6 @@ public class GeoPortalService extends ResourceConfig{ ImplementationProvider.get().setEngine(new PostgisConnectionProvider(), PostgisDBManagerI.class); ImplementationProvider.get().setEngine(new SingleISResourceUCDProvider(), ProfileMap.class); ImplementationProvider.get().setEngine(new PluginManager(), PluginManager.PluginMap.class); - ImplementationProvider.get().setEngine(new StorageHubProvider(), StorageHubClient.class); ImplementationProvider.get().setEngine(new UCDManager(),UCDManagerI.class); ImplementationProvider.get().setEngine(new ConfigurationCache(), ConfigurationCache.ConfigurationMap.class); diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java index 1691198..7f2a6aa 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java @@ -13,7 +13,7 @@ import lombok.extern.slf4j.Slf4j; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.utils.Files; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.application.geoportal.service.model.internal.db.PostgisTable; import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; import org.gcube.common.storagehub.client.dsl.FileContainer; @@ -29,7 +29,6 @@ import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; import java.io.File; import java.net.MalformedURLException; import java.net.URL; -import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java index fa8f72e..0f05931 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java @@ -17,9 +17,9 @@ import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.cms.implementations.ImplementationProvider; import org.gcube.application.geoportal.service.engine.materialization.SDIManager; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; -import org.gcube.application.geoportal.service.engine.WorkspaceManager.FileOptions; -import org.gcube.application.geoportal.service.engine.WorkspaceManager.FolderOptions; +import org.gcube.application.cms.implementations.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager.FileOptions; +import org.gcube.application.cms.implementations.WorkspaceManager.FolderOptions; import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; import org.gcube.application.geoportal.service.model.internal.faults.*; import org.gcube.application.cms.serialization.Serialization; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java index 7ac71ee..c35b353 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ProfiledMongoManager.java @@ -47,10 +47,9 @@ import org.gcube.application.geoportal.common.model.JSONPathWrapper; import org.gcube.application.geoportal.common.utils.ContextUtils; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.cms.implementations.ImplementationProvider; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.application.geoportal.common.model.rest.ConfigurationException; import org.gcube.application.geoportal.service.engine.providers.PluginManager; -import org.gcube.application.geoportal.service.engine.providers.ucd.ProfileMap; import org.gcube.application.geoportal.service.model.internal.faults.*; import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.geoportal.service.utils.UserUtils; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ucd/SingleISResourceUCDProvider.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ucd/SingleISResourceUCDProvider.java index 24fc3e6..0bff6ce 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ucd/SingleISResourceUCDProvider.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/ucd/SingleISResourceUCDProvider.java @@ -10,7 +10,7 @@ import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; import org.gcube.application.geoportal.common.model.rest.ConfigurationException; import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.storagehub.client.dsl.FolderContainer; @@ -24,7 +24,6 @@ import java.io.StringReader; import java.io.StringWriter; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; import java.time.Duration; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java index 662b450..cf95775 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java @@ -1,7 +1,6 @@ package org.gcube.application.geoportal.service.ws; import org.gcube.application.cms.tests.TokenSetter; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/MoveWSFolder.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/MoveWSFolder.java index 3e582af..cb3ad29 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/MoveWSFolder.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/MoveWSFolder.java @@ -1,7 +1,7 @@ package org.gcube.application.geoportal.service.ws; import org.gcube.application.cms.tests.TokenSetter; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java index c6a7682..50e561d 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java @@ -1,7 +1,7 @@ package org.gcube.application.geoportal.service.ws; import org.gcube.application.cms.tests.TokenSetter; -import org.gcube.application.geoportal.service.engine.WorkspaceManager; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java index 24fa9e5..466525c 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -76,25 +76,25 @@ public class SDIManager { } - protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){ - // ******** IDENTIFY LAYER NAME correct layer name - // Must be unique under same WS - // equal to shp base name - - String baseName= ""; - - // Chose layer name first identifying filename - for(Object p:fileset.getPayloads()){ - RegisteredFile f = Serialization.convert(p,RegisteredFile.class); - String name=f.getName(); - if(name.endsWith(".shp")) { - log.debug("SHP is {}",name); - baseName= Files.fixFilename(name.substring(0,name.lastIndexOf('.'))); - break; - } - } - return baseName; - } +// protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){ +// // ******** IDENTIFY LAYER NAME correct layer name +// // Must be unique under same WS +// // equal to shp base name +// +// String baseName= ""; +// +// // Chose layer name first identifying filename +// for(Object p:fileset.getPayloads()){ +// RegisteredFile f = Serialization.convert(p,RegisteredFile.class); +// String name=f.getName(); +// if(name.endsWith(".shp")||name.endsWith(".tif")) { +// log.debug("File for layer is {}",name); +// baseName= Files.fixFilename(name.substring(0,name.lastIndexOf('.'))); +// break; +// } +// } +// return baseName; +// } protected String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { String storeName=encoder.getName(); diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java index 06a3466..bfbde05 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java @@ -5,12 +5,16 @@ import it.geosolutions.geoserver.rest.GeoServerRESTReader; import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; import it.geosolutions.geoserver.rest.decoder.RESTLayer; import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; +import it.geosolutions.geoserver.rest.encoder.coverage.GSCoverageEncoder; import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; import lombok.extern.slf4j.Slf4j; import org.bson.Document; +import org.gcube.application.cms.implementations.ImplementationProvider; +import org.gcube.application.cms.implementations.WorkspaceManager; import org.gcube.application.cms.plugins.requests.BaseExecutionRequest; import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder; +import org.gcube.application.cms.sdi.model.SupportedFormat; import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer; import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; @@ -19,16 +23,21 @@ import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoSer import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.common.utils.StorageUtils; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.faults.*; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; +import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.List; +import static it.geosolutions.geoserver.rest.encoder.GSResourceEncoder.ProjectionPolicy.REPROJECT_TO_DECLARED; import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.EPSG_4326; import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.WGS84_FULL; @@ -70,11 +79,37 @@ public class SDIManagerWrapper extends SDIManager{ layerBuilder.setHost(getGeoserverHostName()); + + // Evaluate layer data in filesets + // TODO optimize cicles + List toCheckFormats=SupportedFormat.getByExtension(".tif",".shp"); + SupportedFormat selectedFormat = null; + String baseName=null; + for(SupportedFormat format: toCheckFormats){ + log.debug("Checking for {}",format); + for (Object o : fileSet.getPayloads()) { + RegisteredFile file = Serialization.convert(o, RegisteredFile.class); + format.consider(file); + } + if(format.getIsProposedFilesetValid()){ + log.debug("Selected format is {}. Fileset is {}",format,format.getToUseFileSet()); + selectedFormat = format; + // get basename for layer + baseName = format.getToUseFileSet().get(0).getName(); + baseName= Files.fixFilename(baseName.substring(0,baseName.lastIndexOf('.'))); + break; + } + } + + if(selectedFormat==null) + throw new SDIInteractionException("Unable to identify layer format. Configured formats are "+toCheckFormats); + + // Evaluate Layer Name - String baseName = getToUseBaseLayerName(fileSet); log.debug("Base layer name is {}, checking conflicts.. ",baseName); String toSetLayerName = baseName; - //Check if layer already exists + + //Checking if layer already exists int count = 0; GeoServerRESTReader gsReader = getCurrentGeoserver().getReader(); while (gsReader.getLayer(workspace, toSetLayerName) != null) { @@ -86,41 +121,7 @@ public class SDIManagerWrapper extends SDIManager{ layerBuilder.setLayerName(toSetLayerName); - String folderRelativePath = basePersistencePAth + "/" + documentID + "/" + fileSet.getUUID() + "/" + toSetLayerName; - log.debug("GS Relative destination path is {}", folderRelativePath); - layerBuilder.setPersistencePath(folderRelativePath); - List filenames = new ArrayList<>(); - - String absolutePath = null; - - for (Object o : fileSet.getPayloads()) { - RegisteredFile file = Serialization.convert(o, RegisteredFile.class); - log.info("Sending {} to GS {} at {} ", file, getGeoserverHostName(), folderRelativePath); - String completeFilename = Files.fixFilename(file.getName()); - completeFilename = completeFilename.replaceAll(baseName, toSetLayerName); - - - Destination destination = new Destination(completeFilename); - destination.setCreateSubfolders(true); - destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); - destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); - - destination.setPersistenceId("geoserver"); - destination.setSubFolder(folderRelativePath); - - log.debug("Sending {} to {}", file, destination); - TransferResult result = getDtGeoServer().httpSource(new URL(file.getLink()), destination); - log.debug("Transferred " + result); - - - filenames.add(completeFilename); - // NB Clash con subfolder is APPEND, thus FOLDER is expected to be the one specified by caller - //geoserverInfo.put(""result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); - absolutePath = result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/")); - } - - layerBuilder.setFiles(filenames); // Publishing layer in GS String storeName = toSetLayerName + "_store"; @@ -131,31 +132,51 @@ public class SDIManagerWrapper extends SDIManager{ createWorkspace(workspace); - log.debug("Publishing remote folder " + absolutePath); - // - URL directoryPath = new URL("file:" + absolutePath + "/" + toSetLayerName + ".tif"); + // Actually publishing files into GS - //TODO Evaluate SRS + Boolean published = null; - boolean published = publisher.publishShp( - workspace, - storeName, - null, - toSetLayerName, - // UploadMethod.FILE, // neeeds zip - GeoServerRESTPublisher.UploadMethod.EXTERNAL, // needs shp - directoryPath.toURI(), - EPSG_4326, //SRS - ""); // default style + switch(selectedFormat.getFileExtension()){ + case ".tif" : { + RegisteredFile f = selectedFormat.getToUseFileSet().get(0); + File temp= null; + try { + temp = Files.downloadFromUrl(f.getName(),f.getLink()); + published = publisher.publishGeoTIFF(workspace, storeName, toSetLayerName,temp,EPSG_4326,REPROJECT_TO_DECLARED,"raster"); + }finally{ if (temp!=null) java.nio.file.Files.deleteIfExists(temp.toPath());} + break; + } + case ".shp" :{ + //TODO Evaluate SRS + String absolutePath = transferFilesetToGS(basePersistencePAth,documentID,selectedFormat.getToUseFileSet(), + fileSet.getUUID(),toSetLayerName,baseName,layerBuilder); + URL directoryPath = new URL("file:" + absolutePath + "/" + toSetLayerName + ".shp"); + published = publisher.publishShp( + workspace, + storeName, + null, + toSetLayerName, + // UploadMethod.FILE, // neeeds zip + GeoServerRESTPublisher.UploadMethod.EXTERNAL, // needs shp + directoryPath.toURI(), + EPSG_4326, //SRS + ""); + break; + } + default: { + throw new SDIInteractionException("Unsupported data format"); + } + } + // check if success if (!published) { throw new SDIInteractionException("Unable to publish layer " + toSetLayerName + " under " + workspace + ". Unknown Geoserver fault."); } - RESTLayer l = gsReader.getLayer(workspace, toSetLayerName); - RESTFeatureType f = gsReader.getFeatureType(l); +// RESTLayer l = gsReader.getLayer(workspace, toSetLayerName); +// RESTFeatureType f = gsReader.getFeatureType(l); GCubeSDILayer materialization = layerBuilder.getLayer(); @@ -175,6 +196,46 @@ public class SDIManagerWrapper extends SDIManager{ } } + private String transferFilesetToGS(String basePersistencePAth,String documentID,List fileSet, + String fileSetUUID, + String toSetLayerName, String baseName, GCubeSDILayerBuilder layerBuilder) + throws MalformedURLException, InvalidSourceException, SourceNotSetException, InvalidDestinationException, + DestinationNotSetException, InitializationException, FailedTransferException { + String folderRelativePath = basePersistencePAth + "/" + documentID + "/" + fileSetUUID + "/" + toSetLayerName; + log.debug("GS Relative destination path is {}", folderRelativePath); + layerBuilder.setPersistencePath(folderRelativePath); + + List filenames = new ArrayList<>(); + + String absolutePath = null; + + for (RegisteredFile file : fileSet) { + log.info("Sending {} to GS {} at {} ", file, getGeoserverHostName(), folderRelativePath); + String completeFilename = Files.fixFilename(file.getName()); + completeFilename = completeFilename.replaceAll(baseName, toSetLayerName); + + + Destination destination = new Destination(completeFilename); + destination.setCreateSubfolders(true); + destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + + destination.setPersistenceId("geoserver"); + destination.setSubFolder(folderRelativePath); + + log.debug("Sending {} to {}", file, destination); + TransferResult result = getDtGeoServer().httpSource(new URL(file.getLink()), destination); + log.debug("Transferred " + result); + + + filenames.add(completeFilename); + // NB Clash con subfolder is APPEND, thus FOLDER is expected to be the one specified by caller + //geoserverInfo.put(""result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); + absolutePath = result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/")); + } + layerBuilder.setFiles(filenames); + return absolutePath; + } public void deleteLayer(GCubeSDILayer toDelete) throws SDIInteractionException { log.trace("Deleting {}",toDelete); diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/SupportedFormat.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/SupportedFormat.java new file mode 100644 index 0000000..9ae1324 --- /dev/null +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/SupportedFormat.java @@ -0,0 +1,62 @@ +package org.gcube.application.cms.sdi.model; + +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; + +import java.util.ArrayList; +@Getter +@RequiredArgsConstructor +public abstract class SupportedFormat { + + public static ArrayList getByExtension(String... extensions){ + ArrayList toReturn = new ArrayList<>(); + + for (String extension : extensions) { + extension=extension.toLowerCase(); + switch (extension){ + case ".tif" : {toReturn.add(new SupportedFormat(".tif") { + @Override + public void consider(RegisteredFile f) { + if (f.getName().endsWith(getFileExtension())){ + getToUseFileSet().add(f); + isProposedFilesetValid=true; + } + } + }); + break; + } + case ".shp":{toReturn.add(new SupportedFormat(".shp") { + @Override + public void consider(RegisteredFile f) { + getToUseFileSet().add(f); + if (f.getName().endsWith(getFileExtension())){ + isProposedFilesetValid=true; + } + } + }); + break; + + } + } + } + + return toReturn; + } + + @NonNull + private String fileExtension; + + protected Boolean isProposedFilesetValid=false; + + private ArrayList toUseFileSet = new ArrayList<>(); + + public abstract void consider(RegisteredFile f); + + + @Override + public String toString() { + return fileExtension ; + } +} diff --git a/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/MaterializerTests.java b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/MaterializerTests.java index 5ca54e9..0b3e88f 100644 --- a/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/MaterializerTests.java +++ b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/MaterializerTests.java @@ -2,6 +2,7 @@ package org.gcube.application.cms.sdi.plugins; import com.fasterxml.jackson.core.JsonProcessingException; import org.bson.Document; +import org.gcube.application.cms.implementations.ImplementationProvider; import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException; import org.gcube.application.cms.plugins.faults.MaterializationException; import org.gcube.application.cms.plugins.faults.PluginExecutionException; @@ -12,10 +13,20 @@ import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.cms.tests.TestDocuments; import org.gcube.application.cms.tests.TestProfiles; import org.gcube.application.cms.tests.plugins.BasicPluginTest; +import org.gcube.application.geoportal.common.faults.StorageException; +import org.gcube.application.geoportal.common.model.JSONPathWrapper; import org.gcube.application.geoportal.common.model.document.Project; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import org.junit.Test; +import java.io.File; +import java.io.IOException; + import static junit.framework.TestCase.assertTrue; import static org.junit.Assume.assumeTrue; @@ -24,14 +35,30 @@ public class MaterializerTests extends BasicPluginTest { @Test public void testShape() throws JsonProcessingException, PluginExecutionException { assumeTrue(GCubeTest.isTestInfrastructureEnabled()); - Project doc= TestDocuments.documentMap.get("registeredFile.json"); + Project doc= TestDocuments.documentMap.get("registeredGeoTif.json"); // doc.setProfileID("sdi-tests"); + + + + materialize(doc); } - + @Test + public void download() throws ConfigurationException, IOException, StorageException { + assumeTrue(GCubeTest.isTestInfrastructureEnabled()); + Project doc= TestDocuments.documentMap.get("registeredGeoTif.json"); + for(Object o : new JSONPathWrapper(doc.getTheDocument().toJson()).getByPath("$..fileset")){ + RegisteredFileSet fs = Serialization.convert(o,RegisteredFileSet.class); + for(Object oo: fs.getPayloads()){ + RegisteredFile f = Serialization.convert(oo,RegisteredFile.class); + File file = Files.downloadFromUrl(f.getName(),f.getLink()); +// Files.deleteIfExists(file.toPath()); + } + } + } diff --git a/test-data/profiledDocuments/profiles/basicProfile.json b/test-data/profiledDocuments/profiles/basicProfile.json index 1084ee0..6c7364c 100644 --- a/test-data/profiledDocuments/profiles/basicProfile.json +++ b/test-data/profiledDocuments/profiles/basicProfile.json @@ -36,6 +36,14 @@ {"STEP" : "PUBLISH", "roles" :[]} ] } + }, { + "_id" : "SDI-Default-Materializer", + "_type" : "Materializer", + "_configuration" : { + "registeredFileSetPaths" : [ + {"schemaField" : "section._children[?(@.fileset)]","documentPath" : "section.fileset"} + ] + } }, { "_id" : "DEFAULT-SINGLE-STEP", diff --git a/test-data/profiledDocuments/registeredGeoTif.json b/test-data/profiledDocuments/registeredGeoTif.json new file mode 100644 index 0000000..ba62174 --- /dev/null +++ b/test-data/profiledDocuments/registeredGeoTif.json @@ -0,0 +1,103 @@ +{ + "_id": "62ea45d8270ce23808c7fe4c", + "_version": "1.0.1", + "_info": + { + "_creationInfo": + { + "_user": + { + "_username": "fabio.sinibaldi" + }, + "_context": + { + "_id": "/gcube/devsec/devVRE", + "_name": "/devVRE" + }, + "_instant": "2022-08-03T11:54:26.546" + }, + "_lastEditInfo": + { + "_user": + { + "_username": "fabio.sinibaldi" + }, + "_context": + { + "_id": "/gcube/devsec/devVRE", + "_name": "/devVRE" + }, + "_instant": "2022-08-03T11:56:00.987" + }, + "_access": + { + "_policy": "OPEN", + "_license": "" + } + }, + "_profileID": "basic", + "_profileVersion": "1.0.0", + "_lifecycleInformation": + { + "_phase": "DRAFT", + "_lastInvokedStep": null, + "_lastOperationStatus": "OK", + "_errorMessages": + [], + "_warningMessages": + [], + "_triggeredEvents": + [ + { + "event": "INIT_DOCUMENT", + "lastOperationStatus": "OK", + "errorMessages": null, + "warningMessages": null + } + ], + "_notes": null + }, + "_relationships": null, + "_identificationReferences": null, + "_theDocument": + { + "section": + { + "title": "My Title", + "fileset": + { + "customField": "customFieldValue", + "_uuid": "12fac10b-fada-4f43-b491-76a77ab02209", + "_creationInfo": + { + "_user": + { + "_username": "fabio.sinibaldi" + }, + "_context": + { + "_id": "/gcube/devsec/devVRE", + "_name": "/devVRE" + }, + "_instant": "2022-08-03T11:55:18.888" + }, + "_access": + { + "_policy": "OPEN", + "_license": "" + }, + "_folderID": "b751a1b6-4cf0-43ff-82b2-e2f301edf245", + "_payloads": + [ + { + "_mimetype": "image/tiff", + "_storageID": "06ff5d37-5015-46f8-91a2-527ffab9a30f", + "_link": "https://data-d.d4science.org/shub/E_NGw3SXNwRlA5cHc0T1JQN2RZeTFUTEt3TmtRckxxZ1pIZXFtaWJld1RybzlSem9oREplWXRxdnFyQTVtejVUaA==", + "_name": "sample.tif" + } + ] + } + } + }, + "_lock": null +} \ No newline at end of file