From 1459d8b64c1891d8f444bb8977567954c78dbb3d Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 11:08:57 +0100 Subject: [PATCH 1/6] Use of EPSG dataset --- CHANGELOG.md | 4 + pom.xml | 23 +++-- src/.gitignore | 1 + .../thredds/ISOMetadataMarshalling.java | 31 ++++++- .../plugins/thredds/sis/SISPluginFactory.java | 2 +- .../transfer/plugins/sis/PublishTest.java | 84 +++++++++++++++++++ .../transfer/plugins/sis/TestGetMetadata.java | 46 +++++----- .../transfer/plugins/sis/TokenSetter.java | 33 ++++++++ 8 files changed, 196 insertions(+), 28 deletions(-) create mode 100644 src/.gitignore create mode 100644 src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java create mode 100644 src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 195f775..26f14b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.data.transfer.sis-geotk-plugin +## [v1.3.0-SNAPSHOT] 2020-09-07 +Use of EPSG Dataset +Proxy support + ## [v1.2.0] 2020-09-07 Upgrade to apache-sis 1.0 diff --git a/pom.xml b/pom.xml index e0905ab..793c41f 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.data.transfer sis-geotk-plugin - 1.2.0 + 1.3.0-SNAPSHOT Sis/GeoToolkit plugin Apache Sis/Geotk plugin for data-transfer-service @@ -79,7 +79,8 @@ org.gcube.spatial.data sdi-library - [1.0.0,1.0.2] + + [1.0.0,2.0) @@ -120,10 +121,10 @@ - - com.esri.geometry - esri-geometry-api - + + + + @@ -133,6 +134,16 @@ + + + + org.apache.sis.non-free + sis-embedded-data + ${sis.version} + runtime + + + diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..9bb88d3 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +/.DS_Store diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java index 05a71f7..43b4c39 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java @@ -14,17 +14,46 @@ import org.apache.sis.xml.XML; import org.opengis.metadata.Metadata; import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class ISOMetadataMarshalling { + private static MarshallerPool pool=null; +// static { +// +// File temp=new File(System.getProperty("java.io.tmpdir")+"/EPSG"); +// temp.mkdirs(); +// +// EmbeddedConnectionPoolDataSource ds = new EmbeddedConnectionPoolDataSource(); +// ds.setConnectionAttributes("create=true"); +// +// ds.setDatabaseName(temp.getAbsolutePath()+"/SpatialMetadata"); +// +//// ds.setDataSourceName(arg0); +//// ds.setServerName("someHost"); +//// ds.setPortNumber("1527"); +//// ds.setDatabaseName("someDB"); +// +// Configuration.current().setDatabase(new Supplier() { +// @Override +// public DataSource get() { +// return ds; +// } +// }); +// +// +// } + + @Synchronized private static MarshallerPool getPool() throws JAXBException { if(pool==null) pool=new MarshallerPool( - singletonMap(XML.METADATA_VERSION,"2007")); + singletonMap(XML.METADATA_VERSION,"2016")); return pool; } diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java index a88c283..fe8a9a0 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java @@ -20,7 +20,7 @@ public class SISPluginFactory extends AbstractPluginFactory { - static final String PLUGIN_ID="SIS/GEOTK"; + public static final String PLUGIN_ID="SIS/GEOTK"; public static final String SOURCE_PARAMETER="SOURCE_FILE"; public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY"; diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java b/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java new file mode 100644 index 0000000..ef5fd6f --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java @@ -0,0 +1,84 @@ +package org.gcube.data.transfer.plugins.sis; + +import java.io.File; +import java.io.FilenameFilter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Set; + +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory; +import org.gcube.smartgears.Constants; +import org.gcube.spatial.data.clients.model.engine.Engine; +import org.gcube.spatial.data.sdi.interfaces.Metadata; +import org.gcube.spatial.data.sdi.model.faults.RemoteException; +import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; + + +public class PublishTest { + + public static void main(String[] args) throws MalformedURLException, RemoteException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + + TokenSetter.set("/pred4s/preprod/preVRE"); + + + + String hostname= + new URL(SDIAbstractPlugin.management().build().getConfiguration().getByEngine(Engine.TH_ENGINE).get(0).getBaseEndpoint()).getHost(); + + DataTransferClient client=DataTransferClient.getInstanceByEndpoint("https://"+hostname); + + + File metadata=Paths.get("src/test/resources/toPublishMeta").toFile(); +// for(File f:metadata.listFiles(new FilenameFilter() { +// @Override +// public boolean accept(File dir, String name) { +// return name.endsWith(".xml"); +// } +// })) { +// for(File f:new File[] { +// new File("/Users/FabioISTI/Downloads/NASA_Surface_Air_Temperature_1950_2100_rcp45.nc") +// }) { + + for(URL url:new URL[] { + new URL("https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp45.nc") + }) { + + // if not present, generate with sis/geotk + Destination toSetDestination=new Destination(); + toSetDestination.setCreateSubfolders(true); + toSetDestination.setDestinationFileName("another_dataset"); + toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + toSetDestination.setPersistenceId("thredds"); + + //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH + String fileLocation="public/netcdf/test_"+System.currentTimeMillis(); + toSetDestination.setSubFolder(fileLocation); + + PluginInvocation inv=new PluginInvocation(SISPluginFactory.PLUGIN_ID); + inv.setParameters(Collections.singletonMap(SISPluginFactory.VALIDATE_PARAMETER, "false")); + + System.out.println( +// client.localFile(f, toSetDestination, Collections.singleton(new PluginInvocation(SISPluginFactory.PLUGIN_ID)))); + client.httpSource(url, toSetDestination, inv)); + + } + + } + + + +} diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java index 27dbbba..dfa31ff 100644 --- a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java @@ -16,10 +16,11 @@ import javax.xml.bind.JAXBException; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.UnsupportedStorageException; -import org.apache.sis.xml.XML; import org.gcube.data.transfer.plugins.thredds.ISOMetadataMarshalling; import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin; -import org.opengis.metadata.Metadata; + +import ucar.nc2.NetcdfFile; +import ucar.nc2.dataset.NetcdfDataset; public class TestGetMetadata { @@ -39,38 +40,43 @@ public class TestGetMetadata { // *********** NETCDF 3 - "/Users/FabioISTI/Downloads/Aaptos_aaptos.nc" + "/Users/FabioISTI/Downloads/Aaptos_aaptos.nc", + "/Users/FabioISTI/Downloads/dataset-armor-3d-rep-monthly_20181115T1200Z_P20190301T0000Z.nc", + //"/Users/FabioISTI/Downloads/data_retro_Run_10026_911.nc", + //"/Users/FabioISTI/Downloads/2BPR6_2021_a_NRT_Gosud_V3.nc", + "/Users/FabioISTI/Downloads/GO_WTEP_2015_TRAJ.nc" + }; for(String f:toCheckFiles) { System.out.println("checking "+f); try{ check(false,f); - }catch(UnsupportedStorageException e) { + }catch(Exception e) { System.err.println(e.getMessage()); -// System.out.println("Trying opening "+f+" manually"); -// open(f); + System.out.println("Trying opening "+f+" manually"); + open(f); } } System.out.println("Done"); } -// private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException { -// System.out.println("Opening "+path); -// NetcdfFile f=NetcdfFiles.open(path); -// System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId()); + private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException { + System.out.println("Opening "+path); + NetcdfFile f=NetcdfFile.open(path); + System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId()); + + + NetcdfDataset ncDs=NetcdfDataset.openDataset(path); +// NcMLReaderNew. // -// -// NetcdfDataset ncDs=NetcdfDatasets.openDataset(path); -//// NcMLReaderNew. -//// -// System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId()); -// -//// NetcdfDatasets.openFile(path,null); -// -// -// } + System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId()); + +// NetcdfDatasets.openFile(path,null); + + + } private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException, JAXBException { diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java new file mode 100644 index 0000000..a050ac9 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java @@ -0,0 +1,33 @@ +package org.gcube.data.transfer.plugins.sis; + +import java.util.Properties; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +import lombok.extern.slf4j.Slf4j; +@Slf4j +public class TokenSetter { + + +private static Properties props=new Properties(); + + static{ + try { + props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); + } catch (Exception e) { + throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); + } + } + + + public static void set(String scope){ + try{ + if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope); + SecurityTokenProvider.instance.set(props.getProperty(scope)); + }catch(Throwable e){ + log.warn("Unable to set token for scope "+scope,e); + } + ScopeProvider.instance.set(scope); + } +} From 8271d0b973f06fcabffd0ade545ce4f749034bff Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 11:09:20 +0100 Subject: [PATCH 2/6] Proxy support --- .../plugins/thredds/ThreddsInstanceManager.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java index f7c8daa..9f36a47 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java @@ -71,7 +71,18 @@ public class ThreddsInstanceManager { protected ThreddsInstanceManager(DataTransferContext context) { log.warn("Instance Creation. Should happen only once. Loading information from context.."); this.ctx=context; - currentHostname=ctx.getCtx().container().configuration().hostname(); + try { + log.info("Loading proxy configuration.."); + currentHostname=ctx.getCtx().configuration().proxyAddress().hostname(); + if(currentHostname==null||currentHostname.isEmpty()) throw new Exception("Proxy is : "+currentHostname); + }catch(Exception e) { + log.info("Unable to get proxy..",e); + currentHostname=ctx.getCtx().container().configuration().hostname(); + } + log.info("Hostname to be used is "+currentHostname); + + + currentGHNId=ctx.getCtx().container().id(); String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml"; From 4d7b97c5e18cbb76e6064ff6b49c80d7096053eb Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 11:56:58 +0100 Subject: [PATCH 3/6] derby dependency --- pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 793c41f..111d27d 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,11 @@ ${sis.version} runtime - + + org.apache.derby + derby + runtime + From c917eb76a413cfb42f6e66301c60f23ca7871e7b Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 12:16:13 +0100 Subject: [PATCH 4/6] restricted sdi-library range --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 111d27d..5cb5216 100644 --- a/pom.xml +++ b/pom.xml @@ -79,8 +79,8 @@ org.gcube.spatial.data sdi-library - - [1.0.0,2.0) + [1.0.0,1.0.2] + From 00413916f73c29052bf27b2214dc465352b46e8b Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 12:40:31 +0100 Subject: [PATCH 5/6] Force 2007 XML format --- .../data/transfer/plugins/thredds/ISOMetadataMarshalling.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java index 43b4c39..75aee3e 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java @@ -53,7 +53,7 @@ public class ISOMetadataMarshalling { private static MarshallerPool getPool() throws JAXBException { if(pool==null) pool=new MarshallerPool( - singletonMap(XML.METADATA_VERSION,"2016")); + singletonMap(XML.METADATA_VERSION,"2007")); return pool; } From a1f284eae038d68bf19d45b981bbf1112c1485b8 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 25 Mar 2021 12:54:40 +0100 Subject: [PATCH 6/6] release --- CHANGELOG.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f14b8..8ee9ab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.data.transfer.sis-geotk-plugin -## [v1.3.0-SNAPSHOT] 2020-09-07 +## [v1.3.0] 2021-03-25 Use of EPSG Dataset Proxy support diff --git a/pom.xml b/pom.xml index 5cb5216..84acf9f 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.data.transfer sis-geotk-plugin - 1.3.0-SNAPSHOT + 1.3.0 Sis/GeoToolkit plugin Apache Sis/Geotk plugin for data-transfer-service