diff --git a/CHANGELOG.md b/CHANGELOG.md index 195f775..8ee9ab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.data.transfer.sis-geotk-plugin +## [v1.3.0] 2021-03-25 +Use of EPSG Dataset +Proxy support + ## [v1.2.0] 2020-09-07 Upgrade to apache-sis 1.0 diff --git a/pom.xml b/pom.xml index e0905ab..84acf9f 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.data.transfer sis-geotk-plugin - 1.2.0 + 1.3.0 Sis/GeoToolkit plugin Apache Sis/Geotk plugin for data-transfer-service @@ -80,6 +80,7 @@ org.gcube.spatial.data sdi-library [1.0.0,1.0.2] + @@ -120,10 +121,10 @@ - - com.esri.geometry - esri-geometry-api - + + + + @@ -133,6 +134,20 @@ + + + + org.apache.sis.non-free + sis-embedded-data + ${sis.version} + runtime + + + org.apache.derby + derby + runtime + + diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..9bb88d3 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +/.DS_Store diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java index 05a71f7..75aee3e 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java @@ -14,11 +14,40 @@ import org.apache.sis.xml.XML; import org.opengis.metadata.Metadata; import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class ISOMetadataMarshalling { + private static MarshallerPool pool=null; +// static { +// +// File temp=new File(System.getProperty("java.io.tmpdir")+"/EPSG"); +// temp.mkdirs(); +// +// EmbeddedConnectionPoolDataSource ds = new EmbeddedConnectionPoolDataSource(); +// ds.setConnectionAttributes("create=true"); +// +// ds.setDatabaseName(temp.getAbsolutePath()+"/SpatialMetadata"); +// +//// ds.setDataSourceName(arg0); +//// ds.setServerName("someHost"); +//// ds.setPortNumber("1527"); +//// ds.setDatabaseName("someDB"); +// +// Configuration.current().setDatabase(new Supplier() { +// @Override +// public DataSource get() { +// return ds; +// } +// }); +// +// +// } + + @Synchronized private static MarshallerPool getPool() throws JAXBException { diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java index f7c8daa..9f36a47 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java @@ -71,7 +71,18 @@ public class ThreddsInstanceManager { protected ThreddsInstanceManager(DataTransferContext context) { log.warn("Instance Creation. Should happen only once. Loading information from context.."); this.ctx=context; - currentHostname=ctx.getCtx().container().configuration().hostname(); + try { + log.info("Loading proxy configuration.."); + currentHostname=ctx.getCtx().configuration().proxyAddress().hostname(); + if(currentHostname==null||currentHostname.isEmpty()) throw new Exception("Proxy is : "+currentHostname); + }catch(Exception e) { + log.info("Unable to get proxy..",e); + currentHostname=ctx.getCtx().container().configuration().hostname(); + } + log.info("Hostname to be used is "+currentHostname); + + + currentGHNId=ctx.getCtx().container().id(); String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml"; diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java index a88c283..fe8a9a0 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java @@ -20,7 +20,7 @@ public class SISPluginFactory extends AbstractPluginFactory { - static final String PLUGIN_ID="SIS/GEOTK"; + public static final String PLUGIN_ID="SIS/GEOTK"; public static final String SOURCE_PARAMETER="SOURCE_FILE"; public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY"; diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java b/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java new file mode 100644 index 0000000..ef5fd6f --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java @@ -0,0 +1,84 @@ +package org.gcube.data.transfer.plugins.sis; + +import java.io.File; +import java.io.FilenameFilter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Set; + +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory; +import org.gcube.smartgears.Constants; +import org.gcube.spatial.data.clients.model.engine.Engine; +import org.gcube.spatial.data.sdi.interfaces.Metadata; +import org.gcube.spatial.data.sdi.model.faults.RemoteException; +import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; + + +public class PublishTest { + + public static void main(String[] args) throws MalformedURLException, RemoteException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + + TokenSetter.set("/pred4s/preprod/preVRE"); + + + + String hostname= + new URL(SDIAbstractPlugin.management().build().getConfiguration().getByEngine(Engine.TH_ENGINE).get(0).getBaseEndpoint()).getHost(); + + DataTransferClient client=DataTransferClient.getInstanceByEndpoint("https://"+hostname); + + + File metadata=Paths.get("src/test/resources/toPublishMeta").toFile(); +// for(File f:metadata.listFiles(new FilenameFilter() { +// @Override +// public boolean accept(File dir, String name) { +// return name.endsWith(".xml"); +// } +// })) { +// for(File f:new File[] { +// new File("/Users/FabioISTI/Downloads/NASA_Surface_Air_Temperature_1950_2100_rcp45.nc") +// }) { + + for(URL url:new URL[] { + new URL("https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp45.nc") + }) { + + // if not present, generate with sis/geotk + Destination toSetDestination=new Destination(); + toSetDestination.setCreateSubfolders(true); + toSetDestination.setDestinationFileName("another_dataset"); + toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + toSetDestination.setPersistenceId("thredds"); + + //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH + String fileLocation="public/netcdf/test_"+System.currentTimeMillis(); + toSetDestination.setSubFolder(fileLocation); + + PluginInvocation inv=new PluginInvocation(SISPluginFactory.PLUGIN_ID); + inv.setParameters(Collections.singletonMap(SISPluginFactory.VALIDATE_PARAMETER, "false")); + + System.out.println( +// client.localFile(f, toSetDestination, Collections.singleton(new PluginInvocation(SISPluginFactory.PLUGIN_ID)))); + client.httpSource(url, toSetDestination, inv)); + + } + + } + + + +} diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java index 27dbbba..dfa31ff 100644 --- a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java @@ -16,10 +16,11 @@ import javax.xml.bind.JAXBException; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.UnsupportedStorageException; -import org.apache.sis.xml.XML; import org.gcube.data.transfer.plugins.thredds.ISOMetadataMarshalling; import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin; -import org.opengis.metadata.Metadata; + +import ucar.nc2.NetcdfFile; +import ucar.nc2.dataset.NetcdfDataset; public class TestGetMetadata { @@ -39,38 +40,43 @@ public class TestGetMetadata { // *********** NETCDF 3 - "/Users/FabioISTI/Downloads/Aaptos_aaptos.nc" + "/Users/FabioISTI/Downloads/Aaptos_aaptos.nc", + "/Users/FabioISTI/Downloads/dataset-armor-3d-rep-monthly_20181115T1200Z_P20190301T0000Z.nc", + //"/Users/FabioISTI/Downloads/data_retro_Run_10026_911.nc", + //"/Users/FabioISTI/Downloads/2BPR6_2021_a_NRT_Gosud_V3.nc", + "/Users/FabioISTI/Downloads/GO_WTEP_2015_TRAJ.nc" + }; for(String f:toCheckFiles) { System.out.println("checking "+f); try{ check(false,f); - }catch(UnsupportedStorageException e) { + }catch(Exception e) { System.err.println(e.getMessage()); -// System.out.println("Trying opening "+f+" manually"); -// open(f); + System.out.println("Trying opening "+f+" manually"); + open(f); } } System.out.println("Done"); } -// private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException { -// System.out.println("Opening "+path); -// NetcdfFile f=NetcdfFiles.open(path); -// System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId()); + private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException { + System.out.println("Opening "+path); + NetcdfFile f=NetcdfFile.open(path); + System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId()); + + + NetcdfDataset ncDs=NetcdfDataset.openDataset(path); +// NcMLReaderNew. // -// -// NetcdfDataset ncDs=NetcdfDatasets.openDataset(path); -//// NcMLReaderNew. -//// -// System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId()); -// -//// NetcdfDatasets.openFile(path,null); -// -// -// } + System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId()); + +// NetcdfDatasets.openFile(path,null); + + + } private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException, JAXBException { diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java new file mode 100644 index 0000000..a050ac9 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java @@ -0,0 +1,33 @@ +package org.gcube.data.transfer.plugins.sis; + +import java.util.Properties; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +import lombok.extern.slf4j.Slf4j; +@Slf4j +public class TokenSetter { + + +private static Properties props=new Properties(); + + static{ + try { + props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); + } catch (Exception e) { + throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); + } + } + + + public static void set(String scope){ + try{ + if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope); + SecurityTokenProvider.instance.set(props.getProperty(scope)); + }catch(Throwable e){ + log.warn("Unable to set token for scope "+scope,e); + } + ScopeProvider.instance.set(scope); + } +}