Merge pull request '1.3.0' (#5) from 1.3.0 into master
This commit is contained in:
commit
95a837ba48
|
@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data.transfer.sis-geotk-plugin
|
# Changelog for org.gcube.data.transfer.sis-geotk-plugin
|
||||||
|
|
||||||
|
## [v1.3.0] 2021-03-25
|
||||||
|
Use of EPSG Dataset
|
||||||
|
Proxy support
|
||||||
|
|
||||||
## [v1.2.0] 2020-09-07
|
## [v1.2.0] 2020-09-07
|
||||||
Upgrade to apache-sis 1.0
|
Upgrade to apache-sis 1.0
|
||||||
|
|
||||||
|
|
25
pom.xml
25
pom.xml
|
@ -9,7 +9,7 @@
|
||||||
</parent>
|
</parent>
|
||||||
<groupId>org.gcube.data.transfer</groupId>
|
<groupId>org.gcube.data.transfer</groupId>
|
||||||
<artifactId>sis-geotk-plugin</artifactId>
|
<artifactId>sis-geotk-plugin</artifactId>
|
||||||
<version>1.2.0</version>
|
<version>1.3.0</version>
|
||||||
<name>Sis/GeoToolkit plugin</name>
|
<name>Sis/GeoToolkit plugin</name>
|
||||||
<description>Apache Sis/Geotk plugin for data-transfer-service</description>
|
<description>Apache Sis/Geotk plugin for data-transfer-service</description>
|
||||||
|
|
||||||
|
@ -80,6 +80,7 @@
|
||||||
<groupId>org.gcube.spatial.data</groupId>
|
<groupId>org.gcube.spatial.data</groupId>
|
||||||
<artifactId>sdi-library</artifactId>
|
<artifactId>sdi-library</artifactId>
|
||||||
<version>[1.0.0,1.0.2]</version>
|
<version>[1.0.0,1.0.2]</version>
|
||||||
|
<!-- <version>[1.0.0,2.0)</version> -->
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- SMARTGEARS -->
|
<!-- SMARTGEARS -->
|
||||||
|
@ -120,10 +121,10 @@
|
||||||
|
|
||||||
<!-- SIS EXTERNALS -->
|
<!-- SIS EXTERNALS -->
|
||||||
<!-- https://mvnrepository.com/artifact/com.esri.geometry/esri-geometry-api -->
|
<!-- https://mvnrepository.com/artifact/com.esri.geometry/esri-geometry-api -->
|
||||||
<dependency>
|
<!-- <dependency> -->
|
||||||
<groupId>com.esri.geometry</groupId>
|
<!-- <groupId>com.esri.geometry</groupId> -->
|
||||||
<artifactId>esri-geometry-api</artifactId>
|
<!-- <artifactId>esri-geometry-api</artifactId> -->
|
||||||
</dependency>
|
<!-- </dependency> -->
|
||||||
|
|
||||||
<!-- https://mvnrepository.com/artifact/edu.ucar/cdm -->
|
<!-- https://mvnrepository.com/artifact/edu.ucar/cdm -->
|
||||||
<dependency>
|
<dependency>
|
||||||
|
@ -133,6 +134,20 @@
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
<!-- EPSG DATASET -->
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.sis.non-free</groupId>
|
||||||
|
<artifactId>sis-embedded-data</artifactId>
|
||||||
|
<version>${sis.version}</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.derby</groupId>
|
||||||
|
<artifactId>derby</artifactId>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<!-- TEST -->
|
<!-- TEST -->
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
/.DS_Store
|
|
@ -14,11 +14,40 @@ import org.apache.sis.xml.XML;
|
||||||
import org.opengis.metadata.Metadata;
|
import org.opengis.metadata.Metadata;
|
||||||
|
|
||||||
import lombok.Synchronized;
|
import lombok.Synchronized;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
public class ISOMetadataMarshalling {
|
public class ISOMetadataMarshalling {
|
||||||
|
|
||||||
|
|
||||||
private static MarshallerPool pool=null;
|
private static MarshallerPool pool=null;
|
||||||
|
|
||||||
|
// static {
|
||||||
|
//
|
||||||
|
// File temp=new File(System.getProperty("java.io.tmpdir")+"/EPSG");
|
||||||
|
// temp.mkdirs();
|
||||||
|
//
|
||||||
|
// EmbeddedConnectionPoolDataSource ds = new EmbeddedConnectionPoolDataSource();
|
||||||
|
// ds.setConnectionAttributes("create=true");
|
||||||
|
//
|
||||||
|
// ds.setDatabaseName(temp.getAbsolutePath()+"/SpatialMetadata");
|
||||||
|
//
|
||||||
|
//// ds.setDataSourceName(arg0);
|
||||||
|
//// ds.setServerName("someHost");
|
||||||
|
//// ds.setPortNumber("1527");
|
||||||
|
//// ds.setDatabaseName("someDB");
|
||||||
|
//
|
||||||
|
// Configuration.current().setDatabase(new Supplier<DataSource>() {
|
||||||
|
// @Override
|
||||||
|
// public DataSource get() {
|
||||||
|
// return ds;
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
private static MarshallerPool getPool() throws JAXBException {
|
private static MarshallerPool getPool() throws JAXBException {
|
||||||
|
|
|
@ -71,7 +71,18 @@ public class ThreddsInstanceManager {
|
||||||
protected ThreddsInstanceManager(DataTransferContext context) {
|
protected ThreddsInstanceManager(DataTransferContext context) {
|
||||||
log.warn("Instance Creation. Should happen only once. Loading information from context..");
|
log.warn("Instance Creation. Should happen only once. Loading information from context..");
|
||||||
this.ctx=context;
|
this.ctx=context;
|
||||||
|
try {
|
||||||
|
log.info("Loading proxy configuration..");
|
||||||
|
currentHostname=ctx.getCtx().configuration().proxyAddress().hostname();
|
||||||
|
if(currentHostname==null||currentHostname.isEmpty()) throw new Exception("Proxy is : "+currentHostname);
|
||||||
|
}catch(Exception e) {
|
||||||
|
log.info("Unable to get proxy..",e);
|
||||||
currentHostname=ctx.getCtx().container().configuration().hostname();
|
currentHostname=ctx.getCtx().container().configuration().hostname();
|
||||||
|
}
|
||||||
|
log.info("Hostname to be used is "+currentHostname);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
currentGHNId=ctx.getCtx().container().id();
|
currentGHNId=ctx.getCtx().container().id();
|
||||||
String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml";
|
String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml";
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,7 @@ public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static final String PLUGIN_ID="SIS/GEOTK";
|
public static final String PLUGIN_ID="SIS/GEOTK";
|
||||||
|
|
||||||
public static final String SOURCE_PARAMETER="SOURCE_FILE";
|
public static final String SOURCE_PARAMETER="SOURCE_FILE";
|
||||||
public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY";
|
public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY";
|
||||||
|
|
|
@ -0,0 +1,84 @@
|
||||||
|
package org.gcube.data.transfer.plugins.sis;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FilenameFilter;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.DataTransferClient;
|
||||||
|
import org.gcube.data.transfer.library.TransferResult;
|
||||||
|
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||||
|
import org.gcube.data.transfer.model.PluginInvocation;
|
||||||
|
import org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory;
|
||||||
|
import org.gcube.smartgears.Constants;
|
||||||
|
import org.gcube.spatial.data.clients.model.engine.Engine;
|
||||||
|
import org.gcube.spatial.data.sdi.interfaces.Metadata;
|
||||||
|
import org.gcube.spatial.data.sdi.model.faults.RemoteException;
|
||||||
|
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
|
||||||
|
|
||||||
|
|
||||||
|
public class PublishTest {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws MalformedURLException, RemoteException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
|
||||||
|
|
||||||
|
TokenSetter.set("/pred4s/preprod/preVRE");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
String hostname=
|
||||||
|
new URL(SDIAbstractPlugin.management().build().getConfiguration().getByEngine(Engine.TH_ENGINE).get(0).getBaseEndpoint()).getHost();
|
||||||
|
|
||||||
|
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("https://"+hostname);
|
||||||
|
|
||||||
|
|
||||||
|
File metadata=Paths.get("src/test/resources/toPublishMeta").toFile();
|
||||||
|
// for(File f:metadata.listFiles(new FilenameFilter() {
|
||||||
|
// @Override
|
||||||
|
// public boolean accept(File dir, String name) {
|
||||||
|
// return name.endsWith(".xml");
|
||||||
|
// }
|
||||||
|
// })) {
|
||||||
|
// for(File f:new File[] {
|
||||||
|
// new File("/Users/FabioISTI/Downloads/NASA_Surface_Air_Temperature_1950_2100_rcp45.nc")
|
||||||
|
// }) {
|
||||||
|
|
||||||
|
for(URL url:new URL[] {
|
||||||
|
new URL("https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp45.nc")
|
||||||
|
}) {
|
||||||
|
|
||||||
|
// if not present, generate with sis/geotk
|
||||||
|
Destination toSetDestination=new Destination();
|
||||||
|
toSetDestination.setCreateSubfolders(true);
|
||||||
|
toSetDestination.setDestinationFileName("another_dataset");
|
||||||
|
toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||||
|
toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
||||||
|
toSetDestination.setPersistenceId("thredds");
|
||||||
|
|
||||||
|
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
|
||||||
|
String fileLocation="public/netcdf/test_"+System.currentTimeMillis();
|
||||||
|
toSetDestination.setSubFolder(fileLocation);
|
||||||
|
|
||||||
|
PluginInvocation inv=new PluginInvocation(SISPluginFactory.PLUGIN_ID);
|
||||||
|
inv.setParameters(Collections.singletonMap(SISPluginFactory.VALIDATE_PARAMETER, "false"));
|
||||||
|
|
||||||
|
System.out.println(
|
||||||
|
// client.localFile(f, toSetDestination, Collections.singleton(new PluginInvocation(SISPluginFactory.PLUGIN_ID))));
|
||||||
|
client.httpSource(url, toSetDestination, inv));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -16,10 +16,11 @@ import javax.xml.bind.JAXBException;
|
||||||
|
|
||||||
import org.apache.sis.storage.DataStoreException;
|
import org.apache.sis.storage.DataStoreException;
|
||||||
import org.apache.sis.storage.UnsupportedStorageException;
|
import org.apache.sis.storage.UnsupportedStorageException;
|
||||||
import org.apache.sis.xml.XML;
|
|
||||||
import org.gcube.data.transfer.plugins.thredds.ISOMetadataMarshalling;
|
import org.gcube.data.transfer.plugins.thredds.ISOMetadataMarshalling;
|
||||||
import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin;
|
import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin;
|
||||||
import org.opengis.metadata.Metadata;
|
|
||||||
|
import ucar.nc2.NetcdfFile;
|
||||||
|
import ucar.nc2.dataset.NetcdfDataset;
|
||||||
|
|
||||||
public class TestGetMetadata {
|
public class TestGetMetadata {
|
||||||
|
|
||||||
|
@ -39,38 +40,43 @@ public class TestGetMetadata {
|
||||||
|
|
||||||
// *********** NETCDF 3
|
// *********** NETCDF 3
|
||||||
|
|
||||||
"/Users/FabioISTI/Downloads/Aaptos_aaptos.nc"
|
"/Users/FabioISTI/Downloads/Aaptos_aaptos.nc",
|
||||||
|
"/Users/FabioISTI/Downloads/dataset-armor-3d-rep-monthly_20181115T1200Z_P20190301T0000Z.nc",
|
||||||
|
//"/Users/FabioISTI/Downloads/data_retro_Run_10026_911.nc",
|
||||||
|
//"/Users/FabioISTI/Downloads/2BPR6_2021_a_NRT_Gosud_V3.nc",
|
||||||
|
"/Users/FabioISTI/Downloads/GO_WTEP_2015_TRAJ.nc"
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
for(String f:toCheckFiles) {
|
for(String f:toCheckFiles) {
|
||||||
System.out.println("checking "+f);
|
System.out.println("checking "+f);
|
||||||
try{
|
try{
|
||||||
check(false,f);
|
check(false,f);
|
||||||
}catch(UnsupportedStorageException e) {
|
}catch(Exception e) {
|
||||||
System.err.println(e.getMessage());
|
System.err.println(e.getMessage());
|
||||||
// System.out.println("Trying opening "+f+" manually");
|
System.out.println("Trying opening "+f+" manually");
|
||||||
// open(f);
|
open(f);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
System.out.println("Done");
|
System.out.println("Done");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException {
|
private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException {
|
||||||
// System.out.println("Opening "+path);
|
System.out.println("Opening "+path);
|
||||||
// NetcdfFile f=NetcdfFiles.open(path);
|
NetcdfFile f=NetcdfFile.open(path);
|
||||||
// System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId());
|
System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId());
|
||||||
|
|
||||||
|
|
||||||
|
NetcdfDataset ncDs=NetcdfDataset.openDataset(path);
|
||||||
|
// NcMLReaderNew.
|
||||||
//
|
//
|
||||||
//
|
System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId());
|
||||||
// NetcdfDataset ncDs=NetcdfDatasets.openDataset(path);
|
|
||||||
//// NcMLReaderNew.
|
// NetcdfDatasets.openFile(path,null);
|
||||||
////
|
|
||||||
// System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId());
|
|
||||||
//
|
}
|
||||||
//// NetcdfDatasets.openFile(path,null);
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException, JAXBException {
|
private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException, JAXBException {
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
package org.gcube.data.transfer.plugins.sis;
|
||||||
|
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
@Slf4j
|
||||||
|
public class TokenSetter {
|
||||||
|
|
||||||
|
|
||||||
|
private static Properties props=new Properties();
|
||||||
|
|
||||||
|
static{
|
||||||
|
try {
|
||||||
|
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void set(String scope){
|
||||||
|
try{
|
||||||
|
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||||
|
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
||||||
|
}catch(Throwable e){
|
||||||
|
log.warn("Unable to set token for scope "+scope,e);
|
||||||
|
}
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue