gCubeSystem
/
sis-geotk-plugin
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request '1.3.0' (#5) from 1.3.0 into master

Browse Source
20760_https r5.1.0
Fabio Sinibaldi 3 years ago
parent fed8b3cbef a1f284eae0
commit 95a837ba48
9 changed files with 210 additions and 27 deletions
Show Stats Download Patch File Download Diff File

4
CHANGELOG.md
Unescape Escape View File

@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data.transfer.sis-geotk-plugin
## [v1.3.0] 2021-03-25
Use of EPSG Dataset
Proxy support
## [v1.2.0] 2020-09-07
Upgrade to apache-sis 1.0

25
pom.xml
Unescape Escape View File

@ -9,7 +9,7 @@
</parent>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>sis-geotk-plugin</artifactId>
<version>1.2.0</version>
<version>1.3.0</version>
<name>Sis/GeoToolkit plugin</name>
<description>Apache Sis/Geotk plugin for data-transfer-service</description>
@ -80,6 +80,7 @@
<groupId>org.gcube.spatial.data</groupId>
<artifactId>sdi-library</artifactId>
<version>[1.0.0,1.0.2]</version>
<!-- <version>[1.0.0,2.0)</version> -->
</dependency>
<!-- SMARTGEARS -->
@ -120,10 +121,10 @@
<!-- SIS EXTERNALS -->
<!-- https://mvnrepository.com/artifact/com.esri.geometry/esri-geometry-api -->
<dependency>
<groupId>com.esri.geometry</groupId>
<artifactId>esri-geometry-api</artifactId>
</dependency>
<!-- <dependency> -->
<!-- <groupId>com.esri.geometry</groupId> -->
<!-- <artifactId>esri-geometry-api</artifactId> -->
<!-- </dependency> -->
<!-- https://mvnrepository.com/artifact/edu.ucar/cdm -->
<dependency>
@ -133,6 +134,20 @@
</dependency>
<!-- EPSG DATASET -->
<dependency>
<groupId>org.apache.sis.non-free</groupId>
<artifactId>sis-embedded-data</artifactId>
<version>${sis.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<scope>runtime</scope>
</dependency>
<!-- TEST -->

1
src/.gitignore vendored
Unescape Escape View File

@ -0,0 +1 @@
/.DS_Store

29
src/main/java/org/gcube/data/transfer/plugins/thredds/ISOMetadataMarshalling.java
Unescape Escape View File

@ -14,11 +14,40 @@ import org.apache.sis.xml.XML;
import org.opengis.metadata.Metadata;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ISOMetadataMarshalling {
private static MarshallerPool pool=null;
// static {
//
// File temp=new File(System.getProperty("java.io.tmpdir")+"/EPSG");
// temp.mkdirs();
//
// EmbeddedConnectionPoolDataSource ds = new EmbeddedConnectionPoolDataSource();
// ds.setConnectionAttributes("create=true");
//
// ds.setDatabaseName(temp.getAbsolutePath()+"/SpatialMetadata");
//
//// ds.setDataSourceName(arg0);
//// ds.setServerName("someHost");
//// ds.setPortNumber("1527");
//// ds.setDatabaseName("someDB");
//
// Configuration.current().setDatabase(new Supplier<DataSource>() {
// @Override
// public DataSource get() {
// return ds;
// }
// });
//
//
// }
@Synchronized
private static MarshallerPool getPool() throws JAXBException {

13
src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java
Unescape Escape View File

@ -71,7 +71,18 @@ public class ThreddsInstanceManager {
protected ThreddsInstanceManager(DataTransferContext context) {
log.warn("Instance Creation. Should happen only once. Loading information from context..");
this.ctx=context;
currentHostname=ctx.getCtx().container().configuration().hostname();
try {
log.info("Loading proxy configuration..");
currentHostname=ctx.getCtx().configuration().proxyAddress().hostname();
if(currentHostname==null||currentHostname.isEmpty()) throw new Exception("Proxy is : "+currentHostname);
}catch(Exception e) {
log.info("Unable to get proxy..",e);
currentHostname=ctx.getCtx().container().configuration().hostname();
}
log.info("Hostname to be used is "+currentHostname);
currentGHNId=ctx.getCtx().container().id();
String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml";

2
src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java
Unescape Escape View File

@ -20,7 +20,7 @@ public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
static final String PLUGIN_ID="SIS/GEOTK";
public static final String PLUGIN_ID="SIS/GEOTK";
public static final String SOURCE_PARAMETER="SOURCE_FILE";
public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY";

84
src/test/java/org/gcube/data/transfer/plugins/sis/PublishTest.java
Unescape Escape View File

@ -0,0 +1,84 @@
package org.gcube.data.transfer.plugins.sis;
import java.io.File;
import java.io.FilenameFilter;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory;
import org.gcube.smartgears.Constants;
import org.gcube.spatial.data.clients.model.engine.Engine;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.faults.RemoteException;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
public class PublishTest {
public static void main(String[] args) throws MalformedURLException, RemoteException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
TokenSetter.set("/pred4s/preprod/preVRE");
String hostname=
new URL(SDIAbstractPlugin.management().build().getConfiguration().getByEngine(Engine.TH_ENGINE).get(0).getBaseEndpoint()).getHost();
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("https://"+hostname);
File metadata=Paths.get("src/test/resources/toPublishMeta").toFile();
// for(File f:metadata.listFiles(new FilenameFilter() {
// @Override
// public boolean accept(File dir, String name) {
// return name.endsWith(".xml");
// }
// })) {
// for(File f:new File[] {
// new File("/Users/FabioISTI/Downloads/NASA_Surface_Air_Temperature_1950_2100_rcp45.nc")
// }) {
for(URL url:new URL[] {
new URL("https://thredds.d4science.org/thredds/fileServer/public/netcdf/ClimateChange/NASA_Precipitations_1950_2100_rcp45.nc")
}) {
// if not present, generate with sis/geotk
Destination toSetDestination=new Destination();
toSetDestination.setCreateSubfolders(true);
toSetDestination.setDestinationFileName("another_dataset");
toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
toSetDestination.setPersistenceId("thredds");
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
String fileLocation="public/netcdf/test_"+System.currentTimeMillis();
toSetDestination.setSubFolder(fileLocation);
PluginInvocation inv=new PluginInvocation(SISPluginFactory.PLUGIN_ID);
inv.setParameters(Collections.singletonMap(SISPluginFactory.VALIDATE_PARAMETER, "false"));
System.out.println(
// client.localFile(f, toSetDestination, Collections.singleton(new PluginInvocation(SISPluginFactory.PLUGIN_ID))));
client.httpSource(url, toSetDestination, inv));
}
}
}

46
src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java
Unescape Escape View File

@ -16,10 +16,11 @@ import javax.xml.bind.JAXBException;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.UnsupportedStorageException;
import org.apache.sis.xml.XML;
import org.gcube.data.transfer.plugins.thredds.ISOMetadataMarshalling;
import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin;
import org.opengis.metadata.Metadata;
import ucar.nc2.NetcdfFile;
import ucar.nc2.dataset.NetcdfDataset;
public class TestGetMetadata {
@ -39,38 +40,43 @@ public class TestGetMetadata {
// *********** NETCDF 3
"/Users/FabioISTI/Downloads/Aaptos_aaptos.nc"
"/Users/FabioISTI/Downloads/Aaptos_aaptos.nc",
"/Users/FabioISTI/Downloads/dataset-armor-3d-rep-monthly_20181115T1200Z_P20190301T0000Z.nc",
//"/Users/FabioISTI/Downloads/data_retro_Run_10026_911.nc",
//"/Users/FabioISTI/Downloads/2BPR6_2021_a_NRT_Gosud_V3.nc",
"/Users/FabioISTI/Downloads/GO_WTEP_2015_TRAJ.nc"
};
for(String f:toCheckFiles) {
System.out.println("checking "+f);
try{
check(false,f);
}catch(UnsupportedStorageException e) {
}catch(Exception e) {
System.err.println(e.getMessage());
// System.out.println("Trying opening "+f+" manually");
// open(f);
System.out.println("Trying opening "+f+" manually");
open(f);
}
}
System.out.println("Done");
}
// private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException {
// System.out.println("Opening "+path);
// NetcdfFile f=NetcdfFiles.open(path);
// System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId());
//
//
// NetcdfDataset ncDs=NetcdfDatasets.openDataset(path);
//// NcMLReaderNew.
////
// System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId());
//
//// NetcdfDatasets.openFile(path,null);
//
private static void open(String path) throws IOException, UnsupportedStorageException, DataStoreException {
System.out.println("Opening "+path);
NetcdfFile f=NetcdfFile.open(path);
System.out.println(f.getFileTypeDescription()+"\t"+f.getFileTypeId());
NetcdfDataset ncDs=NetcdfDataset.openDataset(path);
// NcMLReaderNew.
//
// }
System.out.println(ncDs.getFileTypeDescription()+"\t"+ncDs.getFileTypeId());
// NetcdfDatasets.openFile(path,null);
}
private static void check(boolean checkCopy,String src) throws UnsupportedStorageException, DataStoreException, IOException, NoSuchAlgorithmException, JAXBException {

33
src/test/java/org/gcube/data/transfer/plugins/sis/TokenSetter.java
Unescape Escape View File

@ -0,0 +1,33 @@
package org.gcube.data.transfer.plugins.sis;
import java.util.Properties;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TokenSetter {
private static Properties props=new Properties();
static{
try {
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
} catch (Exception e) {
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
}
}
public static void set(String scope){
try{
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
SecurityTokenProvider.instance.set(props.getProperty(scope));
}catch(Throwable e){
log.warn("Unable to set token for scope "+scope,e);
}
ScopeProvider.instance.set(scope);
}
}
Write Preview
Loading…
Cancel
Save