Initial import.
git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@131226 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
f924eed16d
commit
7ac5508a1b
|
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>data-transfer-library</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=false
|
||||
version=1
|
||||
|
|
@ -0,0 +1 @@
|
|||
${gcube.license}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
The gCube System - ${name}
|
||||
--------------------------------------------------
|
||||
|
||||
${description}
|
||||
|
||||
|
||||
${gcube.description}
|
||||
|
||||
${gcube.funding}
|
||||
|
||||
|
||||
Version
|
||||
--------------------------------------------------
|
||||
|
||||
${version} (${buildDate})
|
||||
|
||||
Please see the file named "changelog.xml" in this directory for the release notes.
|
||||
|
||||
|
||||
|
||||
Authors
|
||||
--------------------------------------------------
|
||||
|
||||
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
||||
|
||||
Maintainers
|
||||
-----------
|
||||
|
||||
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
||||
|
||||
|
||||
Download information
|
||||
--------------------------------------------------
|
||||
|
||||
Source code is available from SVN:
|
||||
${scm.url}
|
||||
|
||||
Binaries can be downloaded from the gCube website:
|
||||
${gcube.website}
|
||||
|
||||
Installation
|
||||
--------------------------------------------------
|
||||
|
||||
Installation documentation is available on-line in the gCube Wiki:
|
||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||
|
||||
Documentation
|
||||
--------------------------------------------------
|
||||
|
||||
Documentation is available on-line in the gCube Wiki:
|
||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||
|
||||
|
||||
Support
|
||||
--------------------------------------------------
|
||||
|
||||
Bugs and support requests can be reported in the gCube issue tracking tool:
|
||||
${gcube.issueTracking}
|
||||
|
||||
|
||||
Licensing
|
||||
--------------------------------------------------
|
||||
|
||||
This software is licensed under the terms you may find in the file named "LICENSE" in this directory.
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<ReleaseNotes>
|
||||
<Changeset component="data-transfer-library-1.0.0" date="2016-02-22">
|
||||
<Change>First Release</Change>
|
||||
</Changeset>
|
||||
</ReleaseNotes>
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
<id>servicearchive</id>
|
||||
<formats>
|
||||
<format>tar.gz</format>
|
||||
</formats>
|
||||
<baseDirectory>/</baseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>${distroDirectory}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<useDefaultExcludes>true</useDefaultExcludes>
|
||||
<includes>
|
||||
<include>README</include>
|
||||
<include>LICENSE</include>
|
||||
<include>changelog.xml</include>
|
||||
</includes>
|
||||
<fileMode>755</fileMode>
|
||||
<filtered>true</filtered>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<files>
|
||||
<file>
|
||||
<source>target/${build.finalName}.${project.packaging}</source>
|
||||
<outputDirectory>/${artifactId}</outputDirectory>
|
||||
</file>
|
||||
</files>
|
||||
</assembly>
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Resource>
|
||||
<ID></ID>
|
||||
<Type>Service</Type>
|
||||
<Profile>
|
||||
<Description>${description}</Description>
|
||||
<Class>DataTransfer</Class>
|
||||
<Name>${artifactId}</Name>
|
||||
<Version>1.0.0</Version>
|
||||
<Packages>
|
||||
<Software>
|
||||
<Description>${description}</Description>
|
||||
<Name>${artifactId}</Name>
|
||||
<Version>${version}</Version>
|
||||
<MavenCoordinates>
|
||||
<groupId>${groupId}</groupId>
|
||||
<artifactId>${artifactId}</artifactId>
|
||||
<version>${version}</version>
|
||||
</MavenCoordinates>
|
||||
<Type>library</Type>
|
||||
<Files>
|
||||
<File>${build.finalName}.jar</File>
|
||||
</Files>
|
||||
</Software>
|
||||
</Packages>
|
||||
</Profile>
|
||||
</Resource>
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.tools</groupId>
|
||||
<artifactId>maven-parent</artifactId>
|
||||
<version>LATEST</version>
|
||||
</parent>
|
||||
<groupId>org.gcube.data.transfer</groupId>
|
||||
<artifactId>data-transfer-library</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<name>DataTransfer-library</name>
|
||||
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.distribution</groupId>
|
||||
<artifactId>gcube-bom</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<!-- GCUBE -->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>authorization-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.core</groupId>
|
||||
<artifactId>common-gcube-calls</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.transfer</groupId>
|
||||
<artifactId>data-transfer-model</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<!-- STORAGE -->
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-core</artifactId>
|
||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-wrapper</artifactId>
|
||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
<!-- EXTERNAL -->
|
||||
<!-- lombok -->
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>0.11.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- JERSEY -->
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.core</groupId>
|
||||
<artifactId>jersey-client</artifactId>
|
||||
<version>2.22.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.media</groupId>
|
||||
<artifactId>jersey-media-jaxb</artifactId>
|
||||
<version>2.18</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- TEST -->
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.10</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Test log binding -->
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.0.13</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||
import org.gcube.data.transfer.library.transferers.Transferer;
|
||||
import org.gcube.data.transfer.library.transferers.TransfererBuilder;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
|
||||
import lombok.Synchronized;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class DataTransferClient {
|
||||
|
||||
private Transferer transferer=null;
|
||||
|
||||
private DataTransferClient(Transferer transferer) {
|
||||
this.transferer=transferer;
|
||||
}
|
||||
|
||||
public static DataTransferClient getInstanceByEndpoint(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
||||
log.debug("Getting transferer for endpoint : "+endpoint);
|
||||
return new DataTransferClient(TransfererBuilder.getTransfererByHost(endpoint));
|
||||
}
|
||||
|
||||
public static DataTransferClient getInstanceByNodeId(String id) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
||||
log.debug("Getting transferer for nodeId : "+id);
|
||||
return new DataTransferClient(TransfererBuilder.getTransfererByhostingNodeId(id));
|
||||
}
|
||||
|
||||
|
||||
public TransferCapabilities getDestinationCapabilities() throws InitializationException{
|
||||
return this.transferer.getDestinationCapabilities();
|
||||
}
|
||||
|
||||
|
||||
public TransferResult localFile(String path,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
return localFile(path,dest,Collections.<PluginInvocation> emptySet());
|
||||
}
|
||||
|
||||
public TransferResult localFile(File file,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
return localFile(file,dest,Collections.<PluginInvocation> emptySet());
|
||||
}
|
||||
|
||||
public TransferResult httpSource(String url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
return httpSource(url,dest,Collections.<PluginInvocation> emptySet());
|
||||
}
|
||||
|
||||
public TransferResult httpSource(URL url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
return httpSource(url,dest,Collections.<PluginInvocation> emptySet());
|
||||
}
|
||||
|
||||
public TransferResult storageId(String id,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
return storageId(id,dest,Collections.<PluginInvocation> emptySet());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Synchronized("transferer")
|
||||
public TransferResult localFile(String path,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||
log.debug("Sending local file {} to {} : {} - {}",path,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||
transferer.localFile(path);
|
||||
return doTheTransfer(dest, pluginInvocations);
|
||||
}
|
||||
|
||||
@Synchronized("transferer")
|
||||
public TransferResult localFile(File file,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||
log.debug("Sending local file {} to {} : {} - {}",file.getAbsolutePath(),transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||
transferer.localFile(file);
|
||||
return doTheTransfer(dest, pluginInvocations);
|
||||
}
|
||||
|
||||
@Synchronized("transferer")
|
||||
public TransferResult httpSource(String url,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||
log.debug("Passed url string : "+url);
|
||||
try{
|
||||
return this.httpSource(new URL(url),dest,pluginInvocations);
|
||||
}catch(MalformedURLException e){
|
||||
throw new InvalidSourceException("Invalid url : "+url);
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized("transferer")
|
||||
public TransferResult httpSource(URL url,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||
log.debug("Sending from url {} to {} : {} - {}",url,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||
transferer.fromURL(url);
|
||||
return doTheTransfer(dest, pluginInvocations);
|
||||
}
|
||||
|
||||
@Synchronized("transferer")
|
||||
public TransferResult storageId(String id,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||
log.debug("Sending from storage id {} to {} : {} - {}",id,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||
transferer.storageFileId(id);
|
||||
return doTheTransfer(dest, pluginInvocations);
|
||||
}
|
||||
|
||||
private TransferResult doTheTransfer(Destination dest,Set<PluginInvocation> pluginInvocations) throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
try{
|
||||
transferer.setDestination(dest);
|
||||
transferer.setInvocations(pluginInvocations);
|
||||
return transferer.transfer();
|
||||
}finally{
|
||||
transferer.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
|
||||
import org.gcube.data.transfer.library.model.Source;
|
||||
|
||||
@Data
|
||||
public class TransferResult {
|
||||
|
||||
@NonNull
|
||||
private Source source;
|
||||
@NonNull
|
||||
private String destinationHostName;
|
||||
@NonNull
|
||||
private Long elapsedTime;
|
||||
@NonNull
|
||||
private Long transferedBytes;
|
||||
@NonNull
|
||||
private String remotePath;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package org.gcube.data.transfer.library.caches;
|
||||
|
||||
import lombok.Synchronized;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.gcube.data.transfer.library.client.Client;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
|
||||
@Slf4j
|
||||
public class CapabilitiesCache extends TTLCache<TransferCapabilities> {
|
||||
|
||||
private static CapabilitiesCache instance=null;
|
||||
|
||||
// @Synchronized
|
||||
// public static CapabilitiesCache getInstance(){
|
||||
// if(instance==null)instance=new CapabilitiesCache();
|
||||
// return instance;
|
||||
// }
|
||||
|
||||
private CapabilitiesCache(){
|
||||
super(5*60*1000l,2*60*1000l,"Capabilities");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected TransferCapabilities getNew(String id) throws Exception{
|
||||
log.debug("Getting capabilties for host "+id);
|
||||
return new Client(id).getCapabilties();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
package org.gcube.data.transfer.library.caches;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import lombok.Synchronized;
|
||||
import lombok.ToString;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@ToString
|
||||
@Slf4j
|
||||
public abstract class TTLCache<T> {
|
||||
|
||||
|
||||
// STATIC
|
||||
|
||||
// private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1,new ThreadFactory(){
|
||||
// public Thread newThread(Runnable r) {
|
||||
// Thread t = Executors.defaultThreadFactory().newThread(r);
|
||||
// t.setDaemon(true);
|
||||
// return t;
|
||||
// }
|
||||
// });
|
||||
private static final HashSet<TTLCache<?>> createdMaps=new HashSet<TTLCache<?>>();
|
||||
|
||||
|
||||
static {
|
||||
// scheduler.scheduleAtFixedRate(new Runnable() {
|
||||
//
|
||||
//
|
||||
//
|
||||
// @Override
|
||||
// public void run() {
|
||||
// log.debug("Running Maps Cleaner, maps count : "+createdMaps.size());
|
||||
// int removed=0;
|
||||
// for(TTLCache<?> theMap:createdMaps)
|
||||
// theMap.purgeItems();
|
||||
// log.debug("Removed "+removed+" old tickets");
|
||||
//
|
||||
// }
|
||||
// }, 3, 3, TimeUnit.MINUTES);
|
||||
//
|
||||
}
|
||||
|
||||
|
||||
//************************* CACHE
|
||||
|
||||
|
||||
|
||||
protected TTLCache(Long cacheKeepAliveTime, Long objectTTL, String cacheName) {
|
||||
this.cacheKeepAliveTime = cacheKeepAliveTime;
|
||||
this.objectTTL = objectTTL;
|
||||
this.cacheName = cacheName;
|
||||
createdMaps.add(this);
|
||||
log.debug("Created Cache "+this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private ConcurrentHashMap<String,TTLContainer<T>> theMap=new ConcurrentHashMap<>();
|
||||
|
||||
private Long cacheKeepAliveTime;
|
||||
private Long objectTTL;
|
||||
private String cacheName;
|
||||
|
||||
@Synchronized
|
||||
public T getObject(String id) throws Exception{
|
||||
if(!theMap.contains(id)||System.currentTimeMillis()-theMap.get(id).getCreationTime()>objectTTL)
|
||||
theMap.put(id, new TTLContainer<T>(getNew(id)));
|
||||
return theMap.get(id).getTheObject();
|
||||
}
|
||||
|
||||
|
||||
private int purgeItems(){
|
||||
log.debug(cacheName+" Purging objects, keep alive time is "+cacheKeepAliveTime);
|
||||
int totalCount=theMap.size();
|
||||
int removed=0;
|
||||
for(Entry<String,TTLContainer<T>> entry:theMap.entrySet())
|
||||
if(System.currentTimeMillis()-entry.getValue().getLastUsageTime()>cacheKeepAliveTime){
|
||||
theMap.remove(entry.getKey());
|
||||
removed++;
|
||||
}
|
||||
log.debug(cacheName+" Removed "+removed+" out of "+totalCount);
|
||||
return removed;
|
||||
}
|
||||
|
||||
protected abstract T getNew(String id) throws Exception;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package org.gcube.data.transfer.library.caches;
|
||||
|
||||
public class TTLContainer<T> {
|
||||
|
||||
private long lastUsageTime=System.currentTimeMillis();
|
||||
private long creationTime=System.currentTimeMillis();
|
||||
|
||||
private T theObject;
|
||||
public TTLContainer(T theObject) {
|
||||
this.theObject = theObject;
|
||||
}
|
||||
|
||||
|
||||
private void update(){
|
||||
lastUsageTime=System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public T getTheObject(){
|
||||
update();
|
||||
return theObject;
|
||||
}
|
||||
|
||||
public long getLastUsageTime() {
|
||||
return lastUsageTime;
|
||||
}
|
||||
|
||||
public long getCreationTime() {
|
||||
return creationTime;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package org.gcube.data.transfer.library.client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import javax.ws.rs.client.ClientRequestContext;
|
||||
import javax.ws.rs.client.ClientRequestFilter;
|
||||
|
||||
import org.gcube.common.calls.Call;
|
||||
import org.gcube.common.calls.Interceptors;
|
||||
import org.gcube.common.calls.Request;
|
||||
import org.gcube.data.transfer.library.utils.ScopeUtils;
|
||||
|
||||
public class AuthorizationFilter implements ClientRequestFilter {
|
||||
|
||||
|
||||
@Override
|
||||
public void filter(final ClientRequestContext rc) throws IOException {
|
||||
if (ScopeUtils.getCurrentScope()!=null){
|
||||
Request requestContext = Interceptors.executeRequestChain(new Call());
|
||||
|
||||
for (Entry<String, String> entry: requestContext.getHeaders()){
|
||||
rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package org.gcube.data.transfer.library.client;
|
||||
|
||||
import javax.ws.rs.client.ClientBuilder;
|
||||
import javax.ws.rs.client.Entity;
|
||||
import javax.ws.rs.client.WebTarget;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.CommunicationException;
|
||||
import org.gcube.data.transfer.library.faults.DataTransferException;
|
||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.model.ServiceConstants;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
import org.gcube.data.transfer.model.TransferRequest;
|
||||
import org.gcube.data.transfer.model.TransferTicket;
|
||||
import org.glassfish.jersey.client.ClientConfig;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class Client {
|
||||
|
||||
private static ClientConfig config=null;
|
||||
|
||||
static{
|
||||
log.debug("Creating configuration ..");
|
||||
config=new ClientConfig();
|
||||
config.register(AuthorizationFilter.class);
|
||||
}
|
||||
|
||||
private String endpoint;
|
||||
|
||||
private WebTarget rootTarget;
|
||||
|
||||
public Client(String endpoint) throws ServiceNotFoundException{
|
||||
try{
|
||||
log.debug("Creating client for base "+endpoint);
|
||||
this.endpoint=endpoint+"";
|
||||
rootTarget= ClientBuilder.newClient(config).target(endpoint).path("data-transfer-service").path(ServiceConstants.APPLICATION_PATH);
|
||||
// checkResponse(rootTarget.request().get());
|
||||
|
||||
log.debug("Root Taget IS {} ",rootTarget.getUri());
|
||||
}catch(Exception e){
|
||||
throw new ServiceNotFoundException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public String getEndpoint() {
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
public TransferCapabilities getCapabilties() throws CommunicationException{
|
||||
WebTarget capabilitiesTarget=rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME);
|
||||
log.debug("Getting capabilities from {}, path is {} ",endpoint,capabilitiesTarget.getUri());
|
||||
try{
|
||||
Response resp=capabilitiesTarget.request().accept(MediaType.APPLICATION_XML_TYPE).get();
|
||||
checkResponse(resp);
|
||||
return resp.readEntity(TransferCapabilities.class);
|
||||
}catch(Exception e){
|
||||
throw new CommunicationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public TransferTicket submit(TransferRequest request) throws RemoteServiceException{
|
||||
log.debug("Sending request {} to {}",request,endpoint);
|
||||
try{
|
||||
Response resp=rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML));
|
||||
checkResponse(resp);
|
||||
return resp.readEntity(TransferTicket.class);
|
||||
}catch(Exception e){
|
||||
throw new RemoteServiceException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public TransferTicket getTransferStatus(String transferId) throws RemoteServiceException{
|
||||
log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint);
|
||||
try{
|
||||
Response resp=rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get();
|
||||
checkResponse(resp);
|
||||
return resp.readEntity(TransferTicket.class);
|
||||
}catch(Exception e){
|
||||
throw new RemoteServiceException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void checkResponse(Response toCheck) throws Exception{
|
||||
switch(toCheck.getStatusInfo().getFamily()){
|
||||
case SUCCESSFUL : break;
|
||||
default : throw new Exception("Unexpected Response code : "+toCheck.getStatus());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class CommunicationException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public CommunicationException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CommunicationException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CommunicationException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CommunicationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CommunicationException(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public abstract class DataTransferException extends Exception {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 8488351139669552406L;
|
||||
|
||||
public DataTransferException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DataTransferException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DataTransferException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DataTransferException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DataTransferException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class DestinationNotSetException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 4792958203546912779L;
|
||||
|
||||
public DestinationNotSetException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DestinationNotSetException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DestinationNotSetException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DestinationNotSetException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public DestinationNotSetException(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class FailedTransferException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -807695677781676381L;
|
||||
|
||||
public FailedTransferException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public FailedTransferException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public FailedTransferException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public FailedTransferException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public FailedTransferException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class HostingNodeNotFoundException extends InitializationException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 5786009015981659074L;
|
||||
|
||||
public HostingNodeNotFoundException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public HostingNodeNotFoundException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public HostingNodeNotFoundException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public HostingNodeNotFoundException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public HostingNodeNotFoundException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class InitializationException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 2204462227670167203L;
|
||||
|
||||
public InitializationException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InitializationException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InitializationException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InitializationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InitializationException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class InvalidDestinationException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -1882334325516885722L;
|
||||
|
||||
public InvalidDestinationException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidDestinationException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidDestinationException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidDestinationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidDestinationException(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class InvalidSourceException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 7631518288856917202L;
|
||||
|
||||
public InvalidSourceException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidSourceException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidSourceException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidSourceException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidSourceException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class RemoteServiceException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 5320978791087129997L;
|
||||
|
||||
public RemoteServiceException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public RemoteServiceException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public RemoteServiceException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public RemoteServiceException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public RemoteServiceException(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class ServiceNotFoundException extends InitializationException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -3494011839783891174L;
|
||||
|
||||
public ServiceNotFoundException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public ServiceNotFoundException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public ServiceNotFoundException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public ServiceNotFoundException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public ServiceNotFoundException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class SourceNotSetException extends DataTransferException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -5868191252361823341L;
|
||||
|
||||
public SourceNotSetException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SourceNotSetException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SourceNotSetException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SourceNotSetException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SourceNotSetException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package org.gcube.data.transfer.library.faults;
|
||||
|
||||
public class UnreachableNodeException extends InitializationException {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 8739586712271226827L;
|
||||
|
||||
public UnreachableNodeException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public UnreachableNodeException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public UnreachableNodeException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public UnreachableNodeException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public UnreachableNodeException(String message, Throwable cause,
|
||||
boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package org.gcube.data.transfer.library.model;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
|
||||
public class LocalSource extends Source<File> {
|
||||
|
||||
private File theFile=null;
|
||||
|
||||
public LocalSource(File theFile) throws InvalidSourceException {
|
||||
super();
|
||||
if(theFile==null)throw new InvalidSourceException("File cannot be null");
|
||||
this.theFile = theFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validate() throws InvalidSourceException {
|
||||
if(!theFile.exists()) throw new InvalidSourceException("File "+theFile.getAbsolutePath()+"doesn't exist");
|
||||
if(!theFile.canRead()) throw new InvalidSourceException("Unable to read from file "+theFile.getAbsolutePath());
|
||||
if(theFile.isDirectory()) throw new InvalidSourceException("Transfer of directory is not yet supported");
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepare() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clean() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getTheSource() {
|
||||
return theFile;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package org.gcube.data.transfer.library.model;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public abstract class Source<T>{
|
||||
|
||||
public abstract boolean validate() throws InvalidSourceException;
|
||||
|
||||
public abstract void prepare();
|
||||
|
||||
public abstract void clean();
|
||||
|
||||
public abstract T getTheSource();
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package org.gcube.data.transfer.library.model;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||
|
||||
public class StorageSource extends Source<String> {
|
||||
|
||||
private String id=null;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public StorageSource(String id) throws InvalidSourceException {
|
||||
super();
|
||||
if(id==null) throw new InvalidSourceException("Storage id cannot be null");
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validate() throws InvalidSourceException {
|
||||
try{
|
||||
if(!StorageUtils.checkStorageId(id)) throw new Exception("Not valid");
|
||||
}catch(Exception e){
|
||||
throw new InvalidSourceException("Invalid storage ID "+id);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepare() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clean() {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTheSource() {
|
||||
return id;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
package org.gcube.data.transfer.library.model;
|
||||
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
|
||||
public class URLSource extends Source<URL> {
|
||||
|
||||
|
||||
private URL theURL=null;
|
||||
|
||||
|
||||
public URLSource(URL theURL) throws InvalidSourceException {
|
||||
super();
|
||||
if(theURL==null) throw new InvalidSourceException("URL Source cannot be null");
|
||||
this.theURL = theURL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validate() throws InvalidSourceException {
|
||||
try{
|
||||
HttpURLConnection conn=(HttpURLConnection) theURL.openConnection();
|
||||
conn.setConnectTimeout(3000);
|
||||
conn.setRequestMethod("HEAD");
|
||||
int responseCode = conn.getResponseCode();
|
||||
return (200 <= responseCode && responseCode <= 399);
|
||||
}catch(Exception e){
|
||||
throw new InvalidSourceException("Unable to contact URL "+theURL,e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepare() {
|
||||
// nothing to do
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clean() {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.lang.Object#toString()
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("URLSource [theURL=");
|
||||
builder.append(theURL);
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL getTheSource() {
|
||||
return theURL;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package org.gcube.data.transfer.library.transferers;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
import org.gcube.data.transfer.library.client.Client;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.model.LocalSource;
|
||||
import org.gcube.data.transfer.library.model.StorageSource;
|
||||
import org.gcube.data.transfer.library.model.URLSource;
|
||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||
import org.gcube.data.transfer.model.TransferRequest;
|
||||
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
||||
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
||||
|
||||
@Slf4j
|
||||
public class HTTPTransferer extends Transferer {
|
||||
|
||||
HTTPTransferer(Client client) {
|
||||
super(client);
|
||||
}
|
||||
|
||||
private URL link;
|
||||
|
||||
private String toDeleteStorageId=null;
|
||||
|
||||
@Override
|
||||
protected TransferRequest prepareRequest()throws InitializationException {
|
||||
return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT),this.destination,this.invocations);
|
||||
}
|
||||
|
||||
|
||||
private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{
|
||||
if(source instanceof LocalSource){
|
||||
toDeleteStorageId=StorageUtils.putOntoStorage(((LocalSource)source).getTheSource());
|
||||
return new URL(StorageUtils.getUrlById(toDeleteStorageId));
|
||||
}else if (source instanceof StorageSource){
|
||||
return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource()));
|
||||
}else if (source instanceof URLSource){
|
||||
return ((URLSource)source).getTheSource();
|
||||
}else throw new InvalidSourceException("Source cannot be handled "+source);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void prepare() throws InitializationException {
|
||||
try{
|
||||
link=getHttpLink();
|
||||
log.debug("Obtained link "+link);
|
||||
super.prepare();
|
||||
}catch(RemoteBackendException e){
|
||||
throw new InitializationException(e);
|
||||
}catch(FileNotFoundException e){
|
||||
throw new InitializationException(e);
|
||||
}catch(InvalidSourceException e){
|
||||
throw new InitializationException(e);
|
||||
}catch (MalformedURLException e){
|
||||
throw new InitializationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void clean() {
|
||||
// TODO Auto-generated method stub
|
||||
super.clean();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
package org.gcube.data.transfer.library.transferers;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.library.client.Client;
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.library.model.LocalSource;
|
||||
import org.gcube.data.transfer.library.model.Source;
|
||||
import org.gcube.data.transfer.library.model.StorageSource;
|
||||
import org.gcube.data.transfer.library.model.URLSource;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
import org.gcube.data.transfer.model.TransferRequest;
|
||||
import org.gcube.data.transfer.model.TransferTicket;
|
||||
import org.gcube.data.transfer.model.TransferTicket.Status;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public abstract class Transferer {
|
||||
|
||||
protected Client client;
|
||||
|
||||
protected Transferer(Client client){
|
||||
this.client=client;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
protected Source source=null;
|
||||
protected boolean prepared=false;
|
||||
protected Destination destination=null;
|
||||
protected Set<PluginInvocation> invocations=null;
|
||||
|
||||
|
||||
public void setDestination(Destination destination) {
|
||||
this.destination = destination;
|
||||
}
|
||||
|
||||
public void setInvocations(Set<PluginInvocation> invocations) {
|
||||
this.invocations = invocations;
|
||||
}
|
||||
|
||||
public Transferer localFile(File f) throws InvalidSourceException{
|
||||
source=new LocalSource(f);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Transferer localFile(String path) throws InvalidSourceException{
|
||||
try{
|
||||
File f=new File(path);
|
||||
return localFile(f);
|
||||
}catch(NullPointerException e){
|
||||
throw new InvalidSourceException("Path is null",e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Transferer storageFileId(String fileId) throws InvalidSourceException{
|
||||
source=new StorageSource(fileId);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Transferer fromURL(URL sourceURL) throws InvalidSourceException{
|
||||
source=new URLSource(sourceURL);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
public void reset(){
|
||||
this.destination=null;
|
||||
this.source=null;
|
||||
this.invocations=null;
|
||||
}
|
||||
|
||||
public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
try{
|
||||
checkDestination();
|
||||
checkSource();
|
||||
checkPluginInvocations();
|
||||
prepare();
|
||||
TransferRequest request=prepareRequest();
|
||||
log.debug("Request is {}, sending it to {}",request,client.getEndpoint());
|
||||
TransferResult result=doTheTransfer(request);
|
||||
return result;
|
||||
}finally{
|
||||
clean();
|
||||
}
|
||||
}
|
||||
|
||||
protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{
|
||||
try{
|
||||
TransferTicket submissionResponse= client.submit(request);
|
||||
boolean continuePolling=true;
|
||||
TransferTicket ticket=null;
|
||||
do{
|
||||
ticket=client.getTransferStatus(submissionResponse.getId());
|
||||
System.out.println("Status : "+ticket);
|
||||
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
||||
try{
|
||||
Thread.sleep(500);
|
||||
}catch(InterruptedException e){}
|
||||
}while(continuePolling);
|
||||
if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage());
|
||||
if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage());
|
||||
long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis();
|
||||
return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
|
||||
}catch(RemoteServiceException e){
|
||||
throw new FailedTransferException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkDestination() throws InvalidDestinationException,DestinationNotSetException{
|
||||
if(destination==null) throw new DestinationNotSetException();
|
||||
// perform checks
|
||||
}
|
||||
|
||||
protected void checkSource() throws SourceNotSetException, InvalidSourceException{
|
||||
if(source==null) throw new SourceNotSetException();
|
||||
source.validate();
|
||||
}
|
||||
|
||||
protected void checkPluginInvocations(){
|
||||
|
||||
}
|
||||
|
||||
protected abstract TransferRequest prepareRequest() throws InitializationException;
|
||||
|
||||
|
||||
protected void prepare() throws InitializationException{
|
||||
prepared=true;
|
||||
}
|
||||
protected void clean(){
|
||||
|
||||
}
|
||||
|
||||
public TransferCapabilities getDestinationCapabilities() throws InitializationException {
|
||||
try{
|
||||
return client.getCapabilties();
|
||||
}catch(Exception e){
|
||||
throw new InitializationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package org.gcube.data.transfer.library.transferers;
|
||||
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.gcube.common.resources.gcore.HostingNode;
|
||||
import org.gcube.data.transfer.library.caches.CapabilitiesCache;
|
||||
import org.gcube.data.transfer.library.client.Client;
|
||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
@Slf4j
|
||||
public class TransfererBuilder {
|
||||
|
||||
private static final int timeout=10*1000;
|
||||
|
||||
|
||||
//e.g. http://pc-fabio.isti.cnr.it:8080/data-transfer-service/gcube/service
|
||||
|
||||
public static Transferer getTransfererByHost(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
||||
log.debug("Get transferer by Host "+endpoint);
|
||||
try{
|
||||
URL url=new URL(endpoint);
|
||||
|
||||
String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort();
|
||||
//TODO Implement checks
|
||||
|
||||
// if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout);
|
||||
// String finalHost=host;
|
||||
// if(!finalHost.endsWith(ServiceConstants.APPLICATION_PATH)){
|
||||
// // adjust host
|
||||
// finalHost=finalHost+(host.endsWith("/")?"":"/")+"data-transfer-service"+ServiceConstants.APPLICATION_PATH;
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost);
|
||||
// log.debug("Host is ok, getting targetCapabilities");
|
||||
// TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl);
|
||||
|
||||
// TODO determine method by capabilities checking
|
||||
|
||||
return new HTTPTransferer(new Client(baseUrl));
|
||||
}catch(Exception e){
|
||||
throw new ServiceNotFoundException(e);
|
||||
}
|
||||
}
|
||||
public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
||||
String hostname=retrieveHostnameByNodeId(hostId);
|
||||
return getTransfererByHost(hostname);
|
||||
}
|
||||
|
||||
|
||||
private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{
|
||||
SimpleQuery query = queryFor(HostingNode.class);
|
||||
|
||||
query.addCondition("$resource/ID/text() eq '"+nodeId+"'");
|
||||
|
||||
|
||||
List<HostingNode> found= clientFor(HostingNode.class).submit(query);
|
||||
|
||||
if(found.isEmpty()) throw new HostingNodeNotFoundException("No Hosting node with the id "+nodeId);
|
||||
|
||||
return "http://"+found.get(0).profile().description().name();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package org.gcube.data.transfer.library.utils;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class ScopeUtils {
|
||||
|
||||
public static String getCurrentScope(){
|
||||
try{
|
||||
String token=SecurityTokenProvider.instance.get();
|
||||
log.debug("Token is : "+token);
|
||||
if(token==null) throw new Exception("Security Token is null");
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
return entry.getContext();
|
||||
}catch(Exception e ){
|
||||
log.debug("Unable to resolve token, checking scope provider..",e);
|
||||
return ScopeProvider.instance.get();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package org.gcube.data.transfer.library.utils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.bson.types.ObjectId;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||
|
||||
@Slf4j
|
||||
public class StorageUtils {
|
||||
|
||||
public static final IClient getClient(){
|
||||
return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient();
|
||||
}
|
||||
|
||||
|
||||
//return Id
|
||||
public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{
|
||||
IClient client=getClient();
|
||||
log.debug("Uploading local file "+source.getAbsolutePath());
|
||||
return client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString());
|
||||
}
|
||||
|
||||
public static final boolean checkStorageId(String id){
|
||||
return getClient().getHttpUrl().RFile(id)!=null;
|
||||
}
|
||||
|
||||
public static final String getUrlById(String id){
|
||||
IClient client=getClient();
|
||||
log.debug("Id is "+id);
|
||||
return client.getHttpUrl().RFile(id);
|
||||
}
|
||||
|
||||
public static final void removeById(String id){
|
||||
IClient client=getClient();
|
||||
client.remove().RFile(id);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package org.gcube.data.transfer.library.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.UUID;
|
||||
|
||||
public class Utils {
|
||||
|
||||
/**
|
||||
* Pings a HTTP URL. This effectively sends a HEAD request and returns <code>true</code> if the response code is in
|
||||
* the 200-399 range.
|
||||
* @param url The HTTP URL to be pinged.
|
||||
* @param timeout The timeout in millis for both the connection timeout and the response read timeout. Note that
|
||||
* the total timeout is effectively two times the given timeout.
|
||||
* @return <code>true</code> if the given HTTP URL has returned response code 200-399 on a HEAD request within the
|
||||
* given timeout, otherwise <code>false</code>.
|
||||
*/
|
||||
public static boolean pingURL(String url, int timeout) {
|
||||
url = url.replaceFirst("^https", "http"); // Otherwise an exception may be thrown on invalid SSL certificates.
|
||||
|
||||
try {
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
connection.setConnectTimeout(timeout);
|
||||
connection.setReadTimeout(timeout);
|
||||
connection.setRequestMethod("HEAD");
|
||||
int responseCode = connection.getResponseCode();
|
||||
return (200 <= responseCode && responseCode <= 399);
|
||||
} catch (IOException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static final String getUniqueString(){
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
|
||||
public class ErrorReport extends TransferReport {
|
||||
|
||||
String id;
|
||||
|
||||
|
||||
public ErrorReport(String id) {
|
||||
super(null);
|
||||
this.id=id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String print() {
|
||||
return "ERROR : "+id;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
import org.gcube.data.transfer.library.TransferReport.ReportType;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||
|
||||
import ch.qos.logback.core.util.ExecutorServiceUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class MultipleTransferBenchmark {
|
||||
|
||||
|
||||
static List<String> endpoints;
|
||||
static List<String> nodeIds;
|
||||
|
||||
|
||||
static List<String> files;
|
||||
static List<String> uris;
|
||||
static List<String> storageIds;
|
||||
|
||||
static ExecutorService executor=ExecutorServiceUtil.newExecutorService();
|
||||
|
||||
|
||||
static CountDownLatch doneSignal;
|
||||
|
||||
static ArrayList<TransferReport> reports=new ArrayList<>();
|
||||
|
||||
|
||||
static String scope="/gcube/devsec/devVRE";
|
||||
|
||||
static{
|
||||
endpoints=Arrays.asList(new String[]{
|
||||
// "http://node4-d-d4s.d4science.org:80/data-transfer-service/gcube/service",
|
||||
"http://node3-d-d4s.d4science.org:80/data-transfer-service/gcube/service"
|
||||
});
|
||||
doneSignal=new CountDownLatch(endpoints.size());
|
||||
|
||||
|
||||
|
||||
nodeIds=new ArrayList<>();
|
||||
files=new ArrayList<>();
|
||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||
uris=new ArrayList<>();
|
||||
uris.add("http://goo.gl/r5jFZ9");
|
||||
storageIds=new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) throws InitializationException{
|
||||
TokenSetter.set(scope);
|
||||
Map<String,DataTransferClient> clients=new HashMap<>();
|
||||
for(String endpoint:endpoints)
|
||||
clients.put(endpoint,DataTransferClient.getInstanceByEndpoint(endpoint));
|
||||
for(String id:nodeIds)
|
||||
clients.put(id,DataTransferClient.getInstanceByEndpoint(id));
|
||||
|
||||
|
||||
|
||||
|
||||
HashSet<DataTransferClient> startedTests=new HashSet<>();
|
||||
for(final Entry<String,DataTransferClient> entry:clients.entrySet()){
|
||||
if(!startedTests.contains(entry.getValue())){
|
||||
startedTests.add(entry.getValue());
|
||||
|
||||
executor.execute(new Runnable(){
|
||||
|
||||
final DataTransferClient dt=entry.getValue();
|
||||
final String id=entry.getKey();
|
||||
@Override
|
||||
public void run() {
|
||||
try{
|
||||
TransferReport report=new TransferReport(dt.getDestinationCapabilities());
|
||||
TokenSetter.set(scope);
|
||||
|
||||
log.debug("Sending files to {} ",dt.getDestinationCapabilities());
|
||||
|
||||
Destination dest=new Destination("banchmarkoutputFile");
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
|
||||
for(String f:files){
|
||||
try {
|
||||
TransferResult res=dt.localFile(f,dest);
|
||||
report.addReport(ReportType.local,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||
} catch (InvalidSourceException | SourceNotSetException
|
||||
| FailedTransferException
|
||||
| InitializationException e) {
|
||||
log.error("Unable to send file {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("Sending uris to {}",dt.getDestinationCapabilities());
|
||||
for(String f:uris){
|
||||
try {
|
||||
TransferResult res=dt.httpSource(f,dest);
|
||||
report.addReport(ReportType.uri,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||
} catch (InvalidSourceException | SourceNotSetException
|
||||
| FailedTransferException
|
||||
| InitializationException e) {
|
||||
log.error("Unable to send uri {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("Sending storageIds to {}",dt.getDestinationCapabilities());
|
||||
for(String f:storageIds){
|
||||
try {
|
||||
TransferResult res=dt.storageId(f,dest);
|
||||
report.addReport(ReportType.storage,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||
} catch (InvalidSourceException | SourceNotSetException
|
||||
| FailedTransferException
|
||||
| InitializationException e) {
|
||||
log.error("Unable to send storageId {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||
}
|
||||
}
|
||||
reports.add(report);
|
||||
}catch(Exception e){
|
||||
reports.add(new ErrorReport(id));
|
||||
}finally{
|
||||
doneSignal.countDown();
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
doneSignal.await();
|
||||
} catch (InterruptedException e) {
|
||||
|
||||
}
|
||||
System.out.println("*****************************");
|
||||
for(TransferReport r:reports)System.out.println(r.print());
|
||||
|
||||
ExecutorServiceUtil.shutdown(executor);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||
|
||||
public class StorageTest {
|
||||
|
||||
static String scope="/gcube/devsec";
|
||||
|
||||
public static void main(String[] args) throws RemoteBackendException, FileNotFoundException {
|
||||
ScopeProvider.instance.set(scope);
|
||||
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
||||
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
||||
System.out.println(StorageUtils.getUrlById(id));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
||||
import javax.ws.rs.client.ClientBuilder;
|
||||
import javax.ws.rs.client.WebTarget;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.data.transfer.library.client.AuthorizationFilter;
|
||||
import org.gcube.data.transfer.library.client.Client;
|
||||
import org.gcube.data.transfer.library.faults.CommunicationException;
|
||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||
import org.gcube.data.transfer.model.ServiceConstants;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
import org.gcube.data.transfer.model.TransferRequest;
|
||||
import org.gcube.data.transfer.model.TransferTicket;
|
||||
import org.gcube.data.transfer.model.TransferTicket.Status;
|
||||
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
||||
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
||||
import org.glassfish.jersey.client.ClientConfig;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
public class TestClientCalls {
|
||||
|
||||
static String hostname="http://node3-d-d4s.d4science.org:80";
|
||||
// static String hostname="http://pc-fabio.isti.cnr.it:8080";
|
||||
static String scope="/gcube/devNext";
|
||||
static Client client;
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void init() throws ServiceNotFoundException{
|
||||
ScopeProvider.instance.set(scope);
|
||||
client=new Client(hostname);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void getCapabilties() throws CommunicationException{
|
||||
System.out.println(client.getCapabilties());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void doTheTransfer() throws MalformedURLException, RemoteServiceException{
|
||||
Destination dest=new Destination("outputFile");
|
||||
dest.setCreateSubfolders(true);
|
||||
dest.setSubFolder("bla/bla/bllaaa");
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.ADD_SUFFIX);
|
||||
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
||||
TransferRequest request= new TransferRequest("", new HttpDownloadSettings(new URL("http://goo.gl/oLP7zG"), HttpDownloadOptions.DEFAULT),dest);
|
||||
System.out.println("Submitting "+request);
|
||||
TransferTicket ticket=client.submit(request);
|
||||
System.out.println("Ticket is "+ticket);
|
||||
|
||||
boolean continuePolling=true;
|
||||
do{
|
||||
ticket=client.getTransferStatus(ticket.getId());
|
||||
System.out.println("Status : "+ticket);
|
||||
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
||||
try{
|
||||
Thread.sleep(1000);
|
||||
}catch(InterruptedException e){}
|
||||
}while(continuePolling);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void directCall(){
|
||||
javax.ws.rs.client.Client client = ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class));
|
||||
WebTarget target=client.target(hostname+"/data-transfer-service"+ServiceConstants.APPLICATION_PATH+"Capabilities");
|
||||
// WebTarget target=client.target(hostname+"/data-transfer-service/gcube/service/Capabilities");
|
||||
System.out.println("Asking capabilities to target : "+target.getUri());
|
||||
System.out.println(target.
|
||||
request(MediaType.APPLICATION_XML).get(TransferCapabilities.class));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
public class TokenSetter {
|
||||
|
||||
|
||||
|
||||
private static Properties props=new Properties();
|
||||
|
||||
static{
|
||||
try {
|
||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void set(String scope){
|
||||
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
||||
ScopeProvider.instance.set(scope);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.gcube.data.transfer.library.model.Source;
|
||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.ToString;
|
||||
|
||||
public class TransferReport {
|
||||
|
||||
public static enum ReportType{
|
||||
local,uri,storage
|
||||
}
|
||||
|
||||
|
||||
private static final char end='\n';
|
||||
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
@ToString
|
||||
private static class ReportItem{
|
||||
String source;
|
||||
long size;
|
||||
long elapsed;
|
||||
|
||||
|
||||
// bytes/msec
|
||||
public long getAvgSpeed(){
|
||||
return size/elapsed;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
protected TransferCapabilities host;
|
||||
|
||||
private Map<ReportType,ArrayList<ReportItem>> reports=new HashMap<>();
|
||||
|
||||
|
||||
|
||||
public TransferReport(TransferCapabilities host) {
|
||||
super();
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
|
||||
public void addReport(ReportType sourceType,String theSource,long size,long elapsed){
|
||||
if(!reports.containsKey(sourceType)) reports.put(sourceType, new ArrayList<ReportItem>());
|
||||
reports.get(sourceType).add(new ReportItem(theSource, size, elapsed));
|
||||
}
|
||||
|
||||
|
||||
public String print(){
|
||||
StringBuilder builder=new StringBuilder(" Report for : "+host+end);
|
||||
for(Entry<ReportType,ArrayList<ReportItem>> entry:reports.entrySet()){
|
||||
builder.append("Source type "+entry.getKey()+end);
|
||||
ReportItem maxSizeItem=null;
|
||||
ReportItem maxAvgSpeedItem=null;
|
||||
ReportItem minSizeItem=null;
|
||||
ReportItem minAvgSpeedItem=null;
|
||||
long currentAvgSpeedCounter=0l;
|
||||
|
||||
for(ReportItem item:entry.getValue()){
|
||||
if(maxSizeItem==null||maxSizeItem.getSize()<item.getSize()) maxSizeItem=item;
|
||||
if(maxAvgSpeedItem==null||maxAvgSpeedItem.getAvgSpeed()<item.getAvgSpeed()) maxAvgSpeedItem=item;
|
||||
if(minSizeItem==null||minSizeItem.getSize()>item.getSize()) minSizeItem=item;
|
||||
if(minAvgSpeedItem==null||minAvgSpeedItem.getAvgSpeed()>item.getAvgSpeed()) minAvgSpeedItem=item;
|
||||
currentAvgSpeedCounter+=item.getAvgSpeed();
|
||||
}
|
||||
|
||||
builder.append("Max Size Item : "+maxSizeItem+end);
|
||||
builder.append("Max Avg Speed Item : "+maxAvgSpeedItem+end);
|
||||
builder.append("Min Size Item : "+minSizeItem+end);
|
||||
builder.append("Min Avg Speed Item : "+minAvgSpeedItem+end);
|
||||
builder.append("Total avg speed : "+currentAvgSpeedCounter/entry.getValue().size()+end);
|
||||
}
|
||||
|
||||
return builder.toString();
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package org.gcube.data.transfer.library;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TransfererTest {
|
||||
|
||||
static String hostname="http://node3-d-d4s.d4science.org:80";
|
||||
static String nodeId="462b68c5-463f-4295-86da-37d6c0abc7ea";
|
||||
static String scope="/gcube/devsec";
|
||||
|
||||
static DataTransferClient client;
|
||||
|
||||
@BeforeClass
|
||||
public static void init() throws UnreachableNodeException, ServiceNotFoundException, HostingNodeNotFoundException{
|
||||
ScopeProvider.instance.set(scope);
|
||||
client=DataTransferClient.getInstanceByEndpoint(hostname);
|
||||
// client=DataTransferClient.getInstanceByNodeId(nodeId);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void localFile() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
for(int i=0;i<3;i++){
|
||||
String localFile="/home/fabio/Dropbox/Mindless/01- COMA - Mindless.mp3";
|
||||
String transferredFileName="Mindless.mp3";
|
||||
Destination dest=new Destination(transferredFileName);
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
TransferResult res=client.localFile(localFile,dest);
|
||||
String remotePath=res.getRemotePath();
|
||||
Assert.assertEquals(transferredFileName, remotePath.substring(remotePath.lastIndexOf(File.separatorChar)+1));
|
||||
System.out.println(res);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void httpUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
String link="http://goo.gl/oLP7zG";
|
||||
System.out.println(client.httpSource(link,new Destination("Some import")));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException, InvalidDestinationException, DestinationNotSetException{
|
||||
ScopeProvider.instance.set(scope);
|
||||
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
||||
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
||||
Destination dest=new Destination("some/where","My Pdf.pdf");
|
||||
TransferResult res=client.storageId(id,dest);
|
||||
Assert.assertTrue(res.getRemotePath().contains(dest.getSubFolder()+File.separatorChar+dest.getDestinationFileName()));
|
||||
System.out.println(client.storageId(id,dest));
|
||||
}
|
||||
|
||||
@Test(expected=InvalidSourceException.class)
|
||||
public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
System.out.println(client.storageId("13245780t",new Destination("my file")));
|
||||
}
|
||||
|
||||
@Test(expected=InvalidSourceException.class)
|
||||
public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
String localFile="/home/fabio/Downloads/123045689.mp3";
|
||||
System.out.println(client.localFile(localFile,new Destination("12345")));
|
||||
}
|
||||
|
||||
@Test(expected=InvalidSourceException.class)
|
||||
public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||
String link="https://www.some.where.com/over/theRainbow.txt";
|
||||
System.out.println(client.httpSource(link,new Destination("oz")));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue