Initial import.
git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@131226 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
f924eed16d
commit
7ac5508a1b
|
@ -0,0 +1,36 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<classpath>
|
||||||
|
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="optional" value="true"/>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="optional" value="true"/>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||||
|
<attributes>
|
||||||
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
</attributes>
|
||||||
|
</classpathentry>
|
||||||
|
<classpathentry kind="output" path="target/classes"/>
|
||||||
|
</classpath>
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<projectDescription>
|
||||||
|
<name>data-transfer-library</name>
|
||||||
|
<comment></comment>
|
||||||
|
<projects>
|
||||||
|
</projects>
|
||||||
|
<buildSpec>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
</buildSpec>
|
||||||
|
<natures>
|
||||||
|
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||||
|
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||||
|
</natures>
|
||||||
|
</projectDescription>
|
|
@ -0,0 +1,6 @@
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
encoding//src/main/java=UTF-8
|
||||||
|
encoding//src/main/resources=UTF-8
|
||||||
|
encoding//src/test/java=UTF-8
|
||||||
|
encoding//src/test/resources=UTF-8
|
||||||
|
encoding/<project>=UTF-8
|
|
@ -0,0 +1,5 @@
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||||
|
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||||
|
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||||
|
org.eclipse.jdt.core.compiler.source=1.7
|
|
@ -0,0 +1,4 @@
|
||||||
|
activeProfiles=
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
resolveWorkspaceProjects=false
|
||||||
|
version=1
|
|
@ -0,0 +1 @@
|
||||||
|
${gcube.license}
|
|
@ -0,0 +1,65 @@
|
||||||
|
The gCube System - ${name}
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
${description}
|
||||||
|
|
||||||
|
|
||||||
|
${gcube.description}
|
||||||
|
|
||||||
|
${gcube.funding}
|
||||||
|
|
||||||
|
|
||||||
|
Version
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
${version} (${buildDate})
|
||||||
|
|
||||||
|
Please see the file named "changelog.xml" in this directory for the release notes.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Authors
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
||||||
|
|
||||||
|
Maintainers
|
||||||
|
-----------
|
||||||
|
|
||||||
|
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
||||||
|
|
||||||
|
|
||||||
|
Download information
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
Source code is available from SVN:
|
||||||
|
${scm.url}
|
||||||
|
|
||||||
|
Binaries can be downloaded from the gCube website:
|
||||||
|
${gcube.website}
|
||||||
|
|
||||||
|
Installation
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
Installation documentation is available on-line in the gCube Wiki:
|
||||||
|
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||||
|
|
||||||
|
Documentation
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
Documentation is available on-line in the gCube Wiki:
|
||||||
|
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||||
|
${gcube.wikiRoot}/Tabular_Data_Manager
|
||||||
|
|
||||||
|
|
||||||
|
Support
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
Bugs and support requests can be reported in the gCube issue tracking tool:
|
||||||
|
${gcube.issueTracking}
|
||||||
|
|
||||||
|
|
||||||
|
Licensing
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
This software is licensed under the terms you may find in the file named "LICENSE" in this directory.
|
|
@ -0,0 +1,5 @@
|
||||||
|
<ReleaseNotes>
|
||||||
|
<Changeset component="data-transfer-library-1.0.0" date="2016-02-22">
|
||||||
|
<Change>First Release</Change>
|
||||||
|
</Changeset>
|
||||||
|
</ReleaseNotes>
|
|
@ -0,0 +1,30 @@
|
||||||
|
<assembly
|
||||||
|
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>servicearchive</id>
|
||||||
|
<formats>
|
||||||
|
<format>tar.gz</format>
|
||||||
|
</formats>
|
||||||
|
<baseDirectory>/</baseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${distroDirectory}</directory>
|
||||||
|
<outputDirectory>/</outputDirectory>
|
||||||
|
<useDefaultExcludes>true</useDefaultExcludes>
|
||||||
|
<includes>
|
||||||
|
<include>README</include>
|
||||||
|
<include>LICENSE</include>
|
||||||
|
<include>changelog.xml</include>
|
||||||
|
</includes>
|
||||||
|
<fileMode>755</fileMode>
|
||||||
|
<filtered>true</filtered>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<files>
|
||||||
|
<file>
|
||||||
|
<source>target/${build.finalName}.${project.packaging}</source>
|
||||||
|
<outputDirectory>/${artifactId}</outputDirectory>
|
||||||
|
</file>
|
||||||
|
</files>
|
||||||
|
</assembly>
|
|
@ -0,0 +1,29 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<Resource>
|
||||||
|
<ID></ID>
|
||||||
|
<Type>Service</Type>
|
||||||
|
<Profile>
|
||||||
|
<Description>${description}</Description>
|
||||||
|
<Class>DataTransfer</Class>
|
||||||
|
<Name>${artifactId}</Name>
|
||||||
|
<Version>1.0.0</Version>
|
||||||
|
<Packages>
|
||||||
|
<Software>
|
||||||
|
<Description>${description}</Description>
|
||||||
|
<Name>${artifactId}</Name>
|
||||||
|
<Version>${version}</Version>
|
||||||
|
<MavenCoordinates>
|
||||||
|
<groupId>${groupId}</groupId>
|
||||||
|
<artifactId>${artifactId}</artifactId>
|
||||||
|
<version>${version}</version>
|
||||||
|
</MavenCoordinates>
|
||||||
|
<Type>library</Type>
|
||||||
|
<Files>
|
||||||
|
<File>${build.finalName}.jar</File>
|
||||||
|
</Files>
|
||||||
|
</Software>
|
||||||
|
</Packages>
|
||||||
|
</Profile>
|
||||||
|
</Resource>
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,108 @@
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.gcube.tools</groupId>
|
||||||
|
<artifactId>maven-parent</artifactId>
|
||||||
|
<version>LATEST</version>
|
||||||
|
</parent>
|
||||||
|
<groupId>org.gcube.data.transfer</groupId>
|
||||||
|
<artifactId>data-transfer-library</artifactId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
<name>DataTransfer-library</name>
|
||||||
|
|
||||||
|
|
||||||
|
<dependencyManagement>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.distribution</groupId>
|
||||||
|
<artifactId>gcube-bom</artifactId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
<type>pom</type>
|
||||||
|
<scope>import</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</dependencyManagement>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
|
||||||
|
<!-- GCUBE -->
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common</groupId>
|
||||||
|
<artifactId>authorization-client</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.core</groupId>
|
||||||
|
<artifactId>common-gcube-calls</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.data.transfer</groupId>
|
||||||
|
<artifactId>data-transfer-model</artifactId>
|
||||||
|
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- STORAGE -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.contentmanagement</groupId>
|
||||||
|
<artifactId>storage-manager-core</artifactId>
|
||||||
|
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.contentmanagement</groupId>
|
||||||
|
<artifactId>storage-manager-wrapper</artifactId>
|
||||||
|
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<!-- EXTERNAL -->
|
||||||
|
<!-- lombok -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
<version>0.11.6</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- JERSEY -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.glassfish.jersey.core</groupId>
|
||||||
|
<artifactId>jersey-client</artifactId>
|
||||||
|
<version>2.22.2</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.glassfish.jersey.media</groupId>
|
||||||
|
<artifactId>jersey-media-jaxb</artifactId>
|
||||||
|
<version>2.18</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- TEST -->
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.10</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<!-- Test log binding -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
<version>1.0.13</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
|
||||||
|
</project>
|
|
@ -0,0 +1,126 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||||
|
import org.gcube.data.transfer.library.transferers.Transferer;
|
||||||
|
import org.gcube.data.transfer.library.transferers.TransfererBuilder;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.PluginInvocation;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
|
||||||
|
import lombok.Synchronized;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class DataTransferClient {
|
||||||
|
|
||||||
|
private Transferer transferer=null;
|
||||||
|
|
||||||
|
private DataTransferClient(Transferer transferer) {
|
||||||
|
this.transferer=transferer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DataTransferClient getInstanceByEndpoint(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
||||||
|
log.debug("Getting transferer for endpoint : "+endpoint);
|
||||||
|
return new DataTransferClient(TransfererBuilder.getTransfererByHost(endpoint));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DataTransferClient getInstanceByNodeId(String id) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
||||||
|
log.debug("Getting transferer for nodeId : "+id);
|
||||||
|
return new DataTransferClient(TransfererBuilder.getTransfererByhostingNodeId(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public TransferCapabilities getDestinationCapabilities() throws InitializationException{
|
||||||
|
return this.transferer.getDestinationCapabilities();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public TransferResult localFile(String path,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
return localFile(path,dest,Collections.<PluginInvocation> emptySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferResult localFile(File file,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
return localFile(file,dest,Collections.<PluginInvocation> emptySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferResult httpSource(String url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
return httpSource(url,dest,Collections.<PluginInvocation> emptySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferResult httpSource(URL url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
return httpSource(url,dest,Collections.<PluginInvocation> emptySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferResult storageId(String id,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
return storageId(id,dest,Collections.<PluginInvocation> emptySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Synchronized("transferer")
|
||||||
|
public TransferResult localFile(String path,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||||
|
log.debug("Sending local file {} to {} : {} - {}",path,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||||
|
transferer.localFile(path);
|
||||||
|
return doTheTransfer(dest, pluginInvocations);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized("transferer")
|
||||||
|
public TransferResult localFile(File file,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||||
|
log.debug("Sending local file {} to {} : {} - {}",file.getAbsolutePath(),transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||||
|
transferer.localFile(file);
|
||||||
|
return doTheTransfer(dest, pluginInvocations);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized("transferer")
|
||||||
|
public TransferResult httpSource(String url,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||||
|
log.debug("Passed url string : "+url);
|
||||||
|
try{
|
||||||
|
return this.httpSource(new URL(url),dest,pluginInvocations);
|
||||||
|
}catch(MalformedURLException e){
|
||||||
|
throw new InvalidSourceException("Invalid url : "+url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized("transferer")
|
||||||
|
public TransferResult httpSource(URL url,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||||
|
log.debug("Sending from url {} to {} : {} - {}",url,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||||
|
transferer.fromURL(url);
|
||||||
|
return doTheTransfer(dest, pluginInvocations);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized("transferer")
|
||||||
|
public TransferResult storageId(String id,Destination dest,Set<PluginInvocation> pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
||||||
|
log.debug("Sending from storage id {} to {} : {} - {}",id,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations);
|
||||||
|
transferer.storageFileId(id);
|
||||||
|
return doTheTransfer(dest, pluginInvocations);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TransferResult doTheTransfer(Destination dest,Set<PluginInvocation> pluginInvocations) throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
try{
|
||||||
|
transferer.setDestination(dest);
|
||||||
|
transferer.setInvocations(pluginInvocations);
|
||||||
|
return transferer.transfer();
|
||||||
|
}finally{
|
||||||
|
transferer.reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,22 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NonNull;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.model.Source;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class TransferResult {
|
||||||
|
|
||||||
|
@NonNull
|
||||||
|
private Source source;
|
||||||
|
@NonNull
|
||||||
|
private String destinationHostName;
|
||||||
|
@NonNull
|
||||||
|
private Long elapsedTime;
|
||||||
|
@NonNull
|
||||||
|
private Long transferedBytes;
|
||||||
|
@NonNull
|
||||||
|
private String remotePath;
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
package org.gcube.data.transfer.library.caches;
|
||||||
|
|
||||||
|
import lombok.Synchronized;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.client.Client;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class CapabilitiesCache extends TTLCache<TransferCapabilities> {
|
||||||
|
|
||||||
|
private static CapabilitiesCache instance=null;
|
||||||
|
|
||||||
|
// @Synchronized
|
||||||
|
// public static CapabilitiesCache getInstance(){
|
||||||
|
// if(instance==null)instance=new CapabilitiesCache();
|
||||||
|
// return instance;
|
||||||
|
// }
|
||||||
|
|
||||||
|
private CapabilitiesCache(){
|
||||||
|
super(5*60*1000l,2*60*1000l,"Capabilities");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected TransferCapabilities getNew(String id) throws Exception{
|
||||||
|
log.debug("Getting capabilties for host "+id);
|
||||||
|
return new Client(id).getCapabilties();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,94 @@
|
||||||
|
package org.gcube.data.transfer.library.caches;
|
||||||
|
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.ThreadFactory;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import lombok.Synchronized;
|
||||||
|
import lombok.ToString;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@ToString
|
||||||
|
@Slf4j
|
||||||
|
public abstract class TTLCache<T> {
|
||||||
|
|
||||||
|
|
||||||
|
// STATIC
|
||||||
|
|
||||||
|
// private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1,new ThreadFactory(){
|
||||||
|
// public Thread newThread(Runnable r) {
|
||||||
|
// Thread t = Executors.defaultThreadFactory().newThread(r);
|
||||||
|
// t.setDaemon(true);
|
||||||
|
// return t;
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
private static final HashSet<TTLCache<?>> createdMaps=new HashSet<TTLCache<?>>();
|
||||||
|
|
||||||
|
|
||||||
|
static {
|
||||||
|
// scheduler.scheduleAtFixedRate(new Runnable() {
|
||||||
|
//
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// @Override
|
||||||
|
// public void run() {
|
||||||
|
// log.debug("Running Maps Cleaner, maps count : "+createdMaps.size());
|
||||||
|
// int removed=0;
|
||||||
|
// for(TTLCache<?> theMap:createdMaps)
|
||||||
|
// theMap.purgeItems();
|
||||||
|
// log.debug("Removed "+removed+" old tickets");
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
// }, 3, 3, TimeUnit.MINUTES);
|
||||||
|
//
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//************************* CACHE
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
protected TTLCache(Long cacheKeepAliveTime, Long objectTTL, String cacheName) {
|
||||||
|
this.cacheKeepAliveTime = cacheKeepAliveTime;
|
||||||
|
this.objectTTL = objectTTL;
|
||||||
|
this.cacheName = cacheName;
|
||||||
|
createdMaps.add(this);
|
||||||
|
log.debug("Created Cache "+this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private ConcurrentHashMap<String,TTLContainer<T>> theMap=new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
private Long cacheKeepAliveTime;
|
||||||
|
private Long objectTTL;
|
||||||
|
private String cacheName;
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
public T getObject(String id) throws Exception{
|
||||||
|
if(!theMap.contains(id)||System.currentTimeMillis()-theMap.get(id).getCreationTime()>objectTTL)
|
||||||
|
theMap.put(id, new TTLContainer<T>(getNew(id)));
|
||||||
|
return theMap.get(id).getTheObject();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private int purgeItems(){
|
||||||
|
log.debug(cacheName+" Purging objects, keep alive time is "+cacheKeepAliveTime);
|
||||||
|
int totalCount=theMap.size();
|
||||||
|
int removed=0;
|
||||||
|
for(Entry<String,TTLContainer<T>> entry:theMap.entrySet())
|
||||||
|
if(System.currentTimeMillis()-entry.getValue().getLastUsageTime()>cacheKeepAliveTime){
|
||||||
|
theMap.remove(entry.getKey());
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
log.debug(cacheName+" Removed "+removed+" out of "+totalCount);
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract T getNew(String id) throws Exception;
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,30 @@
|
||||||
|
package org.gcube.data.transfer.library.caches;
|
||||||
|
|
||||||
|
public class TTLContainer<T> {
|
||||||
|
|
||||||
|
private long lastUsageTime=System.currentTimeMillis();
|
||||||
|
private long creationTime=System.currentTimeMillis();
|
||||||
|
|
||||||
|
private T theObject;
|
||||||
|
public TTLContainer(T theObject) {
|
||||||
|
this.theObject = theObject;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void update(){
|
||||||
|
lastUsageTime=System.currentTimeMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
public T getTheObject(){
|
||||||
|
update();
|
||||||
|
return theObject;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getLastUsageTime() {
|
||||||
|
return lastUsageTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getCreationTime() {
|
||||||
|
return creationTime;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,28 @@
|
||||||
|
package org.gcube.data.transfer.library.client;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
|
import javax.ws.rs.client.ClientRequestContext;
|
||||||
|
import javax.ws.rs.client.ClientRequestFilter;
|
||||||
|
|
||||||
|
import org.gcube.common.calls.Call;
|
||||||
|
import org.gcube.common.calls.Interceptors;
|
||||||
|
import org.gcube.common.calls.Request;
|
||||||
|
import org.gcube.data.transfer.library.utils.ScopeUtils;
|
||||||
|
|
||||||
|
public class AuthorizationFilter implements ClientRequestFilter {
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void filter(final ClientRequestContext rc) throws IOException {
|
||||||
|
if (ScopeUtils.getCurrentScope()!=null){
|
||||||
|
Request requestContext = Interceptors.executeRequestChain(new Call());
|
||||||
|
|
||||||
|
for (Entry<String, String> entry: requestContext.getHeaders()){
|
||||||
|
rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,96 @@
|
||||||
|
package org.gcube.data.transfer.library.client;
|
||||||
|
|
||||||
|
import javax.ws.rs.client.ClientBuilder;
|
||||||
|
import javax.ws.rs.client.Entity;
|
||||||
|
import javax.ws.rs.client.WebTarget;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.CommunicationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.DataTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||||
|
import org.gcube.data.transfer.model.ServiceConstants;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
import org.gcube.data.transfer.model.TransferRequest;
|
||||||
|
import org.gcube.data.transfer.model.TransferTicket;
|
||||||
|
import org.glassfish.jersey.client.ClientConfig;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class Client {
|
||||||
|
|
||||||
|
private static ClientConfig config=null;
|
||||||
|
|
||||||
|
static{
|
||||||
|
log.debug("Creating configuration ..");
|
||||||
|
config=new ClientConfig();
|
||||||
|
config.register(AuthorizationFilter.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String endpoint;
|
||||||
|
|
||||||
|
private WebTarget rootTarget;
|
||||||
|
|
||||||
|
public Client(String endpoint) throws ServiceNotFoundException{
|
||||||
|
try{
|
||||||
|
log.debug("Creating client for base "+endpoint);
|
||||||
|
this.endpoint=endpoint+"";
|
||||||
|
rootTarget= ClientBuilder.newClient(config).target(endpoint).path("data-transfer-service").path(ServiceConstants.APPLICATION_PATH);
|
||||||
|
// checkResponse(rootTarget.request().get());
|
||||||
|
|
||||||
|
log.debug("Root Taget IS {} ",rootTarget.getUri());
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new ServiceNotFoundException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public String getEndpoint() {
|
||||||
|
return endpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferCapabilities getCapabilties() throws CommunicationException{
|
||||||
|
WebTarget capabilitiesTarget=rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME);
|
||||||
|
log.debug("Getting capabilities from {}, path is {} ",endpoint,capabilitiesTarget.getUri());
|
||||||
|
try{
|
||||||
|
Response resp=capabilitiesTarget.request().accept(MediaType.APPLICATION_XML_TYPE).get();
|
||||||
|
checkResponse(resp);
|
||||||
|
return resp.readEntity(TransferCapabilities.class);
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new CommunicationException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public TransferTicket submit(TransferRequest request) throws RemoteServiceException{
|
||||||
|
log.debug("Sending request {} to {}",request,endpoint);
|
||||||
|
try{
|
||||||
|
Response resp=rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML));
|
||||||
|
checkResponse(resp);
|
||||||
|
return resp.readEntity(TransferTicket.class);
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new RemoteServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferTicket getTransferStatus(String transferId) throws RemoteServiceException{
|
||||||
|
log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint);
|
||||||
|
try{
|
||||||
|
Response resp=rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get();
|
||||||
|
checkResponse(resp);
|
||||||
|
return resp.readEntity(TransferTicket.class);
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new RemoteServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected void checkResponse(Response toCheck) throws Exception{
|
||||||
|
switch(toCheck.getStatusInfo().getFamily()){
|
||||||
|
case SUCCESSFUL : break;
|
||||||
|
default : throw new Exception("Unexpected Response code : "+toCheck.getStatus());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class CommunicationException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
public CommunicationException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommunicationException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommunicationException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommunicationException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public CommunicationException(String message, Throwable cause, boolean enableSuppression,
|
||||||
|
boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public abstract class DataTransferException extends Exception {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 8488351139669552406L;
|
||||||
|
|
||||||
|
public DataTransferException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataTransferException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataTransferException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataTransferException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataTransferException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class DestinationNotSetException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 4792958203546912779L;
|
||||||
|
|
||||||
|
public DestinationNotSetException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DestinationNotSetException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DestinationNotSetException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DestinationNotSetException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public DestinationNotSetException(String message, Throwable cause, boolean enableSuppression,
|
||||||
|
boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class FailedTransferException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -807695677781676381L;
|
||||||
|
|
||||||
|
public FailedTransferException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public FailedTransferException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public FailedTransferException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public FailedTransferException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public FailedTransferException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class HostingNodeNotFoundException extends InitializationException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 5786009015981659074L;
|
||||||
|
|
||||||
|
public HostingNodeNotFoundException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public HostingNodeNotFoundException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public HostingNodeNotFoundException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public HostingNodeNotFoundException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public HostingNodeNotFoundException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class InitializationException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 2204462227670167203L;
|
||||||
|
|
||||||
|
public InitializationException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InitializationException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InitializationException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InitializationException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InitializationException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class InvalidDestinationException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -1882334325516885722L;
|
||||||
|
|
||||||
|
public InvalidDestinationException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidDestinationException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidDestinationException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidDestinationException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidDestinationException(String message, Throwable cause, boolean enableSuppression,
|
||||||
|
boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class InvalidSourceException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 7631518288856917202L;
|
||||||
|
|
||||||
|
public InvalidSourceException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidSourceException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidSourceException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidSourceException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public InvalidSourceException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class RemoteServiceException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 5320978791087129997L;
|
||||||
|
|
||||||
|
public RemoteServiceException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteServiceException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteServiceException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteServiceException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteServiceException(String message, Throwable cause, boolean enableSuppression,
|
||||||
|
boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class ServiceNotFoundException extends InitializationException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -3494011839783891174L;
|
||||||
|
|
||||||
|
public ServiceNotFoundException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServiceNotFoundException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServiceNotFoundException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServiceNotFoundException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServiceNotFoundException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class SourceNotSetException extends DataTransferException {
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = -5868191252361823341L;
|
||||||
|
|
||||||
|
public SourceNotSetException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public SourceNotSetException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public SourceNotSetException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public SourceNotSetException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public SourceNotSetException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,37 @@
|
||||||
|
package org.gcube.data.transfer.library.faults;
|
||||||
|
|
||||||
|
public class UnreachableNodeException extends InitializationException {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static final long serialVersionUID = 8739586712271226827L;
|
||||||
|
|
||||||
|
public UnreachableNodeException() {
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public UnreachableNodeException(String message) {
|
||||||
|
super(message);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public UnreachableNodeException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public UnreachableNodeException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
public UnreachableNodeException(String message, Throwable cause,
|
||||||
|
boolean enableSuppression, boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
// TODO Auto-generated constructor stub
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,41 @@
|
||||||
|
package org.gcube.data.transfer.library.model;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
|
||||||
|
public class LocalSource extends Source<File> {
|
||||||
|
|
||||||
|
private File theFile=null;
|
||||||
|
|
||||||
|
public LocalSource(File theFile) throws InvalidSourceException {
|
||||||
|
super();
|
||||||
|
if(theFile==null)throw new InvalidSourceException("File cannot be null");
|
||||||
|
this.theFile = theFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean validate() throws InvalidSourceException {
|
||||||
|
if(!theFile.exists()) throw new InvalidSourceException("File "+theFile.getAbsolutePath()+"doesn't exist");
|
||||||
|
if(!theFile.canRead()) throw new InvalidSourceException("Unable to read from file "+theFile.getAbsolutePath());
|
||||||
|
if(theFile.isDirectory()) throw new InvalidSourceException("Transfer of directory is not yet supported");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void prepare() {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clean() {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public File getTheSource() {
|
||||||
|
return theFile;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package org.gcube.data.transfer.library.model;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public abstract class Source<T>{
|
||||||
|
|
||||||
|
public abstract boolean validate() throws InvalidSourceException;
|
||||||
|
|
||||||
|
public abstract void prepare();
|
||||||
|
|
||||||
|
public abstract void clean();
|
||||||
|
|
||||||
|
public abstract T getTheSource();
|
||||||
|
}
|
|
@ -0,0 +1,47 @@
|
||||||
|
package org.gcube.data.transfer.library.model;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||||
|
|
||||||
|
public class StorageSource extends Source<String> {
|
||||||
|
|
||||||
|
private String id=null;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public StorageSource(String id) throws InvalidSourceException {
|
||||||
|
super();
|
||||||
|
if(id==null) throw new InvalidSourceException("Storage id cannot be null");
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean validate() throws InvalidSourceException {
|
||||||
|
try{
|
||||||
|
if(!StorageUtils.checkStorageId(id)) throw new Exception("Not valid");
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new InvalidSourceException("Invalid storage ID "+id);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void prepare() {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clean() {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTheSource() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,62 @@
|
||||||
|
package org.gcube.data.transfer.library.model;
|
||||||
|
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
|
||||||
|
public class URLSource extends Source<URL> {
|
||||||
|
|
||||||
|
|
||||||
|
private URL theURL=null;
|
||||||
|
|
||||||
|
|
||||||
|
public URLSource(URL theURL) throws InvalidSourceException {
|
||||||
|
super();
|
||||||
|
if(theURL==null) throw new InvalidSourceException("URL Source cannot be null");
|
||||||
|
this.theURL = theURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean validate() throws InvalidSourceException {
|
||||||
|
try{
|
||||||
|
HttpURLConnection conn=(HttpURLConnection) theURL.openConnection();
|
||||||
|
conn.setConnectTimeout(3000);
|
||||||
|
conn.setRequestMethod("HEAD");
|
||||||
|
int responseCode = conn.getResponseCode();
|
||||||
|
return (200 <= responseCode && responseCode <= 399);
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new InvalidSourceException("Unable to contact URL "+theURL,e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void prepare() {
|
||||||
|
// nothing to do
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clean() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see java.lang.Object#toString()
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
builder.append("URLSource [theURL=");
|
||||||
|
builder.append(theURL);
|
||||||
|
builder.append("]");
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public URL getTheSource() {
|
||||||
|
return theURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,71 @@
|
||||||
|
package org.gcube.data.transfer.library.transferers;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||||
|
import org.gcube.data.transfer.library.client.Client;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.model.LocalSource;
|
||||||
|
import org.gcube.data.transfer.library.model.StorageSource;
|
||||||
|
import org.gcube.data.transfer.library.model.URLSource;
|
||||||
|
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||||
|
import org.gcube.data.transfer.model.TransferRequest;
|
||||||
|
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
||||||
|
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class HTTPTransferer extends Transferer {
|
||||||
|
|
||||||
|
HTTPTransferer(Client client) {
|
||||||
|
super(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
private URL link;
|
||||||
|
|
||||||
|
private String toDeleteStorageId=null;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected TransferRequest prepareRequest()throws InitializationException {
|
||||||
|
return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT),this.destination,this.invocations);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{
|
||||||
|
if(source instanceof LocalSource){
|
||||||
|
toDeleteStorageId=StorageUtils.putOntoStorage(((LocalSource)source).getTheSource());
|
||||||
|
return new URL(StorageUtils.getUrlById(toDeleteStorageId));
|
||||||
|
}else if (source instanceof StorageSource){
|
||||||
|
return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource()));
|
||||||
|
}else if (source instanceof URLSource){
|
||||||
|
return ((URLSource)source).getTheSource();
|
||||||
|
}else throw new InvalidSourceException("Source cannot be handled "+source);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void prepare() throws InitializationException {
|
||||||
|
try{
|
||||||
|
link=getHttpLink();
|
||||||
|
log.debug("Obtained link "+link);
|
||||||
|
super.prepare();
|
||||||
|
}catch(RemoteBackendException e){
|
||||||
|
throw new InitializationException(e);
|
||||||
|
}catch(FileNotFoundException e){
|
||||||
|
throw new InitializationException(e);
|
||||||
|
}catch(InvalidSourceException e){
|
||||||
|
throw new InitializationException(e);
|
||||||
|
}catch (MalformedURLException e){
|
||||||
|
throw new InitializationException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void clean() {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
super.clean();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,157 @@
|
||||||
|
package org.gcube.data.transfer.library.transferers;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.TransferResult;
|
||||||
|
import org.gcube.data.transfer.library.client.Client;
|
||||||
|
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.model.LocalSource;
|
||||||
|
import org.gcube.data.transfer.library.model.Source;
|
||||||
|
import org.gcube.data.transfer.library.model.StorageSource;
|
||||||
|
import org.gcube.data.transfer.library.model.URLSource;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.PluginInvocation;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
import org.gcube.data.transfer.model.TransferRequest;
|
||||||
|
import org.gcube.data.transfer.model.TransferTicket;
|
||||||
|
import org.gcube.data.transfer.model.TransferTicket.Status;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public abstract class Transferer {
|
||||||
|
|
||||||
|
protected Client client;
|
||||||
|
|
||||||
|
protected Transferer(Client client){
|
||||||
|
this.client=client;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
protected Source source=null;
|
||||||
|
protected boolean prepared=false;
|
||||||
|
protected Destination destination=null;
|
||||||
|
protected Set<PluginInvocation> invocations=null;
|
||||||
|
|
||||||
|
|
||||||
|
public void setDestination(Destination destination) {
|
||||||
|
this.destination = destination;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setInvocations(Set<PluginInvocation> invocations) {
|
||||||
|
this.invocations = invocations;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Transferer localFile(File f) throws InvalidSourceException{
|
||||||
|
source=new LocalSource(f);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Transferer localFile(String path) throws InvalidSourceException{
|
||||||
|
try{
|
||||||
|
File f=new File(path);
|
||||||
|
return localFile(f);
|
||||||
|
}catch(NullPointerException e){
|
||||||
|
throw new InvalidSourceException("Path is null",e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public Transferer storageFileId(String fileId) throws InvalidSourceException{
|
||||||
|
source=new StorageSource(fileId);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Transferer fromURL(URL sourceURL) throws InvalidSourceException{
|
||||||
|
source=new URLSource(sourceURL);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void reset(){
|
||||||
|
this.destination=null;
|
||||||
|
this.source=null;
|
||||||
|
this.invocations=null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
try{
|
||||||
|
checkDestination();
|
||||||
|
checkSource();
|
||||||
|
checkPluginInvocations();
|
||||||
|
prepare();
|
||||||
|
TransferRequest request=prepareRequest();
|
||||||
|
log.debug("Request is {}, sending it to {}",request,client.getEndpoint());
|
||||||
|
TransferResult result=doTheTransfer(request);
|
||||||
|
return result;
|
||||||
|
}finally{
|
||||||
|
clean();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{
|
||||||
|
try{
|
||||||
|
TransferTicket submissionResponse= client.submit(request);
|
||||||
|
boolean continuePolling=true;
|
||||||
|
TransferTicket ticket=null;
|
||||||
|
do{
|
||||||
|
ticket=client.getTransferStatus(submissionResponse.getId());
|
||||||
|
System.out.println("Status : "+ticket);
|
||||||
|
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
||||||
|
try{
|
||||||
|
Thread.sleep(500);
|
||||||
|
}catch(InterruptedException e){}
|
||||||
|
}while(continuePolling);
|
||||||
|
if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage());
|
||||||
|
if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage());
|
||||||
|
long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis();
|
||||||
|
return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
|
||||||
|
}catch(RemoteServiceException e){
|
||||||
|
throw new FailedTransferException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void checkDestination() throws InvalidDestinationException,DestinationNotSetException{
|
||||||
|
if(destination==null) throw new DestinationNotSetException();
|
||||||
|
// perform checks
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void checkSource() throws SourceNotSetException, InvalidSourceException{
|
||||||
|
if(source==null) throw new SourceNotSetException();
|
||||||
|
source.validate();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void checkPluginInvocations(){
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract TransferRequest prepareRequest() throws InitializationException;
|
||||||
|
|
||||||
|
|
||||||
|
protected void prepare() throws InitializationException{
|
||||||
|
prepared=true;
|
||||||
|
}
|
||||||
|
protected void clean(){
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public TransferCapabilities getDestinationCapabilities() throws InitializationException {
|
||||||
|
try{
|
||||||
|
return client.getCapabilties();
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new InitializationException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,74 @@
|
||||||
|
package org.gcube.data.transfer.library.transferers;
|
||||||
|
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||||
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||||
|
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import org.gcube.common.resources.gcore.HostingNode;
|
||||||
|
import org.gcube.data.transfer.library.caches.CapabilitiesCache;
|
||||||
|
import org.gcube.data.transfer.library.client.Client;
|
||||||
|
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
@Slf4j
|
||||||
|
public class TransfererBuilder {
|
||||||
|
|
||||||
|
private static final int timeout=10*1000;
|
||||||
|
|
||||||
|
|
||||||
|
//e.g. http://pc-fabio.isti.cnr.it:8080/data-transfer-service/gcube/service
|
||||||
|
|
||||||
|
public static Transferer getTransfererByHost(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
||||||
|
log.debug("Get transferer by Host "+endpoint);
|
||||||
|
try{
|
||||||
|
URL url=new URL(endpoint);
|
||||||
|
|
||||||
|
String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort();
|
||||||
|
//TODO Implement checks
|
||||||
|
|
||||||
|
// if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout);
|
||||||
|
// String finalHost=host;
|
||||||
|
// if(!finalHost.endsWith(ServiceConstants.APPLICATION_PATH)){
|
||||||
|
// // adjust host
|
||||||
|
// finalHost=finalHost+(host.endsWith("/")?"":"/")+"data-transfer-service"+ServiceConstants.APPLICATION_PATH;
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost);
|
||||||
|
// log.debug("Host is ok, getting targetCapabilities");
|
||||||
|
// TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl);
|
||||||
|
|
||||||
|
// TODO determine method by capabilities checking
|
||||||
|
|
||||||
|
return new HTTPTransferer(new Client(baseUrl));
|
||||||
|
}catch(Exception e){
|
||||||
|
throw new ServiceNotFoundException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
||||||
|
String hostname=retrieveHostnameByNodeId(hostId);
|
||||||
|
return getTransfererByHost(hostname);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{
|
||||||
|
SimpleQuery query = queryFor(HostingNode.class);
|
||||||
|
|
||||||
|
query.addCondition("$resource/ID/text() eq '"+nodeId+"'");
|
||||||
|
|
||||||
|
|
||||||
|
List<HostingNode> found= clientFor(HostingNode.class).submit(query);
|
||||||
|
|
||||||
|
if(found.isEmpty()) throw new HostingNodeNotFoundException("No Hosting node with the id "+nodeId);
|
||||||
|
|
||||||
|
return "http://"+found.get(0).profile().description().name();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,27 @@
|
||||||
|
package org.gcube.data.transfer.library.utils;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
|
||||||
|
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class ScopeUtils {
|
||||||
|
|
||||||
|
public static String getCurrentScope(){
|
||||||
|
try{
|
||||||
|
String token=SecurityTokenProvider.instance.get();
|
||||||
|
log.debug("Token is : "+token);
|
||||||
|
if(token==null) throw new Exception("Security Token is null");
|
||||||
|
AuthorizationEntry entry = authorizationService().get(token);
|
||||||
|
return entry.getContext();
|
||||||
|
}catch(Exception e ){
|
||||||
|
log.debug("Unable to resolve token, checking scope provider..",e);
|
||||||
|
return ScopeProvider.instance.get();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,46 @@
|
||||||
|
package org.gcube.data.transfer.library.utils;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import org.bson.types.ObjectId;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||||
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||||
|
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||||
|
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||||
|
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class StorageUtils {
|
||||||
|
|
||||||
|
public static final IClient getClient(){
|
||||||
|
return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//return Id
|
||||||
|
public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{
|
||||||
|
IClient client=getClient();
|
||||||
|
log.debug("Uploading local file "+source.getAbsolutePath());
|
||||||
|
return client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final boolean checkStorageId(String id){
|
||||||
|
return getClient().getHttpUrl().RFile(id)!=null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final String getUrlById(String id){
|
||||||
|
IClient client=getClient();
|
||||||
|
log.debug("Id is "+id);
|
||||||
|
return client.getHttpUrl().RFile(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final void removeById(String id){
|
||||||
|
IClient client=getClient();
|
||||||
|
client.remove().RFile(id);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,40 @@
|
||||||
|
package org.gcube.data.transfer.library.utils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
public class Utils {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pings a HTTP URL. This effectively sends a HEAD request and returns <code>true</code> if the response code is in
|
||||||
|
* the 200-399 range.
|
||||||
|
* @param url The HTTP URL to be pinged.
|
||||||
|
* @param timeout The timeout in millis for both the connection timeout and the response read timeout. Note that
|
||||||
|
* the total timeout is effectively two times the given timeout.
|
||||||
|
* @return <code>true</code> if the given HTTP URL has returned response code 200-399 on a HEAD request within the
|
||||||
|
* given timeout, otherwise <code>false</code>.
|
||||||
|
*/
|
||||||
|
public static boolean pingURL(String url, int timeout) {
|
||||||
|
url = url.replaceFirst("^https", "http"); // Otherwise an exception may be thrown on invalid SSL certificates.
|
||||||
|
|
||||||
|
try {
|
||||||
|
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||||
|
connection.setConnectTimeout(timeout);
|
||||||
|
connection.setReadTimeout(timeout);
|
||||||
|
connection.setRequestMethod("HEAD");
|
||||||
|
int responseCode = connection.getResponseCode();
|
||||||
|
return (200 <= responseCode && responseCode <= 399);
|
||||||
|
} catch (IOException exception) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public static final String getUniqueString(){
|
||||||
|
return UUID.randomUUID().toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
|
||||||
|
public class ErrorReport extends TransferReport {
|
||||||
|
|
||||||
|
String id;
|
||||||
|
|
||||||
|
|
||||||
|
public ErrorReport(String id) {
|
||||||
|
super(null);
|
||||||
|
this.id=id;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String print() {
|
||||||
|
return "ERROR : "+id;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,161 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.TransferReport.ReportType;
|
||||||
|
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||||
|
|
||||||
|
import ch.qos.logback.core.util.ExecutorServiceUtil;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class MultipleTransferBenchmark {
|
||||||
|
|
||||||
|
|
||||||
|
static List<String> endpoints;
|
||||||
|
static List<String> nodeIds;
|
||||||
|
|
||||||
|
|
||||||
|
static List<String> files;
|
||||||
|
static List<String> uris;
|
||||||
|
static List<String> storageIds;
|
||||||
|
|
||||||
|
static ExecutorService executor=ExecutorServiceUtil.newExecutorService();
|
||||||
|
|
||||||
|
|
||||||
|
static CountDownLatch doneSignal;
|
||||||
|
|
||||||
|
static ArrayList<TransferReport> reports=new ArrayList<>();
|
||||||
|
|
||||||
|
|
||||||
|
static String scope="/gcube/devsec/devVRE";
|
||||||
|
|
||||||
|
static{
|
||||||
|
endpoints=Arrays.asList(new String[]{
|
||||||
|
// "http://node4-d-d4s.d4science.org:80/data-transfer-service/gcube/service",
|
||||||
|
"http://node3-d-d4s.d4science.org:80/data-transfer-service/gcube/service"
|
||||||
|
});
|
||||||
|
doneSignal=new CountDownLatch(endpoints.size());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
nodeIds=new ArrayList<>();
|
||||||
|
files=new ArrayList<>();
|
||||||
|
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||||
|
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||||
|
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||||
|
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
||||||
|
uris=new ArrayList<>();
|
||||||
|
uris.add("http://goo.gl/r5jFZ9");
|
||||||
|
storageIds=new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(String[] args) throws InitializationException{
|
||||||
|
TokenSetter.set(scope);
|
||||||
|
Map<String,DataTransferClient> clients=new HashMap<>();
|
||||||
|
for(String endpoint:endpoints)
|
||||||
|
clients.put(endpoint,DataTransferClient.getInstanceByEndpoint(endpoint));
|
||||||
|
for(String id:nodeIds)
|
||||||
|
clients.put(id,DataTransferClient.getInstanceByEndpoint(id));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
HashSet<DataTransferClient> startedTests=new HashSet<>();
|
||||||
|
for(final Entry<String,DataTransferClient> entry:clients.entrySet()){
|
||||||
|
if(!startedTests.contains(entry.getValue())){
|
||||||
|
startedTests.add(entry.getValue());
|
||||||
|
|
||||||
|
executor.execute(new Runnable(){
|
||||||
|
|
||||||
|
final DataTransferClient dt=entry.getValue();
|
||||||
|
final String id=entry.getKey();
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
try{
|
||||||
|
TransferReport report=new TransferReport(dt.getDestinationCapabilities());
|
||||||
|
TokenSetter.set(scope);
|
||||||
|
|
||||||
|
log.debug("Sending files to {} ",dt.getDestinationCapabilities());
|
||||||
|
|
||||||
|
Destination dest=new Destination("banchmarkoutputFile");
|
||||||
|
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||||
|
|
||||||
|
for(String f:files){
|
||||||
|
try {
|
||||||
|
TransferResult res=dt.localFile(f,dest);
|
||||||
|
report.addReport(ReportType.local,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||||
|
} catch (InvalidSourceException | SourceNotSetException
|
||||||
|
| FailedTransferException
|
||||||
|
| InitializationException e) {
|
||||||
|
log.error("Unable to send file {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug("Sending uris to {}",dt.getDestinationCapabilities());
|
||||||
|
for(String f:uris){
|
||||||
|
try {
|
||||||
|
TransferResult res=dt.httpSource(f,dest);
|
||||||
|
report.addReport(ReportType.uri,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||||
|
} catch (InvalidSourceException | SourceNotSetException
|
||||||
|
| FailedTransferException
|
||||||
|
| InitializationException e) {
|
||||||
|
log.error("Unable to send uri {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug("Sending storageIds to {}",dt.getDestinationCapabilities());
|
||||||
|
for(String f:storageIds){
|
||||||
|
try {
|
||||||
|
TransferResult res=dt.storageId(f,dest);
|
||||||
|
report.addReport(ReportType.storage,f, res.getTransferedBytes(), res.getElapsedTime());
|
||||||
|
} catch (InvalidSourceException | SourceNotSetException
|
||||||
|
| FailedTransferException
|
||||||
|
| InitializationException e) {
|
||||||
|
log.error("Unable to send storageId {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reports.add(report);
|
||||||
|
}catch(Exception e){
|
||||||
|
reports.add(new ErrorReport(id));
|
||||||
|
}finally{
|
||||||
|
doneSignal.countDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
doneSignal.await();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
|
}
|
||||||
|
System.out.println("*****************************");
|
||||||
|
for(TransferReport r:reports)System.out.println(r.print());
|
||||||
|
|
||||||
|
ExecutorServiceUtil.shutdown(executor);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||||
|
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||||
|
|
||||||
|
public class StorageTest {
|
||||||
|
|
||||||
|
static String scope="/gcube/devsec";
|
||||||
|
|
||||||
|
public static void main(String[] args) throws RemoteBackendException, FileNotFoundException {
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
||||||
|
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
||||||
|
System.out.println(StorageUtils.getUrlById(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,86 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
import javax.ws.rs.client.ClientBuilder;
|
||||||
|
import javax.ws.rs.client.WebTarget;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.data.transfer.library.client.AuthorizationFilter;
|
||||||
|
import org.gcube.data.transfer.library.client.Client;
|
||||||
|
import org.gcube.data.transfer.library.faults.CommunicationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||||
|
import org.gcube.data.transfer.model.ServiceConstants;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
import org.gcube.data.transfer.model.TransferRequest;
|
||||||
|
import org.gcube.data.transfer.model.TransferTicket;
|
||||||
|
import org.gcube.data.transfer.model.TransferTicket.Status;
|
||||||
|
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
||||||
|
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
||||||
|
import org.glassfish.jersey.client.ClientConfig;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
|
||||||
|
public class TestClientCalls {
|
||||||
|
|
||||||
|
static String hostname="http://node3-d-d4s.d4science.org:80";
|
||||||
|
// static String hostname="http://pc-fabio.isti.cnr.it:8080";
|
||||||
|
static String scope="/gcube/devNext";
|
||||||
|
static Client client;
|
||||||
|
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws ServiceNotFoundException{
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
client=new Client(hostname);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getCapabilties() throws CommunicationException{
|
||||||
|
System.out.println(client.getCapabilties());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void doTheTransfer() throws MalformedURLException, RemoteServiceException{
|
||||||
|
Destination dest=new Destination("outputFile");
|
||||||
|
dest.setCreateSubfolders(true);
|
||||||
|
dest.setSubFolder("bla/bla/bllaaa");
|
||||||
|
dest.setOnExistingFileName(DestinationClashPolicy.ADD_SUFFIX);
|
||||||
|
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
||||||
|
TransferRequest request= new TransferRequest("", new HttpDownloadSettings(new URL("http://goo.gl/oLP7zG"), HttpDownloadOptions.DEFAULT),dest);
|
||||||
|
System.out.println("Submitting "+request);
|
||||||
|
TransferTicket ticket=client.submit(request);
|
||||||
|
System.out.println("Ticket is "+ticket);
|
||||||
|
|
||||||
|
boolean continuePolling=true;
|
||||||
|
do{
|
||||||
|
ticket=client.getTransferStatus(ticket.getId());
|
||||||
|
System.out.println("Status : "+ticket);
|
||||||
|
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
||||||
|
try{
|
||||||
|
Thread.sleep(1000);
|
||||||
|
}catch(InterruptedException e){}
|
||||||
|
}while(continuePolling);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void directCall(){
|
||||||
|
javax.ws.rs.client.Client client = ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class));
|
||||||
|
WebTarget target=client.target(hostname+"/data-transfer-service"+ServiceConstants.APPLICATION_PATH+"Capabilities");
|
||||||
|
// WebTarget target=client.target(hostname+"/data-transfer-service/gcube/service/Capabilities");
|
||||||
|
System.out.println("Asking capabilities to target : "+target.getUri());
|
||||||
|
System.out.println(target.
|
||||||
|
request(MediaType.APPLICATION_XML).get(TransferCapabilities.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
|
||||||
|
public class TokenSetter {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private static Properties props=new Properties();
|
||||||
|
|
||||||
|
static{
|
||||||
|
try {
|
||||||
|
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void set(String scope){
|
||||||
|
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||||
|
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,89 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
|
import org.gcube.data.transfer.library.model.Source;
|
||||||
|
import org.gcube.data.transfer.model.TransferCapabilities;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.ToString;
|
||||||
|
|
||||||
|
public class TransferReport {
|
||||||
|
|
||||||
|
public static enum ReportType{
|
||||||
|
local,uri,storage
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static final char end='\n';
|
||||||
|
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Getter
|
||||||
|
@ToString
|
||||||
|
private static class ReportItem{
|
||||||
|
String source;
|
||||||
|
long size;
|
||||||
|
long elapsed;
|
||||||
|
|
||||||
|
|
||||||
|
// bytes/msec
|
||||||
|
public long getAvgSpeed(){
|
||||||
|
return size/elapsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected TransferCapabilities host;
|
||||||
|
|
||||||
|
private Map<ReportType,ArrayList<ReportItem>> reports=new HashMap<>();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public TransferReport(TransferCapabilities host) {
|
||||||
|
super();
|
||||||
|
this.host = host;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void addReport(ReportType sourceType,String theSource,long size,long elapsed){
|
||||||
|
if(!reports.containsKey(sourceType)) reports.put(sourceType, new ArrayList<ReportItem>());
|
||||||
|
reports.get(sourceType).add(new ReportItem(theSource, size, elapsed));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public String print(){
|
||||||
|
StringBuilder builder=new StringBuilder(" Report for : "+host+end);
|
||||||
|
for(Entry<ReportType,ArrayList<ReportItem>> entry:reports.entrySet()){
|
||||||
|
builder.append("Source type "+entry.getKey()+end);
|
||||||
|
ReportItem maxSizeItem=null;
|
||||||
|
ReportItem maxAvgSpeedItem=null;
|
||||||
|
ReportItem minSizeItem=null;
|
||||||
|
ReportItem minAvgSpeedItem=null;
|
||||||
|
long currentAvgSpeedCounter=0l;
|
||||||
|
|
||||||
|
for(ReportItem item:entry.getValue()){
|
||||||
|
if(maxSizeItem==null||maxSizeItem.getSize()<item.getSize()) maxSizeItem=item;
|
||||||
|
if(maxAvgSpeedItem==null||maxAvgSpeedItem.getAvgSpeed()<item.getAvgSpeed()) maxAvgSpeedItem=item;
|
||||||
|
if(minSizeItem==null||minSizeItem.getSize()>item.getSize()) minSizeItem=item;
|
||||||
|
if(minAvgSpeedItem==null||minAvgSpeedItem.getAvgSpeed()>item.getAvgSpeed()) minAvgSpeedItem=item;
|
||||||
|
currentAvgSpeedCounter+=item.getAvgSpeed();
|
||||||
|
}
|
||||||
|
|
||||||
|
builder.append("Max Size Item : "+maxSizeItem+end);
|
||||||
|
builder.append("Max Avg Speed Item : "+maxAvgSpeedItem+end);
|
||||||
|
builder.append("Min Size Item : "+minSizeItem+end);
|
||||||
|
builder.append("Min Avg Speed Item : "+minAvgSpeedItem+end);
|
||||||
|
builder.append("Total avg speed : "+currentAvgSpeedCounter/entry.getValue().size()+end);
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.toString();
|
||||||
|
};
|
||||||
|
}
|
|
@ -0,0 +1,89 @@
|
||||||
|
package org.gcube.data.transfer.library;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||||
|
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||||
|
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||||
|
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||||
|
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||||
|
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||||
|
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||||
|
import org.gcube.data.transfer.library.utils.StorageUtils;
|
||||||
|
import org.gcube.data.transfer.model.Destination;
|
||||||
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TransfererTest {
|
||||||
|
|
||||||
|
static String hostname="http://node3-d-d4s.d4science.org:80";
|
||||||
|
static String nodeId="462b68c5-463f-4295-86da-37d6c0abc7ea";
|
||||||
|
static String scope="/gcube/devsec";
|
||||||
|
|
||||||
|
static DataTransferClient client;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws UnreachableNodeException, ServiceNotFoundException, HostingNodeNotFoundException{
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
client=DataTransferClient.getInstanceByEndpoint(hostname);
|
||||||
|
// client=DataTransferClient.getInstanceByNodeId(nodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void localFile() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
for(int i=0;i<3;i++){
|
||||||
|
String localFile="/home/fabio/Dropbox/Mindless/01- COMA - Mindless.mp3";
|
||||||
|
String transferredFileName="Mindless.mp3";
|
||||||
|
Destination dest=new Destination(transferredFileName);
|
||||||
|
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||||
|
TransferResult res=client.localFile(localFile,dest);
|
||||||
|
String remotePath=res.getRemotePath();
|
||||||
|
Assert.assertEquals(transferredFileName, remotePath.substring(remotePath.lastIndexOf(File.separatorChar)+1));
|
||||||
|
System.out.println(res);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void httpUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
String link="http://goo.gl/oLP7zG";
|
||||||
|
System.out.println(client.httpSource(link,new Destination("Some import")));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
||||||
|
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
||||||
|
Destination dest=new Destination("some/where","My Pdf.pdf");
|
||||||
|
TransferResult res=client.storageId(id,dest);
|
||||||
|
Assert.assertTrue(res.getRemotePath().contains(dest.getSubFolder()+File.separatorChar+dest.getDestinationFileName()));
|
||||||
|
System.out.println(client.storageId(id,dest));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=InvalidSourceException.class)
|
||||||
|
public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
System.out.println(client.storageId("13245780t",new Destination("my file")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=InvalidSourceException.class)
|
||||||
|
public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
String localFile="/home/fabio/Downloads/123045689.mp3";
|
||||||
|
System.out.println(client.localFile(localFile,new Destination("12345")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=InvalidSourceException.class)
|
||||||
|
public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{
|
||||||
|
String link="https://www.some.where.com/over/theRainbow.txt";
|
||||||
|
System.out.println(client.httpSource(link,new Destination("oz")));
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue