Initial import.
git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@131225 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
60b717d1f6
commit
f924eed16d
36
.classpath
36
.classpath
|
@ -1,36 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<classpath>
|
|
||||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="optional" value="true"/>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="optional" value="true"/>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
|
||||||
<attributes>
|
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
|
||||||
</attributes>
|
|
||||||
</classpathentry>
|
|
||||||
<classpathentry kind="output" path="target/classes"/>
|
|
||||||
</classpath>
|
|
23
.project
23
.project
|
@ -1,23 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>data-transfer-library</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
|
@ -1,6 +0,0 @@
|
||||||
eclipse.preferences.version=1
|
|
||||||
encoding//src/main/java=UTF-8
|
|
||||||
encoding//src/main/resources=UTF-8
|
|
||||||
encoding//src/test/java=UTF-8
|
|
||||||
encoding//src/test/resources=UTF-8
|
|
||||||
encoding/<project>=UTF-8
|
|
|
@ -1,5 +0,0 @@
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
|
||||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.7
|
|
|
@ -1,4 +0,0 @@
|
||||||
activeProfiles=
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
resolveWorkspaceProjects=false
|
|
||||||
version=1
|
|
|
@ -1 +0,0 @@
|
||||||
${gcube.license}
|
|
|
@ -1,65 +0,0 @@
|
||||||
The gCube System - ${name}
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
${description}
|
|
||||||
|
|
||||||
|
|
||||||
${gcube.description}
|
|
||||||
|
|
||||||
${gcube.funding}
|
|
||||||
|
|
||||||
|
|
||||||
Version
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
${version} (${buildDate})
|
|
||||||
|
|
||||||
Please see the file named "changelog.xml" in this directory for the release notes.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Authors
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
|
||||||
|
|
||||||
Maintainers
|
|
||||||
-----------
|
|
||||||
|
|
||||||
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
|
|
||||||
|
|
||||||
|
|
||||||
Download information
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
Source code is available from SVN:
|
|
||||||
${scm.url}
|
|
||||||
|
|
||||||
Binaries can be downloaded from the gCube website:
|
|
||||||
${gcube.website}
|
|
||||||
|
|
||||||
Installation
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
Installation documentation is available on-line in the gCube Wiki:
|
|
||||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
|
||||||
|
|
||||||
Documentation
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
Documentation is available on-line in the gCube Wiki:
|
|
||||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
|
||||||
${gcube.wikiRoot}/Tabular_Data_Manager
|
|
||||||
|
|
||||||
|
|
||||||
Support
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
Bugs and support requests can be reported in the gCube issue tracking tool:
|
|
||||||
${gcube.issueTracking}
|
|
||||||
|
|
||||||
|
|
||||||
Licensing
|
|
||||||
--------------------------------------------------
|
|
||||||
|
|
||||||
This software is licensed under the terms you may find in the file named "LICENSE" in this directory.
|
|
|
@ -1,5 +0,0 @@
|
||||||
<ReleaseNotes>
|
|
||||||
<Changeset component="data-transfer-library-1.0.0" date="2016-02-22">
|
|
||||||
<Change>First Release</Change>
|
|
||||||
</Changeset>
|
|
||||||
</ReleaseNotes>
|
|
|
@ -1,30 +0,0 @@
|
||||||
<assembly
|
|
||||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
|
||||||
<id>servicearchive</id>
|
|
||||||
<formats>
|
|
||||||
<format>tar.gz</format>
|
|
||||||
</formats>
|
|
||||||
<baseDirectory>/</baseDirectory>
|
|
||||||
<fileSets>
|
|
||||||
<fileSet>
|
|
||||||
<directory>${distroDirectory}</directory>
|
|
||||||
<outputDirectory>/</outputDirectory>
|
|
||||||
<useDefaultExcludes>true</useDefaultExcludes>
|
|
||||||
<includes>
|
|
||||||
<include>README</include>
|
|
||||||
<include>LICENSE</include>
|
|
||||||
<include>changelog.xml</include>
|
|
||||||
</includes>
|
|
||||||
<fileMode>755</fileMode>
|
|
||||||
<filtered>true</filtered>
|
|
||||||
</fileSet>
|
|
||||||
</fileSets>
|
|
||||||
<files>
|
|
||||||
<file>
|
|
||||||
<source>target/${build.finalName}.${project.packaging}</source>
|
|
||||||
<outputDirectory>/${artifactId}</outputDirectory>
|
|
||||||
</file>
|
|
||||||
</files>
|
|
||||||
</assembly>
|
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<Resource>
|
|
||||||
<ID></ID>
|
|
||||||
<Type>Service</Type>
|
|
||||||
<Profile>
|
|
||||||
<Description>${description}</Description>
|
|
||||||
<Class>DataTransfer</Class>
|
|
||||||
<Name>${artifactId}</Name>
|
|
||||||
<Version>1.0.0</Version>
|
|
||||||
<Packages>
|
|
||||||
<Software>
|
|
||||||
<Description>${description}</Description>
|
|
||||||
<Name>${artifactId}</Name>
|
|
||||||
<Version>${version}</Version>
|
|
||||||
<MavenCoordinates>
|
|
||||||
<groupId>${groupId}</groupId>
|
|
||||||
<artifactId>${artifactId}</artifactId>
|
|
||||||
<version>${version}</version>
|
|
||||||
</MavenCoordinates>
|
|
||||||
<Type>library</Type>
|
|
||||||
<Files>
|
|
||||||
<File>${build.finalName}.jar</File>
|
|
||||||
</Files>
|
|
||||||
</Software>
|
|
||||||
</Packages>
|
|
||||||
</Profile>
|
|
||||||
</Resource>
|
|
||||||
|
|
||||||
|
|
80
pom.xml
80
pom.xml
|
@ -1,80 +0,0 @@
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
<parent>
|
|
||||||
<groupId>org.gcube.tools</groupId>
|
|
||||||
<artifactId>maven-parent</artifactId>
|
|
||||||
<version>LATEST</version>
|
|
||||||
</parent>
|
|
||||||
<groupId>org.gcube.data.transfer</groupId>
|
|
||||||
<artifactId>data-transfer-library</artifactId>
|
|
||||||
<version>0.0.1-SNAPSHOT</version>
|
|
||||||
<name>DataTransfer-library</name>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.core</groupId>
|
|
||||||
<artifactId>common-gcube-calls</artifactId>
|
|
||||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.data.transfer</groupId>
|
|
||||||
<artifactId>data-transfer-model</artifactId>
|
|
||||||
<version>1.0.0-SNAPSHOT</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-api</artifactId>
|
|
||||||
<version>1.7.5</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>junit</groupId>
|
|
||||||
<artifactId>junit</artifactId>
|
|
||||||
<version>4.10</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- STORAGE -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.contentmanagement</groupId>
|
|
||||||
<artifactId>storage-manager-core</artifactId>
|
|
||||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.contentmanagement</groupId>
|
|
||||||
<artifactId>storage-manager-wrapper</artifactId>
|
|
||||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- lombok -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.projectlombok</groupId>
|
|
||||||
<artifactId>lombok</artifactId>
|
|
||||||
<version>0.11.6</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- JERSEY -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.glassfish.jersey.core</groupId>
|
|
||||||
<artifactId>jersey-client</artifactId>
|
|
||||||
<version>2.22.2</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.glassfish.jersey.media</groupId>
|
|
||||||
<artifactId>jersey-media-jaxb</artifactId>
|
|
||||||
<version>2.18</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- Test log binding -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>ch.qos.logback</groupId>
|
|
||||||
<artifactId>logback-classic</artifactId>
|
|
||||||
<version>1.0.13</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
|
|
||||||
|
|
||||||
</project>
|
|
|
@ -1,87 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
import lombok.Synchronized;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
|
||||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
|
||||||
import org.gcube.data.transfer.library.transferers.Transferer;
|
|
||||||
import org.gcube.data.transfer.library.transferers.TransfererBuilder;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class DataTransferClient {
|
|
||||||
|
|
||||||
private Transferer transferer=null;
|
|
||||||
|
|
||||||
private DataTransferClient(Transferer transferer) {
|
|
||||||
this.transferer=transferer;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static DataTransferClient getInstanceByEndpoint(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
|
||||||
log.debug("Getting transferer for endpoint : "+endpoint);
|
|
||||||
return new DataTransferClient(TransfererBuilder.getTransfererByHost(endpoint));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static DataTransferClient getInstanceByNodeId(String id) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
|
||||||
log.debug("Getting transferer for nodeId : "+id);
|
|
||||||
return new DataTransferClient(TransfererBuilder.getTransfererByhostingNodeId(id));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public TransferCapabilities getDestinationCapabilities() throws InitializationException{
|
|
||||||
return this.transferer.getDestinationCapabilities();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Synchronized("transferer")
|
|
||||||
public TransferResult localFile(String path) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
|
||||||
log.debug("Sending local file {} to {}",path,transferer.getDestinationCapabilities().getHostName());
|
|
||||||
transferer.localFile(path);
|
|
||||||
return transferer.transfer();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Synchronized("transferer")
|
|
||||||
public TransferResult localFile(File file) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
|
||||||
log.debug("Sending local file {} to {}",file.getAbsolutePath(),transferer.getDestinationCapabilities().getHostName());
|
|
||||||
transferer.localFile(file);
|
|
||||||
return transferer.transfer();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Synchronized("transferer")
|
|
||||||
public TransferResult httpSource(String url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
|
||||||
log.debug("Passed url string : "+url);
|
|
||||||
try{
|
|
||||||
return this.httpSource(new URL(url));
|
|
||||||
}catch(MalformedURLException e){
|
|
||||||
throw new InvalidSourceException("Invalid url : "+url);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Synchronized("transferer")
|
|
||||||
public TransferResult httpSource(URL url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
|
||||||
log.debug("Sending from url {} to {}",url,transferer.getDestinationCapabilities().getHostName());
|
|
||||||
transferer.fromURL(url);
|
|
||||||
return transferer.transfer();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Synchronized("transferer")
|
|
||||||
public TransferResult storageId(String id) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
|
|
||||||
log.debug("Sending from storage id {} to {}",id,transferer.getDestinationCapabilities().getHostName());
|
|
||||||
transferer.storageFileId(id);
|
|
||||||
return transferer.transfer();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,22 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NonNull;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.model.Source;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public class TransferResult {
|
|
||||||
|
|
||||||
@NonNull
|
|
||||||
private Source source;
|
|
||||||
@NonNull
|
|
||||||
private String destinationHostName;
|
|
||||||
@NonNull
|
|
||||||
private Long elapsedTime;
|
|
||||||
@NonNull
|
|
||||||
private Long transferedBytes;
|
|
||||||
@NonNull
|
|
||||||
private String remotePath;
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,31 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.caches;
|
|
||||||
|
|
||||||
import lombok.Synchronized;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.client.Client;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class CapabilitiesCache extends TTLCache<TransferCapabilities> {
|
|
||||||
|
|
||||||
private static CapabilitiesCache instance=null;
|
|
||||||
|
|
||||||
// @Synchronized
|
|
||||||
// public static CapabilitiesCache getInstance(){
|
|
||||||
// if(instance==null)instance=new CapabilitiesCache();
|
|
||||||
// return instance;
|
|
||||||
// }
|
|
||||||
|
|
||||||
private CapabilitiesCache(){
|
|
||||||
super(5*60*1000l,2*60*1000l,"Capabilities");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected TransferCapabilities getNew(String id) throws Exception{
|
|
||||||
log.debug("Getting capabilties for host "+id);
|
|
||||||
return new Client(id).getCapabilties();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,94 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.caches;
|
|
||||||
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
|
||||||
import java.util.concurrent.ThreadFactory;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import lombok.Synchronized;
|
|
||||||
import lombok.ToString;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@ToString
|
|
||||||
@Slf4j
|
|
||||||
public abstract class TTLCache<T> {
|
|
||||||
|
|
||||||
|
|
||||||
// STATIC
|
|
||||||
|
|
||||||
// private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1,new ThreadFactory(){
|
|
||||||
// public Thread newThread(Runnable r) {
|
|
||||||
// Thread t = Executors.defaultThreadFactory().newThread(r);
|
|
||||||
// t.setDaemon(true);
|
|
||||||
// return t;
|
|
||||||
// }
|
|
||||||
// });
|
|
||||||
private static final HashSet<TTLCache<?>> createdMaps=new HashSet<TTLCache<?>>();
|
|
||||||
|
|
||||||
|
|
||||||
static {
|
|
||||||
// scheduler.scheduleAtFixedRate(new Runnable() {
|
|
||||||
//
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// @Override
|
|
||||||
// public void run() {
|
|
||||||
// log.debug("Running Maps Cleaner, maps count : "+createdMaps.size());
|
|
||||||
// int removed=0;
|
|
||||||
// for(TTLCache<?> theMap:createdMaps)
|
|
||||||
// theMap.purgeItems();
|
|
||||||
// log.debug("Removed "+removed+" old tickets");
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// }, 3, 3, TimeUnit.MINUTES);
|
|
||||||
//
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
//************************* CACHE
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
protected TTLCache(Long cacheKeepAliveTime, Long objectTTL, String cacheName) {
|
|
||||||
this.cacheKeepAliveTime = cacheKeepAliveTime;
|
|
||||||
this.objectTTL = objectTTL;
|
|
||||||
this.cacheName = cacheName;
|
|
||||||
createdMaps.add(this);
|
|
||||||
log.debug("Created Cache "+this);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private ConcurrentHashMap<String,TTLContainer<T>> theMap=new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
private Long cacheKeepAliveTime;
|
|
||||||
private Long objectTTL;
|
|
||||||
private String cacheName;
|
|
||||||
|
|
||||||
@Synchronized
|
|
||||||
public T getObject(String id) throws Exception{
|
|
||||||
if(!theMap.contains(id)||System.currentTimeMillis()-theMap.get(id).getCreationTime()>objectTTL)
|
|
||||||
theMap.put(id, new TTLContainer<T>(getNew(id)));
|
|
||||||
return theMap.get(id).getTheObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private int purgeItems(){
|
|
||||||
log.debug(cacheName+" Purging objects, keep alive time is "+cacheKeepAliveTime);
|
|
||||||
int totalCount=theMap.size();
|
|
||||||
int removed=0;
|
|
||||||
for(Entry<String,TTLContainer<T>> entry:theMap.entrySet())
|
|
||||||
if(System.currentTimeMillis()-entry.getValue().getLastUsageTime()>cacheKeepAliveTime){
|
|
||||||
theMap.remove(entry.getKey());
|
|
||||||
removed++;
|
|
||||||
}
|
|
||||||
log.debug(cacheName+" Removed "+removed+" out of "+totalCount);
|
|
||||||
return removed;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract T getNew(String id) throws Exception;
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,30 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.caches;
|
|
||||||
|
|
||||||
public class TTLContainer<T> {
|
|
||||||
|
|
||||||
private long lastUsageTime=System.currentTimeMillis();
|
|
||||||
private long creationTime=System.currentTimeMillis();
|
|
||||||
|
|
||||||
private T theObject;
|
|
||||||
public TTLContainer(T theObject) {
|
|
||||||
this.theObject = theObject;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void update(){
|
|
||||||
lastUsageTime=System.currentTimeMillis();
|
|
||||||
}
|
|
||||||
|
|
||||||
public T getTheObject(){
|
|
||||||
update();
|
|
||||||
return theObject;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getLastUsageTime() {
|
|
||||||
return lastUsageTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getCreationTime() {
|
|
||||||
return creationTime;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,29 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.client;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
|
|
||||||
import javax.ws.rs.client.ClientRequestContext;
|
|
||||||
import javax.ws.rs.client.ClientRequestFilter;
|
|
||||||
|
|
||||||
import org.gcube.common.calls.Call;
|
|
||||||
import org.gcube.common.calls.Interceptors;
|
|
||||||
import org.gcube.common.calls.Request;
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
|
|
||||||
public class AuthorizationFilter implements ClientRequestFilter {
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void filter(final ClientRequestContext rc) throws IOException {
|
|
||||||
if (ScopeProvider.instance.get()!=null){
|
|
||||||
Request requestContext = Interceptors.executeRequestChain(new Call());
|
|
||||||
|
|
||||||
for (Entry<String, String> entry: requestContext.getHeaders()){
|
|
||||||
System.out.println("setting "+entry.getKey()+" "+entry.getValue());
|
|
||||||
rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,96 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.client;
|
|
||||||
|
|
||||||
import javax.ws.rs.client.ClientBuilder;
|
|
||||||
import javax.ws.rs.client.Entity;
|
|
||||||
import javax.ws.rs.client.WebTarget;
|
|
||||||
import javax.ws.rs.core.MediaType;
|
|
||||||
import javax.ws.rs.core.Response;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.CommunicationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.DataTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.model.ServiceConstants;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
import org.gcube.data.transfer.model.TransferRequest;
|
|
||||||
import org.gcube.data.transfer.model.TransferTicket;
|
|
||||||
import org.glassfish.jersey.client.ClientConfig;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class Client {
|
|
||||||
|
|
||||||
private static ClientConfig config=null;
|
|
||||||
|
|
||||||
static{
|
|
||||||
log.debug("Creating configuration ..");
|
|
||||||
config=new ClientConfig();
|
|
||||||
config.register(AuthorizationFilter.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String endpoint;
|
|
||||||
|
|
||||||
private WebTarget rootTarget;
|
|
||||||
|
|
||||||
public Client(String endpoint) throws ServiceNotFoundException{
|
|
||||||
try{
|
|
||||||
log.debug("Creating client for base "+endpoint);
|
|
||||||
this.endpoint=endpoint+"";
|
|
||||||
rootTarget= ClientBuilder.newClient(config).target(endpoint).path("data-transfer-service").path(ServiceConstants.APPLICATION_PATH);
|
|
||||||
// checkResponse(rootTarget.request().get());
|
|
||||||
|
|
||||||
log.debug("Root Taget IS {} ",rootTarget.getUri());
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new ServiceNotFoundException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public String getEndpoint() {
|
|
||||||
return endpoint;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransferCapabilities getCapabilties() throws CommunicationException{
|
|
||||||
WebTarget capabilitiesTarget=rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME);
|
|
||||||
log.debug("Getting capabilities from {}, path is {} ",endpoint,capabilitiesTarget.getUri());
|
|
||||||
try{
|
|
||||||
Response resp=capabilitiesTarget.request().accept(MediaType.APPLICATION_XML_TYPE).get();
|
|
||||||
checkResponse(resp);
|
|
||||||
return resp.readEntity(TransferCapabilities.class);
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new CommunicationException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public TransferTicket submit(TransferRequest request) throws RemoteServiceException{
|
|
||||||
log.debug("Sending request {} to {}",request,endpoint);
|
|
||||||
try{
|
|
||||||
Response resp=rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML));
|
|
||||||
checkResponse(resp);
|
|
||||||
return resp.readEntity(TransferTicket.class);
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new RemoteServiceException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransferTicket getTransferStatus(String transferId) throws RemoteServiceException{
|
|
||||||
log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint);
|
|
||||||
try{
|
|
||||||
Response resp=rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get();
|
|
||||||
checkResponse(resp);
|
|
||||||
return resp.readEntity(TransferTicket.class);
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new RemoteServiceException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected void checkResponse(Response toCheck) throws Exception{
|
|
||||||
switch(toCheck.getStatusInfo().getFamily()){
|
|
||||||
case SUCCESSFUL : break;
|
|
||||||
default : throw new Exception("Unexpected Response code : "+toCheck.getStatus());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class CommunicationException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 1L;
|
|
||||||
|
|
||||||
public CommunicationException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public CommunicationException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public CommunicationException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public CommunicationException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public CommunicationException(String message, Throwable cause, boolean enableSuppression,
|
|
||||||
boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public abstract class DataTransferException extends Exception {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 8488351139669552406L;
|
|
||||||
|
|
||||||
public DataTransferException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataTransferException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataTransferException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataTransferException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataTransferException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class FailedTransferException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = -807695677781676381L;
|
|
||||||
|
|
||||||
public FailedTransferException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public FailedTransferException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public FailedTransferException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public FailedTransferException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public FailedTransferException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class HostingNodeNotFoundException extends InitializationException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 5786009015981659074L;
|
|
||||||
|
|
||||||
public HostingNodeNotFoundException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public HostingNodeNotFoundException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public HostingNodeNotFoundException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public HostingNodeNotFoundException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public HostingNodeNotFoundException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class InitializationException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 2204462227670167203L;
|
|
||||||
|
|
||||||
public InitializationException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InitializationException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InitializationException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InitializationException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InitializationException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class InvalidSourceException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 7631518288856917202L;
|
|
||||||
|
|
||||||
public InvalidSourceException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InvalidSourceException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InvalidSourceException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InvalidSourceException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public InvalidSourceException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class RemoteServiceException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 5320978791087129997L;
|
|
||||||
|
|
||||||
public RemoteServiceException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public RemoteServiceException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public RemoteServiceException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public RemoteServiceException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public RemoteServiceException(String message, Throwable cause, boolean enableSuppression,
|
|
||||||
boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class ServiceNotFoundException extends InitializationException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = -3494011839783891174L;
|
|
||||||
|
|
||||||
public ServiceNotFoundException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public ServiceNotFoundException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public ServiceNotFoundException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public ServiceNotFoundException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public ServiceNotFoundException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class SourceNotSetException extends DataTransferException {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = -5868191252361823341L;
|
|
||||||
|
|
||||||
public SourceNotSetException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public SourceNotSetException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public SourceNotSetException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public SourceNotSetException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public SourceNotSetException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,37 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.faults;
|
|
||||||
|
|
||||||
public class UnreachableNodeException extends InitializationException {
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
private static final long serialVersionUID = 8739586712271226827L;
|
|
||||||
|
|
||||||
public UnreachableNodeException() {
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public UnreachableNodeException(String message) {
|
|
||||||
super(message);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public UnreachableNodeException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public UnreachableNodeException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
public UnreachableNodeException(String message, Throwable cause,
|
|
||||||
boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
// TODO Auto-generated constructor stub
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,41 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.model;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
|
|
||||||
public class LocalSource extends Source<File> {
|
|
||||||
|
|
||||||
private File theFile=null;
|
|
||||||
|
|
||||||
public LocalSource(File theFile) throws InvalidSourceException {
|
|
||||||
super();
|
|
||||||
if(theFile==null)throw new InvalidSourceException("File cannot be null");
|
|
||||||
this.theFile = theFile;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean validate() throws InvalidSourceException {
|
|
||||||
if(!theFile.exists()) throw new InvalidSourceException("File "+theFile.getAbsolutePath()+"doesn't exist");
|
|
||||||
if(!theFile.canRead()) throw new InvalidSourceException("Unable to read from file "+theFile.getAbsolutePath());
|
|
||||||
if(theFile.isDirectory()) throw new InvalidSourceException("Transfer of directory is not yet supported");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void prepare() {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void clean() {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public File getTheSource() {
|
|
||||||
return theFile;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,17 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.model;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public abstract class Source<T>{
|
|
||||||
|
|
||||||
public abstract boolean validate() throws InvalidSourceException;
|
|
||||||
|
|
||||||
public abstract void prepare();
|
|
||||||
|
|
||||||
public abstract void clean();
|
|
||||||
|
|
||||||
public abstract T getTheSource();
|
|
||||||
}
|
|
|
@ -1,47 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.model;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
|
||||||
|
|
||||||
public class StorageSource extends Source<String> {
|
|
||||||
|
|
||||||
private String id=null;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public StorageSource(String id) throws InvalidSourceException {
|
|
||||||
super();
|
|
||||||
if(id==null) throw new InvalidSourceException("Storage id cannot be null");
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean validate() throws InvalidSourceException {
|
|
||||||
try{
|
|
||||||
if(!StorageUtils.checkStorageId(id)) throw new Exception("Not valid");
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new InvalidSourceException("Invalid storage ID "+id);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void prepare() {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void clean() {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getTheSource() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,62 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.model;
|
|
||||||
|
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
|
|
||||||
public class URLSource extends Source<URL> {
|
|
||||||
|
|
||||||
|
|
||||||
private URL theURL=null;
|
|
||||||
|
|
||||||
|
|
||||||
public URLSource(URL theURL) throws InvalidSourceException {
|
|
||||||
super();
|
|
||||||
if(theURL==null) throw new InvalidSourceException("URL Source cannot be null");
|
|
||||||
this.theURL = theURL;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean validate() throws InvalidSourceException {
|
|
||||||
try{
|
|
||||||
HttpURLConnection conn=(HttpURLConnection) theURL.openConnection();
|
|
||||||
conn.setConnectTimeout(3000);
|
|
||||||
conn.setRequestMethod("HEAD");
|
|
||||||
int responseCode = conn.getResponseCode();
|
|
||||||
return (200 <= responseCode && responseCode <= 399);
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new InvalidSourceException("Unable to contact URL "+theURL,e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void prepare() {
|
|
||||||
// nothing to do
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void clean() {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
|
|
||||||
/* (non-Javadoc)
|
|
||||||
* @see java.lang.Object#toString()
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
StringBuilder builder = new StringBuilder();
|
|
||||||
builder.append("URLSource [theURL=");
|
|
||||||
builder.append(theURL);
|
|
||||||
builder.append("]");
|
|
||||||
return builder.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public URL getTheSource() {
|
|
||||||
return theURL;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,71 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.transferers;
|
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
||||||
import org.gcube.data.transfer.library.client.Client;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.model.LocalSource;
|
|
||||||
import org.gcube.data.transfer.library.model.StorageSource;
|
|
||||||
import org.gcube.data.transfer.library.model.URLSource;
|
|
||||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
|
||||||
import org.gcube.data.transfer.model.TransferRequest;
|
|
||||||
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
|
||||||
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class HTTPTransferer extends Transferer {
|
|
||||||
|
|
||||||
HTTPTransferer(Client client) {
|
|
||||||
super(client);
|
|
||||||
}
|
|
||||||
|
|
||||||
private URL link;
|
|
||||||
|
|
||||||
private String toDeleteStorageId=null;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected TransferRequest prepareRequest()throws InitializationException {
|
|
||||||
return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{
|
|
||||||
if(source instanceof LocalSource){
|
|
||||||
toDeleteStorageId=StorageUtils.putOntoStorage(((LocalSource)source).getTheSource());
|
|
||||||
return new URL(StorageUtils.getUrlById(toDeleteStorageId));
|
|
||||||
}else if (source instanceof StorageSource){
|
|
||||||
return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource()));
|
|
||||||
}else if (source instanceof URLSource){
|
|
||||||
return ((URLSource)source).getTheSource();
|
|
||||||
}else throw new InvalidSourceException("Source cannot be handled "+source);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void prepare() throws InitializationException {
|
|
||||||
try{
|
|
||||||
link=getHttpLink();
|
|
||||||
log.debug("Obtained link "+link);
|
|
||||||
super.prepare();
|
|
||||||
}catch(RemoteBackendException e){
|
|
||||||
throw new InitializationException(e);
|
|
||||||
}catch(FileNotFoundException e){
|
|
||||||
throw new InitializationException(e);
|
|
||||||
}catch(InvalidSourceException e){
|
|
||||||
throw new InitializationException(e);
|
|
||||||
}catch (MalformedURLException e){
|
|
||||||
throw new InitializationException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void clean() {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
super.clean();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,125 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.transferers;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.TransferResult;
|
|
||||||
import org.gcube.data.transfer.library.client.Client;
|
|
||||||
import org.gcube.data.transfer.library.faults.CommunicationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
|
||||||
import org.gcube.data.transfer.library.model.LocalSource;
|
|
||||||
import org.gcube.data.transfer.library.model.Source;
|
|
||||||
import org.gcube.data.transfer.library.model.StorageSource;
|
|
||||||
import org.gcube.data.transfer.library.model.URLSource;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
import org.gcube.data.transfer.model.TransferRequest;
|
|
||||||
import org.gcube.data.transfer.model.TransferTicket;
|
|
||||||
import org.gcube.data.transfer.model.TransferTicket.Status;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public abstract class Transferer {
|
|
||||||
|
|
||||||
protected Client client;
|
|
||||||
|
|
||||||
protected Transferer(Client client){
|
|
||||||
this.client=client;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
protected Source source=null;
|
|
||||||
protected boolean prepared=false;
|
|
||||||
|
|
||||||
public Transferer localFile(File f) throws InvalidSourceException{
|
|
||||||
source=new LocalSource(f);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Transferer localFile(String path) throws InvalidSourceException{
|
|
||||||
try{
|
|
||||||
File f=new File(path);
|
|
||||||
return localFile(f);
|
|
||||||
}catch(NullPointerException e){
|
|
||||||
throw new InvalidSourceException("Path is null",e);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public Transferer storageFileId(String fileId) throws InvalidSourceException{
|
|
||||||
source=new StorageSource(fileId);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Transferer fromURL(URL sourceURL) throws InvalidSourceException{
|
|
||||||
source=new URLSource(sourceURL);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException{
|
|
||||||
try{
|
|
||||||
checkSource();
|
|
||||||
prepare();
|
|
||||||
TransferRequest request=prepareRequest();
|
|
||||||
log.debug("Request is {}, sending it to {}",request,client.getEndpoint());
|
|
||||||
TransferResult result=doTheTransfer(request);
|
|
||||||
return result;
|
|
||||||
}finally{
|
|
||||||
clean();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{
|
|
||||||
try{
|
|
||||||
TransferTicket submissionResponse= client.submit(request);
|
|
||||||
boolean continuePolling=true;
|
|
||||||
TransferTicket ticket=null;
|
|
||||||
do{
|
|
||||||
ticket=client.getTransferStatus(submissionResponse.getId());
|
|
||||||
System.out.println("Status : "+ticket);
|
|
||||||
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
|
||||||
try{
|
|
||||||
Thread.sleep(500);
|
|
||||||
}catch(InterruptedException e){}
|
|
||||||
}while(continuePolling);
|
|
||||||
if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage());
|
|
||||||
if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage());
|
|
||||||
long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis();
|
|
||||||
return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
|
|
||||||
}catch(RemoteServiceException e){
|
|
||||||
throw new FailedTransferException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void checkSource() throws SourceNotSetException, InvalidSourceException{
|
|
||||||
if(source==null) throw new SourceNotSetException();
|
|
||||||
source.validate();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract TransferRequest prepareRequest() throws InitializationException;
|
|
||||||
|
|
||||||
|
|
||||||
protected void prepare() throws InitializationException{
|
|
||||||
prepared=true;
|
|
||||||
}
|
|
||||||
protected void clean(){
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransferCapabilities getDestinationCapabilities() throws InitializationException {
|
|
||||||
try{
|
|
||||||
return client.getCapabilties();
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new InitializationException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,74 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.transferers;
|
|
||||||
|
|
||||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|
||||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.gcube.common.resources.gcore.HostingNode;
|
|
||||||
import org.gcube.data.transfer.library.caches.CapabilitiesCache;
|
|
||||||
import org.gcube.data.transfer.library.client.Client;
|
|
||||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
||||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|
||||||
@Slf4j
|
|
||||||
public class TransfererBuilder {
|
|
||||||
|
|
||||||
private static final int timeout=10*1000;
|
|
||||||
|
|
||||||
|
|
||||||
//e.g. http://pc-fabio.isti.cnr.it:8080/data-transfer-service/gcube/service
|
|
||||||
|
|
||||||
public static Transferer getTransfererByHost(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{
|
|
||||||
log.debug("Get transferer by Host "+endpoint);
|
|
||||||
try{
|
|
||||||
URL url=new URL(endpoint);
|
|
||||||
|
|
||||||
String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort();
|
|
||||||
//TODO Implement checks
|
|
||||||
|
|
||||||
// if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout);
|
|
||||||
// String finalHost=host;
|
|
||||||
// if(!finalHost.endsWith(ServiceConstants.APPLICATION_PATH)){
|
|
||||||
// // adjust host
|
|
||||||
// finalHost=finalHost+(host.endsWith("/")?"":"/")+"data-transfer-service"+ServiceConstants.APPLICATION_PATH;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost);
|
|
||||||
// log.debug("Host is ok, getting targetCapabilities");
|
|
||||||
// TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl);
|
|
||||||
|
|
||||||
// TODO determine method by capabilities checking
|
|
||||||
|
|
||||||
return new HTTPTransferer(new Client(baseUrl));
|
|
||||||
}catch(Exception e){
|
|
||||||
throw new ServiceNotFoundException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
|
|
||||||
String hostname=retrieveHostnameByNodeId(hostId);
|
|
||||||
return getTransfererByHost(hostname);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{
|
|
||||||
SimpleQuery query = queryFor(HostingNode.class);
|
|
||||||
|
|
||||||
query.addCondition("$resource/ID/text() eq '"+nodeId+"'");
|
|
||||||
|
|
||||||
|
|
||||||
List<HostingNode> found= clientFor(HostingNode.class).submit(query);
|
|
||||||
|
|
||||||
if(found.isEmpty()) throw new HostingNodeNotFoundException("No Hosting node with the id "+nodeId);
|
|
||||||
|
|
||||||
return "http://"+found.get(0).profile().description().name();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,47 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.utils;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileInputStream;
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import org.bson.types.ObjectId;
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
|
||||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
||||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
|
||||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
|
||||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class StorageUtils {
|
|
||||||
|
|
||||||
public static final IClient getClient(){
|
|
||||||
return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
//return Id
|
|
||||||
public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{
|
|
||||||
IClient client=getClient();
|
|
||||||
log.debug("Uploading local file "+source.getAbsolutePath());
|
|
||||||
return client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final boolean checkStorageId(String id){
|
|
||||||
ScopeProvider.instance.get();
|
|
||||||
return getClient().getHttpUrl().RFile(id)!=null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final String getUrlById(String id){
|
|
||||||
IClient client=getClient();
|
|
||||||
log.debug("Id is "+id);
|
|
||||||
return client.getHttpUrl().RFile(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final void removeById(String id){
|
|
||||||
IClient client=getClient();
|
|
||||||
client.remove().RFile(id);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,39 +0,0 @@
|
||||||
package org.gcube.data.transfer.library.utils;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
public class Utils {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pings a HTTP URL. This effectively sends a HEAD request and returns <code>true</code> if the response code is in
|
|
||||||
* the 200-399 range.
|
|
||||||
* @param url The HTTP URL to be pinged.
|
|
||||||
* @param timeout The timeout in millis for both the connection timeout and the response read timeout. Note that
|
|
||||||
* the total timeout is effectively two times the given timeout.
|
|
||||||
* @return <code>true</code> if the given HTTP URL has returned response code 200-399 on a HEAD request within the
|
|
||||||
* given timeout, otherwise <code>false</code>.
|
|
||||||
*/
|
|
||||||
public static boolean pingURL(String url, int timeout) {
|
|
||||||
url = url.replaceFirst("^https", "http"); // Otherwise an exception may be thrown on invalid SSL certificates.
|
|
||||||
|
|
||||||
try {
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
|
||||||
connection.setConnectTimeout(timeout);
|
|
||||||
connection.setReadTimeout(timeout);
|
|
||||||
connection.setRequestMethod("HEAD");
|
|
||||||
int responseCode = connection.getResponseCode();
|
|
||||||
return (200 <= responseCode && responseCode <= 399);
|
|
||||||
} catch (IOException exception) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static final String getUniqueString(){
|
|
||||||
return UUID.randomUUID().toString();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,19 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
|
|
||||||
public class ErrorReport extends TransferReport {
|
|
||||||
|
|
||||||
String id;
|
|
||||||
|
|
||||||
|
|
||||||
public ErrorReport(String id) {
|
|
||||||
super(null);
|
|
||||||
this.id=id;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String print() {
|
|
||||||
return "ERROR : "+id;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,156 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.data.transfer.library.TransferReport.ReportType;
|
|
||||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
|
||||||
|
|
||||||
import ch.qos.logback.core.util.ExecutorServiceUtil;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class MultipleTransferBenchmark {
|
|
||||||
|
|
||||||
|
|
||||||
static List<String> endpoints;
|
|
||||||
static List<String> nodeIds;
|
|
||||||
|
|
||||||
|
|
||||||
static List<String> files;
|
|
||||||
static List<String> uris;
|
|
||||||
static List<String> storageIds;
|
|
||||||
|
|
||||||
static ExecutorService executor=ExecutorServiceUtil.newExecutorService();
|
|
||||||
|
|
||||||
|
|
||||||
static CountDownLatch doneSignal;
|
|
||||||
|
|
||||||
static ArrayList<TransferReport> reports=new ArrayList<>();
|
|
||||||
|
|
||||||
|
|
||||||
static String scope="/gcube/devsec";
|
|
||||||
|
|
||||||
static{
|
|
||||||
endpoints=Arrays.asList(new String[]{
|
|
||||||
"http://node4-d-d4s.d4science.org:80/data-transfer-service/gcube/service",
|
|
||||||
"http://node3-d-d4s.d4science.org:80/data-transfer-service/gcube/service"
|
|
||||||
});
|
|
||||||
doneSignal=new CountDownLatch(endpoints.size());
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
nodeIds=new ArrayList<>();
|
|
||||||
files=new ArrayList<>();
|
|
||||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
|
||||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
|
||||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
|
||||||
files.add("/home/fabio/Documents/Pictures/web_trend_map.png");
|
|
||||||
uris=new ArrayList<>();
|
|
||||||
uris.add("http://goo.gl/r5jFZ9");
|
|
||||||
storageIds=new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static void main(String[] args) throws InitializationException{
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
Map<String,DataTransferClient> clients=new HashMap<>();
|
|
||||||
for(String endpoint:endpoints)
|
|
||||||
clients.put(endpoint,DataTransferClient.getInstanceByEndpoint(endpoint));
|
|
||||||
for(String id:nodeIds)
|
|
||||||
clients.put(id,DataTransferClient.getInstanceByEndpoint(id));
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
HashSet<DataTransferClient> startedTests=new HashSet<>();
|
|
||||||
for(final Entry<String,DataTransferClient> entry:clients.entrySet()){
|
|
||||||
if(!startedTests.contains(entry.getValue())){
|
|
||||||
startedTests.add(entry.getValue());
|
|
||||||
|
|
||||||
executor.execute(new Runnable(){
|
|
||||||
|
|
||||||
final DataTransferClient dt=entry.getValue();
|
|
||||||
final String id=entry.getKey();
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
try{
|
|
||||||
TransferReport report=new TransferReport(dt.getDestinationCapabilities());
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
|
|
||||||
log.debug("Sending files to {} ",dt.getDestinationCapabilities());
|
|
||||||
for(String f:files){
|
|
||||||
try {
|
|
||||||
TransferResult res=dt.localFile(f);
|
|
||||||
report.addReport(ReportType.local,f, res.getTransferedBytes(), res.getElapsedTime());
|
|
||||||
} catch (InvalidSourceException | SourceNotSetException
|
|
||||||
| FailedTransferException
|
|
||||||
| InitializationException e) {
|
|
||||||
log.error("Unable to send file {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug("Sending uris to {}",dt.getDestinationCapabilities());
|
|
||||||
for(String f:uris){
|
|
||||||
try {
|
|
||||||
TransferResult res=dt.httpSource(f);
|
|
||||||
report.addReport(ReportType.uri,f, res.getTransferedBytes(), res.getElapsedTime());
|
|
||||||
} catch (InvalidSourceException | SourceNotSetException
|
|
||||||
| FailedTransferException
|
|
||||||
| InitializationException e) {
|
|
||||||
log.error("Unable to send uri {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug("Sending storageIds to {}",dt.getDestinationCapabilities());
|
|
||||||
for(String f:storageIds){
|
|
||||||
try {
|
|
||||||
TransferResult res=dt.storageId(f);
|
|
||||||
report.addReport(ReportType.storage,f, res.getTransferedBytes(), res.getElapsedTime());
|
|
||||||
} catch (InvalidSourceException | SourceNotSetException
|
|
||||||
| FailedTransferException
|
|
||||||
| InitializationException e) {
|
|
||||||
log.error("Unable to send storageId {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reports.add(report);
|
|
||||||
}catch(Exception e){
|
|
||||||
reports.add(new ErrorReport(id));
|
|
||||||
}finally{
|
|
||||||
doneSignal.countDown();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
doneSignal.await();
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
|
|
||||||
}
|
|
||||||
System.out.println("*****************************");
|
|
||||||
for(TransferReport r:reports)System.out.println(r.print());
|
|
||||||
|
|
||||||
ExecutorServiceUtil.shutdown(executor);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,21 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
||||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
|
||||||
|
|
||||||
public class StorageTest {
|
|
||||||
|
|
||||||
static String scope="/gcube/devsec";
|
|
||||||
|
|
||||||
public static void main(String[] args) throws RemoteBackendException, FileNotFoundException {
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
|
||||||
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
|
||||||
System.out.println(StorageUtils.getUrlById(id));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,79 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
import javax.ws.rs.client.ClientBuilder;
|
|
||||||
import javax.ws.rs.client.WebTarget;
|
|
||||||
import javax.ws.rs.core.MediaType;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.data.transfer.library.client.AuthorizationFilter;
|
|
||||||
import org.gcube.data.transfer.library.client.Client;
|
|
||||||
import org.gcube.data.transfer.library.faults.CommunicationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.model.ServiceConstants;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
import org.gcube.data.transfer.model.TransferRequest;
|
|
||||||
import org.gcube.data.transfer.model.TransferTicket;
|
|
||||||
import org.gcube.data.transfer.model.TransferTicket.Status;
|
|
||||||
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
|
|
||||||
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
|
|
||||||
import org.glassfish.jersey.client.ClientConfig;
|
|
||||||
import org.junit.BeforeClass;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
|
|
||||||
public class TestClientCalls {
|
|
||||||
|
|
||||||
static String hostname="http://node3-d-d4s.d4science.org:80";
|
|
||||||
// static String hostname="http://pc-fabio.isti.cnr.it:8080";
|
|
||||||
static String scope="/gcube/devNext";
|
|
||||||
static Client client;
|
|
||||||
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void init() throws ServiceNotFoundException{
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
client=new Client(hostname);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void getCapabilties() throws CommunicationException{
|
|
||||||
System.out.println(client.getCapabilties());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void doTheTransfer() throws MalformedURLException, RemoteServiceException{
|
|
||||||
TransferRequest request= new TransferRequest("", new HttpDownloadSettings(new URL("http://goo.gl/oLP7zG"), HttpDownloadOptions.DEFAULT));
|
|
||||||
System.out.println("Submitting "+request);
|
|
||||||
TransferTicket ticket=client.submit(request);
|
|
||||||
System.out.println("Ticket is "+ticket);
|
|
||||||
|
|
||||||
boolean continuePolling=true;
|
|
||||||
do{
|
|
||||||
ticket=client.getTransferStatus(ticket.getId());
|
|
||||||
System.out.println("Status : "+ticket);
|
|
||||||
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
|
|
||||||
try{
|
|
||||||
Thread.sleep(1000);
|
|
||||||
}catch(InterruptedException e){}
|
|
||||||
}while(continuePolling);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void directCall(){
|
|
||||||
javax.ws.rs.client.Client client = ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class));
|
|
||||||
WebTarget target=client.target(hostname+"/data-transfer-service"+ServiceConstants.APPLICATION_PATH+"Capabilities");
|
|
||||||
// WebTarget target=client.target(hostname+"/data-transfer-service/gcube/service/Capabilities");
|
|
||||||
System.out.println("Asking capabilities to target : "+target.getUri());
|
|
||||||
System.out.println(target.
|
|
||||||
request(MediaType.APPLICATION_XML).get(TransferCapabilities.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,89 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
|
|
||||||
import org.gcube.data.transfer.library.model.Source;
|
|
||||||
import org.gcube.data.transfer.model.TransferCapabilities;
|
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.ToString;
|
|
||||||
|
|
||||||
public class TransferReport {
|
|
||||||
|
|
||||||
public static enum ReportType{
|
|
||||||
local,uri,storage
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static final char end='\n';
|
|
||||||
|
|
||||||
@AllArgsConstructor
|
|
||||||
@Getter
|
|
||||||
@ToString
|
|
||||||
private static class ReportItem{
|
|
||||||
String source;
|
|
||||||
long size;
|
|
||||||
long elapsed;
|
|
||||||
|
|
||||||
|
|
||||||
// bytes/msec
|
|
||||||
public long getAvgSpeed(){
|
|
||||||
return size/elapsed;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected TransferCapabilities host;
|
|
||||||
|
|
||||||
private Map<ReportType,ArrayList<ReportItem>> reports=new HashMap<>();
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public TransferReport(TransferCapabilities host) {
|
|
||||||
super();
|
|
||||||
this.host = host;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void addReport(ReportType sourceType,String theSource,long size,long elapsed){
|
|
||||||
if(!reports.containsKey(sourceType)) reports.put(sourceType, new ArrayList<ReportItem>());
|
|
||||||
reports.get(sourceType).add(new ReportItem(theSource, size, elapsed));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public String print(){
|
|
||||||
StringBuilder builder=new StringBuilder(" Report for : "+host+end);
|
|
||||||
for(Entry<ReportType,ArrayList<ReportItem>> entry:reports.entrySet()){
|
|
||||||
builder.append("Source type "+entry.getKey()+end);
|
|
||||||
ReportItem maxSizeItem=null;
|
|
||||||
ReportItem maxAvgSpeedItem=null;
|
|
||||||
ReportItem minSizeItem=null;
|
|
||||||
ReportItem minAvgSpeedItem=null;
|
|
||||||
long currentAvgSpeedCounter=0l;
|
|
||||||
|
|
||||||
for(ReportItem item:entry.getValue()){
|
|
||||||
if(maxSizeItem==null||maxSizeItem.getSize()<item.getSize()) maxSizeItem=item;
|
|
||||||
if(maxAvgSpeedItem==null||maxAvgSpeedItem.getAvgSpeed()<item.getAvgSpeed()) maxAvgSpeedItem=item;
|
|
||||||
if(minSizeItem==null||minSizeItem.getSize()>item.getSize()) minSizeItem=item;
|
|
||||||
if(minAvgSpeedItem==null||minAvgSpeedItem.getAvgSpeed()>item.getAvgSpeed()) minAvgSpeedItem=item;
|
|
||||||
currentAvgSpeedCounter+=item.getAvgSpeed();
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.append("Max Size Item : "+maxSizeItem+end);
|
|
||||||
builder.append("Max Avg Speed Item : "+maxAvgSpeedItem+end);
|
|
||||||
builder.append("Min Size Item : "+minSizeItem+end);
|
|
||||||
builder.append("Min Avg Speed Item : "+minAvgSpeedItem+end);
|
|
||||||
builder.append("Total avg speed : "+currentAvgSpeedCounter/entry.getValue().size()+end);
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder.toString();
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -1,74 +0,0 @@
|
||||||
package org.gcube.data.transfer.library;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
||||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
|
||||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
|
||||||
import org.gcube.data.transfer.library.utils.StorageUtils;
|
|
||||||
import org.junit.BeforeClass;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
public class TransfererTest {
|
|
||||||
|
|
||||||
static String hostname="http://node3-d-d4s.d4science.org:80";
|
|
||||||
static String nodeId="462b68c5-463f-4295-86da-37d6c0abc7ea";
|
|
||||||
static String scope="/gcube/devsec";
|
|
||||||
|
|
||||||
static DataTransferClient client;
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void init() throws UnreachableNodeException, ServiceNotFoundException, HostingNodeNotFoundException{
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
client=DataTransferClient.getInstanceByEndpoint(hostname);
|
|
||||||
// client=DataTransferClient.getInstanceByNodeId(nodeId);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void localFile() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
String localFile="/home/fabio/Dropbox/Mindless/01- COMA - Mindless.mp3";
|
|
||||||
System.out.println(client.localFile(localFile));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void httpUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
String link="http://goo.gl/oLP7zG";
|
|
||||||
System.out.println(client.httpSource(link));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException{
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf";
|
|
||||||
String id=StorageUtils.putOntoStorage(new File(toUpload));
|
|
||||||
System.out.println(client.storageId(id));
|
|
||||||
// System.out.println(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(expected=InvalidSourceException.class)
|
|
||||||
public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
System.out.println(client.storageId("13245780t"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(expected=InvalidSourceException.class)
|
|
||||||
public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
String localFile="/home/fabio/Downloads/123045689.mp3";
|
|
||||||
System.out.println(client.localFile(localFile));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(expected=InvalidSourceException.class)
|
|
||||||
public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
|
|
||||||
String link="https://www.some.where.com/over/theRainbow.txt";
|
|
||||||
System.out.println(client.httpSource(link));
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue