From f924eed16d9ad88b197d2d87bd837757e24ce348 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 8 Sep 2016 13:43:57 +0000 Subject: [PATCH] Initial import. git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@131225 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 36 ---- .project | 23 --- .settings/org.eclipse.core.resources.prefs | 6 - .settings/org.eclipse.jdt.core.prefs | 5 - .settings/org.eclipse.m2e.core.prefs | 4 - distro/LICENSE | 1 - distro/README | 65 -------- distro/changelog.xml | 5 - distro/descriptor.xml | 30 ---- distro/profile.xml | 29 ---- pom.xml | 80 --------- .../transfer/library/DataTransferClient.java | 87 ---------- .../data/transfer/library/TransferResult.java | 22 --- .../library/caches/CapabilitiesCache.java | 31 ---- .../transfer/library/caches/TTLCache.java | 94 ----------- .../transfer/library/caches/TTLContainer.java | 30 ---- .../library/client/AuthorizationFilter.java | 29 ---- .../data/transfer/library/client/Client.java | 96 ----------- .../faults/CommunicationException.java | 35 ---- .../library/faults/DataTransferException.java | 35 ---- .../faults/FailedTransferException.java | 35 ---- .../faults/HostingNodeNotFoundException.java | 35 ---- .../faults/InitializationException.java | 35 ---- .../faults/InvalidSourceException.java | 35 ---- .../faults/RemoteServiceException.java | 35 ---- .../faults/ServiceNotFoundException.java | 35 ---- .../library/faults/SourceNotSetException.java | 35 ---- .../faults/UnreachableNodeException.java | 37 ----- .../transfer/library/model/LocalSource.java | 41 ----- .../data/transfer/library/model/Source.java | 17 -- .../transfer/library/model/StorageSource.java | 47 ------ .../transfer/library/model/URLSource.java | 62 ------- .../library/transferers/HTTPTransferer.java | 71 -------- .../library/transferers/Transferer.java | 125 -------------- .../transferers/TransfererBuilder.java | 74 --------- .../transfer/library/utils/StorageUtils.java | 47 ------ .../data/transfer/library/utils/Utils.java | 39 ----- .../data/transfer/library/ErrorReport.java | 19 --- .../library/MultipleTransferBenchmark.java | 156 ------------------ .../data/transfer/library/StorageTest.java | 21 --- .../transfer/library/TestClientCalls.java | 79 --------- .../data/transfer/library/TransferReport.java | 89 ---------- .../data/transfer/library/TransfererTest.java | 74 --------- 43 files changed, 1986 deletions(-) delete mode 100644 .classpath delete mode 100644 .project delete mode 100644 .settings/org.eclipse.core.resources.prefs delete mode 100644 .settings/org.eclipse.jdt.core.prefs delete mode 100644 .settings/org.eclipse.m2e.core.prefs delete mode 100644 distro/LICENSE delete mode 100644 distro/README delete mode 100644 distro/changelog.xml delete mode 100644 distro/descriptor.xml delete mode 100644 distro/profile.xml delete mode 100644 pom.xml delete mode 100644 src/main/java/org/gcube/data/transfer/library/DataTransferClient.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/TransferResult.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/client/Client.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/model/LocalSource.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/model/Source.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/model/StorageSource.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/model/URLSource.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java delete mode 100644 src/main/java/org/gcube/data/transfer/library/utils/Utils.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/ErrorReport.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/StorageTest.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/TestClientCalls.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/TransferReport.java delete mode 100644 src/test/java/org/gcube/data/transfer/library/TransfererTest.java diff --git a/.classpath b/.classpath deleted file mode 100644 index e43402f..0000000 --- a/.classpath +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index cdb05cd..0000000 --- a/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - data-transfer-library - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 29abf99..0000000 --- a/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,6 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding//src/test/resources=UTF-8 -encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index ec4300d..0000000 --- a/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,5 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index ff7698f..0000000 --- a/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=false -version=1 diff --git a/distro/LICENSE b/distro/LICENSE deleted file mode 100644 index 2d9616a..0000000 --- a/distro/LICENSE +++ /dev/null @@ -1 +0,0 @@ -${gcube.license} \ No newline at end of file diff --git a/distro/README b/distro/README deleted file mode 100644 index 9ac47a3..0000000 --- a/distro/README +++ /dev/null @@ -1,65 +0,0 @@ -The gCube System - ${name} --------------------------------------------------- - -${description} - - -${gcube.description} - -${gcube.funding} - - -Version --------------------------------------------------- - -${version} (${buildDate}) - -Please see the file named "changelog.xml" in this directory for the release notes. - - - -Authors --------------------------------------------------- - -* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). - -Maintainers ------------ - -* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). - - -Download information --------------------------------------------------- - -Source code is available from SVN: - ${scm.url} - -Binaries can be downloaded from the gCube website: - ${gcube.website} - -Installation --------------------------------------------------- - -Installation documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot}/Tabular_Data_Manager - -Documentation --------------------------------------------------- - -Documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot}/Tabular_Data_Manager - ${gcube.wikiRoot}/Tabular_Data_Manager - - -Support --------------------------------------------------- - -Bugs and support requests can be reported in the gCube issue tracking tool: - ${gcube.issueTracking} - - -Licensing --------------------------------------------------- - -This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/changelog.xml b/distro/changelog.xml deleted file mode 100644 index 993d50e..0000000 --- a/distro/changelog.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - First Release - - \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml deleted file mode 100644 index dc46ade..0000000 --- a/distro/descriptor.xml +++ /dev/null @@ -1,30 +0,0 @@ - - servicearchive - - tar.gz - - / - - - ${distroDirectory} - / - true - - README - LICENSE - changelog.xml - - 755 - true - - - - - target/${build.finalName}.${project.packaging} - /${artifactId} - - - \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml deleted file mode 100644 index 429a22e..0000000 --- a/distro/profile.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - Service - - ${description} - DataTransfer - ${artifactId} - 1.0.0 - - - ${description} - ${artifactId} - ${version} - - ${groupId} - ${artifactId} - ${version} - - library - - ${build.finalName}.jar - - - - - - - diff --git a/pom.xml b/pom.xml deleted file mode 100644 index de41280..0000000 --- a/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - 4.0.0 - - org.gcube.tools - maven-parent - LATEST - - org.gcube.data.transfer - data-transfer-library - 0.0.1-SNAPSHOT - DataTransfer-library - - - - - org.gcube.core - common-gcube-calls - [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) - - - org.gcube.data.transfer - data-transfer-model - 1.0.0-SNAPSHOT - - - org.slf4j - slf4j-api - 1.7.5 - - - junit - junit - 4.10 - test - - - - - org.gcube.contentmanagement - storage-manager-core - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) - - - org.gcube.contentmanagement - storage-manager-wrapper - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) - - - - - org.projectlombok - lombok - 0.11.6 - - - - - org.glassfish.jersey.core - jersey-client - 2.22.2 - - - - org.glassfish.jersey.media - jersey-media-jaxb - 2.18 - - - - - ch.qos.logback - logback-classic - 1.0.13 - test - - - - - \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java b/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java deleted file mode 100644 index 63aaace..0000000 --- a/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; - -import lombok.Synchronized; -import lombok.extern.slf4j.Slf4j; - -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; -import org.gcube.data.transfer.library.faults.UnreachableNodeException; -import org.gcube.data.transfer.library.transferers.Transferer; -import org.gcube.data.transfer.library.transferers.TransfererBuilder; -import org.gcube.data.transfer.model.TransferCapabilities; - -@Slf4j -public class DataTransferClient { - - private Transferer transferer=null; - - private DataTransferClient(Transferer transferer) { - this.transferer=transferer; - } - - public static DataTransferClient getInstanceByEndpoint(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{ - log.debug("Getting transferer for endpoint : "+endpoint); - return new DataTransferClient(TransfererBuilder.getTransfererByHost(endpoint)); - } - - public static DataTransferClient getInstanceByNodeId(String id) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{ - log.debug("Getting transferer for nodeId : "+id); - return new DataTransferClient(TransfererBuilder.getTransfererByhostingNodeId(id)); - } - - - public TransferCapabilities getDestinationCapabilities() throws InitializationException{ - return this.transferer.getDestinationCapabilities(); - } - - @Synchronized("transferer") - public TransferResult localFile(String path) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); - log.debug("Sending local file {} to {}",path,transferer.getDestinationCapabilities().getHostName()); - transferer.localFile(path); - return transferer.transfer(); - } - - @Synchronized("transferer") - public TransferResult localFile(File file) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); - log.debug("Sending local file {} to {}",file.getAbsolutePath(),transferer.getDestinationCapabilities().getHostName()); - transferer.localFile(file); - return transferer.transfer(); - } - - @Synchronized("transferer") - public TransferResult httpSource(String url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); - log.debug("Passed url string : "+url); - try{ - return this.httpSource(new URL(url)); - }catch(MalformedURLException e){ - throw new InvalidSourceException("Invalid url : "+url); - } - } - - @Synchronized("transferer") - public TransferResult httpSource(URL url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); - log.debug("Sending from url {} to {}",url,transferer.getDestinationCapabilities().getHostName()); - transferer.fromURL(url); - return transferer.transfer(); - } - - @Synchronized("transferer") - public TransferResult storageId(String id) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); - log.debug("Sending from storage id {} to {}",id,transferer.getDestinationCapabilities().getHostName()); - transferer.storageFileId(id); - return transferer.transfer(); - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/TransferResult.java b/src/main/java/org/gcube/data/transfer/library/TransferResult.java deleted file mode 100644 index 97b89fd..0000000 --- a/src/main/java/org/gcube/data/transfer/library/TransferResult.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.gcube.data.transfer.library; - -import lombok.Data; -import lombok.NonNull; - -import org.gcube.data.transfer.library.model.Source; - -@Data -public class TransferResult { - - @NonNull - private Source source; - @NonNull - private String destinationHostName; - @NonNull - private Long elapsedTime; - @NonNull - private Long transferedBytes; - @NonNull - private String remotePath; - -} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java b/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java deleted file mode 100644 index 0d41613..0000000 --- a/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.gcube.data.transfer.library.caches; - -import lombok.Synchronized; -import lombok.extern.slf4j.Slf4j; - -import org.gcube.data.transfer.library.client.Client; -import org.gcube.data.transfer.model.TransferCapabilities; - -@Slf4j -public class CapabilitiesCache extends TTLCache { - - private static CapabilitiesCache instance=null; - -// @Synchronized -// public static CapabilitiesCache getInstance(){ -// if(instance==null)instance=new CapabilitiesCache(); -// return instance; -// } - - private CapabilitiesCache(){ - super(5*60*1000l,2*60*1000l,"Capabilities"); - } - - - @Override - protected TransferCapabilities getNew(String id) throws Exception{ - log.debug("Getting capabilties for host "+id); - return new Client(id).getCapabilties(); - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java b/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java deleted file mode 100644 index d9abd44..0000000 --- a/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.gcube.data.transfer.library.caches; - -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import lombok.Synchronized; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; - -@ToString -@Slf4j -public abstract class TTLCache { - - - // STATIC - -// private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1,new ThreadFactory(){ -// public Thread newThread(Runnable r) { -// Thread t = Executors.defaultThreadFactory().newThread(r); -// t.setDaemon(true); -// return t; -// } -// }); - private static final HashSet> createdMaps=new HashSet>(); - - - static { -// scheduler.scheduleAtFixedRate(new Runnable() { -// -// -// -// @Override -// public void run() { -// log.debug("Running Maps Cleaner, maps count : "+createdMaps.size()); -// int removed=0; -// for(TTLCache theMap:createdMaps) -// theMap.purgeItems(); -// log.debug("Removed "+removed+" old tickets"); -// -// } -// }, 3, 3, TimeUnit.MINUTES); -// - } - - - //************************* CACHE - - - - protected TTLCache(Long cacheKeepAliveTime, Long objectTTL, String cacheName) { - this.cacheKeepAliveTime = cacheKeepAliveTime; - this.objectTTL = objectTTL; - this.cacheName = cacheName; - createdMaps.add(this); - log.debug("Created Cache "+this); - } - - - - private ConcurrentHashMap> theMap=new ConcurrentHashMap<>(); - - private Long cacheKeepAliveTime; - private Long objectTTL; - private String cacheName; - - @Synchronized - public T getObject(String id) throws Exception{ - if(!theMap.contains(id)||System.currentTimeMillis()-theMap.get(id).getCreationTime()>objectTTL) - theMap.put(id, new TTLContainer(getNew(id))); - return theMap.get(id).getTheObject(); - } - - - private int purgeItems(){ - log.debug(cacheName+" Purging objects, keep alive time is "+cacheKeepAliveTime); - int totalCount=theMap.size(); - int removed=0; - for(Entry> entry:theMap.entrySet()) - if(System.currentTimeMillis()-entry.getValue().getLastUsageTime()>cacheKeepAliveTime){ - theMap.remove(entry.getKey()); - removed++; - } - log.debug(cacheName+" Removed "+removed+" out of "+totalCount); - return removed; - } - - protected abstract T getNew(String id) throws Exception; - -} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java b/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java deleted file mode 100644 index e12db1e..0000000 --- a/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.gcube.data.transfer.library.caches; - -public class TTLContainer { - - private long lastUsageTime=System.currentTimeMillis(); - private long creationTime=System.currentTimeMillis(); - - private T theObject; - public TTLContainer(T theObject) { - this.theObject = theObject; - } - - - private void update(){ - lastUsageTime=System.currentTimeMillis(); - } - - public T getTheObject(){ - update(); - return theObject; - } - - public long getLastUsageTime() { - return lastUsageTime; - } - - public long getCreationTime() { - return creationTime; - } -} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java b/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java deleted file mode 100644 index 7ed4f54..0000000 --- a/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.data.transfer.library.client; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map.Entry; - -import javax.ws.rs.client.ClientRequestContext; -import javax.ws.rs.client.ClientRequestFilter; - -import org.gcube.common.calls.Call; -import org.gcube.common.calls.Interceptors; -import org.gcube.common.calls.Request; -import org.gcube.common.scope.api.ScopeProvider; - -public class AuthorizationFilter implements ClientRequestFilter { - - - @Override - public void filter(final ClientRequestContext rc) throws IOException { - if (ScopeProvider.instance.get()!=null){ - Request requestContext = Interceptors.executeRequestChain(new Call()); - - for (Entry entry: requestContext.getHeaders()){ - System.out.println("setting "+entry.getKey()+" "+entry.getValue()); - rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue())); - } - } - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/client/Client.java b/src/main/java/org/gcube/data/transfer/library/client/Client.java deleted file mode 100644 index b70fa5b..0000000 --- a/src/main/java/org/gcube/data/transfer/library/client/Client.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.gcube.data.transfer.library.client; - -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import org.gcube.data.transfer.library.faults.CommunicationException; -import org.gcube.data.transfer.library.faults.DataTransferException; -import org.gcube.data.transfer.library.faults.RemoteServiceException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.model.ServiceConstants; -import org.gcube.data.transfer.model.TransferCapabilities; -import org.gcube.data.transfer.model.TransferRequest; -import org.gcube.data.transfer.model.TransferTicket; -import org.glassfish.jersey.client.ClientConfig; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class Client { - - private static ClientConfig config=null; - - static{ - log.debug("Creating configuration .."); - config=new ClientConfig(); - config.register(AuthorizationFilter.class); - } - - private String endpoint; - - private WebTarget rootTarget; - - public Client(String endpoint) throws ServiceNotFoundException{ - try{ - log.debug("Creating client for base "+endpoint); - this.endpoint=endpoint+""; - rootTarget= ClientBuilder.newClient(config).target(endpoint).path("data-transfer-service").path(ServiceConstants.APPLICATION_PATH); -// checkResponse(rootTarget.request().get()); - - log.debug("Root Taget IS {} ",rootTarget.getUri()); - }catch(Exception e){ - throw new ServiceNotFoundException(e); - } - } - - - public String getEndpoint() { - return endpoint; - } - - public TransferCapabilities getCapabilties() throws CommunicationException{ - WebTarget capabilitiesTarget=rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME); - log.debug("Getting capabilities from {}, path is {} ",endpoint,capabilitiesTarget.getUri()); - try{ - Response resp=capabilitiesTarget.request().accept(MediaType.APPLICATION_XML_TYPE).get(); - checkResponse(resp); - return resp.readEntity(TransferCapabilities.class); - }catch(Exception e){ - throw new CommunicationException(e); - } - } - - - public TransferTicket submit(TransferRequest request) throws RemoteServiceException{ - log.debug("Sending request {} to {}",request,endpoint); - try{ - Response resp=rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML)); - checkResponse(resp); - return resp.readEntity(TransferTicket.class); - }catch(Exception e){ - throw new RemoteServiceException(e); - } - } - - public TransferTicket getTransferStatus(String transferId) throws RemoteServiceException{ - log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint); - try{ - Response resp=rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get(); - checkResponse(resp); - return resp.readEntity(TransferTicket.class); - }catch(Exception e){ - throw new RemoteServiceException(e); - } - } - - - protected void checkResponse(Response toCheck) throws Exception{ - switch(toCheck.getStatusInfo().getFamily()){ - case SUCCESSFUL : break; - default : throw new Exception("Unexpected Response code : "+toCheck.getStatus()); - } - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java b/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java deleted file mode 100644 index f0b51a0..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class CommunicationException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = 1L; - - public CommunicationException() { - // TODO Auto-generated constructor stub - } - - public CommunicationException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public CommunicationException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public CommunicationException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public CommunicationException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java b/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java deleted file mode 100644 index 069e3e9..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public abstract class DataTransferException extends Exception { - - /** - * - */ - private static final long serialVersionUID = 8488351139669552406L; - - public DataTransferException() { - // TODO Auto-generated constructor stub - } - - public DataTransferException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public DataTransferException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public DataTransferException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public DataTransferException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java b/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java deleted file mode 100644 index 28ceda8..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class FailedTransferException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = -807695677781676381L; - - public FailedTransferException() { - // TODO Auto-generated constructor stub - } - - public FailedTransferException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public FailedTransferException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public FailedTransferException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public FailedTransferException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java b/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java deleted file mode 100644 index 8e547fa..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class HostingNodeNotFoundException extends InitializationException { - - /** - * - */ - private static final long serialVersionUID = 5786009015981659074L; - - public HostingNodeNotFoundException() { - // TODO Auto-generated constructor stub - } - - public HostingNodeNotFoundException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public HostingNodeNotFoundException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public HostingNodeNotFoundException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public HostingNodeNotFoundException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java b/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java deleted file mode 100644 index d074a9c..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class InitializationException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = 2204462227670167203L; - - public InitializationException() { - // TODO Auto-generated constructor stub - } - - public InitializationException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public InitializationException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public InitializationException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public InitializationException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java b/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java deleted file mode 100644 index 93e7b09..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class InvalidSourceException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = 7631518288856917202L; - - public InvalidSourceException() { - // TODO Auto-generated constructor stub - } - - public InvalidSourceException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public InvalidSourceException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public InvalidSourceException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public InvalidSourceException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java b/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java deleted file mode 100644 index f8c52dc..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class RemoteServiceException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = 5320978791087129997L; - - public RemoteServiceException() { - // TODO Auto-generated constructor stub - } - - public RemoteServiceException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public RemoteServiceException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public RemoteServiceException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public RemoteServiceException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java b/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java deleted file mode 100644 index adf0395..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class ServiceNotFoundException extends InitializationException { - - /** - * - */ - private static final long serialVersionUID = -3494011839783891174L; - - public ServiceNotFoundException() { - // TODO Auto-generated constructor stub - } - - public ServiceNotFoundException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public ServiceNotFoundException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public ServiceNotFoundException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public ServiceNotFoundException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java b/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java deleted file mode 100644 index 12733f4..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class SourceNotSetException extends DataTransferException { - - /** - * - */ - private static final long serialVersionUID = -5868191252361823341L; - - public SourceNotSetException() { - // TODO Auto-generated constructor stub - } - - public SourceNotSetException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public SourceNotSetException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public SourceNotSetException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public SourceNotSetException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java b/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java deleted file mode 100644 index a01a338..0000000 --- a/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.data.transfer.library.faults; - -public class UnreachableNodeException extends InitializationException { - - - - /** - * - */ - private static final long serialVersionUID = 8739586712271226827L; - - public UnreachableNodeException() { - // TODO Auto-generated constructor stub - } - - public UnreachableNodeException(String message) { - super(message); - // TODO Auto-generated constructor stub - } - - public UnreachableNodeException(Throwable cause) { - super(cause); - // TODO Auto-generated constructor stub - } - - public UnreachableNodeException(String message, Throwable cause) { - super(message, cause); - // TODO Auto-generated constructor stub - } - - public UnreachableNodeException(String message, Throwable cause, - boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - // TODO Auto-generated constructor stub - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java b/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java deleted file mode 100644 index 18909c7..0000000 --- a/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.gcube.data.transfer.library.model; - -import java.io.File; - -import org.gcube.data.transfer.library.faults.InvalidSourceException; - -public class LocalSource extends Source { - - private File theFile=null; - - public LocalSource(File theFile) throws InvalidSourceException { - super(); - if(theFile==null)throw new InvalidSourceException("File cannot be null"); - this.theFile = theFile; - } - - @Override - public boolean validate() throws InvalidSourceException { - if(!theFile.exists()) throw new InvalidSourceException("File "+theFile.getAbsolutePath()+"doesn't exist"); - if(!theFile.canRead()) throw new InvalidSourceException("Unable to read from file "+theFile.getAbsolutePath()); - if(theFile.isDirectory()) throw new InvalidSourceException("Transfer of directory is not yet supported"); - return true; - } - - @Override - public void prepare() { - // TODO Auto-generated method stub - - } - - @Override - public void clean() { - // TODO Auto-generated method stub - - } - - @Override - public File getTheSource() { - return theFile; - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/model/Source.java b/src/main/java/org/gcube/data/transfer/library/model/Source.java deleted file mode 100644 index dee9add..0000000 --- a/src/main/java/org/gcube/data/transfer/library/model/Source.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.data.transfer.library.model; - -import org.gcube.data.transfer.library.faults.InvalidSourceException; - -import lombok.Data; - -@Data -public abstract class Source{ - - public abstract boolean validate() throws InvalidSourceException; - - public abstract void prepare(); - - public abstract void clean(); - - public abstract T getTheSource(); -} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java b/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java deleted file mode 100644 index 4c60357..0000000 --- a/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.gcube.data.transfer.library.model; - -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.utils.StorageUtils; - -public class StorageSource extends Source { - - private String id=null; - - - - - - public StorageSource(String id) throws InvalidSourceException { - super(); - if(id==null) throw new InvalidSourceException("Storage id cannot be null"); - this.id = id; - } - - @Override - public boolean validate() throws InvalidSourceException { - try{ - if(!StorageUtils.checkStorageId(id)) throw new Exception("Not valid"); - }catch(Exception e){ - throw new InvalidSourceException("Invalid storage ID "+id); - } - return true; - } - - @Override - public void prepare() { - // TODO Auto-generated method stub - - } - - @Override - public void clean() { - // TODO Auto-generated method stub - - } - - @Override - public String getTheSource() { - return id; - } - -} diff --git a/src/main/java/org/gcube/data/transfer/library/model/URLSource.java b/src/main/java/org/gcube/data/transfer/library/model/URLSource.java deleted file mode 100644 index 61109c8..0000000 --- a/src/main/java/org/gcube/data/transfer/library/model/URLSource.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.gcube.data.transfer.library.model; - -import java.net.HttpURLConnection; -import java.net.URL; - -import org.gcube.data.transfer.library.faults.InvalidSourceException; - -public class URLSource extends Source { - - - private URL theURL=null; - - - public URLSource(URL theURL) throws InvalidSourceException { - super(); - if(theURL==null) throw new InvalidSourceException("URL Source cannot be null"); - this.theURL = theURL; - } - - @Override - public boolean validate() throws InvalidSourceException { - try{ - HttpURLConnection conn=(HttpURLConnection) theURL.openConnection(); - conn.setConnectTimeout(3000); - conn.setRequestMethod("HEAD"); - int responseCode = conn.getResponseCode(); - return (200 <= responseCode && responseCode <= 399); - }catch(Exception e){ - throw new InvalidSourceException("Unable to contact URL "+theURL,e); - } - } - - @Override - public void prepare() { - // nothing to do - - } - - @Override - public void clean() { - // nothing to do - } - - /* (non-Javadoc) - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("URLSource [theURL="); - builder.append(theURL); - builder.append("]"); - return builder.toString(); - } - - @Override - public URL getTheSource() { - return theURL; - } - - -} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java deleted file mode 100644 index 6b5b098..0000000 --- a/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.gcube.data.transfer.library.transferers; - -import java.io.FileNotFoundException; -import java.net.MalformedURLException; -import java.net.URL; - -import lombok.extern.slf4j.Slf4j; - -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; -import org.gcube.data.transfer.library.client.Client; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.model.LocalSource; -import org.gcube.data.transfer.library.model.StorageSource; -import org.gcube.data.transfer.library.model.URLSource; -import org.gcube.data.transfer.library.utils.StorageUtils; -import org.gcube.data.transfer.model.TransferRequest; -import org.gcube.data.transfer.model.options.HttpDownloadOptions; -import org.gcube.data.transfer.model.settings.HttpDownloadSettings; - -@Slf4j -public class HTTPTransferer extends Transferer { - - HTTPTransferer(Client client) { - super(client); - } - - private URL link; - - private String toDeleteStorageId=null; - - @Override - protected TransferRequest prepareRequest()throws InitializationException { - return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT)); - } - - - private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{ - if(source instanceof LocalSource){ - toDeleteStorageId=StorageUtils.putOntoStorage(((LocalSource)source).getTheSource()); - return new URL(StorageUtils.getUrlById(toDeleteStorageId)); - }else if (source instanceof StorageSource){ - return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource())); - }else if (source instanceof URLSource){ - return ((URLSource)source).getTheSource(); - }else throw new InvalidSourceException("Source cannot be handled "+source); - } - - @Override - protected void prepare() throws InitializationException { - try{ - link=getHttpLink(); - log.debug("Obtained link "+link); - super.prepare(); - }catch(RemoteBackendException e){ - throw new InitializationException(e); - }catch(FileNotFoundException e){ - throw new InitializationException(e); - }catch(InvalidSourceException e){ - throw new InitializationException(e); - }catch (MalformedURLException e){ - throw new InitializationException(e); - } - } - - @Override - protected void clean() { - // TODO Auto-generated method stub - super.clean(); - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java deleted file mode 100644 index 0061043..0000000 --- a/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.gcube.data.transfer.library.transferers; - -import java.io.File; -import java.net.URL; - -import lombok.extern.slf4j.Slf4j; - -import org.gcube.data.transfer.library.TransferResult; -import org.gcube.data.transfer.library.client.Client; -import org.gcube.data.transfer.library.faults.CommunicationException; -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.RemoteServiceException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; -import org.gcube.data.transfer.library.model.LocalSource; -import org.gcube.data.transfer.library.model.Source; -import org.gcube.data.transfer.library.model.StorageSource; -import org.gcube.data.transfer.library.model.URLSource; -import org.gcube.data.transfer.model.TransferCapabilities; -import org.gcube.data.transfer.model.TransferRequest; -import org.gcube.data.transfer.model.TransferTicket; -import org.gcube.data.transfer.model.TransferTicket.Status; - -@Slf4j -public abstract class Transferer { - - protected Client client; - - protected Transferer(Client client){ - this.client=client; - } - - - - - protected Source source=null; - protected boolean prepared=false; - - public Transferer localFile(File f) throws InvalidSourceException{ - source=new LocalSource(f); - return this; - } - - public Transferer localFile(String path) throws InvalidSourceException{ - try{ - File f=new File(path); - return localFile(f); - }catch(NullPointerException e){ - throw new InvalidSourceException("Path is null",e); - } - - } - - public Transferer storageFileId(String fileId) throws InvalidSourceException{ - source=new StorageSource(fileId); - return this; - } - - public Transferer fromURL(URL sourceURL) throws InvalidSourceException{ - source=new URLSource(sourceURL); - return this; - } - - - public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException{ - try{ - checkSource(); - prepare(); - TransferRequest request=prepareRequest(); - log.debug("Request is {}, sending it to {}",request,client.getEndpoint()); - TransferResult result=doTheTransfer(request); - return result; - }finally{ - clean(); - } - } - - protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{ - try{ - TransferTicket submissionResponse= client.submit(request); - boolean continuePolling=true; - TransferTicket ticket=null; - do{ - ticket=client.getTransferStatus(submissionResponse.getId()); - System.out.println("Status : "+ticket); - continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING); - try{ - Thread.sleep(500); - }catch(InterruptedException e){} - }while(continuePolling); - if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage()); - if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage()); - long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis(); - return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName()); - }catch(RemoteServiceException e){ - throw new FailedTransferException(e); - } - } - - protected void checkSource() throws SourceNotSetException, InvalidSourceException{ - if(source==null) throw new SourceNotSetException(); - source.validate(); - } - - protected abstract TransferRequest prepareRequest() throws InitializationException; - - - protected void prepare() throws InitializationException{ - prepared=true; - } - protected void clean(){ - - } - - public TransferCapabilities getDestinationCapabilities() throws InitializationException { - try{ - return client.getCapabilties(); - }catch(Exception e){ - throw new InitializationException(e); - } - } - - -} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java b/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java deleted file mode 100644 index 044a982..0000000 --- a/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.gcube.data.transfer.library.transferers; - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import org.gcube.common.resources.gcore.HostingNode; -import org.gcube.data.transfer.library.caches.CapabilitiesCache; -import org.gcube.data.transfer.library.client.Client; -import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.library.faults.UnreachableNodeException; -import org.gcube.data.transfer.model.TransferCapabilities; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -@Slf4j -public class TransfererBuilder { - - private static final int timeout=10*1000; - - - //e.g. http://pc-fabio.isti.cnr.it:8080/data-transfer-service/gcube/service - - public static Transferer getTransfererByHost(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{ - log.debug("Get transferer by Host "+endpoint); - try{ - URL url=new URL(endpoint); - - String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort(); - //TODO Implement checks - - // if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout); - // String finalHost=host; - // if(!finalHost.endsWith(ServiceConstants.APPLICATION_PATH)){ - // // adjust host - // finalHost=finalHost+(host.endsWith("/")?"":"/")+"data-transfer-service"+ServiceConstants.APPLICATION_PATH; - // } - // - // - // if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost); - // log.debug("Host is ok, getting targetCapabilities"); -// TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl); - - // TODO determine method by capabilities checking - - return new HTTPTransferer(new Client(baseUrl)); - }catch(Exception e){ - throw new ServiceNotFoundException(e); - } - } - public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{ - String hostname=retrieveHostnameByNodeId(hostId); - return getTransfererByHost(hostname); - } - - - private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{ - SimpleQuery query = queryFor(HostingNode.class); - - query.addCondition("$resource/ID/text() eq '"+nodeId+"'"); - - - List found= clientFor(HostingNode.class).submit(query); - - if(found.isEmpty()) throw new HostingNodeNotFoundException("No Hosting node with the id "+nodeId); - - return "http://"+found.get(0).profile().description().name(); - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java b/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java deleted file mode 100644 index a04572a..0000000 --- a/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.gcube.data.transfer.library.utils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; - -import lombok.extern.slf4j.Slf4j; - -import org.bson.types.ObjectId; -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.contentmanagement.blobstorage.service.IClient; -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; -import org.gcube.contentmanager.storageclient.wrapper.AccessType; -import org.gcube.contentmanager.storageclient.wrapper.MemoryType; -import org.gcube.contentmanager.storageclient.wrapper.StorageClient; - -@Slf4j -public class StorageUtils { - - public static final IClient getClient(){ - return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient(); - } - - - //return Id - public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{ - IClient client=getClient(); - log.debug("Uploading local file "+source.getAbsolutePath()); - return client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString()); - } - - public static final boolean checkStorageId(String id){ - ScopeProvider.instance.get(); - return getClient().getHttpUrl().RFile(id)!=null; - } - - public static final String getUrlById(String id){ - IClient client=getClient(); - log.debug("Id is "+id); - return client.getHttpUrl().RFile(id); - } - - public static final void removeById(String id){ - IClient client=getClient(); - client.remove().RFile(id); - } -} diff --git a/src/main/java/org/gcube/data/transfer/library/utils/Utils.java b/src/main/java/org/gcube/data/transfer/library/utils/Utils.java deleted file mode 100644 index 8d135c5..0000000 --- a/src/main/java/org/gcube/data/transfer/library/utils/Utils.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.gcube.data.transfer.library.utils; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.UUID; - -public class Utils { - - /** - * Pings a HTTP URL. This effectively sends a HEAD request and returns true if the response code is in - * the 200-399 range. - * @param url The HTTP URL to be pinged. - * @param timeout The timeout in millis for both the connection timeout and the response read timeout. Note that - * the total timeout is effectively two times the given timeout. - * @return true if the given HTTP URL has returned response code 200-399 on a HEAD request within the - * given timeout, otherwise false. - */ - public static boolean pingURL(String url, int timeout) { - url = url.replaceFirst("^https", "http"); // Otherwise an exception may be thrown on invalid SSL certificates. - - try { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setConnectTimeout(timeout); - connection.setReadTimeout(timeout); - connection.setRequestMethod("HEAD"); - int responseCode = connection.getResponseCode(); - return (200 <= responseCode && responseCode <= 399); - } catch (IOException exception) { - return false; - } - } - - - - public static final String getUniqueString(){ - return UUID.randomUUID().toString(); - } -} diff --git a/src/test/java/org/gcube/data/transfer/library/ErrorReport.java b/src/test/java/org/gcube/data/transfer/library/ErrorReport.java deleted file mode 100644 index ccf2b57..0000000 --- a/src/test/java/org/gcube/data/transfer/library/ErrorReport.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.gcube.data.transfer.library; - -import org.gcube.data.transfer.model.TransferCapabilities; - -public class ErrorReport extends TransferReport { - - String id; - - - public ErrorReport(String id) { - super(null); - this.id=id; - } - - @Override - public String print() { - return "ERROR : "+id; - } -} diff --git a/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java b/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java deleted file mode 100644 index 322b68c..0000000 --- a/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data.transfer.library.TransferReport.ReportType; -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; - -import ch.qos.logback.core.util.ExecutorServiceUtil; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MultipleTransferBenchmark { - - - static List endpoints; - static List nodeIds; - - - static List files; - static List uris; - static List storageIds; - - static ExecutorService executor=ExecutorServiceUtil.newExecutorService(); - - - static CountDownLatch doneSignal; - - static ArrayList reports=new ArrayList<>(); - - - static String scope="/gcube/devsec"; - - static{ - endpoints=Arrays.asList(new String[]{ - "http://node4-d-d4s.d4science.org:80/data-transfer-service/gcube/service", - "http://node3-d-d4s.d4science.org:80/data-transfer-service/gcube/service" - }); - doneSignal=new CountDownLatch(endpoints.size()); - - - - nodeIds=new ArrayList<>(); - files=new ArrayList<>(); - files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); - files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); - files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); - files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); - uris=new ArrayList<>(); - uris.add("http://goo.gl/r5jFZ9"); - storageIds=new ArrayList<>(); - } - - - - - public static void main(String[] args) throws InitializationException{ - ScopeProvider.instance.set(scope); - Map clients=new HashMap<>(); - for(String endpoint:endpoints) - clients.put(endpoint,DataTransferClient.getInstanceByEndpoint(endpoint)); - for(String id:nodeIds) - clients.put(id,DataTransferClient.getInstanceByEndpoint(id)); - - - - - HashSet startedTests=new HashSet<>(); - for(final Entry entry:clients.entrySet()){ - if(!startedTests.contains(entry.getValue())){ - startedTests.add(entry.getValue()); - - executor.execute(new Runnable(){ - - final DataTransferClient dt=entry.getValue(); - final String id=entry.getKey(); - @Override - public void run() { - try{ - TransferReport report=new TransferReport(dt.getDestinationCapabilities()); - ScopeProvider.instance.set(scope); - - log.debug("Sending files to {} ",dt.getDestinationCapabilities()); - for(String f:files){ - try { - TransferResult res=dt.localFile(f); - report.addReport(ReportType.local,f, res.getTransferedBytes(), res.getElapsedTime()); - } catch (InvalidSourceException | SourceNotSetException - | FailedTransferException - | InitializationException e) { - log.error("Unable to send file {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); - } - } - - log.debug("Sending uris to {}",dt.getDestinationCapabilities()); - for(String f:uris){ - try { - TransferResult res=dt.httpSource(f); - report.addReport(ReportType.uri,f, res.getTransferedBytes(), res.getElapsedTime()); - } catch (InvalidSourceException | SourceNotSetException - | FailedTransferException - | InitializationException e) { - log.error("Unable to send uri {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); - } - } - - log.debug("Sending storageIds to {}",dt.getDestinationCapabilities()); - for(String f:storageIds){ - try { - TransferResult res=dt.storageId(f); - report.addReport(ReportType.storage,f, res.getTransferedBytes(), res.getElapsedTime()); - } catch (InvalidSourceException | SourceNotSetException - | FailedTransferException - | InitializationException e) { - log.error("Unable to send storageId {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); - } - } - reports.add(report); - }catch(Exception e){ - reports.add(new ErrorReport(id)); - }finally{ - doneSignal.countDown(); - } - - } - }); - } - } - - try { - doneSignal.await(); - } catch (InterruptedException e) { - - } - System.out.println("*****************************"); - for(TransferReport r:reports)System.out.println(r.print()); - - ExecutorServiceUtil.shutdown(executor); - } - - - - - -} diff --git a/src/test/java/org/gcube/data/transfer/library/StorageTest.java b/src/test/java/org/gcube/data/transfer/library/StorageTest.java deleted file mode 100644 index b3a2c1b..0000000 --- a/src/test/java/org/gcube/data/transfer/library/StorageTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.io.File; -import java.io.FileNotFoundException; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; -import org.gcube.data.transfer.library.utils.StorageUtils; - -public class StorageTest { - - static String scope="/gcube/devsec"; - - public static void main(String[] args) throws RemoteBackendException, FileNotFoundException { - ScopeProvider.instance.set(scope); - String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf"; - String id=StorageUtils.putOntoStorage(new File(toUpload)); - System.out.println(StorageUtils.getUrlById(id)); - } - -} diff --git a/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java b/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java deleted file mode 100644 index 9edce7e..0000000 --- a/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.net.MalformedURLException; -import java.net.URL; - -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data.transfer.library.client.AuthorizationFilter; -import org.gcube.data.transfer.library.client.Client; -import org.gcube.data.transfer.library.faults.CommunicationException; -import org.gcube.data.transfer.library.faults.RemoteServiceException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.model.ServiceConstants; -import org.gcube.data.transfer.model.TransferCapabilities; -import org.gcube.data.transfer.model.TransferRequest; -import org.gcube.data.transfer.model.TransferTicket; -import org.gcube.data.transfer.model.TransferTicket.Status; -import org.gcube.data.transfer.model.options.HttpDownloadOptions; -import org.gcube.data.transfer.model.settings.HttpDownloadSettings; -import org.glassfish.jersey.client.ClientConfig; -import org.junit.BeforeClass; -import org.junit.Test; - - -public class TestClientCalls { - - static String hostname="http://node3-d-d4s.d4science.org:80"; -// static String hostname="http://pc-fabio.isti.cnr.it:8080"; - static String scope="/gcube/devNext"; - static Client client; - - - @BeforeClass - public static void init() throws ServiceNotFoundException{ - ScopeProvider.instance.set(scope); - client=new Client(hostname); - } - - - @Test - public void getCapabilties() throws CommunicationException{ - System.out.println(client.getCapabilties()); - } - - - @Test - public void doTheTransfer() throws MalformedURLException, RemoteServiceException{ - TransferRequest request= new TransferRequest("", new HttpDownloadSettings(new URL("http://goo.gl/oLP7zG"), HttpDownloadOptions.DEFAULT)); - System.out.println("Submitting "+request); - TransferTicket ticket=client.submit(request); - System.out.println("Ticket is "+ticket); - - boolean continuePolling=true; - do{ - ticket=client.getTransferStatus(ticket.getId()); - System.out.println("Status : "+ticket); - continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING); - try{ - Thread.sleep(1000); - }catch(InterruptedException e){} - }while(continuePolling); - } - - - - @Test - public void directCall(){ - javax.ws.rs.client.Client client = ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class)); - WebTarget target=client.target(hostname+"/data-transfer-service"+ServiceConstants.APPLICATION_PATH+"Capabilities"); -// WebTarget target=client.target(hostname+"/data-transfer-service/gcube/service/Capabilities"); - System.out.println("Asking capabilities to target : "+target.getUri()); - System.out.println(target. - request(MediaType.APPLICATION_XML).get(TransferCapabilities.class)); - } - -} diff --git a/src/test/java/org/gcube/data/transfer/library/TransferReport.java b/src/test/java/org/gcube/data/transfer/library/TransferReport.java deleted file mode 100644 index d61b1b2..0000000 --- a/src/test/java/org/gcube/data/transfer/library/TransferReport.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import org.gcube.data.transfer.library.model.Source; -import org.gcube.data.transfer.model.TransferCapabilities; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; - -public class TransferReport { - - public static enum ReportType{ - local,uri,storage - } - - - private static final char end='\n'; - - @AllArgsConstructor - @Getter - @ToString - private static class ReportItem{ - String source; - long size; - long elapsed; - - - // bytes/msec - public long getAvgSpeed(){ - return size/elapsed; - } - - - - - } - - - protected TransferCapabilities host; - - private Map> reports=new HashMap<>(); - - - - public TransferReport(TransferCapabilities host) { - super(); - this.host = host; - } - - - public void addReport(ReportType sourceType,String theSource,long size,long elapsed){ - if(!reports.containsKey(sourceType)) reports.put(sourceType, new ArrayList()); - reports.get(sourceType).add(new ReportItem(theSource, size, elapsed)); - } - - - public String print(){ - StringBuilder builder=new StringBuilder(" Report for : "+host+end); - for(Entry> entry:reports.entrySet()){ - builder.append("Source type "+entry.getKey()+end); - ReportItem maxSizeItem=null; - ReportItem maxAvgSpeedItem=null; - ReportItem minSizeItem=null; - ReportItem minAvgSpeedItem=null; - long currentAvgSpeedCounter=0l; - - for(ReportItem item:entry.getValue()){ - if(maxSizeItem==null||maxSizeItem.getSize()item.getSize()) minSizeItem=item; - if(minAvgSpeedItem==null||minAvgSpeedItem.getAvgSpeed()>item.getAvgSpeed()) minAvgSpeedItem=item; - currentAvgSpeedCounter+=item.getAvgSpeed(); - } - - builder.append("Max Size Item : "+maxSizeItem+end); - builder.append("Max Avg Speed Item : "+maxAvgSpeedItem+end); - builder.append("Min Size Item : "+minSizeItem+end); - builder.append("Min Avg Speed Item : "+minAvgSpeedItem+end); - builder.append("Total avg speed : "+currentAvgSpeedCounter/entry.getValue().size()+end); - } - - return builder.toString(); - }; -} diff --git a/src/test/java/org/gcube/data/transfer/library/TransfererTest.java b/src/test/java/org/gcube/data/transfer/library/TransfererTest.java deleted file mode 100644 index 7ffa873..0000000 --- a/src/test/java/org/gcube/data/transfer/library/TransfererTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.gcube.data.transfer.library; - -import java.io.File; -import java.io.FileNotFoundException; -import java.net.MalformedURLException; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; -import org.gcube.data.transfer.library.faults.UnreachableNodeException; -import org.gcube.data.transfer.library.utils.StorageUtils; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TransfererTest { - - static String hostname="http://node3-d-d4s.d4science.org:80"; - static String nodeId="462b68c5-463f-4295-86da-37d6c0abc7ea"; - static String scope="/gcube/devsec"; - - static DataTransferClient client; - - @BeforeClass - public static void init() throws UnreachableNodeException, ServiceNotFoundException, HostingNodeNotFoundException{ - ScopeProvider.instance.set(scope); - client=DataTransferClient.getInstanceByEndpoint(hostname); -// client=DataTransferClient.getInstanceByNodeId(nodeId); - } - - @Test - public void localFile() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - String localFile="/home/fabio/Dropbox/Mindless/01- COMA - Mindless.mp3"; - System.out.println(client.localFile(localFile)); - } - - - @Test - public void httpUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - String link="http://goo.gl/oLP7zG"; - System.out.println(client.httpSource(link)); - } - - - @Test - public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException{ - ScopeProvider.instance.set(scope); - String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf"; - String id=StorageUtils.putOntoStorage(new File(toUpload)); - System.out.println(client.storageId(id)); -// System.out.println(id); - } - - @Test(expected=InvalidSourceException.class) - public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - System.out.println(client.storageId("13245780t")); - } - - @Test(expected=InvalidSourceException.class) - public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - String localFile="/home/fabio/Downloads/123045689.mp3"; - System.out.println(client.localFile(localFile)); - } - - @Test(expected=InvalidSourceException.class) - public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - String link="https://www.some.where.com/over/theRainbow.txt"; - System.out.println(client.httpSource(link)); - } -}