diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..e43402f --- /dev/null +++ b/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..cdb05cd --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + data-transfer-library + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..ec4300d --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..ff7698f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=false +version=1 diff --git a/distro/LICENSE b/distro/LICENSE new file mode 100644 index 0000000..2d9616a --- /dev/null +++ b/distro/LICENSE @@ -0,0 +1 @@ +${gcube.license} \ No newline at end of file diff --git a/distro/README b/distro/README new file mode 100644 index 0000000..9ac47a3 --- /dev/null +++ b/distro/README @@ -0,0 +1,65 @@ +The gCube System - ${name} +-------------------------------------------------- + +${description} + + +${gcube.description} + +${gcube.funding} + + +Version +-------------------------------------------------- + +${version} (${buildDate}) + +Please see the file named "changelog.xml" in this directory for the release notes. + + + +Authors +-------------------------------------------------- + +* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). + +Maintainers +----------- + +* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). + + +Download information +-------------------------------------------------- + +Source code is available from SVN: + ${scm.url} + +Binaries can be downloaded from the gCube website: + ${gcube.website} + +Installation +-------------------------------------------------- + +Installation documentation is available on-line in the gCube Wiki: + ${gcube.wikiRoot}/Tabular_Data_Manager + +Documentation +-------------------------------------------------- + +Documentation is available on-line in the gCube Wiki: + ${gcube.wikiRoot}/Tabular_Data_Manager + ${gcube.wikiRoot}/Tabular_Data_Manager + + +Support +-------------------------------------------------- + +Bugs and support requests can be reported in the gCube issue tracking tool: + ${gcube.issueTracking} + + +Licensing +-------------------------------------------------- + +This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/changelog.xml b/distro/changelog.xml new file mode 100644 index 0000000..993d50e --- /dev/null +++ b/distro/changelog.xml @@ -0,0 +1,5 @@ + + + First Release + + \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml new file mode 100644 index 0000000..dc46ade --- /dev/null +++ b/distro/descriptor.xml @@ -0,0 +1,30 @@ + + servicearchive + + tar.gz + + / + + + ${distroDirectory} + / + true + + README + LICENSE + changelog.xml + + 755 + true + + + + + target/${build.finalName}.${project.packaging} + /${artifactId} + + + \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml new file mode 100644 index 0000000..429a22e --- /dev/null +++ b/distro/profile.xml @@ -0,0 +1,29 @@ + + + + Service + + ${description} + DataTransfer + ${artifactId} + 1.0.0 + + + ${description} + ${artifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + + library + + ${build.finalName}.jar + + + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6de00d8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,108 @@ + + 4.0.0 + + org.gcube.tools + maven-parent + LATEST + + org.gcube.data.transfer + data-transfer-library + 1.0.0-SNAPSHOT + DataTransfer-library + + + + + + org.gcube.distribution + gcube-bom + 1.0.0-SNAPSHOT + pom + import + + + + + + + + + + org.gcube.common + authorization-client + + + + org.gcube.core + common-gcube-calls + + + org.gcube.data.transfer + data-transfer-model + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + org.gcube.contentmanagement + storage-manager-core + [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + + + org.gcube.contentmanagement + storage-manager-wrapper + [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + + + + + + + + org.projectlombok + lombok + 0.11.6 + + + + + org.glassfish.jersey.core + jersey-client + 2.22.2 + + + + org.glassfish.jersey.media + jersey-media-jaxb + 2.18 + + + + org.slf4j + slf4j-api + + + + + + junit + junit + 4.10 + test + + + + + + + + ch.qos.logback + logback-classic + 1.0.13 + test + + + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java b/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java new file mode 100644 index 0000000..4811fe3 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java @@ -0,0 +1,126 @@ +package org.gcube.data.transfer.library; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collections; +import java.util.Set; + +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.library.transferers.Transferer; +import org.gcube.data.transfer.library.transferers.TransfererBuilder; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.TransferCapabilities; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DataTransferClient { + + private Transferer transferer=null; + + private DataTransferClient(Transferer transferer) { + this.transferer=transferer; + } + + public static DataTransferClient getInstanceByEndpoint(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{ + log.debug("Getting transferer for endpoint : "+endpoint); + return new DataTransferClient(TransfererBuilder.getTransfererByHost(endpoint)); + } + + public static DataTransferClient getInstanceByNodeId(String id) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{ + log.debug("Getting transferer for nodeId : "+id); + return new DataTransferClient(TransfererBuilder.getTransfererByhostingNodeId(id)); + } + + + public TransferCapabilities getDestinationCapabilities() throws InitializationException{ + return this.transferer.getDestinationCapabilities(); + } + + + public TransferResult localFile(String path,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + return localFile(path,dest,Collections. emptySet()); + } + + public TransferResult localFile(File file,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + return localFile(file,dest,Collections. emptySet()); + } + + public TransferResult httpSource(String url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + return httpSource(url,dest,Collections. emptySet()); + } + + public TransferResult httpSource(URL url,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + return httpSource(url,dest,Collections. emptySet()); + } + + public TransferResult storageId(String id,Destination dest)throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + return storageId(id,dest,Collections. emptySet()); + } + + + + @Synchronized("transferer") + public TransferResult localFile(String path,Destination dest,Set pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); + log.debug("Sending local file {} to {} : {} - {}",path,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations); + transferer.localFile(path); + return doTheTransfer(dest, pluginInvocations); + } + + @Synchronized("transferer") + public TransferResult localFile(File file,Destination dest,Set pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); + log.debug("Sending local file {} to {} : {} - {}",file.getAbsolutePath(),transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations); + transferer.localFile(file); + return doTheTransfer(dest, pluginInvocations); + } + + @Synchronized("transferer") + public TransferResult httpSource(String url,Destination dest,Set pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); + log.debug("Passed url string : "+url); + try{ + return this.httpSource(new URL(url),dest,pluginInvocations); + }catch(MalformedURLException e){ + throw new InvalidSourceException("Invalid url : "+url); + } + } + + @Synchronized("transferer") + public TransferResult httpSource(URL url,Destination dest,Set pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); + log.debug("Sending from url {} to {} : {} - {}",url,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations); + transferer.fromURL(url); + return doTheTransfer(dest, pluginInvocations); + } + + @Synchronized("transferer") + public TransferResult storageId(String id,Destination dest,Set pluginInvocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); + log.debug("Sending from storage id {} to {} : {} - {}",id,transferer.getDestinationCapabilities().getHostName(),dest,pluginInvocations); + transferer.storageFileId(id); + return doTheTransfer(dest, pluginInvocations); + } + + private TransferResult doTheTransfer(Destination dest,Set pluginInvocations) throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + try{ + transferer.setDestination(dest); + transferer.setInvocations(pluginInvocations); + return transferer.transfer(); + }finally{ + transferer.reset(); + } + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/TransferResult.java b/src/main/java/org/gcube/data/transfer/library/TransferResult.java new file mode 100644 index 0000000..97b89fd --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/TransferResult.java @@ -0,0 +1,22 @@ +package org.gcube.data.transfer.library; + +import lombok.Data; +import lombok.NonNull; + +import org.gcube.data.transfer.library.model.Source; + +@Data +public class TransferResult { + + @NonNull + private Source source; + @NonNull + private String destinationHostName; + @NonNull + private Long elapsedTime; + @NonNull + private Long transferedBytes; + @NonNull + private String remotePath; + +} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java b/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java new file mode 100644 index 0000000..0d41613 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/caches/CapabilitiesCache.java @@ -0,0 +1,31 @@ +package org.gcube.data.transfer.library.caches; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +import org.gcube.data.transfer.library.client.Client; +import org.gcube.data.transfer.model.TransferCapabilities; + +@Slf4j +public class CapabilitiesCache extends TTLCache { + + private static CapabilitiesCache instance=null; + +// @Synchronized +// public static CapabilitiesCache getInstance(){ +// if(instance==null)instance=new CapabilitiesCache(); +// return instance; +// } + + private CapabilitiesCache(){ + super(5*60*1000l,2*60*1000l,"Capabilities"); + } + + + @Override + protected TransferCapabilities getNew(String id) throws Exception{ + log.debug("Getting capabilties for host "+id); + return new Client(id).getCapabilties(); + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java b/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java new file mode 100644 index 0000000..d9abd44 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/caches/TTLCache.java @@ -0,0 +1,94 @@ +package org.gcube.data.transfer.library.caches; + +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import lombok.Synchronized; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@ToString +@Slf4j +public abstract class TTLCache { + + + // STATIC + +// private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1,new ThreadFactory(){ +// public Thread newThread(Runnable r) { +// Thread t = Executors.defaultThreadFactory().newThread(r); +// t.setDaemon(true); +// return t; +// } +// }); + private static final HashSet> createdMaps=new HashSet>(); + + + static { +// scheduler.scheduleAtFixedRate(new Runnable() { +// +// +// +// @Override +// public void run() { +// log.debug("Running Maps Cleaner, maps count : "+createdMaps.size()); +// int removed=0; +// for(TTLCache theMap:createdMaps) +// theMap.purgeItems(); +// log.debug("Removed "+removed+" old tickets"); +// +// } +// }, 3, 3, TimeUnit.MINUTES); +// + } + + + //************************* CACHE + + + + protected TTLCache(Long cacheKeepAliveTime, Long objectTTL, String cacheName) { + this.cacheKeepAliveTime = cacheKeepAliveTime; + this.objectTTL = objectTTL; + this.cacheName = cacheName; + createdMaps.add(this); + log.debug("Created Cache "+this); + } + + + + private ConcurrentHashMap> theMap=new ConcurrentHashMap<>(); + + private Long cacheKeepAliveTime; + private Long objectTTL; + private String cacheName; + + @Synchronized + public T getObject(String id) throws Exception{ + if(!theMap.contains(id)||System.currentTimeMillis()-theMap.get(id).getCreationTime()>objectTTL) + theMap.put(id, new TTLContainer(getNew(id))); + return theMap.get(id).getTheObject(); + } + + + private int purgeItems(){ + log.debug(cacheName+" Purging objects, keep alive time is "+cacheKeepAliveTime); + int totalCount=theMap.size(); + int removed=0; + for(Entry> entry:theMap.entrySet()) + if(System.currentTimeMillis()-entry.getValue().getLastUsageTime()>cacheKeepAliveTime){ + theMap.remove(entry.getKey()); + removed++; + } + log.debug(cacheName+" Removed "+removed+" out of "+totalCount); + return removed; + } + + protected abstract T getNew(String id) throws Exception; + +} diff --git a/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java b/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java new file mode 100644 index 0000000..e12db1e --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/caches/TTLContainer.java @@ -0,0 +1,30 @@ +package org.gcube.data.transfer.library.caches; + +public class TTLContainer { + + private long lastUsageTime=System.currentTimeMillis(); + private long creationTime=System.currentTimeMillis(); + + private T theObject; + public TTLContainer(T theObject) { + this.theObject = theObject; + } + + + private void update(){ + lastUsageTime=System.currentTimeMillis(); + } + + public T getTheObject(){ + update(); + return theObject; + } + + public long getLastUsageTime() { + return lastUsageTime; + } + + public long getCreationTime() { + return creationTime; + } +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java b/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java new file mode 100644 index 0000000..115972c --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/client/AuthorizationFilter.java @@ -0,0 +1,28 @@ +package org.gcube.data.transfer.library.client; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map.Entry; + +import javax.ws.rs.client.ClientRequestContext; +import javax.ws.rs.client.ClientRequestFilter; + +import org.gcube.common.calls.Call; +import org.gcube.common.calls.Interceptors; +import org.gcube.common.calls.Request; +import org.gcube.data.transfer.library.utils.ScopeUtils; + +public class AuthorizationFilter implements ClientRequestFilter { + + + @Override + public void filter(final ClientRequestContext rc) throws IOException { + if (ScopeUtils.getCurrentScope()!=null){ + Request requestContext = Interceptors.executeRequestChain(new Call()); + + for (Entry entry: requestContext.getHeaders()){ + rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue())); + } + } + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/client/Client.java b/src/main/java/org/gcube/data/transfer/library/client/Client.java new file mode 100644 index 0000000..b70fa5b --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/client/Client.java @@ -0,0 +1,96 @@ +package org.gcube.data.transfer.library.client; + +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.gcube.data.transfer.library.faults.CommunicationException; +import org.gcube.data.transfer.library.faults.DataTransferException; +import org.gcube.data.transfer.library.faults.RemoteServiceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.model.ServiceConstants; +import org.gcube.data.transfer.model.TransferCapabilities; +import org.gcube.data.transfer.model.TransferRequest; +import org.gcube.data.transfer.model.TransferTicket; +import org.glassfish.jersey.client.ClientConfig; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Client { + + private static ClientConfig config=null; + + static{ + log.debug("Creating configuration .."); + config=new ClientConfig(); + config.register(AuthorizationFilter.class); + } + + private String endpoint; + + private WebTarget rootTarget; + + public Client(String endpoint) throws ServiceNotFoundException{ + try{ + log.debug("Creating client for base "+endpoint); + this.endpoint=endpoint+""; + rootTarget= ClientBuilder.newClient(config).target(endpoint).path("data-transfer-service").path(ServiceConstants.APPLICATION_PATH); +// checkResponse(rootTarget.request().get()); + + log.debug("Root Taget IS {} ",rootTarget.getUri()); + }catch(Exception e){ + throw new ServiceNotFoundException(e); + } + } + + + public String getEndpoint() { + return endpoint; + } + + public TransferCapabilities getCapabilties() throws CommunicationException{ + WebTarget capabilitiesTarget=rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME); + log.debug("Getting capabilities from {}, path is {} ",endpoint,capabilitiesTarget.getUri()); + try{ + Response resp=capabilitiesTarget.request().accept(MediaType.APPLICATION_XML_TYPE).get(); + checkResponse(resp); + return resp.readEntity(TransferCapabilities.class); + }catch(Exception e){ + throw new CommunicationException(e); + } + } + + + public TransferTicket submit(TransferRequest request) throws RemoteServiceException{ + log.debug("Sending request {} to {}",request,endpoint); + try{ + Response resp=rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML)); + checkResponse(resp); + return resp.readEntity(TransferTicket.class); + }catch(Exception e){ + throw new RemoteServiceException(e); + } + } + + public TransferTicket getTransferStatus(String transferId) throws RemoteServiceException{ + log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint); + try{ + Response resp=rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get(); + checkResponse(resp); + return resp.readEntity(TransferTicket.class); + }catch(Exception e){ + throw new RemoteServiceException(e); + } + } + + + protected void checkResponse(Response toCheck) throws Exception{ + switch(toCheck.getStatusInfo().getFamily()){ + case SUCCESSFUL : break; + default : throw new Exception("Unexpected Response code : "+toCheck.getStatus()); + } + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java b/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java new file mode 100644 index 0000000..f0b51a0 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/CommunicationException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class CommunicationException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = 1L; + + public CommunicationException() { + // TODO Auto-generated constructor stub + } + + public CommunicationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CommunicationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CommunicationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CommunicationException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java b/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java new file mode 100644 index 0000000..069e3e9 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/DataTransferException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public abstract class DataTransferException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 8488351139669552406L; + + public DataTransferException() { + // TODO Auto-generated constructor stub + } + + public DataTransferException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public DataTransferException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public DataTransferException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public DataTransferException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/DestinationNotSetException.java b/src/main/java/org/gcube/data/transfer/library/faults/DestinationNotSetException.java new file mode 100644 index 0000000..51fd838 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/DestinationNotSetException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class DestinationNotSetException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = 4792958203546912779L; + + public DestinationNotSetException() { + // TODO Auto-generated constructor stub + } + + public DestinationNotSetException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public DestinationNotSetException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public DestinationNotSetException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public DestinationNotSetException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java b/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java new file mode 100644 index 0000000..28ceda8 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/FailedTransferException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class FailedTransferException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = -807695677781676381L; + + public FailedTransferException() { + // TODO Auto-generated constructor stub + } + + public FailedTransferException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public FailedTransferException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public FailedTransferException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public FailedTransferException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java b/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java new file mode 100644 index 0000000..8e547fa --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/HostingNodeNotFoundException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class HostingNodeNotFoundException extends InitializationException { + + /** + * + */ + private static final long serialVersionUID = 5786009015981659074L; + + public HostingNodeNotFoundException() { + // TODO Auto-generated constructor stub + } + + public HostingNodeNotFoundException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public HostingNodeNotFoundException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public HostingNodeNotFoundException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public HostingNodeNotFoundException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java b/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java new file mode 100644 index 0000000..d074a9c --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/InitializationException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class InitializationException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = 2204462227670167203L; + + public InitializationException() { + // TODO Auto-generated constructor stub + } + + public InitializationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InitializationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public InitializationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InitializationException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/InvalidDestinationException.java b/src/main/java/org/gcube/data/transfer/library/faults/InvalidDestinationException.java new file mode 100644 index 0000000..d77a541 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/InvalidDestinationException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class InvalidDestinationException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = -1882334325516885722L; + + public InvalidDestinationException() { + // TODO Auto-generated constructor stub + } + + public InvalidDestinationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InvalidDestinationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public InvalidDestinationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InvalidDestinationException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java b/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java new file mode 100644 index 0000000..93e7b09 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/InvalidSourceException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class InvalidSourceException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = 7631518288856917202L; + + public InvalidSourceException() { + // TODO Auto-generated constructor stub + } + + public InvalidSourceException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InvalidSourceException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public InvalidSourceException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InvalidSourceException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java b/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java new file mode 100644 index 0000000..f8c52dc --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/RemoteServiceException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class RemoteServiceException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = 5320978791087129997L; + + public RemoteServiceException() { + // TODO Auto-generated constructor stub + } + + public RemoteServiceException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public RemoteServiceException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public RemoteServiceException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public RemoteServiceException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java b/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java new file mode 100644 index 0000000..adf0395 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/ServiceNotFoundException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class ServiceNotFoundException extends InitializationException { + + /** + * + */ + private static final long serialVersionUID = -3494011839783891174L; + + public ServiceNotFoundException() { + // TODO Auto-generated constructor stub + } + + public ServiceNotFoundException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public ServiceNotFoundException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public ServiceNotFoundException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public ServiceNotFoundException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java b/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java new file mode 100644 index 0000000..12733f4 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/SourceNotSetException.java @@ -0,0 +1,35 @@ +package org.gcube.data.transfer.library.faults; + +public class SourceNotSetException extends DataTransferException { + + /** + * + */ + private static final long serialVersionUID = -5868191252361823341L; + + public SourceNotSetException() { + // TODO Auto-generated constructor stub + } + + public SourceNotSetException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public SourceNotSetException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public SourceNotSetException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public SourceNotSetException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java b/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java new file mode 100644 index 0000000..a01a338 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/faults/UnreachableNodeException.java @@ -0,0 +1,37 @@ +package org.gcube.data.transfer.library.faults; + +public class UnreachableNodeException extends InitializationException { + + + + /** + * + */ + private static final long serialVersionUID = 8739586712271226827L; + + public UnreachableNodeException() { + // TODO Auto-generated constructor stub + } + + public UnreachableNodeException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public UnreachableNodeException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public UnreachableNodeException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public UnreachableNodeException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java b/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java new file mode 100644 index 0000000..18909c7 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/model/LocalSource.java @@ -0,0 +1,41 @@ +package org.gcube.data.transfer.library.model; + +import java.io.File; + +import org.gcube.data.transfer.library.faults.InvalidSourceException; + +public class LocalSource extends Source { + + private File theFile=null; + + public LocalSource(File theFile) throws InvalidSourceException { + super(); + if(theFile==null)throw new InvalidSourceException("File cannot be null"); + this.theFile = theFile; + } + + @Override + public boolean validate() throws InvalidSourceException { + if(!theFile.exists()) throw new InvalidSourceException("File "+theFile.getAbsolutePath()+"doesn't exist"); + if(!theFile.canRead()) throw new InvalidSourceException("Unable to read from file "+theFile.getAbsolutePath()); + if(theFile.isDirectory()) throw new InvalidSourceException("Transfer of directory is not yet supported"); + return true; + } + + @Override + public void prepare() { + // TODO Auto-generated method stub + + } + + @Override + public void clean() { + // TODO Auto-generated method stub + + } + + @Override + public File getTheSource() { + return theFile; + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/model/Source.java b/src/main/java/org/gcube/data/transfer/library/model/Source.java new file mode 100644 index 0000000..dee9add --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/model/Source.java @@ -0,0 +1,17 @@ +package org.gcube.data.transfer.library.model; + +import org.gcube.data.transfer.library.faults.InvalidSourceException; + +import lombok.Data; + +@Data +public abstract class Source{ + + public abstract boolean validate() throws InvalidSourceException; + + public abstract void prepare(); + + public abstract void clean(); + + public abstract T getTheSource(); +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java b/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java new file mode 100644 index 0000000..4c60357 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java @@ -0,0 +1,47 @@ +package org.gcube.data.transfer.library.model; + +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.utils.StorageUtils; + +public class StorageSource extends Source { + + private String id=null; + + + + + + public StorageSource(String id) throws InvalidSourceException { + super(); + if(id==null) throw new InvalidSourceException("Storage id cannot be null"); + this.id = id; + } + + @Override + public boolean validate() throws InvalidSourceException { + try{ + if(!StorageUtils.checkStorageId(id)) throw new Exception("Not valid"); + }catch(Exception e){ + throw new InvalidSourceException("Invalid storage ID "+id); + } + return true; + } + + @Override + public void prepare() { + // TODO Auto-generated method stub + + } + + @Override + public void clean() { + // TODO Auto-generated method stub + + } + + @Override + public String getTheSource() { + return id; + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/model/URLSource.java b/src/main/java/org/gcube/data/transfer/library/model/URLSource.java new file mode 100644 index 0000000..61109c8 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/model/URLSource.java @@ -0,0 +1,62 @@ +package org.gcube.data.transfer.library.model; + +import java.net.HttpURLConnection; +import java.net.URL; + +import org.gcube.data.transfer.library.faults.InvalidSourceException; + +public class URLSource extends Source { + + + private URL theURL=null; + + + public URLSource(URL theURL) throws InvalidSourceException { + super(); + if(theURL==null) throw new InvalidSourceException("URL Source cannot be null"); + this.theURL = theURL; + } + + @Override + public boolean validate() throws InvalidSourceException { + try{ + HttpURLConnection conn=(HttpURLConnection) theURL.openConnection(); + conn.setConnectTimeout(3000); + conn.setRequestMethod("HEAD"); + int responseCode = conn.getResponseCode(); + return (200 <= responseCode && responseCode <= 399); + }catch(Exception e){ + throw new InvalidSourceException("Unable to contact URL "+theURL,e); + } + } + + @Override + public void prepare() { + // nothing to do + + } + + @Override + public void clean() { + // nothing to do + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("URLSource [theURL="); + builder.append(theURL); + builder.append("]"); + return builder.toString(); + } + + @Override + public URL getTheSource() { + return theURL; + } + + +} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java new file mode 100644 index 0000000..8ed6fbb --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java @@ -0,0 +1,71 @@ +package org.gcube.data.transfer.library.transferers; + +import java.io.FileNotFoundException; +import java.net.MalformedURLException; +import java.net.URL; + +import lombok.extern.slf4j.Slf4j; + +import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import org.gcube.data.transfer.library.client.Client; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.model.LocalSource; +import org.gcube.data.transfer.library.model.StorageSource; +import org.gcube.data.transfer.library.model.URLSource; +import org.gcube.data.transfer.library.utils.StorageUtils; +import org.gcube.data.transfer.model.TransferRequest; +import org.gcube.data.transfer.model.options.HttpDownloadOptions; +import org.gcube.data.transfer.model.settings.HttpDownloadSettings; + +@Slf4j +public class HTTPTransferer extends Transferer { + + HTTPTransferer(Client client) { + super(client); + } + + private URL link; + + private String toDeleteStorageId=null; + + @Override + protected TransferRequest prepareRequest()throws InitializationException { + return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT),this.destination,this.invocations); + } + + + private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{ + if(source instanceof LocalSource){ + toDeleteStorageId=StorageUtils.putOntoStorage(((LocalSource)source).getTheSource()); + return new URL(StorageUtils.getUrlById(toDeleteStorageId)); + }else if (source instanceof StorageSource){ + return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource())); + }else if (source instanceof URLSource){ + return ((URLSource)source).getTheSource(); + }else throw new InvalidSourceException("Source cannot be handled "+source); + } + + @Override + protected void prepare() throws InitializationException { + try{ + link=getHttpLink(); + log.debug("Obtained link "+link); + super.prepare(); + }catch(RemoteBackendException e){ + throw new InitializationException(e); + }catch(FileNotFoundException e){ + throw new InitializationException(e); + }catch(InvalidSourceException e){ + throw new InitializationException(e); + }catch (MalformedURLException e){ + throw new InitializationException(e); + } + } + + @Override + protected void clean() { + // TODO Auto-generated method stub + super.clean(); + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java new file mode 100644 index 0000000..988d2a2 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java @@ -0,0 +1,157 @@ +package org.gcube.data.transfer.library.transferers; + +import java.io.File; +import java.net.URL; +import java.util.Set; + +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.client.Client; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.RemoteServiceException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.model.LocalSource; +import org.gcube.data.transfer.library.model.Source; +import org.gcube.data.transfer.library.model.StorageSource; +import org.gcube.data.transfer.library.model.URLSource; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.TransferCapabilities; +import org.gcube.data.transfer.model.TransferRequest; +import org.gcube.data.transfer.model.TransferTicket; +import org.gcube.data.transfer.model.TransferTicket.Status; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class Transferer { + + protected Client client; + + protected Transferer(Client client){ + this.client=client; + } + + + + + protected Source source=null; + protected boolean prepared=false; + protected Destination destination=null; + protected Set invocations=null; + + + public void setDestination(Destination destination) { + this.destination = destination; + } + + public void setInvocations(Set invocations) { + this.invocations = invocations; + } + + public Transferer localFile(File f) throws InvalidSourceException{ + source=new LocalSource(f); + return this; + } + + public Transferer localFile(String path) throws InvalidSourceException{ + try{ + File f=new File(path); + return localFile(f); + }catch(NullPointerException e){ + throw new InvalidSourceException("Path is null",e); + } + + } + + public Transferer storageFileId(String fileId) throws InvalidSourceException{ + source=new StorageSource(fileId); + return this; + } + + public Transferer fromURL(URL sourceURL) throws InvalidSourceException{ + source=new URLSource(sourceURL); + return this; + } + + + public void reset(){ + this.destination=null; + this.source=null; + this.invocations=null; + } + + public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + try{ + checkDestination(); + checkSource(); + checkPluginInvocations(); + prepare(); + TransferRequest request=prepareRequest(); + log.debug("Request is {}, sending it to {}",request,client.getEndpoint()); + TransferResult result=doTheTransfer(request); + return result; + }finally{ + clean(); + } + } + + protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{ + try{ + TransferTicket submissionResponse= client.submit(request); + boolean continuePolling=true; + TransferTicket ticket=null; + do{ + ticket=client.getTransferStatus(submissionResponse.getId()); + System.out.println("Status : "+ticket); + continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING); + try{ + Thread.sleep(500); + }catch(InterruptedException e){} + }while(continuePolling); + if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage()); + if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage()); + long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis(); + return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName()); + }catch(RemoteServiceException e){ + throw new FailedTransferException(e); + } + } + + protected void checkDestination() throws InvalidDestinationException,DestinationNotSetException{ + if(destination==null) throw new DestinationNotSetException(); + // perform checks + } + + protected void checkSource() throws SourceNotSetException, InvalidSourceException{ + if(source==null) throw new SourceNotSetException(); + source.validate(); + } + + protected void checkPluginInvocations(){ + + } + + protected abstract TransferRequest prepareRequest() throws InitializationException; + + + protected void prepare() throws InitializationException{ + prepared=true; + } + protected void clean(){ + + } + + public TransferCapabilities getDestinationCapabilities() throws InitializationException { + try{ + return client.getCapabilties(); + }catch(Exception e){ + throw new InitializationException(e); + } + } + + +} diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java b/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java new file mode 100644 index 0000000..044a982 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java @@ -0,0 +1,74 @@ +package org.gcube.data.transfer.library.transferers; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.gcube.common.resources.gcore.HostingNode; +import org.gcube.data.transfer.library.caches.CapabilitiesCache; +import org.gcube.data.transfer.library.client.Client; +import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.TransferCapabilities; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +@Slf4j +public class TransfererBuilder { + + private static final int timeout=10*1000; + + + //e.g. http://pc-fabio.isti.cnr.it:8080/data-transfer-service/gcube/service + + public static Transferer getTransfererByHost(String endpoint) throws UnreachableNodeException, ServiceNotFoundException{ + log.debug("Get transferer by Host "+endpoint); + try{ + URL url=new URL(endpoint); + + String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort(); + //TODO Implement checks + + // if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout); + // String finalHost=host; + // if(!finalHost.endsWith(ServiceConstants.APPLICATION_PATH)){ + // // adjust host + // finalHost=finalHost+(host.endsWith("/")?"":"/")+"data-transfer-service"+ServiceConstants.APPLICATION_PATH; + // } + // + // + // if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost); + // log.debug("Host is ok, getting targetCapabilities"); +// TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl); + + // TODO determine method by capabilities checking + + return new HTTPTransferer(new Client(baseUrl)); + }catch(Exception e){ + throw new ServiceNotFoundException(e); + } + } + public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{ + String hostname=retrieveHostnameByNodeId(hostId); + return getTransfererByHost(hostname); + } + + + private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{ + SimpleQuery query = queryFor(HostingNode.class); + + query.addCondition("$resource/ID/text() eq '"+nodeId+"'"); + + + List found= clientFor(HostingNode.class).submit(query); + + if(found.isEmpty()) throw new HostingNodeNotFoundException("No Hosting node with the id "+nodeId); + + return "http://"+found.get(0).profile().description().name(); + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/utils/ScopeUtils.java b/src/main/java/org/gcube/data/transfer/library/utils/ScopeUtils.java new file mode 100644 index 0000000..149d0d7 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/utils/ScopeUtils.java @@ -0,0 +1,27 @@ +package org.gcube.data.transfer.library.utils; + +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ScopeUtils { + + public static String getCurrentScope(){ + try{ + String token=SecurityTokenProvider.instance.get(); + log.debug("Token is : "+token); + if(token==null) throw new Exception("Security Token is null"); + AuthorizationEntry entry = authorizationService().get(token); + return entry.getContext(); + }catch(Exception e ){ + log.debug("Unable to resolve token, checking scope provider..",e); + return ScopeProvider.instance.get(); + } + } + +} diff --git a/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java b/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java new file mode 100644 index 0000000..5c8ad4e --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java @@ -0,0 +1,46 @@ +package org.gcube.data.transfer.library.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + +import lombok.extern.slf4j.Slf4j; + +import org.bson.types.ObjectId; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; + +@Slf4j +public class StorageUtils { + + public static final IClient getClient(){ + return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient(); + } + + + //return Id + public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{ + IClient client=getClient(); + log.debug("Uploading local file "+source.getAbsolutePath()); + return client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString()); + } + + public static final boolean checkStorageId(String id){ + return getClient().getHttpUrl().RFile(id)!=null; + } + + public static final String getUrlById(String id){ + IClient client=getClient(); + log.debug("Id is "+id); + return client.getHttpUrl().RFile(id); + } + + public static final void removeById(String id){ + IClient client=getClient(); + client.remove().RFile(id); + } +} diff --git a/src/main/java/org/gcube/data/transfer/library/utils/Utils.java b/src/main/java/org/gcube/data/transfer/library/utils/Utils.java new file mode 100644 index 0000000..0c7043f --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/library/utils/Utils.java @@ -0,0 +1,40 @@ +package org.gcube.data.transfer.library.utils; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.UUID; + +public class Utils { + + /** + * Pings a HTTP URL. This effectively sends a HEAD request and returns true if the response code is in + * the 200-399 range. + * @param url The HTTP URL to be pinged. + * @param timeout The timeout in millis for both the connection timeout and the response read timeout. Note that + * the total timeout is effectively two times the given timeout. + * @return true if the given HTTP URL has returned response code 200-399 on a HEAD request within the + * given timeout, otherwise false. + */ + public static boolean pingURL(String url, int timeout) { + url = url.replaceFirst("^https", "http"); // Otherwise an exception may be thrown on invalid SSL certificates. + + try { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); + connection.setRequestMethod("HEAD"); + int responseCode = connection.getResponseCode(); + return (200 <= responseCode && responseCode <= 399); + } catch (IOException exception) { + return false; + } + } + + + + public static final String getUniqueString(){ + return UUID.randomUUID().toString(); + } + +} diff --git a/src/test/java/org/gcube/data/transfer/library/ErrorReport.java b/src/test/java/org/gcube/data/transfer/library/ErrorReport.java new file mode 100644 index 0000000..ccf2b57 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/ErrorReport.java @@ -0,0 +1,19 @@ +package org.gcube.data.transfer.library; + +import org.gcube.data.transfer.model.TransferCapabilities; + +public class ErrorReport extends TransferReport { + + String id; + + + public ErrorReport(String id) { + super(null); + this.id=id; + } + + @Override + public String print() { + return "ERROR : "+id; + } +} diff --git a/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java b/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java new file mode 100644 index 0000000..aaa9524 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/MultipleTransferBenchmark.java @@ -0,0 +1,161 @@ +package org.gcube.data.transfer.library; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; + +import org.gcube.data.transfer.library.TransferReport.ReportType; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; + +import ch.qos.logback.core.util.ExecutorServiceUtil; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MultipleTransferBenchmark { + + + static List endpoints; + static List nodeIds; + + + static List files; + static List uris; + static List storageIds; + + static ExecutorService executor=ExecutorServiceUtil.newExecutorService(); + + + static CountDownLatch doneSignal; + + static ArrayList reports=new ArrayList<>(); + + + static String scope="/gcube/devsec/devVRE"; + + static{ + endpoints=Arrays.asList(new String[]{ +// "http://node4-d-d4s.d4science.org:80/data-transfer-service/gcube/service", + "http://node3-d-d4s.d4science.org:80/data-transfer-service/gcube/service" + }); + doneSignal=new CountDownLatch(endpoints.size()); + + + + nodeIds=new ArrayList<>(); + files=new ArrayList<>(); + files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); + files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); + files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); + files.add("/home/fabio/Documents/Pictures/web_trend_map.png"); + uris=new ArrayList<>(); + uris.add("http://goo.gl/r5jFZ9"); + storageIds=new ArrayList<>(); + } + + + + + public static void main(String[] args) throws InitializationException{ + TokenSetter.set(scope); + Map clients=new HashMap<>(); + for(String endpoint:endpoints) + clients.put(endpoint,DataTransferClient.getInstanceByEndpoint(endpoint)); + for(String id:nodeIds) + clients.put(id,DataTransferClient.getInstanceByEndpoint(id)); + + + + + HashSet startedTests=new HashSet<>(); + for(final Entry entry:clients.entrySet()){ + if(!startedTests.contains(entry.getValue())){ + startedTests.add(entry.getValue()); + + executor.execute(new Runnable(){ + + final DataTransferClient dt=entry.getValue(); + final String id=entry.getKey(); + @Override + public void run() { + try{ + TransferReport report=new TransferReport(dt.getDestinationCapabilities()); + TokenSetter.set(scope); + + log.debug("Sending files to {} ",dt.getDestinationCapabilities()); + + Destination dest=new Destination("banchmarkoutputFile"); + dest.setOnExistingFileName(DestinationClashPolicy.REWRITE); + + for(String f:files){ + try { + TransferResult res=dt.localFile(f,dest); + report.addReport(ReportType.local,f, res.getTransferedBytes(), res.getElapsedTime()); + } catch (InvalidSourceException | SourceNotSetException + | FailedTransferException + | InitializationException e) { + log.error("Unable to send file {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); + } + } + + log.debug("Sending uris to {}",dt.getDestinationCapabilities()); + for(String f:uris){ + try { + TransferResult res=dt.httpSource(f,dest); + report.addReport(ReportType.uri,f, res.getTransferedBytes(), res.getElapsedTime()); + } catch (InvalidSourceException | SourceNotSetException + | FailedTransferException + | InitializationException e) { + log.error("Unable to send uri {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); + } + } + + log.debug("Sending storageIds to {}",dt.getDestinationCapabilities()); + for(String f:storageIds){ + try { + TransferResult res=dt.storageId(f,dest); + report.addReport(ReportType.storage,f, res.getTransferedBytes(), res.getElapsedTime()); + } catch (InvalidSourceException | SourceNotSetException + | FailedTransferException + | InitializationException e) { + log.error("Unable to send storageId {} to {}, error message : {}",f,dt.getDestinationCapabilities().getHostName(),e.getMessage()); + } + } + reports.add(report); + }catch(Exception e){ + reports.add(new ErrorReport(id)); + }finally{ + doneSignal.countDown(); + } + + } + }); + } + } + + try { + doneSignal.await(); + } catch (InterruptedException e) { + + } + System.out.println("*****************************"); + for(TransferReport r:reports)System.out.println(r.print()); + + ExecutorServiceUtil.shutdown(executor); + } + + + + + +} diff --git a/src/test/java/org/gcube/data/transfer/library/StorageTest.java b/src/test/java/org/gcube/data/transfer/library/StorageTest.java new file mode 100644 index 0000000..b3a2c1b --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/StorageTest.java @@ -0,0 +1,21 @@ +package org.gcube.data.transfer.library; + +import java.io.File; +import java.io.FileNotFoundException; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import org.gcube.data.transfer.library.utils.StorageUtils; + +public class StorageTest { + + static String scope="/gcube/devsec"; + + public static void main(String[] args) throws RemoteBackendException, FileNotFoundException { + ScopeProvider.instance.set(scope); + String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf"; + String id=StorageUtils.putOntoStorage(new File(toUpload)); + System.out.println(StorageUtils.getUrlById(id)); + } + +} diff --git a/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java b/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java new file mode 100644 index 0000000..f5d3f89 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/TestClientCalls.java @@ -0,0 +1,86 @@ +package org.gcube.data.transfer.library; + +import java.net.MalformedURLException; +import java.net.URL; + +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.data.transfer.library.client.AuthorizationFilter; +import org.gcube.data.transfer.library.client.Client; +import org.gcube.data.transfer.library.faults.CommunicationException; +import org.gcube.data.transfer.library.faults.RemoteServiceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.ServiceConstants; +import org.gcube.data.transfer.model.TransferCapabilities; +import org.gcube.data.transfer.model.TransferRequest; +import org.gcube.data.transfer.model.TransferTicket; +import org.gcube.data.transfer.model.TransferTicket.Status; +import org.gcube.data.transfer.model.options.HttpDownloadOptions; +import org.gcube.data.transfer.model.settings.HttpDownloadSettings; +import org.glassfish.jersey.client.ClientConfig; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class TestClientCalls { + + static String hostname="http://node3-d-d4s.d4science.org:80"; +// static String hostname="http://pc-fabio.isti.cnr.it:8080"; + static String scope="/gcube/devNext"; + static Client client; + + + @BeforeClass + public static void init() throws ServiceNotFoundException{ + ScopeProvider.instance.set(scope); + client=new Client(hostname); + } + + + @Test + public void getCapabilties() throws CommunicationException{ + System.out.println(client.getCapabilties()); + } + + + @Test + public void doTheTransfer() throws MalformedURLException, RemoteServiceException{ + Destination dest=new Destination("outputFile"); + dest.setCreateSubfolders(true); + dest.setSubFolder("bla/bla/bllaaa"); + dest.setOnExistingFileName(DestinationClashPolicy.ADD_SUFFIX); + dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + TransferRequest request= new TransferRequest("", new HttpDownloadSettings(new URL("http://goo.gl/oLP7zG"), HttpDownloadOptions.DEFAULT),dest); + System.out.println("Submitting "+request); + TransferTicket ticket=client.submit(request); + System.out.println("Ticket is "+ticket); + + boolean continuePolling=true; + do{ + ticket=client.getTransferStatus(ticket.getId()); + System.out.println("Status : "+ticket); + continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING); + try{ + Thread.sleep(1000); + }catch(InterruptedException e){} + }while(continuePolling); + } + + + + @Test + public void directCall(){ + javax.ws.rs.client.Client client = ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class)); + WebTarget target=client.target(hostname+"/data-transfer-service"+ServiceConstants.APPLICATION_PATH+"Capabilities"); +// WebTarget target=client.target(hostname+"/data-transfer-service/gcube/service/Capabilities"); + System.out.println("Asking capabilities to target : "+target.getUri()); + System.out.println(target. + request(MediaType.APPLICATION_XML).get(TransferCapabilities.class)); + } + +} diff --git a/src/test/java/org/gcube/data/transfer/library/TokenSetter.java b/src/test/java/org/gcube/data/transfer/library/TokenSetter.java new file mode 100644 index 0000000..132a941 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/TokenSetter.java @@ -0,0 +1,31 @@ +package org.gcube.data.transfer.library; + +import java.util.Properties; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +public class TokenSetter { + + + + private static Properties props=new Properties(); + + static{ + try { + props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); + } catch (Exception e) { + throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); + } + } + + + public static void set(String scope){ + if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope); + SecurityTokenProvider.instance.set(props.getProperty(scope)); + ScopeProvider.instance.set(scope); + } + + +} + diff --git a/src/test/java/org/gcube/data/transfer/library/TransferReport.java b/src/test/java/org/gcube/data/transfer/library/TransferReport.java new file mode 100644 index 0000000..d61b1b2 --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/TransferReport.java @@ -0,0 +1,89 @@ +package org.gcube.data.transfer.library; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.gcube.data.transfer.library.model.Source; +import org.gcube.data.transfer.model.TransferCapabilities; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +public class TransferReport { + + public static enum ReportType{ + local,uri,storage + } + + + private static final char end='\n'; + + @AllArgsConstructor + @Getter + @ToString + private static class ReportItem{ + String source; + long size; + long elapsed; + + + // bytes/msec + public long getAvgSpeed(){ + return size/elapsed; + } + + + + + } + + + protected TransferCapabilities host; + + private Map> reports=new HashMap<>(); + + + + public TransferReport(TransferCapabilities host) { + super(); + this.host = host; + } + + + public void addReport(ReportType sourceType,String theSource,long size,long elapsed){ + if(!reports.containsKey(sourceType)) reports.put(sourceType, new ArrayList()); + reports.get(sourceType).add(new ReportItem(theSource, size, elapsed)); + } + + + public String print(){ + StringBuilder builder=new StringBuilder(" Report for : "+host+end); + for(Entry> entry:reports.entrySet()){ + builder.append("Source type "+entry.getKey()+end); + ReportItem maxSizeItem=null; + ReportItem maxAvgSpeedItem=null; + ReportItem minSizeItem=null; + ReportItem minAvgSpeedItem=null; + long currentAvgSpeedCounter=0l; + + for(ReportItem item:entry.getValue()){ + if(maxSizeItem==null||maxSizeItem.getSize()item.getSize()) minSizeItem=item; + if(minAvgSpeedItem==null||minAvgSpeedItem.getAvgSpeed()>item.getAvgSpeed()) minAvgSpeedItem=item; + currentAvgSpeedCounter+=item.getAvgSpeed(); + } + + builder.append("Max Size Item : "+maxSizeItem+end); + builder.append("Max Avg Speed Item : "+maxAvgSpeedItem+end); + builder.append("Min Size Item : "+minSizeItem+end); + builder.append("Min Avg Speed Item : "+minAvgSpeedItem+end); + builder.append("Total avg speed : "+currentAvgSpeedCounter/entry.getValue().size()+end); + } + + return builder.toString(); + }; +} diff --git a/src/test/java/org/gcube/data/transfer/library/TransfererTest.java b/src/test/java/org/gcube/data/transfer/library/TransfererTest.java new file mode 100644 index 0000000..dda899c --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/library/TransfererTest.java @@ -0,0 +1,89 @@ +package org.gcube.data.transfer.library; + +import java.io.File; +import java.io.FileNotFoundException; +import java.net.MalformedURLException; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.library.utils.StorageUtils; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TransfererTest { + + static String hostname="http://node3-d-d4s.d4science.org:80"; + static String nodeId="462b68c5-463f-4295-86da-37d6c0abc7ea"; + static String scope="/gcube/devsec"; + + static DataTransferClient client; + + @BeforeClass + public static void init() throws UnreachableNodeException, ServiceNotFoundException, HostingNodeNotFoundException{ + ScopeProvider.instance.set(scope); + client=DataTransferClient.getInstanceByEndpoint(hostname); + // client=DataTransferClient.getInstanceByNodeId(nodeId); + } + + @Test + public void localFile() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + for(int i=0;i<3;i++){ + String localFile="/home/fabio/Dropbox/Mindless/01- COMA - Mindless.mp3"; + String transferredFileName="Mindless.mp3"; + Destination dest=new Destination(transferredFileName); + dest.setOnExistingFileName(DestinationClashPolicy.REWRITE); + TransferResult res=client.localFile(localFile,dest); + String remotePath=res.getRemotePath(); + Assert.assertEquals(transferredFileName, remotePath.substring(remotePath.lastIndexOf(File.separatorChar)+1)); + System.out.println(res); + } + } + + + @Test + public void httpUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + String link="http://goo.gl/oLP7zG"; + System.out.println(client.httpSource(link,new Destination("Some import"))); + } + + + @Test + public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException, InvalidDestinationException, DestinationNotSetException{ + ScopeProvider.instance.set(scope); + String toUpload="/home/fabio/Documents/Personal/DND/Incantesimi 3.5 - Mago e Stregone.pdf"; + String id=StorageUtils.putOntoStorage(new File(toUpload)); + Destination dest=new Destination("some/where","My Pdf.pdf"); + TransferResult res=client.storageId(id,dest); + Assert.assertTrue(res.getRemotePath().contains(dest.getSubFolder()+File.separatorChar+dest.getDestinationFileName())); + System.out.println(client.storageId(id,dest)); + } + + @Test(expected=InvalidSourceException.class) + public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + System.out.println(client.storageId("13245780t",new Destination("my file"))); + } + + @Test(expected=InvalidSourceException.class) + public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + String localFile="/home/fabio/Downloads/123045689.mp3"; + System.out.println(client.localFile(localFile,new Destination("12345"))); + } + + @Test(expected=InvalidSourceException.class) + public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException{ + String link="https://www.some.where.com/over/theRainbow.txt"; + System.out.println(client.httpSource(link,new Destination("oz"))); + } +}