From 7eff5dec476fb07372795700791832158fd224d5 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 26 May 2016 16:35:31 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@128868 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../transfer/library/DataTransferClient.java | 9 +++++--- .../data/transfer/library/client/Client.java | 21 ++++++++++++------- .../transfer/library/model/StorageSource.java | 5 +++-- .../library/transferers/HTTPTransferer.java | 6 +++--- .../library/transferers/Transferer.java | 20 +++++++++--------- .../transferers/TransfererBuilder.java | 10 ++++++--- .../transfer/library/utils/StorageUtils.java | 2 +- .../data/transfer/library/TransfererTest.java | 11 +++++----- 8 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java b/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java index 8a19f80..09d21ff 100644 --- a/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java +++ b/src/main/java/org/gcube/data/transfer/library/DataTransferClient.java @@ -20,7 +20,6 @@ import org.gcube.data.transfer.library.transferers.TransfererBuilder; @Slf4j public class DataTransferClient { - private Transferer transferer=null; private DataTransferClient(Transferer transferer) { @@ -54,10 +53,14 @@ public class DataTransferClient { } @Synchronized("transferer") - public TransferResult httpSource(String url) throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ + public TransferResult httpSource(String url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer"); log.debug("Passed url string : "+url); - return this.httpSource(new URL(url)); + try{ + return this.httpSource(new URL(url)); + }catch(MalformedURLException e){ + throw new InvalidSourceException("Invalid url : "+url); + } } @Synchronized("transferer") diff --git a/src/main/java/org/gcube/data/transfer/library/client/Client.java b/src/main/java/org/gcube/data/transfer/library/client/Client.java index 8e02019..eaa95dc 100644 --- a/src/main/java/org/gcube/data/transfer/library/client/Client.java +++ b/src/main/java/org/gcube/data/transfer/library/client/Client.java @@ -26,29 +26,34 @@ public class Client { config.register(JAXBElement.class); } -// private String hostname; + private String endpoint; private WebTarget rootTarget; - public Client(String hostname){ - log.debug("Creating client for "+hostname); - this.hostname=hostname+""; - rootTarget= ClientBuilder.newClient(config).target(hostname).path("/data-transfer-service/").path(ServiceConstants.APPLICATION_PATH); + public Client(String endpoint){ + log.debug("Creating client for base "+endpoint); + this.endpoint=endpoint+""; + rootTarget= ClientBuilder.newClient(config).target(endpoint).path("/data-transfer-service/").path(ServiceConstants.APPLICATION_PATH); + } + + + public String getEndpoint() { + return endpoint; } public TransferCapabilities getCapabilties(){ - log.debug("Getting capabilities to {} ",hostname); + log.debug("Getting capabilities to {} ",endpoint); return rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).get(TransferCapabilities.class); } public TransferTicket submit(TransferRequest request){ - log.debug("Sending request {} to {}",request,hostname); + log.debug("Sending request {} to {}",request,endpoint); return rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML),TransferTicket.class); } public TransferTicket getTransferStatus(String transferId){ - log.debug("Requesting transfer status [id = {}, hostname={}]",transferId,hostname); + log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint); return rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get(TransferTicket.class); } } diff --git a/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java b/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java index 04f5097..2225939 100644 --- a/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java +++ b/src/main/java/org/gcube/data/transfer/library/model/StorageSource.java @@ -1,6 +1,7 @@ package org.gcube.data.transfer.library.model; import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.utils.StorageUtils; public class StorageSource extends Source { @@ -18,8 +19,8 @@ public class StorageSource extends Source { @Override public boolean validate() throws InvalidSourceException { - // TODO Auto-generated method stub - return false; + if(!StorageUtils.checkStorageId(id)) throw new InvalidSourceException("Invalid storage ID "+id); + return true; } @Override diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java index 8de6247..6b5b098 100644 --- a/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java +++ b/src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java @@ -7,13 +7,13 @@ import java.net.URL; import lombok.extern.slf4j.Slf4j; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; +import org.gcube.data.transfer.library.client.Client; import org.gcube.data.transfer.library.faults.InitializationException; import org.gcube.data.transfer.library.faults.InvalidSourceException; import org.gcube.data.transfer.library.model.LocalSource; import org.gcube.data.transfer.library.model.StorageSource; import org.gcube.data.transfer.library.model.URLSource; import org.gcube.data.transfer.library.utils.StorageUtils; -import org.gcube.data.transfer.model.TransferCapabilities; import org.gcube.data.transfer.model.TransferRequest; import org.gcube.data.transfer.model.options.HttpDownloadOptions; import org.gcube.data.transfer.model.settings.HttpDownloadSettings; @@ -21,8 +21,8 @@ import org.gcube.data.transfer.model.settings.HttpDownloadSettings; @Slf4j public class HTTPTransferer extends Transferer { - HTTPTransferer(TransferCapabilities targetCapabilities) { - super(targetCapabilities); + HTTPTransferer(Client client) { + super(client); } private URL link; diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java b/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java index 15c7644..11eee8d 100644 --- a/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java +++ b/src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java @@ -1,13 +1,10 @@ package org.gcube.data.transfer.library.transferers; import java.io.File; -import java.io.FileNotFoundException; -import java.net.MalformedURLException; import java.net.URL; import lombok.extern.slf4j.Slf4j; -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.client.Client; import org.gcube.data.transfer.library.faults.FailedTransferException; @@ -26,12 +23,15 @@ import org.gcube.data.transfer.model.TransferTicket.Status; @Slf4j public abstract class Transferer { - protected TransferCapabilities targetCapabilities; + protected Client client; - protected Transferer(TransferCapabilities targetCapabilities){ - this.targetCapabilities=targetCapabilities; + protected Transferer(Client client){ + this.client=client; } + + + protected Source source=null; protected boolean prepared=false; @@ -66,7 +66,7 @@ public abstract class Transferer { checkSource(); prepare(); TransferRequest request=prepareRequest(); - log.debug("Request is {}, sending it to {}",request,targetCapabilities.getHostName()); + log.debug("Request is {}, sending it to {}",request,client.getEndpoint()); TransferResult result=doTheTransfer(request); return result; }finally{ @@ -75,7 +75,6 @@ public abstract class Transferer { } protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{ - Client client=new Client(targetCapabilities.getHostName()); TransferTicket submissionResponse= client.submit(request); boolean continuePolling=true; TransferTicket ticket=null; @@ -90,7 +89,7 @@ public abstract class Transferer { if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage()); if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage()); long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis(); - return new TransferResult(source, targetCapabilities.getHostName(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName()); + return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName()); } protected void checkSource() throws SourceNotSetException, InvalidSourceException{ @@ -109,7 +108,8 @@ public abstract class Transferer { } public TransferCapabilities getDestinationCapabilities(){ - return targetCapabilities; + return client.getCapabilties(); } + } diff --git a/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java b/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java index 77076d9..9f00d9f 100644 --- a/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java +++ b/src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java @@ -6,6 +6,7 @@ import java.net.URL; import lombok.extern.slf4j.Slf4j; import org.gcube.data.transfer.library.caches.CapabilitiesCache; +import org.gcube.data.transfer.library.client.Client; import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException; import org.gcube.data.transfer.library.faults.ServiceNotFoundException; import org.gcube.data.transfer.library.faults.UnreachableNodeException; @@ -23,7 +24,7 @@ public class TransfererBuilder { try{ URL url=new URL(endpoint); - String hostname=url.getProtocol()+"://"+url.getHost()+":"+url.getPort(); + String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort(); //TODO Implement checks // if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout); @@ -36,8 +37,11 @@ public class TransfererBuilder { // // if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost); // log.debug("Host is ok, getting targetCapabilities"); - TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(hostname); - return new HTTPTransferer(cap); + TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl); + + // TODO determine method by capabilities checking + + return new HTTPTransferer(new Client(baseUrl)); }catch(MalformedURLException e){ throw new ServiceNotFoundException(e); } diff --git a/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java b/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java index 6aefb3d..4ae504b 100644 --- a/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java +++ b/src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java @@ -35,7 +35,7 @@ public class StorageUtils { public static final String getUrlById(String id){ IClient client=getClient(); log.debug("Id is "+id); - return client.getUrl().RFile(id); + return client.getHttpUrl().RFile(id); } public static final void removeById(String id){ diff --git a/src/test/java/org/gcube/data/transfer/library/TransfererTest.java b/src/test/java/org/gcube/data/transfer/library/TransfererTest.java index 0b4f792..ff7f78e 100644 --- a/src/test/java/org/gcube/data/transfer/library/TransfererTest.java +++ b/src/test/java/org/gcube/data/transfer/library/TransfererTest.java @@ -46,23 +46,24 @@ public class TransfererTest { @Test public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException{ String toUpload="/home/fabio/Downloads/Incantesimi3_5.doc"; - System.out.println(client.storageId(StorageUtils.putOntoStorage(new File(toUpload)))); + String id=StorageUtils.putOntoStorage(new File(toUpload)); + System.out.println(client.storageId(id)); } - @Test + @Test(expected=InvalidSourceException.class) public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ System.out.println(client.storageId("13245780t")); } - @Test + @Test(expected=InvalidSourceException.class) public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ String localFile="/home/fabio/Downloads/123045689.mp3"; System.out.println(client.localFile(localFile)); } - @Test + @Test(expected=InvalidSourceException.class) public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{ - String link="https://www.dropbox.com/s/789023450"; + String link="https://www.some.where.com/over/theRainbow.txt"; System.out.println(client.httpSource(link)); } }