gCubeSystem
/
data-transfer-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@128868 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2016-05-26 16:35:31 +00:00
parent edaf385807
commit 7eff5dec47
8 changed files with 49 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/gcube/data/transfer/library/DataTransferClient.java
View File

@ -20,7 +20,6 @@ import org.gcube.data.transfer.library.transferers.TransfererBuilder;
@Slf4j
public class DataTransferClient {
private Transferer transferer=null;
private DataTransferClient(Transferer transferer) {
@ -54,10 +53,14 @@ public class DataTransferClient {
}
@Synchronized("transferer")
public TransferResult httpSource(String url) throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
public TransferResult httpSource(String url) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
if(transferer==null) throw new RuntimeException("Transferer not set, please set destination before trying to transfer");
log.debug("Passed url string : "+url);
return this.httpSource(new URL(url));
try{
return this.httpSource(new URL(url));
}catch(MalformedURLException e){
throw new InvalidSourceException("Invalid url : "+url);
}
}
@Synchronized("transferer")

21
src/main/java/org/gcube/data/transfer/library/client/Client.java
View File

@ -26,29 +26,34 @@ public class Client {
config.register(JAXBElement.class);
}
// private String hostname;
private String endpoint;
private WebTarget rootTarget;
public Client(String hostname){
log.debug("Creating client for "+hostname);
this.hostname=hostname+"";
rootTarget= ClientBuilder.newClient(config).target(hostname).path("/data-transfer-service/").path(ServiceConstants.APPLICATION_PATH);
public Client(String endpoint){
log.debug("Creating client for base "+endpoint);
this.endpoint=endpoint+"";
rootTarget= ClientBuilder.newClient(config).target(endpoint).path("/data-transfer-service/").path(ServiceConstants.APPLICATION_PATH);
}
public String getEndpoint() {
return endpoint;
}
public TransferCapabilities getCapabilties(){
log.debug("Getting capabilities to {} ",hostname);
log.debug("Getting capabilities to {} ",endpoint);
return rootTarget.path(ServiceConstants.CAPABILTIES_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).get(TransferCapabilities.class);
}
public TransferTicket submit(TransferRequest request){
log.debug("Sending request {} to {}",request,hostname);
log.debug("Sending request {} to {}",request,endpoint);
return rootTarget.path(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(request,MediaType.APPLICATION_XML),TransferTicket.class);
}
public TransferTicket getTransferStatus(String transferId){
log.debug("Requesting transfer status [id = {}, hostname={}]",transferId,hostname);
log.debug("Requesting transfer status [id = {}, endpoint={}]",transferId,endpoint);
return rootTarget.path(ServiceConstants.STATUS_SERVLET_NAME).path(transferId).request(MediaType.APPLICATION_XML).get(TransferTicket.class);
}
}

5
src/main/java/org/gcube/data/transfer/library/model/StorageSource.java
View File

@ -1,6 +1,7 @@
package org.gcube.data.transfer.library.model;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.utils.StorageUtils;
public class StorageSource extends Source<String> {
@ -18,8 +19,8 @@ public class StorageSource extends Source<String> {
@Override
public boolean validate() throws InvalidSourceException {
// TODO Auto-generated method stub
return false;
if(!StorageUtils.checkStorageId(id)) throw new InvalidSourceException("Invalid storage ID "+id);
return true;
}
@Override

6
src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java
View File

@ -7,13 +7,13 @@ import java.net.URL;
import lombok.extern.slf4j.Slf4j;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.model.LocalSource;
import org.gcube.data.transfer.library.model.StorageSource;
import org.gcube.data.transfer.library.model.URLSource;
import org.gcube.data.transfer.library.utils.StorageUtils;
import org.gcube.data.transfer.model.TransferCapabilities;
import org.gcube.data.transfer.model.TransferRequest;
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
@ -21,8 +21,8 @@ import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
@Slf4j
public class HTTPTransferer extends Transferer {
HTTPTransferer(TransferCapabilities targetCapabilities) {
super(targetCapabilities);
HTTPTransferer(Client client) {
super(client);
}
private URL link;

20
src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java
View File

@ -1,13 +1,10 @@
package org.gcube.data.transfer.library.transferers;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.net.URL;
import lombok.extern.slf4j.Slf4j;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.faults.FailedTransferException;
@ -26,12 +23,15 @@ import org.gcube.data.transfer.model.TransferTicket.Status;
@Slf4j
public abstract class Transferer {
protected TransferCapabilities targetCapabilities;
protected Client client;
protected Transferer(TransferCapabilities targetCapabilities){
this.targetCapabilities=targetCapabilities;
protected Transferer(Client client){
this.client=client;
}
protected Source source=null;
protected boolean prepared=false;
@ -66,7 +66,7 @@ public abstract class Transferer {
checkSource();
prepare();
TransferRequest request=prepareRequest();
log.debug("Request is {}, sending it to {}",request,targetCapabilities.getHostName());
log.debug("Request is {}, sending it to {}",request,client.getEndpoint());
TransferResult result=doTheTransfer(request);
return result;
}finally{
@ -75,7 +75,6 @@ public abstract class Transferer {
}
protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{
Client client=new Client(targetCapabilities.getHostName());
TransferTicket submissionResponse= client.submit(request);
boolean continuePolling=true;
TransferTicket ticket=null;
@ -90,7 +89,7 @@ public abstract class Transferer {
if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage());
if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage());
long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis();
return new TransferResult(source, targetCapabilities.getHostName(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
return new TransferResult(source, client.getEndpoint(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
}
protected void checkSource() throws SourceNotSetException, InvalidSourceException{
@ -109,7 +108,8 @@ public abstract class Transferer {
}
public TransferCapabilities getDestinationCapabilities(){
return targetCapabilities;
return client.getCapabilties();
}
}

10
src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java
View File

@ -6,6 +6,7 @@ import java.net.URL;
import lombok.extern.slf4j.Slf4j;
import org.gcube.data.transfer.library.caches.CapabilitiesCache;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
@ -23,7 +24,7 @@ public class TransfererBuilder {
try{
URL url=new URL(endpoint);
String hostname=url.getProtocol()+"://"+url.getHost()+":"+url.getPort();
String baseUrl=url.getProtocol()+"://"+url.getHost()+":"+url.getPort();
//TODO Implement checks
// if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout);
@ -36,8 +37,11 @@ public class TransfererBuilder {
//
// if(!Utils.pingURL(finalHost, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+finalHost);
// log.debug("Host is ok, getting targetCapabilities");
TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(hostname);
return new HTTPTransferer(cap);
TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(baseUrl);
// TODO determine method by capabilities checking
return new HTTPTransferer(new Client(baseUrl));
}catch(MalformedURLException e){
throw new ServiceNotFoundException(e);
}

2
src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java
View File

@ -35,7 +35,7 @@ public class StorageUtils {
public static final String getUrlById(String id){
IClient client=getClient();
log.debug("Id is "+id);
return client.getUrl().RFile(id);
return client.getHttpUrl().RFile(id);
}
public static final void removeById(String id){

11
src/test/java/org/gcube/data/transfer/library/TransfererTest.java
View File

@ -46,23 +46,24 @@ public class TransfererTest {
@Test
public void storage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, RemoteBackendException, FileNotFoundException{
String toUpload="/home/fabio/Downloads/Incantesimi3_5.doc";
System.out.println(client.storageId(StorageUtils.putOntoStorage(new File(toUpload))));
String id=StorageUtils.putOntoStorage(new File(toUpload));
System.out.println(client.storageId(id));
}
@Test
@Test(expected=InvalidSourceException.class)
public void wrongStorage() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
System.out.println(client.storageId("13245780t"));
}
@Test
@Test(expected=InvalidSourceException.class)
public void wrongLocal() throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
String localFile="/home/fabio/Downloads/123045689.mp3";
System.out.println(client.localFile(localFile));
}
@Test
@Test(expected=InvalidSourceException.class)
public void wrongUrl() throws MalformedURLException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException{
String link="https://www.dropbox.com/s/789023450";
String link="https://www.some.where.com/over/theRainbow.txt";
System.out.println(client.httpSource(link));
}
}