gCubeSystem
/
data-transfer-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/data-transfer-library@128809 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2016-05-24 16:16:16 +00:00
parent cdea6493d7
commit cac35c3539
11 changed files with 233 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -35,6 +35,18 @@
<scope>test</scope>
</dependency>
<!-- STORAGE -->
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-core</artifactId>
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-wrapper</artifactId>
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
</dependency>
<!-- lombok -->
<dependency>
<groupId>org.projectlombok</groupId>

6
src/main/java/org/gcube/data/transfer/library/model/LocalSource.java
View File

@ -4,7 +4,7 @@ import java.io.File;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
public class LocalSource extends Source {
public class LocalSource extends Source<File> {
private File theFile=null;
@ -34,4 +34,8 @@ public class LocalSource extends Source {
}
@Override
public File getTheSource() {
return theFile;
}
}

8
src/main/java/org/gcube/data/transfer/library/model/Source.java
View File

@ -5,15 +5,13 @@ import org.gcube.data.transfer.library.faults.InvalidSourceException;
import lombok.Data;
@Data
public abstract class Source{
public static enum SourceType{
URI,STORAGE_ID,LOCAL_FILE
}
public abstract class Source<T>{
public abstract boolean validate() throws InvalidSourceException;
public abstract void prepare();
public abstract void clean();
public abstract T getTheSource();
}

7
src/main/java/org/gcube/data/transfer/library/model/StorageSource.java
View File

@ -2,7 +2,7 @@ package org.gcube.data.transfer.library.model;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
public class StorageSource extends Source {
public class StorageSource extends Source<String> {
private String id=null;
@ -34,4 +34,9 @@ public class StorageSource extends Source {
}
@Override
public String getTheSource() {
return id;
}
}

8
src/main/java/org/gcube/data/transfer/library/model/URLSource.java
View File

@ -5,7 +5,7 @@ import java.net.URL;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
public class URLSource extends Source {
public class URLSource extends Source<URL> {
private URL theURL=null;
@ -21,7 +21,7 @@ public class URLSource extends Source {
public boolean validate() throws InvalidSourceException {
try{
HttpURLConnection conn=(HttpURLConnection) theURL.openConnection();
conn.setConnectTimeout(3);
conn.setConnectTimeout(3000);
conn.setRequestMethod("HEAD");
int responseCode = conn.getResponseCode();
return (200 <= responseCode && responseCode <= 399);
@ -53,8 +53,10 @@ public class URLSource extends Source {
return builder.toString();
}
public URL getTheURL() {
@Override
public URL getTheSource() {
return theURL;
}
}

60
src/main/java/org/gcube/data/transfer/library/transferers/HTTPTransferer.java
View File

@ -1,14 +1,62 @@
package org.gcube.data.transfer.library.transferers;
import org.gcube.data.transfer.library.TransferResult;
import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.net.URL;
import lombok.extern.slf4j.Slf4j;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.model.LocalSource;
import org.gcube.data.transfer.library.model.StorageSource;
import org.gcube.data.transfer.library.model.URLSource;
import org.gcube.data.transfer.library.utils.StorageUtils;
import org.gcube.data.transfer.model.TransferCapabilities;
import org.gcube.data.transfer.model.TransferRequest;
import org.gcube.data.transfer.model.options.HttpDownloadOptions;
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
@Slf4j
public class HTTPTransferer extends Transferer {
@Override
protected TransferResult doTheTransfer() {
System.out.println("I'm doing the transfer ahahahahahah");
System.out.println("Yeah... sure..");
return null;
HTTPTransferer(TransferCapabilities targetCapabilities) {
super(targetCapabilities);
}
private URL link;
@Override
protected TransferRequest prepareRequest()throws InitializationException {
return new TransferRequest("", new HttpDownloadSettings(link, HttpDownloadOptions.DEFAULT));
}
private URL getHttpLink() throws RemoteBackendException, FileNotFoundException, InvalidSourceException, MalformedURLException{
if(source instanceof LocalSource){
return new URL(StorageUtils.putOntoStorage(((LocalSource)source).getTheSource()));
}else if (source instanceof StorageSource){
return new URL(StorageUtils.getUrlById(((StorageSource)source).getTheSource()));
}else if (source instanceof URLSource){
return ((URLSource)source).getTheSource();
}else throw new InvalidSourceException("Source cannot be handled "+source);
}
@Override
protected void prepare() throws InitializationException {
try{
link=getHttpLink();
log.debug("Obtained link "+link);
super.prepare();
}catch(RemoteBackendException e){
throw new InitializationException(e);
}catch(FileNotFoundException e){
throw new InitializationException(e);
}catch(InvalidSourceException e){
throw new InitializationException(e);
}catch (MalformedURLException e){
throw new InitializationException(e);
}
}
}

95
src/main/java/org/gcube/data/transfer/library/transferers/Transferer.java
View File

@ -1,30 +1,45 @@
package org.gcube.data.transfer.library.transferers;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.net.URL;
import lombok.extern.slf4j.Slf4j;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.model.LocalSource;
import org.gcube.data.transfer.library.model.Source;
import org.gcube.data.transfer.library.model.StorageSource;
import org.gcube.data.transfer.library.model.URLSource;
import org.gcube.data.transfer.model.TransferCapabilities;
import org.gcube.data.transfer.model.TransferRequest;
import org.gcube.data.transfer.model.TransferTicket;
import org.gcube.data.transfer.model.TransferTicket.Status;
@Slf4j
public abstract class Transferer {
protected TransferCapabilities targetCapabilities;
protected Transferer(TransferCapabilities targetCapabilities){
this.targetCapabilities=targetCapabilities;
}
protected Source source=null;
protected boolean prepared=false;
public Transferer localFile(File f) throws InvalidSourceException{
source=new LocalSource(f);
return this;
}
public Transferer localFile(String path) throws InvalidSourceException{
try{
File f=new File(path);
@ -32,37 +47,65 @@ public abstract class Transferer {
}catch(NullPointerException e){
throw new InvalidSourceException("Path is null",e);
}
}
public Transferer storageFileId(String fileId) throws InvalidSourceException{
source=new StorageSource(fileId);
return this;
}
public Transferer fromURL(URL sourceURL) throws InvalidSourceException{
source=new URLSource(sourceURL);
return this;
}
public TransferResult transfer() throws SourceNotSetException, InvalidSourceException{
checkSource();
source.prepare();
TransferResult result=doTheTransfer();
source.clean();
return result;
public TransferResult transfer() throws SourceNotSetException, InvalidSourceException, FailedTransferException, InitializationException{
try{
checkSource();
prepare();
TransferRequest request=prepareRequest();
log.debug("Request is {}, sending it to {}",request,targetCapabilities.getHostName());
TransferResult result=doTheTransfer(request);
return result;
}finally{
clean();
}
}
protected TransferResult doTheTransfer(TransferRequest request) throws FailedTransferException{
Client client=new Client(targetCapabilities.getHostName());
TransferTicket submissionResponse= client.submit(request);
boolean continuePolling=true;
TransferTicket ticket=null;
do{
ticket=client.getTransferStatus(submissionResponse.getId());
System.out.println("Status : "+ticket);
continuePolling=ticket.getStatus().equals(Status.PENDING)||ticket.getStatus().equals(Status.TRANSFERRING)||ticket.getStatus().equals(Status.WAITING);
try{
Thread.sleep(500);
}catch(InterruptedException e){}
}while(continuePolling);
if(ticket.getStatus().equals(Status.ERROR)) throw new FailedTransferException("Remote Message : "+ticket.getMessage());
if(ticket.getStatus().equals(Status.STOPPED)) throw new FailedTransferException("Stopped transfer : "+ticket.getMessage());
long elapsedTime=System.currentTimeMillis()-ticket.getSubmissionTime().value.getTimeInMillis();
return new TransferResult(source, targetCapabilities.getHostName(), elapsedTime, ticket.getTransferredBytes(), ticket.getDestinationFileName());
}
protected void checkSource() throws SourceNotSetException, InvalidSourceException{
if(source==null) throw new SourceNotSetException();
source.validate();
}
protected abstract TransferResult doTheTransfer();
protected abstract TransferRequest prepareRequest() throws InitializationException;
protected void prepare() throws InitializationException{
prepared=true;
}
protected void clean(){
}
}

14
src/main/java/org/gcube/data/transfer/library/TransfererBuilder.java → src/main/java/org/gcube/data/transfer/library/transferers/TransfererBuilder.java
View File

@ -1,4 +1,4 @@
package org.gcube.data.transfer.library;
package org.gcube.data.transfer.library.transferers;
import lombok.extern.slf4j.Slf4j;
@ -6,8 +6,6 @@ import org.gcube.data.transfer.library.caches.CapabilitiesCache;
import org.gcube.data.transfer.library.faults.HostingNodeNotFoundException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.library.transferers.HTTPTransferer;
import org.gcube.data.transfer.library.transferers.Transferer;
import org.gcube.data.transfer.library.utils.Utils;
import org.gcube.data.transfer.model.ServiceConstants;
import org.gcube.data.transfer.model.TransferCapabilities;
@ -18,21 +16,21 @@ public class TransfererBuilder {
public Transferer getTransfererByHost(String host) throws UnreachableNodeException, ServiceNotFoundException{
public static Transferer getTransfererByHost(String host) throws UnreachableNodeException, ServiceNotFoundException{
log.debug("Get transferer by Host "+host);
if(!Utils.pingURL(host, timeout)) throw new UnreachableNodeException("No response from host in "+timeout);
if(!Utils.pingURL(host+"REST-API/"+ServiceConstants.APPLICATION_PATH, timeout)) throw new ServiceNotFoundException("No DT Service found @ "+host+"REST-API/"+ServiceConstants.APPLICATION_PATH);
log.debug("Host is ok, getting capabilities");
log.debug("Host is ok, getting targetCapabilities");
TransferCapabilities cap=CapabilitiesCache.getInstance().getObject(host);
return new HTTPTransferer();
return new HTTPTransferer(cap);
}
public Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
public static Transferer getTransfererByhostingNodeId(String hostId) throws HostingNodeNotFoundException, UnreachableNodeException, ServiceNotFoundException{
String hostname=retrieveHostnameByNodeId(hostId);
return getTransfererByHost(hostname);
}
private String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{
private static String retrieveHostnameByNodeId(String nodeId)throws HostingNodeNotFoundException{
return null;
}
}

42
src/main/java/org/gcube/data/transfer/library/utils/StorageUtils.java Normal file
View File

@ -0,0 +1,42 @@
package org.gcube.data.transfer.library.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import lombok.extern.slf4j.Slf4j;
import org.bson.types.ObjectId;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
@Slf4j
public class StorageUtils {
public static final IClient getClient(){
return new StorageClient("data-transfer", "data-transfer", "own", AccessType.SHARED, MemoryType.VOLATILE).getClient();
}
//return public link
public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{
IClient client=getClient();
log.debug("Uploading local file "+source.getAbsolutePath());
String id=client.put(true).LFile(new FileInputStream(source)).RFile(Utils.getUniqueString());
log.debug("Id is "+id);
return client.getUrl().RFile(id);
}
public static final boolean checkStorageId(String id){
return ObjectId.isValid(id);
}
public static final String getUrlById(String id){
IClient client=getClient();
log.debug("Id is "+id);
return client.getUrl().RFile(id);
}
}

6
src/main/java/org/gcube/data/transfer/library/utils/Utils.java
View File

@ -3,6 +3,7 @@ package org.gcube.data.transfer.library.utils;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.UUID;
public class Utils {
@ -30,4 +31,9 @@ public class Utils {
}
}
public static final String getUniqueString(){
return UUID.randomUUID().toString();
}
}

25
src/test/java/org/gcube/data/transfer/library/TransfererTest.java Normal file
View File

@ -0,0 +1,25 @@
package org.gcube.data.transfer.library;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.transferers.TransfererBuilder;
import org.junit.Before;
import org.junit.Test;
public class TransfererTest {
String hostname="http://pc-fabio.isti.cnr.it:8080";
String scope="/gcube/devNext";
@Before
public void init(){
ScopeProvider.instance.set(scope);
}
@Test
public void single(){
TransfererBuilder.
}
}