gCubeSystem
/
data-transfer-service
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-transfer/data-transfer-service/2.0@158844 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2017-11-27 16:45:28 +00:00
parent a4a6b11872
commit 01620ea943
8 changed files with 175 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
distro/changelog.xml
View File

@ -10,4 +10,7 @@
<Change>Improved plugin management</Change>
<Change>Endorsed decompress-plugin</Change>
</Changeset>
<Changeset component="data-transfer-service-2.0.3" date="2016-10-29">
<Change>Added REST interface for upload/download/getFileDescriptor</Change>
</Changeset>
</ReleaseNotes>

2
pom.xml
View File

@ -8,7 +8,7 @@
</parent>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-service</artifactId>
<version>2.0.1-SNAPSHOT</version>
<version>2.0.3-SNAPSHOT</version>
<packaging>war</packaging>
<name>DataTransferService</name>
<scm>

112
src/main/java/org/gcube/data/transfer/service/transfers/REST.java
View File

@ -1,13 +1,16 @@
package org.gcube.data.transfer.service.transfers;
import java.io.File;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Set;
import javax.activation.MimetypesFileTypeMap;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@ -15,6 +18,7 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.gcube.data.transfer.model.Destination;
@ -28,43 +32,49 @@ import org.gcube.data.transfer.model.options.HttpDownloadOptions;
import org.gcube.data.transfer.model.options.TransferOptions.TransferMethod;
import org.gcube.data.transfer.model.settings.FileUploadSettings;
import org.gcube.data.transfer.model.settings.HttpDownloadSettings;
import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider;
import org.gcube.data.transfer.service.transfers.engine.RequestManager;
import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import lombok.extern.slf4j.Slf4j;
@Path(ServiceConstants.REST_SERVLET_NAME)
@Path(ServiceConstants.REST_SERVLET_NAME+"/{destinationId}/{subPath : \\S*}")
@Slf4j
public class REST {
@PathParam("destinationId") String destinationID;
@PathParam("subPath") String subPath;
@Inject
RequestManager requests;
@Inject
PersistenceProvider persistence;
@QueryParam(ServiceConstants.DESTINATION_FILE_NAME) String destinationFileName;
@QueryParam(ServiceConstants.CREATE_DIRS) @DefaultValue("true") Boolean createDirs;
@QueryParam(ServiceConstants.ON_EXISTING_FILE) @DefaultValue("ADD_SUFFIX") DestinationClashPolicy onExistingFile;
@QueryParam(ServiceConstants.ON_EXISTING_DIR) @DefaultValue("APPEND") DestinationClashPolicy onExistingDirectory;
@QueryParam(ServiceConstants.SOURCE_ID) String sourceID;
@FormDataParam(ServiceConstants.MULTIPART_FILE) InputStream uploadedFile;
@FormDataParam(ServiceConstants.MULTIPART_FILE) FormDataContentDisposition uploadedFileDetails;
@FormDataParam("plugin-invocations") Set<PluginInvocation> pluginInvocations;
@POST
@Path("/{method}/{destinationId}/{subPath: .*}")
@POST
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_JSON)
public Object serveFileUpload(@PathParam("method") String methodString,
@PathParam("destinationId") String destinationID, @PathParam("subPath") String subPath ){
@Produces(MediaType.APPLICATION_JSON)
public Object serveFileUpload(@QueryParam("method") @DefaultValue("FileUpload")String methodString,
@QueryParam(ServiceConstants.DESTINATION_FILE_NAME) String destinationFileName,
@QueryParam(ServiceConstants.CREATE_DIRS) @DefaultValue("true") Boolean createDirs,
@QueryParam(ServiceConstants.ON_EXISTING_FILE) @DefaultValue("ADD_SUFFIX") DestinationClashPolicy onExistingFile,
@QueryParam(ServiceConstants.ON_EXISTING_DIR) @DefaultValue("APPEND") DestinationClashPolicy onExistingDirectory,
@QueryParam(ServiceConstants.SOURCE_ID) String sourceID,
@FormDataParam(ServiceConstants.MULTIPART_FILE) InputStream uploadedFile,
@FormDataParam(ServiceConstants.MULTIPART_FILE) FormDataContentDisposition uploadedFileDetails,
@FormDataParam("plugin-invocations") Set<PluginInvocation> pluginInvocations){
try{
String pathString="<"+destinationID+">/"+subPath;
log.info("Received POST request at {} ",pathString);
log.debug("Plugin invocation set : {} ",pluginInvocations);
TransferRequest request=formRequestFromREST(methodString, destinationID, subPath,pluginInvocations);
TransferRequest request=formRequestFromREST(methodString, destinationID, subPath,pluginInvocations,
createDirs,onExistingFile,onExistingDirectory,
uploadedFile,uploadedFileDetails,destinationFileName,sourceID);
log.info("Received REST Request {} ",request);
@ -88,26 +98,32 @@ public class REST {
private TransferRequest formRequestFromREST(String methodString,String destinationID,String subPath, Set<PluginInvocation> pluginInvocations){
private TransferRequest formRequestFromREST(String methodString,String destinationID,String subPath, Set<PluginInvocation> pluginInvocations,
Boolean createDirs, DestinationClashPolicy onExistingFile,
DestinationClashPolicy onExistingDirectory,
InputStream uploadedFile,
FormDataContentDisposition uploadedFileDetails,
String destinationFileName,
String sourceID){
log.info("Creating TransferRequest from REST invocation method : {}, dest ID {}, sub Path {} ",methodString,destinationID,subPath);
TransferMethod method=null;
try{
method=TransferMethod.valueOf(methodString);
}catch (Throwable t) {
throw new WebApplicationException("Invalid selected method "+methodString,Status.BAD_REQUEST);}
throw new WebApplicationException("Invalid selected method "+methodString,t,Status.BAD_REQUEST);}
Destination destination=new Destination();
destination.setCreateSubfolders(createDirs);
destination.setPersistenceId(destinationID);
destination.setSubFolder(subPath);
destination.setOnExistingSubFolder(onExistingDirectory);
destination.setOnExistingFileName(onExistingFile);
TransferRequest resultingRequest=new TransferRequest();
resultingRequest.setDestinationSettings(destination);
resultingRequest.setPluginInvocations(pluginInvocations);
@ -132,7 +148,7 @@ public class REST {
resultingRequest.setSettings(settings);
break;
}catch(MalformedURLException e){
throw new WebApplicationException("Source "+sourceID+" is not a valid URL.",Status.BAD_REQUEST);
throw new WebApplicationException("Source "+sourceID+" is not a valid URL.",e,Status.BAD_REQUEST);
}
}
default: throw new WebApplicationException("Unsupported selected method "+methodString,Status.BAD_REQUEST);
@ -140,4 +156,40 @@ public class REST {
return resultingRequest;
}
// @Path("/{destinationId}/{subPath : \\S*}")
@GET
@Produces("*/*")
public Response getFile(@PathParam("destinationId") String destinationId,
@PathParam("subPath") String subPath,
@QueryParam("descriptor") @DefaultValue("false") Boolean getDescriptor) {
String pathString="<"+destinationID+">/"+subPath;
log.info("Received GET request at {} , descriptor option is {} ",pathString,getDescriptor);
try{
if(getDescriptor) return Response.ok(persistence.getDescriptor(destinationId, subPath), MediaType.APPLICATION_JSON_TYPE).build();
File persisted= persistence.getPersistedFile(destinationId, subPath);
if(!persisted.exists()) throw new WebApplicationException("File "+pathString+" doesn't exists.",Status.NOT_FOUND);
if(persisted.isDirectory()) throw new WebApplicationException("The selected path "+pathString+" is a directory.",Status.BAD_REQUEST);
String mt = new MimetypesFileTypeMap().getContentType(persisted);
return Response.ok(persisted, mt).build();
}catch(DestinationAccessException e) {
throw new WebApplicationException("Unable to access selected path "+pathString,e,Status.INTERNAL_SERVER_ERROR);
}
}
// // @Path("/{destinationId}/{subPath : \\\\S*}")
// @HEAD
// @Produces(MediaType.APPLICATION_JSON)
// public RemoteFileDescriptor getFileDescriptor(@PathParam("destinationId") String destinationId,
// @PathParam("subPath") String subPath ) {
// String pathString="<"+destinationID+">/"+subPath;
// log.info("Received HEAD request at {} ",pathString);
// try{
// return persistence.getDescriptor(destinationId, subPath);
// }catch(DestinationAccessException e) {
// throw new WebApplicationException("Unable to access selected path "+pathString,Status.INTERNAL_SERVER_ERROR);
// }
// }
}

6
src/main/java/org/gcube/data/transfer/service/transfers/engine/PersistenceProvider.java
View File

@ -4,6 +4,7 @@ import java.io.File;
import java.util.Set;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException;
public interface PersistenceProvider {
@ -13,4 +14,9 @@ public interface PersistenceProvider {
public Set<String> getAvaileblContextIds();
File prepareDestination(Destination dest) throws DestinationAccessException;
public File getPersistedFile(String persistenceId,String subPath) throws DestinationAccessException;
public RemoteFileDescriptor getDescriptor(String persistenceId,String subPath) throws DestinationAccessException;
}

38
src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java
View File

@ -3,6 +3,8 @@ package org.gcube.data.transfer.service.transfers.engine.impl;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashSet;
import java.util.Set;
@ -10,6 +12,7 @@ import javax.inject.Singleton;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider;
import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException;
import org.gcube.smartgears.ContextProvider;
@ -23,7 +26,40 @@ import lombok.extern.slf4j.Slf4j;
public class PersistenceProviderImpl implements PersistenceProvider {
@Override
public File getPersistedFile(String persistenceId, String subPath) throws DestinationAccessException {
log.debug("Accessing <{}>/{}",persistenceId,subPath);
File persistenceFolder=getPersistenceFolderById(persistenceId);
return new File(persistenceFolder.getAbsolutePath()+"/"+subPath);
}
@Override
public RemoteFileDescriptor getDescriptor(String persistenceId, String subPath) throws DestinationAccessException {
File file=getPersistedFile(persistenceId, subPath);
log.debug("Getting descriptor for {} ",file.getAbsolutePath());
if(!file.exists()) throw new DestinationAccessException("Unable to find "+file.getAbsolutePath());
RemoteFileDescriptor toReturn=new RemoteFileDescriptor();
try{
BasicFileAttributes attributes=Files.readAttributes(file.toPath(), BasicFileAttributes.class,LinkOption.NOFOLLOW_LINKS);
toReturn.setCreationDate(attributes.creationTime().toMillis());
toReturn.setLastUpdate(attributes.lastModifiedTime().toMillis());
toReturn.setSize(attributes.size());
}catch(Throwable t) {
log.warn("Unable to access attributes for {} ",file.getAbsolutePath(),t);
}
toReturn.setAbsolutePath(file.getAbsolutePath());
toReturn.setDirectory(file.isDirectory());
toReturn.setFilename(file.getName());
toReturn.setPersistenceId(persistenceId);
toReturn.setPath(subPath);
log.debug("Returning descriptor {} ",toReturn);
return toReturn;
}
@Override
public File getPersistenceFolderById(String persistenceId) throws DestinationAccessException {
File toReturn=null;

2
src/test/java/org/gcube/data/transfer/service/CapabilitiesProviderFactory.java
View File

@ -26,7 +26,7 @@ public class CapabilitiesProviderFactory implements Factory<CapabilitiesProvider
return new TransferCapabilities("12345", "localhost", 80,
Collections.singleton((TransferOptions)HttpDownloadOptions.DEFAULT),
new HashSet<PluginDescription>(new PluginManagerImpl().getInstalledPlugins().values())
,Collections.<String> emptySet());
,Collections.singleton("data-transfer-service"));
}
};
}

49
src/test/java/org/gcube/data/transfer/service/PersistenceProviderFactory.java
View File

@ -2,38 +2,51 @@ package org.gcube.data.transfer.service;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider;
import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException;
import org.gcube.data.transfer.service.transfers.engine.impl.PersistenceProviderImpl;
import org.glassfish.hk2.api.Factory;
class PersistenceProviderFactory implements Factory<PersistenceProvider>{
@Override
public PersistenceProvider provide() {
return new PersistenceProvider() {
return new PersistenceProviderImpl() {
//
//
@Override
public File getPersistedFile(String persistenceId, String subPath) throws DestinationAccessException {
File persistenceFolder=getPersistenceFolderById(persistenceId);
File toReturn=new File(persistenceFolder.getAbsolutePath()+"/"+subPath);
try {
toReturn.getParentFile().mkdirs();
toReturn.createNewFile();
} catch (IOException e) {
throw new DestinationAccessException(e);
}
return toReturn;
}
//
@Override
public File getPersistenceFolderById(String persistenceId) throws DestinationAccessException {
return new File(System.getProperty("java.io.tmpdir"));
}
@Override
public Set<String> getAvaileblContextIds() {
return Collections.emptySet();
}
@Override
public File prepareDestination(Destination dest) throws DestinationAccessException {
try {
return File.createTempFile("dest", "");
} catch (IOException e) {
return null;
}
}
//
// @Override
// public Set<String> getAvaileblContextIds() {
// return Collections.singleton("data-transfer-service");
// }
//
// @Override
// public File prepareDestination(Destination dest) throws DestinationAccessException {
// try {
// return File.createTempFile("dest", "");
// } catch (IOException e) {
// return null;
// }
// }
};
}

14
src/test/java/org/gcube/data/transfer/service/TestCall.java
View File

@ -11,6 +11,7 @@ import javax.ws.rs.core.Application;
import javax.ws.rs.core.MediaType;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.ServiceConstants;
import org.gcube.data.transfer.model.TransferCapabilities;
import org.gcube.data.transfer.model.TransferRequest;
@ -129,4 +130,17 @@ public class TestCall extends JerseyTest {
private TransferTicket submit(TransferRequest req){
return target(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_JSON).post(Entity.entity(req,MediaType.APPLICATION_JSON),TransferTicket.class);
}
@Test
public void testREAD() {
System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/temp.txt").request().get());
System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/some/folder/inside/temp.txt").request().get());
}
@Test
public void testHEAD() {
System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/temp.txt").request(MediaType.APPLICATION_JSON).head().readEntity(RemoteFileDescriptor.class));
System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/some/folder/inside/temp.txt").request(MediaType.APPLICATION_JSON).head().readEntity(RemoteFileDescriptor.class));
}
}