From 01620ea943088fd77021378408622dc7ecfc4419 Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Mon, 27 Nov 2017 16:45:28 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-transfer/data-transfer-service/2.0@158844 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 3 + pom.xml | 2 +- .../data/transfer/service/transfers/REST.java | 112 +++++++++++++----- .../transfers/engine/PersistenceProvider.java | 6 + .../engine/impl/PersistenceProviderImpl.java | 38 +++++- .../service/CapabilitiesProviderFactory.java | 2 +- .../service/PersistenceProviderFactory.java | 49 +++++--- .../gcube/data/transfer/service/TestCall.java | 14 +++ 8 files changed, 175 insertions(+), 51 deletions(-) diff --git a/distro/changelog.xml b/distro/changelog.xml index 01ec4cf..c9e5fed 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -10,4 +10,7 @@ Improved plugin management Endorsed decompress-plugin + + Added REST interface for upload/download/getFileDescriptor + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6547be0..dc191aa 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.data.transfer data-transfer-service - 2.0.1-SNAPSHOT + 2.0.3-SNAPSHOT war DataTransferService diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/REST.java b/src/main/java/org/gcube/data/transfer/service/transfers/REST.java index fcd26a9..8842cf3 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/REST.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/REST.java @@ -1,13 +1,16 @@ package org.gcube.data.transfer.service.transfers; +import java.io.File; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.util.Set; +import javax.activation.MimetypesFileTypeMap; import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -15,6 +18,7 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import org.gcube.data.transfer.model.Destination; @@ -28,43 +32,49 @@ import org.gcube.data.transfer.model.options.HttpDownloadOptions; import org.gcube.data.transfer.model.options.TransferOptions.TransferMethod; import org.gcube.data.transfer.model.settings.FileUploadSettings; import org.gcube.data.transfer.model.settings.HttpDownloadSettings; +import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider; import org.gcube.data.transfer.service.transfers.engine.RequestManager; +import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; import lombok.extern.slf4j.Slf4j; -@Path(ServiceConstants.REST_SERVLET_NAME) +@Path(ServiceConstants.REST_SERVLET_NAME+"/{destinationId}/{subPath : \\S*}") @Slf4j public class REST { + @PathParam("destinationId") String destinationID; + @PathParam("subPath") String subPath; + + + @Inject RequestManager requests; + @Inject + PersistenceProvider persistence; - - - @QueryParam(ServiceConstants.DESTINATION_FILE_NAME) String destinationFileName; - @QueryParam(ServiceConstants.CREATE_DIRS) @DefaultValue("true") Boolean createDirs; - @QueryParam(ServiceConstants.ON_EXISTING_FILE) @DefaultValue("ADD_SUFFIX") DestinationClashPolicy onExistingFile; - @QueryParam(ServiceConstants.ON_EXISTING_DIR) @DefaultValue("APPEND") DestinationClashPolicy onExistingDirectory; - @QueryParam(ServiceConstants.SOURCE_ID) String sourceID; - - @FormDataParam(ServiceConstants.MULTIPART_FILE) InputStream uploadedFile; - @FormDataParam(ServiceConstants.MULTIPART_FILE) FormDataContentDisposition uploadedFileDetails; - @FormDataParam("plugin-invocations") Set pluginInvocations; - - - @POST - @Path("/{method}/{destinationId}/{subPath: .*}") + @POST @Consumes(MediaType.WILDCARD) - @Produces(MediaType.APPLICATION_JSON) - - public Object serveFileUpload(@PathParam("method") String methodString, - @PathParam("destinationId") String destinationID, @PathParam("subPath") String subPath ){ + @Produces(MediaType.APPLICATION_JSON) + public Object serveFileUpload(@QueryParam("method") @DefaultValue("FileUpload")String methodString, + @QueryParam(ServiceConstants.DESTINATION_FILE_NAME) String destinationFileName, + @QueryParam(ServiceConstants.CREATE_DIRS) @DefaultValue("true") Boolean createDirs, + @QueryParam(ServiceConstants.ON_EXISTING_FILE) @DefaultValue("ADD_SUFFIX") DestinationClashPolicy onExistingFile, + @QueryParam(ServiceConstants.ON_EXISTING_DIR) @DefaultValue("APPEND") DestinationClashPolicy onExistingDirectory, + @QueryParam(ServiceConstants.SOURCE_ID) String sourceID, + + @FormDataParam(ServiceConstants.MULTIPART_FILE) InputStream uploadedFile, + @FormDataParam(ServiceConstants.MULTIPART_FILE) FormDataContentDisposition uploadedFileDetails, + @FormDataParam("plugin-invocations") Set pluginInvocations){ try{ + String pathString="<"+destinationID+">/"+subPath; + log.info("Received POST request at {} ",pathString); log.debug("Plugin invocation set : {} ",pluginInvocations); - TransferRequest request=formRequestFromREST(methodString, destinationID, subPath,pluginInvocations); + TransferRequest request=formRequestFromREST(methodString, destinationID, subPath,pluginInvocations, + createDirs,onExistingFile,onExistingDirectory, + uploadedFile,uploadedFileDetails,destinationFileName,sourceID); log.info("Received REST Request {} ",request); @@ -88,26 +98,32 @@ public class REST { - private TransferRequest formRequestFromREST(String methodString,String destinationID,String subPath, Set pluginInvocations){ + private TransferRequest formRequestFromREST(String methodString,String destinationID,String subPath, Set pluginInvocations, + Boolean createDirs, DestinationClashPolicy onExistingFile, + DestinationClashPolicy onExistingDirectory, + InputStream uploadedFile, + FormDataContentDisposition uploadedFileDetails, + String destinationFileName, + String sourceID){ log.info("Creating TransferRequest from REST invocation method : {}, dest ID {}, sub Path {} ",methodString,destinationID,subPath); TransferMethod method=null; try{ method=TransferMethod.valueOf(methodString); }catch (Throwable t) { - throw new WebApplicationException("Invalid selected method "+methodString,Status.BAD_REQUEST);} + throw new WebApplicationException("Invalid selected method "+methodString,t,Status.BAD_REQUEST);} + + + - - - Destination destination=new Destination(); destination.setCreateSubfolders(createDirs); destination.setPersistenceId(destinationID); destination.setSubFolder(subPath); destination.setOnExistingSubFolder(onExistingDirectory); destination.setOnExistingFileName(onExistingFile); - - - + + + TransferRequest resultingRequest=new TransferRequest(); resultingRequest.setDestinationSettings(destination); resultingRequest.setPluginInvocations(pluginInvocations); @@ -132,7 +148,7 @@ public class REST { resultingRequest.setSettings(settings); break; }catch(MalformedURLException e){ - throw new WebApplicationException("Source "+sourceID+" is not a valid URL.",Status.BAD_REQUEST); + throw new WebApplicationException("Source "+sourceID+" is not a valid URL.",e,Status.BAD_REQUEST); } } default: throw new WebApplicationException("Unsupported selected method "+methodString,Status.BAD_REQUEST); @@ -140,4 +156,40 @@ public class REST { return resultingRequest; } + + // @Path("/{destinationId}/{subPath : \\S*}") + @GET + @Produces("*/*") + public Response getFile(@PathParam("destinationId") String destinationId, + @PathParam("subPath") String subPath, + @QueryParam("descriptor") @DefaultValue("false") Boolean getDescriptor) { + String pathString="<"+destinationID+">/"+subPath; + log.info("Received GET request at {} , descriptor option is {} ",pathString,getDescriptor); + try{ + if(getDescriptor) return Response.ok(persistence.getDescriptor(destinationId, subPath), MediaType.APPLICATION_JSON_TYPE).build(); + + + File persisted= persistence.getPersistedFile(destinationId, subPath); + if(!persisted.exists()) throw new WebApplicationException("File "+pathString+" doesn't exists.",Status.NOT_FOUND); + if(persisted.isDirectory()) throw new WebApplicationException("The selected path "+pathString+" is a directory.",Status.BAD_REQUEST); + String mt = new MimetypesFileTypeMap().getContentType(persisted); + return Response.ok(persisted, mt).build(); + }catch(DestinationAccessException e) { + throw new WebApplicationException("Unable to access selected path "+pathString,e,Status.INTERNAL_SERVER_ERROR); + } + } + +// // @Path("/{destinationId}/{subPath : \\\\S*}") +// @HEAD +// @Produces(MediaType.APPLICATION_JSON) +// public RemoteFileDescriptor getFileDescriptor(@PathParam("destinationId") String destinationId, +// @PathParam("subPath") String subPath ) { +// String pathString="<"+destinationID+">/"+subPath; +// log.info("Received HEAD request at {} ",pathString); +// try{ +// return persistence.getDescriptor(destinationId, subPath); +// }catch(DestinationAccessException e) { +// throw new WebApplicationException("Unable to access selected path "+pathString,Status.INTERNAL_SERVER_ERROR); +// } +// } } diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/engine/PersistenceProvider.java b/src/main/java/org/gcube/data/transfer/service/transfers/engine/PersistenceProvider.java index cac2fed..cd6d602 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/engine/PersistenceProvider.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/engine/PersistenceProvider.java @@ -4,6 +4,7 @@ import java.io.File; import java.util.Set; import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException; public interface PersistenceProvider { @@ -13,4 +14,9 @@ public interface PersistenceProvider { public Set getAvaileblContextIds(); File prepareDestination(Destination dest) throws DestinationAccessException; + + public File getPersistedFile(String persistenceId,String subPath) throws DestinationAccessException; + + public RemoteFileDescriptor getDescriptor(String persistenceId,String subPath) throws DestinationAccessException; + } diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java index 55988ce..2598b7f 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java @@ -3,6 +3,8 @@ package org.gcube.data.transfer.service.transfers.engine.impl; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; @@ -10,6 +12,7 @@ import javax.inject.Singleton; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider; import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException; import org.gcube.smartgears.ContextProvider; @@ -23,7 +26,40 @@ import lombok.extern.slf4j.Slf4j; public class PersistenceProviderImpl implements PersistenceProvider { - + @Override + public File getPersistedFile(String persistenceId, String subPath) throws DestinationAccessException { + log.debug("Accessing <{}>/{}",persistenceId,subPath); + File persistenceFolder=getPersistenceFolderById(persistenceId); + return new File(persistenceFolder.getAbsolutePath()+"/"+subPath); + } + + + @Override + public RemoteFileDescriptor getDescriptor(String persistenceId, String subPath) throws DestinationAccessException { + File file=getPersistedFile(persistenceId, subPath); + log.debug("Getting descriptor for {} ",file.getAbsolutePath()); + if(!file.exists()) throw new DestinationAccessException("Unable to find "+file.getAbsolutePath()); + RemoteFileDescriptor toReturn=new RemoteFileDescriptor(); + try{ + BasicFileAttributes attributes=Files.readAttributes(file.toPath(), BasicFileAttributes.class,LinkOption.NOFOLLOW_LINKS); + toReturn.setCreationDate(attributes.creationTime().toMillis()); + toReturn.setLastUpdate(attributes.lastModifiedTime().toMillis()); + toReturn.setSize(attributes.size()); + }catch(Throwable t) { + log.warn("Unable to access attributes for {} ",file.getAbsolutePath(),t); + } + + toReturn.setAbsolutePath(file.getAbsolutePath()); + toReturn.setDirectory(file.isDirectory()); + toReturn.setFilename(file.getName()); + toReturn.setPersistenceId(persistenceId); + toReturn.setPath(subPath); + log.debug("Returning descriptor {} ",toReturn); + return toReturn; + } + + + @Override public File getPersistenceFolderById(String persistenceId) throws DestinationAccessException { File toReturn=null; diff --git a/src/test/java/org/gcube/data/transfer/service/CapabilitiesProviderFactory.java b/src/test/java/org/gcube/data/transfer/service/CapabilitiesProviderFactory.java index 8eba760..14850e1 100644 --- a/src/test/java/org/gcube/data/transfer/service/CapabilitiesProviderFactory.java +++ b/src/test/java/org/gcube/data/transfer/service/CapabilitiesProviderFactory.java @@ -26,7 +26,7 @@ public class CapabilitiesProviderFactory implements Factory(new PluginManagerImpl().getInstalledPlugins().values()) - ,Collections. emptySet()); + ,Collections.singleton("data-transfer-service")); } }; } diff --git a/src/test/java/org/gcube/data/transfer/service/PersistenceProviderFactory.java b/src/test/java/org/gcube/data/transfer/service/PersistenceProviderFactory.java index 4eded89..aa00034 100644 --- a/src/test/java/org/gcube/data/transfer/service/PersistenceProviderFactory.java +++ b/src/test/java/org/gcube/data/transfer/service/PersistenceProviderFactory.java @@ -2,38 +2,51 @@ package org.gcube.data.transfer.service; import java.io.File; import java.io.IOException; -import java.util.Collections; -import java.util.Set; -import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.service.transfers.engine.PersistenceProvider; import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException; +import org.gcube.data.transfer.service.transfers.engine.impl.PersistenceProviderImpl; import org.glassfish.hk2.api.Factory; class PersistenceProviderFactory implements Factory{ @Override public PersistenceProvider provide() { - return new PersistenceProvider() { + return new PersistenceProviderImpl() { +// +// + @Override + public File getPersistedFile(String persistenceId, String subPath) throws DestinationAccessException { + File persistenceFolder=getPersistenceFolderById(persistenceId); + File toReturn=new File(persistenceFolder.getAbsolutePath()+"/"+subPath); + try { + toReturn.getParentFile().mkdirs(); + toReturn.createNewFile(); + } catch (IOException e) { + throw new DestinationAccessException(e); + } + return toReturn; + } +// @Override public File getPersistenceFolderById(String persistenceId) throws DestinationAccessException { return new File(System.getProperty("java.io.tmpdir")); } - - @Override - public Set getAvaileblContextIds() { - return Collections.emptySet(); - } - - @Override - public File prepareDestination(Destination dest) throws DestinationAccessException { - try { - return File.createTempFile("dest", ""); - } catch (IOException e) { - return null; - } - } +// +// @Override +// public Set getAvaileblContextIds() { +// return Collections.singleton("data-transfer-service"); +// } +// +// @Override +// public File prepareDestination(Destination dest) throws DestinationAccessException { +// try { +// return File.createTempFile("dest", ""); +// } catch (IOException e) { +// return null; +// } +// } }; } diff --git a/src/test/java/org/gcube/data/transfer/service/TestCall.java b/src/test/java/org/gcube/data/transfer/service/TestCall.java index 1352ae9..318abcc 100644 --- a/src/test/java/org/gcube/data/transfer/service/TestCall.java +++ b/src/test/java/org/gcube/data/transfer/service/TestCall.java @@ -11,6 +11,7 @@ import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.model.ServiceConstants; import org.gcube.data.transfer.model.TransferCapabilities; import org.gcube.data.transfer.model.TransferRequest; @@ -129,4 +130,17 @@ public class TestCall extends JerseyTest { private TransferTicket submit(TransferRequest req){ return target(ServiceConstants.REQUESTS_SERVLET_NAME).request(MediaType.APPLICATION_JSON).post(Entity.entity(req,MediaType.APPLICATION_JSON),TransferTicket.class); } + + @Test + public void testREAD() { + System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/temp.txt").request().get()); + System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/some/folder/inside/temp.txt").request().get()); + } + + @Test + public void testHEAD() { + System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/temp.txt").request(MediaType.APPLICATION_JSON).head().readEntity(RemoteFileDescriptor.class)); + System.out.println(target(ServiceConstants.REST_SERVLET_NAME).path("data-transfer-service/some/folder/inside/temp.txt").request(MediaType.APPLICATION_JSON).head().readEntity(RemoteFileDescriptor.class)); + } + }