From 3fd9b5da499d3567859f5aff3e38f40300a3946f Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Fri, 19 May 2017 16:03:11 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-transfer/data-transfer-service/2.0@148914 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 8 +- .../engine/impl/AbstractTicketHandler.java | 75 ++++++++++++++++++- .../engine/impl/PersistenceProviderImpl.java | 18 +++-- .../engine/impl/PluginManagerImpl.java | 1 + src/main/webapp/WEB-INF/README | 2 +- src/main/webapp/WEB-INF/descriptor.xml | 2 +- .../transfer/service/ClashPolicyTest.java | 9 ++- 7 files changed, 96 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 227b8f0..cd0fd1a 100644 --- a/pom.xml +++ b/pom.xml @@ -63,13 +63,7 @@ org.gcube.core - common-smartgears - - - jackson-annotations - com.fasterxml.jackson.core - - + common-smartgears diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/AbstractTicketHandler.java b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/AbstractTicketHandler.java index fbf1985..dc18612 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/AbstractTicketHandler.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/AbstractTicketHandler.java @@ -7,7 +7,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -80,6 +84,14 @@ public abstract class AbstractTicketHandler { log.trace("Request handling started. Ticket is "+ticket); + String destinationFileName=ticket.getDestinationSettings().getDestinationFileName(); + log.debug("Checking destination file name : {} ",destinationFileName); + if(destinationFileName==null||destinationFileName.isEmpty()){ + setDestinationFileName(ticket); + log.trace("Destination filename not specified. Automatically set {} ",ticket.getDestinationSettings().getDestinationFileName()); + } + + onStep("Checking destination",0d,Status.TRANSFERRING,0l); destination =persistenceProvider.prepareDestination(ticket.getDestinationSettings()); ticket.setDestinationFileName(destination.getAbsolutePath()); @@ -172,7 +184,8 @@ public abstract class AbstractTicketHandler { case HTTPDownload:{ try{ HttpDownloadSettings options=(HttpDownloadSettings) (ticket.getSettings()); - return new BufferedInputStream(options.getSource().openStream()); + String sourceUrl=resolveRedirects(options.getSource().toString()); + return new BufferedInputStream(new URL(sourceUrl).openStream()); }catch(Exception e){ log.debug("Unable to open connection ",e); throw new ManagedException("Cannot open connection to source"); @@ -191,4 +204,64 @@ public abstract class AbstractTicketHandler { throw new ManagedException(ticket.getSettings().getOptions().getMethod()+" cannot be managed"); } } + + + private static final void setDestinationFileName(TransferTicket ticket){ + switch(ticket.getSettings().getOptions().getMethod()){ + case HTTPDownload : { + HttpDownloadSettings options=(HttpDownloadSettings) (ticket.getSettings()); + + ticket.getDestinationSettings().setDestinationFileName(retrieveFileName(options.getSource().toString(), ticket.getId())); + break; + } + default : ticket.getDestinationSettings().setDestinationFileName(ticket.getId()); + } + } + + + private static String resolveRedirects(String url) throws IOException{ + log.debug("Resolving redirect for url {} ",url); + URL urlObj=new URL(url); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + int status=connection.getResponseCode(); + if(status>=300&&status<400){ + String newUrl=connection.getHeaderField("Location"); + log.debug("Following redirect from {} to {} ",url,newUrl); + return resolveRedirects(newUrl); + }else return url; + } + + + private static String retrieveFileName(String url,String defaultName){ + + try{ + String fileName=null; + log.debug("Resolving name for url {} ",url); + url=resolveRedirects(url); + URL urlObj = new URL(url); + + HttpURLConnection connection = (HttpURLConnection) urlObj + .openConnection(); + + String contentDisposition = connection + .getHeaderField("Content-Disposition"); + + + + Pattern regex = Pattern.compile("(?<=filename=\").*?(?=\")"); + Matcher regexMatcher = regex.matcher(contentDisposition); + if (regexMatcher.find()) { + fileName = regexMatcher.group(); + } + + if (fileName == null || fileName.isEmpty()) { + throw new Exception ("Filename was null or empty."); + } + + return fileName; + }catch (Throwable t){ + log.debug("Unable to retrieve name from url {}, reverting to default {}.",url,defaultName,t); + return defaultName; + } + } } diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java index 8571fbb..75ffc32 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java @@ -36,7 +36,7 @@ public class PersistenceProviderImpl implements PersistenceProvider { toReturn=new File(context.persistence().location()); }else{ for(ApplicationConfiguration config:ContextProvider.get().container().configuration().apps()){ - if(config.context().equals(persistenceId)){ + if(config.context().equals(persistenceId)||config.context().equals("/"+persistenceId)){ log.debug("Found persistence ID {}",persistenceId); toReturn= new File(config.persistence().location()); break; @@ -68,7 +68,7 @@ public class PersistenceProviderImpl implements PersistenceProvider { public File prepareDestination(Destination dest) throws DestinationAccessException{ File persistenceFolder=getPersistenceFolderById(dest.getPersistenceId()); if(!persistenceFolder.canWrite()) throw new DestinationAccessException("Cannot write to selecte persistenceFolder [ID :"+dest.getPersistenceId()+"]"); - log.debug("Got Persistence folder ID {}, PATH {}",persistenceFolder.getAbsolutePath(),dest.getPersistenceId()); + log.debug("Got Persistence folder PATH {}, ID {}",persistenceFolder.getAbsolutePath(),dest.getPersistenceId()); String subFolderName=dest.getSubFolder(); File subFolder=persistenceFolder; if(subFolderName!=null){ @@ -104,17 +104,21 @@ public class PersistenceProviderImpl implements PersistenceProvider { } } - private static final File manageClash(DestinationClashPolicy policy, File clashing ) throws DestinationAccessException{ + public static final File manageClash(DestinationClashPolicy policy, File clashing ) throws DestinationAccessException{ log.debug("Managing clash for {}, policy is {} ",clashing.getAbsolutePath(),policy); boolean dir=clashing.isDirectory(); try{ switch(policy){ case ADD_SUFFIX : { - String clashingBaseName=clashing.getName(); - + String clashingName=clashing.getName(); + String clashingBaseName=(!dir&&clashingName.contains("."))?clashingName.substring(0, clashingName.lastIndexOf(".")):clashingName; + String destinationExtension=(!dir&&clashingName.contains("."))?clashingName.substring(clashingName.lastIndexOf(".")):""; + + + int counter=1; - while(clashing.exists()){ - clashing=new File(clashing.getParentFile(),clashingBaseName+"("+counter+")"); + while(clashing.exists()){ + clashing=new File(clashing.getParentFile(),clashingBaseName+"("+counter+")"+destinationExtension); counter++; } if(dir)clashing.mkdirs(); diff --git a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PluginManagerImpl.java b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PluginManagerImpl.java index 5bf332b..96b1185 100644 --- a/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PluginManagerImpl.java +++ b/src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PluginManagerImpl.java @@ -69,6 +69,7 @@ public class PluginManagerImpl implements PluginManager { AbstractPlugin plugin=factory.createWorker(invocation); return plugin.execute(); }catch(PluginException e){ + log.error("Unable to execute plguin invocation {} ",invocation,e); throw new PluginExecutionException(e.getMessage(),e); } } diff --git a/src/main/webapp/WEB-INF/README b/src/main/webapp/WEB-INF/README index 8ccf802..dd4c04f 100644 --- a/src/main/webapp/WEB-INF/README +++ b/src/main/webapp/WEB-INF/README @@ -25,7 +25,7 @@ no. 654119), SoBigData (grant no. 654024); Version -------------------------------------------------- -2.0.0-SNAPSHOT (2017-05-17) +2.0.0-SNAPSHOT (2017-05-19) Please see the file named "changelog.xml" in this directory for the release notes. diff --git a/src/main/webapp/WEB-INF/descriptor.xml b/src/main/webapp/WEB-INF/descriptor.xml index 1ce8afc..0dbbe21 100644 --- a/src/main/webapp/WEB-INF/descriptor.xml +++ b/src/main/webapp/WEB-INF/descriptor.xml @@ -9,7 +9,7 @@ / - /home/fabio/workspaces/trunk_workspace/data-transfer-service/distro + /home/fabio/workspaces/trunk_workspace/data-transfer-service-BRANCH/distro / true diff --git a/src/test/java/org/gcube/data/transfer/service/ClashPolicyTest.java b/src/test/java/org/gcube/data/transfer/service/ClashPolicyTest.java index b3c97b8..9cb38e7 100644 --- a/src/test/java/org/gcube/data/transfer/service/ClashPolicyTest.java +++ b/src/test/java/org/gcube/data/transfer/service/ClashPolicyTest.java @@ -2,15 +2,20 @@ package org.gcube.data.transfer.service; import java.io.File; +import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException; +import org.gcube.data.transfer.service.transfers.engine.impl.PersistenceProviderImpl; public class ClashPolicyTest { public static void main(String[] args) throws DestinationAccessException { File dir=new File ("/home/fabio/workspaces/DT_TESTS"); dir.mkdirs(); -// System.out.println(RequestHandler.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File(dir,"clashing.file")).getAbsolutePath()); - + // System.out.println(RequestHandler.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File(dir,"clashing.file")).getAbsolutePath()); + for(int i=0;i<5;i++){ + System.out.println(PersistenceProviderImpl.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File("ciao")).getAbsolutePath()); + System.out.println(PersistenceProviderImpl.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File("ciao.txt")).getAbsolutePath()); + } } }