gCubeSystem
/
data-transfer-service
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-transfer/data-transfer-service/2.0@148914 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2017-05-19 16:03:11 +00:00
parent 31db1805a5
commit 3fd9b5da49
7 changed files with 96 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pom.xml
View File

@ -63,13 +63,7 @@
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<exclusions>
<exclusion>
<artifactId>jackson-annotations</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
<artifactId>common-smartgears</artifactId>
</dependency>
<dependency>

75
src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/AbstractTicketHandler.java
View File

@ -7,7 +7,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@ -80,6 +84,14 @@ public abstract class AbstractTicketHandler {
log.trace("Request handling started. Ticket is "+ticket);
String destinationFileName=ticket.getDestinationSettings().getDestinationFileName();
log.debug("Checking destination file name : {} ",destinationFileName);
if(destinationFileName==null||destinationFileName.isEmpty()){
setDestinationFileName(ticket);
log.trace("Destination filename not specified. Automatically set {} ",ticket.getDestinationSettings().getDestinationFileName());
}
onStep("Checking destination",0d,Status.TRANSFERRING,0l);
destination =persistenceProvider.prepareDestination(ticket.getDestinationSettings());
ticket.setDestinationFileName(destination.getAbsolutePath());
@ -172,7 +184,8 @@ public abstract class AbstractTicketHandler {
case HTTPDownload:{
try{
HttpDownloadSettings options=(HttpDownloadSettings) (ticket.getSettings());
return new BufferedInputStream(options.getSource().openStream());
String sourceUrl=resolveRedirects(options.getSource().toString());
return new BufferedInputStream(new URL(sourceUrl).openStream());
}catch(Exception e){
log.debug("Unable to open connection ",e);
throw new ManagedException("Cannot open connection to source");
@ -191,4 +204,64 @@ public abstract class AbstractTicketHandler {
throw new ManagedException(ticket.getSettings().getOptions().getMethod()+" cannot be managed");
}
}
private static final void setDestinationFileName(TransferTicket ticket){
switch(ticket.getSettings().getOptions().getMethod()){
case HTTPDownload : {
HttpDownloadSettings options=(HttpDownloadSettings) (ticket.getSettings());
ticket.getDestinationSettings().setDestinationFileName(retrieveFileName(options.getSource().toString(), ticket.getId()));
break;
}
default : ticket.getDestinationSettings().setDestinationFileName(ticket.getId());
}
}
private static String resolveRedirects(String url) throws IOException{
log.debug("Resolving redirect for url {} ",url);
URL urlObj=new URL(url);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
int status=connection.getResponseCode();
if(status>=300&&status<400){
String newUrl=connection.getHeaderField("Location");
log.debug("Following redirect from {} to {} ",url,newUrl);
return resolveRedirects(newUrl);
}else return url;
}
private static String retrieveFileName(String url,String defaultName){
try{
String fileName=null;
log.debug("Resolving name for url {} ",url);
url=resolveRedirects(url);
URL urlObj = new URL(url);
HttpURLConnection connection = (HttpURLConnection) urlObj
.openConnection();
String contentDisposition = connection
.getHeaderField("Content-Disposition");
Pattern regex = Pattern.compile("(?<=filename=\").*?(?=\")");
Matcher regexMatcher = regex.matcher(contentDisposition);
if (regexMatcher.find()) {
fileName = regexMatcher.group();
}
if (fileName == null || fileName.isEmpty()) {
throw new Exception ("Filename was null or empty.");
}
return fileName;
}catch (Throwable t){
log.debug("Unable to retrieve name from url {}, reverting to default {}.",url,defaultName,t);
return defaultName;
}
}
}

18
src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PersistenceProviderImpl.java
View File

@ -36,7 +36,7 @@ public class PersistenceProviderImpl implements PersistenceProvider {
toReturn=new File(context.persistence().location());
}else{
for(ApplicationConfiguration config:ContextProvider.get().container().configuration().apps()){
if(config.context().equals(persistenceId)){
if(config.context().equals(persistenceId)||config.context().equals("/"+persistenceId)){
log.debug("Found persistence ID {}",persistenceId);
toReturn= new File(config.persistence().location());
break;
@ -68,7 +68,7 @@ public class PersistenceProviderImpl implements PersistenceProvider {
public File prepareDestination(Destination dest) throws DestinationAccessException{
File persistenceFolder=getPersistenceFolderById(dest.getPersistenceId());
if(!persistenceFolder.canWrite()) throw new DestinationAccessException("Cannot write to selecte persistenceFolder [ID :"+dest.getPersistenceId()+"]");
log.debug("Got Persistence folder ID {}, PATH {}",persistenceFolder.getAbsolutePath(),dest.getPersistenceId());
log.debug("Got Persistence folder PATH {}, ID {}",persistenceFolder.getAbsolutePath(),dest.getPersistenceId());
String subFolderName=dest.getSubFolder();
File subFolder=persistenceFolder;
if(subFolderName!=null){
@ -104,17 +104,21 @@ public class PersistenceProviderImpl implements PersistenceProvider {
}
}
private static final File manageClash(DestinationClashPolicy policy, File clashing ) throws DestinationAccessException{
public static final File manageClash(DestinationClashPolicy policy, File clashing ) throws DestinationAccessException{
log.debug("Managing clash for {}, policy is {} ",clashing.getAbsolutePath(),policy);
boolean dir=clashing.isDirectory();
try{
switch(policy){
case ADD_SUFFIX : {
String clashingBaseName=clashing.getName();
String clashingName=clashing.getName();
String clashingBaseName=(!dir&&clashingName.contains("."))?clashingName.substring(0, clashingName.lastIndexOf(".")):clashingName;
String destinationExtension=(!dir&&clashingName.contains("."))?clashingName.substring(clashingName.lastIndexOf(".")):"";
int counter=1;
while(clashing.exists()){
clashing=new File(clashing.getParentFile(),clashingBaseName+"("+counter+")");
while(clashing.exists()){
clashing=new File(clashing.getParentFile(),clashingBaseName+"("+counter+")"+destinationExtension);
counter++;
}
if(dir)clashing.mkdirs();

1
src/main/java/org/gcube/data/transfer/service/transfers/engine/impl/PluginManagerImpl.java
View File

@ -69,6 +69,7 @@ public class PluginManagerImpl implements PluginManager {
AbstractPlugin plugin=factory.createWorker(invocation);
return plugin.execute();
}catch(PluginException e){
log.error("Unable to execute plguin invocation {} ",invocation,e);
throw new PluginExecutionException(e.getMessage(),e);
}
}

2
src/main/webapp/WEB-INF/README
View File

@ -25,7 +25,7 @@ no. 654119), SoBigData (grant no. 654024);
Version
--------------------------------------------------
2.0.0-SNAPSHOT (2017-05-17)
2.0.0-SNAPSHOT (2017-05-19)
Please see the file named "changelog.xml" in this directory for the release notes.

2
src/main/webapp/WEB-INF/descriptor.xml
View File

@ -9,7 +9,7 @@
<baseDirectory>/</baseDirectory>
<fileSets>
<fileSet>
<directory>/home/fabio/workspaces/trunk_workspace/data-transfer-service/distro</directory>
<directory>/home/fabio/workspaces/trunk_workspace/data-transfer-service-BRANCH/distro</directory>
<outputDirectory>/</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>

9
src/test/java/org/gcube/data/transfer/service/ClashPolicyTest.java
View File

@ -2,15 +2,20 @@ package org.gcube.data.transfer.service;
import java.io.File;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.service.transfers.engine.faults.DestinationAccessException;
import org.gcube.data.transfer.service.transfers.engine.impl.PersistenceProviderImpl;
public class ClashPolicyTest {
public static void main(String[] args) throws DestinationAccessException {
File dir=new File ("/home/fabio/workspaces/DT_TESTS");
dir.mkdirs();
// System.out.println(RequestHandler.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File(dir,"clashing.file")).getAbsolutePath());
// System.out.println(RequestHandler.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File(dir,"clashing.file")).getAbsolutePath());
for(int i=0;i<5;i++){
System.out.println(PersistenceProviderImpl.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File("ciao")).getAbsolutePath());
System.out.println(PersistenceProviderImpl.manageClash(DestinationClashPolicy.ADD_SUFFIX, new File("ciao.txt")).getAbsolutePath());
}
}
}