diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java index adfe0c7..339e6ea 100644 --- a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java @@ -1,19 +1,15 @@ package org.gcube.dataanalysis.executor.nodes.transducers; import java.io.File; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.UUID; import org.apache.commons.io.FileUtils; -import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm; -import org.gcube.dataanalysis.ecoengine.utils.ZipTools; import org.gcube.dataanalysis.executor.util.DataTransferer; import org.gcube.dataanalysis.executor.util.InfraRetrieval; import org.slf4j.Logger; @@ -21,13 +17,14 @@ import org.slf4j.LoggerFactory; public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{ -// private static String MainPageParam = "MainPage"; + //private static String MainPageParam = "MainPage"; + private static final String DATA_TRANSFER_PERSISTENCE_ID = "data-transfer-service"; private static String FileParam = "ZipFile"; private String transferServiceAddress = ""; private int transferServicePort = 0; - + private Logger LOGGER = LoggerFactory.getLogger(WebApplicationPublisher.class); - + @Override public String getDescription() { return "This algorithm publishes a zip file containing a Web site, based on html and javascript in the e-Infrastructure. It generates a public URL to the application that can be shared."; @@ -44,87 +41,46 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{ try { status = 10; - String scope = config.getGcubeScope(); - String username = config.getParam("ServiceUserName"); - String fileAbsolutePath = config.getParam(FileParam); -// String mainPage = config.getParam(MainPageParam); - - LOGGER.debug("scope: {}",scope); - LOGGER.debug("username: {}",username); - LOGGER.debug("fileAbsolutePath: {}",fileAbsolutePath); -// LOGGER.debug("layerTitle: "+mainPage); - - if (scope==null || username==null) - throw new Exception ("Service parameters are not set - please contact the Administrators"); - if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0) - throw new Exception ("No file has been provided to the process"); -// if (mainPage==null || mainPage.trim().length()==0) -// throw new Exception ("Please provide a valid main page"); - - File f = new File(fileAbsolutePath); - String fileName = f.getName(); - - //unzip the file - LOGGER.debug("Package is in file "+fileName); - - boolean mkdir = folder.mkdir(); - LOGGER.debug("Sandbox {} generated: {} ",folder.getAbsolutePath(), mkdir); - LOGGER.debug("Unzipping package into {}",folder.getAbsolutePath()); - ZipTools.unZip(f.getAbsolutePath(), folder.getAbsolutePath()); -// f.delete(); - LOGGER.debug("Package unzipped and original file deleted"); - - File[] webappfiles = folder.listFiles(); - //get all files for the upload - String prefix = "/"+folder.getName()+"/"; - - Map allfiles = getFilesPaths(webappfiles,prefix); - - //discover the dataTransfer service with Apache - getTransferInfo(config.getGcubeScope()); - String remoteFolder = "/var/www/html"; - //upload every file - int nfiles = allfiles.size(); - float step = 80f/(float)nfiles; - float initialStatus=status; - int i=0; - for (String key:allfiles.keySet()){ - status=(float)MathFunctions.roundDecimal(initialStatus+(float)i*step,2); - String subpath = allfiles.get(key); - subpath = subpath.substring(0,subpath.lastIndexOf("/")); - String remotePath = remoteFolder+subpath+"/"; - LOGGER.debug("Uploading {} -> {}",key,remotePath); - - long transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath).getTransferedBytes(); - if (transferout<=0){ - throw new Exception("Error transferring files to the infrastructure "); - } - i++; - } - String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/"; - if (webappfiles.length==1 && webappfiles[0].isDirectory()){ - producedPage = producedPage + webappfiles[0].getName()+"/"; - } - LOGGER.debug("Entry point of the page "+producedPage); - - //get URL - addOutputString("Generated Website - Main URL", producedPage); - }catch(Exception e){ + String scope = config.getGcubeScope(); + String username = config.getParam("ServiceUserName"); + String fileAbsolutePath = config.getParam(FileParam); + // String mainPage = config.getParam(MainPageParam); + + LOGGER.debug("scope: {}",scope); + LOGGER.debug("username: {}",username); + LOGGER.debug("fileAbsolutePath: {}",fileAbsolutePath); + // LOGGER.debug("layerTitle: "+mainPage); + + if (scope==null || username==null) + throw new Exception ("Service parameters are not set - please contact the Administrators"); + if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0) + throw new Exception ("No file has been provided to the process"); + + getTransferInfo(config.getGcubeScope()); + + DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, fileAbsolutePath, "", DATA_TRANSFER_PERSISTENCE_ID , true,uuid); + String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/"; + + LOGGER.debug("Entry point of the page "+producedPage); + + //get URL + addOutputString("Generated Website - Main URL", producedPage); + }catch(Exception e){ LOGGER.error("an error occurred!",e); throw e; }finally{ - //clean everything + //clean everything if (folder.exists()){ LOGGER.debug("Cleaning folder "+folder); FileUtils.cleanDirectory(folder); FileUtils.deleteDirectory(folder); } - } + } LOGGER.debug("Process finished"); status = 100; } - private Map getFilesPaths(File[] toexplore, String prefix){ +/* private Map getFilesPaths(File[] toexplore, String prefix){ Map toreturn = new HashMap (); for (File toex:toexplore){ if (toex.isDirectory()){ @@ -133,43 +89,43 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{ else toreturn.put(toex.getAbsolutePath(),prefix+toex.getName()); } - + return toreturn; - } + }*/ @Override protected void setInputParameters() { - - inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site")); - inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name")); -// addStringInput(MainPageParam, "Main page of the website", "index.html"); - } + + inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site")); + inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name")); + //inputs.add(MainPageParam, "Main page of the website", "index.html"); + } @Override public void shutdown() { LOGGER.debug("WebApplicationPublisher - shutdown"); } - + public void getTransferInfo(String scope) throws Exception{ - + List apacheAddress = InfraRetrieval.retrieveServiceAddress("Application", "Apache Server", scope, "Transect"); - + if (apacheAddress.size()==0) throw new Exception("Apache Server resource is not available in scope "+scope); - + String apacheServiceAddress = apacheAddress.get(0); apacheServiceAddress = apacheServiceAddress.substring(apacheServiceAddress.indexOf("http://")+7); - + LOGGER.debug("Found {} services",apacheAddress.size()); LOGGER.debug("Apache address: {}",apacheServiceAddress); - List dataTransferAddress = InfraRetrieval.retrieveService("agent-service", scope); - + List dataTransferAddress = InfraRetrieval.retrieveService("data-transfer-service", scope); + if (dataTransferAddress.size()==0) throw new Exception("Data Transfer services are not available in scope "+scope); - + LOGGER.debug("Found {} transfer services",dataTransferAddress.size()); - + int apacheDTPort = 9090; boolean found = false; for (String datatransferservice:dataTransferAddress){ @@ -184,15 +140,15 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{ break; } } - + if (!found) throw new Exception("Apache data transfer has not been found in the same scope of the catalog: "+scope); else LOGGER.debug("Transfer service found at address {} : {}",apacheServiceAddress,apacheDTPort); - + transferServiceAddress = apacheServiceAddress; transferServicePort = apacheDTPort; } - + } diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java index dd62d09..81c826f 100644 --- a/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java +++ b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java @@ -1,10 +1,13 @@ package org.gcube.dataanalysis.executor.util; import java.io.File; +import java.util.HashMap; +import java.util.Map; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.PluginInvocation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,8 +20,12 @@ public class DataTransferer { return transferFileToService(scope, username, host, port, fileAbsolutePath, remoteFolder, null); } - // returns the number of transferred bytes public static TransferResult transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder, String destinationId) throws Exception { + return transferFileToService(scope, username, host, port, fileAbsolutePath, remoteFolder, destinationId, false, null ); + } + + // returns the number of transferred bytes + public static TransferResult transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder, String destinationId, boolean decompress, String decompressedFilefolderDestination) throws Exception { logger.debug("Transferring file {} to {}:{} " , fileAbsolutePath, host, port ); DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+host+":"+port); @@ -29,14 +36,24 @@ public class DataTransferer { //String localfolder = localFile.getParent(); String file = localFile.getName(); - + + + Destination dest=new Destination(file); dest.setSubFolder(remoteFolder); if(destinationId!=null) dest.setPersistenceId(destinationId); - TransferResult transferResult = client.localFile(localFile,dest); + TransferResult transferResult = null; + if(decompress){ + Map params=new HashMap<>(); + params.put("DELETE_ARCHIVE", Boolean.FALSE.toString()); + if (decompressedFilefolderDestination!=null) + params.put("DESTINATION", decompressedFilefolderDestination); + transferResult = client.localFile(localFile,dest, new PluginInvocation("DECOMPRESS",params)); + } else + transferResult = client.localFile(localFile,dest); logger.debug("Transferring...");