git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@149352 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
6d617d97d6
commit
92b9382eeb
|
|
@ -1,19 +1,15 @@
|
|||
package org.gcube.dataanalysis.executor.nodes.transducers;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
|
||||
import org.gcube.dataanalysis.executor.util.DataTransferer;
|
||||
import org.gcube.dataanalysis.executor.util.InfraRetrieval;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -21,13 +17,14 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
|
||||
public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
|
||||
// private static String MainPageParam = "MainPage";
|
||||
//private static String MainPageParam = "MainPage";
|
||||
private static final String DATA_TRANSFER_PERSISTENCE_ID = "data-transfer-service";
|
||||
private static String FileParam = "ZipFile";
|
||||
private String transferServiceAddress = "";
|
||||
private int transferServicePort = 0;
|
||||
|
||||
|
||||
private Logger LOGGER = LoggerFactory.getLogger(WebApplicationPublisher.class);
|
||||
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "This algorithm publishes a zip file containing a Web site, based on html and javascript in the e-Infrastructure. It generates a public URL to the application that can be shared.";
|
||||
|
|
@ -44,87 +41,46 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
|
|||
try
|
||||
{
|
||||
status = 10;
|
||||
String scope = config.getGcubeScope();
|
||||
String username = config.getParam("ServiceUserName");
|
||||
String fileAbsolutePath = config.getParam(FileParam);
|
||||
// String mainPage = config.getParam(MainPageParam);
|
||||
|
||||
LOGGER.debug("scope: {}",scope);
|
||||
LOGGER.debug("username: {}",username);
|
||||
LOGGER.debug("fileAbsolutePath: {}",fileAbsolutePath);
|
||||
// LOGGER.debug("layerTitle: "+mainPage);
|
||||
|
||||
if (scope==null || username==null)
|
||||
throw new Exception ("Service parameters are not set - please contact the Administrators");
|
||||
if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0)
|
||||
throw new Exception ("No file has been provided to the process");
|
||||
// if (mainPage==null || mainPage.trim().length()==0)
|
||||
// throw new Exception ("Please provide a valid main page");
|
||||
|
||||
File f = new File(fileAbsolutePath);
|
||||
String fileName = f.getName();
|
||||
|
||||
//unzip the file
|
||||
LOGGER.debug("Package is in file "+fileName);
|
||||
|
||||
boolean mkdir = folder.mkdir();
|
||||
LOGGER.debug("Sandbox {} generated: {} ",folder.getAbsolutePath(), mkdir);
|
||||
LOGGER.debug("Unzipping package into {}",folder.getAbsolutePath());
|
||||
ZipTools.unZip(f.getAbsolutePath(), folder.getAbsolutePath());
|
||||
// f.delete();
|
||||
LOGGER.debug("Package unzipped and original file deleted");
|
||||
|
||||
File[] webappfiles = folder.listFiles();
|
||||
//get all files for the upload
|
||||
String prefix = "/"+folder.getName()+"/";
|
||||
|
||||
Map<String,String> allfiles = getFilesPaths(webappfiles,prefix);
|
||||
|
||||
//discover the dataTransfer service with Apache
|
||||
getTransferInfo(config.getGcubeScope());
|
||||
String remoteFolder = "/var/www/html";
|
||||
//upload every file
|
||||
int nfiles = allfiles.size();
|
||||
float step = 80f/(float)nfiles;
|
||||
float initialStatus=status;
|
||||
int i=0;
|
||||
for (String key:allfiles.keySet()){
|
||||
status=(float)MathFunctions.roundDecimal(initialStatus+(float)i*step,2);
|
||||
String subpath = allfiles.get(key);
|
||||
subpath = subpath.substring(0,subpath.lastIndexOf("/"));
|
||||
String remotePath = remoteFolder+subpath+"/";
|
||||
LOGGER.debug("Uploading {} -> {}",key,remotePath);
|
||||
|
||||
long transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath).getTransferedBytes();
|
||||
if (transferout<=0){
|
||||
throw new Exception("Error transferring files to the infrastructure ");
|
||||
}
|
||||
i++;
|
||||
}
|
||||
String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/";
|
||||
if (webappfiles.length==1 && webappfiles[0].isDirectory()){
|
||||
producedPage = producedPage + webappfiles[0].getName()+"/";
|
||||
}
|
||||
LOGGER.debug("Entry point of the page "+producedPage);
|
||||
|
||||
//get URL
|
||||
addOutputString("Generated Website - Main URL", producedPage);
|
||||
}catch(Exception e){
|
||||
String scope = config.getGcubeScope();
|
||||
String username = config.getParam("ServiceUserName");
|
||||
String fileAbsolutePath = config.getParam(FileParam);
|
||||
// String mainPage = config.getParam(MainPageParam);
|
||||
|
||||
LOGGER.debug("scope: {}",scope);
|
||||
LOGGER.debug("username: {}",username);
|
||||
LOGGER.debug("fileAbsolutePath: {}",fileAbsolutePath);
|
||||
// LOGGER.debug("layerTitle: "+mainPage);
|
||||
|
||||
if (scope==null || username==null)
|
||||
throw new Exception ("Service parameters are not set - please contact the Administrators");
|
||||
if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0)
|
||||
throw new Exception ("No file has been provided to the process");
|
||||
|
||||
getTransferInfo(config.getGcubeScope());
|
||||
|
||||
DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, fileAbsolutePath, "", DATA_TRANSFER_PERSISTENCE_ID , true,uuid);
|
||||
String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/";
|
||||
|
||||
LOGGER.debug("Entry point of the page "+producedPage);
|
||||
|
||||
//get URL
|
||||
addOutputString("Generated Website - Main URL", producedPage);
|
||||
}catch(Exception e){
|
||||
LOGGER.error("an error occurred!",e);
|
||||
throw e;
|
||||
}finally{
|
||||
//clean everything
|
||||
//clean everything
|
||||
if (folder.exists()){
|
||||
LOGGER.debug("Cleaning folder "+folder);
|
||||
FileUtils.cleanDirectory(folder);
|
||||
FileUtils.deleteDirectory(folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
LOGGER.debug("Process finished");
|
||||
status = 100;
|
||||
}
|
||||
|
||||
private Map<String,String> getFilesPaths(File[] toexplore, String prefix){
|
||||
/* private Map<String,String> getFilesPaths(File[] toexplore, String prefix){
|
||||
Map<String,String> toreturn = new HashMap<String,String> ();
|
||||
for (File toex:toexplore){
|
||||
if (toex.isDirectory()){
|
||||
|
|
@ -133,43 +89,43 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
|
|||
else
|
||||
toreturn.put(toex.getAbsolutePath(),prefix+toex.getName());
|
||||
}
|
||||
|
||||
|
||||
return toreturn;
|
||||
}
|
||||
}*/
|
||||
@Override
|
||||
protected void setInputParameters() {
|
||||
|
||||
inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site"));
|
||||
inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name"));
|
||||
// addStringInput(MainPageParam, "Main page of the website", "index.html");
|
||||
}
|
||||
|
||||
inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site"));
|
||||
inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name"));
|
||||
//inputs.add(MainPageParam, "Main page of the website", "index.html");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
LOGGER.debug("WebApplicationPublisher - shutdown");
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void getTransferInfo(String scope) throws Exception{
|
||||
|
||||
|
||||
List<String> apacheAddress = InfraRetrieval.retrieveServiceAddress("Application", "Apache Server", scope, "Transect");
|
||||
|
||||
|
||||
if (apacheAddress.size()==0)
|
||||
throw new Exception("Apache Server resource is not available in scope "+scope);
|
||||
|
||||
|
||||
String apacheServiceAddress = apacheAddress.get(0);
|
||||
apacheServiceAddress = apacheServiceAddress.substring(apacheServiceAddress.indexOf("http://")+7);
|
||||
|
||||
|
||||
LOGGER.debug("Found {} services",apacheAddress.size());
|
||||
LOGGER.debug("Apache address: {}",apacheServiceAddress);
|
||||
|
||||
List<String> dataTransferAddress = InfraRetrieval.retrieveService("agent-service", scope);
|
||||
|
||||
List<String> dataTransferAddress = InfraRetrieval.retrieveService("data-transfer-service", scope);
|
||||
|
||||
if (dataTransferAddress.size()==0)
|
||||
throw new Exception("Data Transfer services are not available in scope "+scope);
|
||||
|
||||
|
||||
LOGGER.debug("Found {} transfer services",dataTransferAddress.size());
|
||||
|
||||
|
||||
int apacheDTPort = 9090;
|
||||
boolean found = false;
|
||||
for (String datatransferservice:dataTransferAddress){
|
||||
|
|
@ -184,15 +140,15 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (!found)
|
||||
throw new Exception("Apache data transfer has not been found in the same scope of the catalog: "+scope);
|
||||
else
|
||||
LOGGER.debug("Transfer service found at address {} : {}",apacheServiceAddress,apacheDTPort);
|
||||
|
||||
|
||||
transferServiceAddress = apacheServiceAddress;
|
||||
transferServicePort = apacheDTPort;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
package org.gcube.dataanalysis.executor.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.gcube.data.transfer.library.DataTransferClient;
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -17,8 +20,12 @@ public class DataTransferer {
|
|||
return transferFileToService(scope, username, host, port, fileAbsolutePath, remoteFolder, null);
|
||||
}
|
||||
|
||||
// returns the number of transferred bytes
|
||||
public static TransferResult transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder, String destinationId) throws Exception {
|
||||
return transferFileToService(scope, username, host, port, fileAbsolutePath, remoteFolder, destinationId, false, null );
|
||||
}
|
||||
|
||||
// returns the number of transferred bytes
|
||||
public static TransferResult transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder, String destinationId, boolean decompress, String decompressedFilefolderDestination) throws Exception {
|
||||
logger.debug("Transferring file {} to {}:{} " , fileAbsolutePath, host, port );
|
||||
|
||||
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+host+":"+port);
|
||||
|
|
@ -29,14 +36,24 @@ public class DataTransferer {
|
|||
|
||||
//String localfolder = localFile.getParent();
|
||||
String file = localFile.getName();
|
||||
|
||||
|
||||
|
||||
|
||||
Destination dest=new Destination(file);
|
||||
dest.setSubFolder(remoteFolder);
|
||||
|
||||
if(destinationId!=null)
|
||||
dest.setPersistenceId(destinationId);
|
||||
|
||||
TransferResult transferResult = client.localFile(localFile,dest);
|
||||
TransferResult transferResult = null;
|
||||
if(decompress){
|
||||
Map<String,String> params=new HashMap<>();
|
||||
params.put("DELETE_ARCHIVE", Boolean.FALSE.toString());
|
||||
if (decompressedFilefolderDestination!=null)
|
||||
params.put("DESTINATION", decompressedFilefolderDestination);
|
||||
transferResult = client.localFile(localFile,dest, new PluginInvocation("DECOMPRESS",params));
|
||||
} else
|
||||
transferResult = client.localFile(localFile,dest);
|
||||
|
||||
logger.debug("Transferring...");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue