WebApplicationPublisher
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@126658 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
5b853000df
commit
ce3160e7ae
|
|
@ -0,0 +1,196 @@
|
|||
package org.gcube.dataanalysis.executor.nodes.transducers;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
|
||||
import org.gcube.dataanalysis.executor.util.DataTransferer;
|
||||
import org.gcube.dataanalysis.executor.util.InfraRetrieval;
|
||||
|
||||
|
||||
public class WebApplicationPublisher extends StandardLocalExternalAlgorithm{
|
||||
// private static String MainPageParam = "MainPage";
|
||||
private static String FileParam = "ZipFile";
|
||||
private String transferServiceAddress = "";
|
||||
private int transferServicePort = 0;
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "This algorithm publishes a zip file containing a Web site, based on html and javascript in the e-Infrastructure. It generates a public URL to the application that can be shared.";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() throws Exception {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void process() throws Exception {
|
||||
String uuid = "webpub_" + UUID.randomUUID();
|
||||
File folder = new File(config.getConfigPath(), uuid);
|
||||
try
|
||||
{
|
||||
status = 10;
|
||||
String scope = config.getGcubeScope();
|
||||
String username = config.getParam("ServiceUserName");
|
||||
String fileAbsolutePath = config.getParam(FileParam);
|
||||
// String mainPage = config.getParam(MainPageParam);
|
||||
|
||||
AnalysisLogger.getLogger().debug("scope: "+scope);
|
||||
AnalysisLogger.getLogger().debug("username: "+username);
|
||||
AnalysisLogger.getLogger().debug("fileAbsolutePath: "+fileAbsolutePath);
|
||||
// AnalysisLogger.getLogger().debug("layerTitle: "+mainPage);
|
||||
|
||||
if (scope==null || username==null)
|
||||
throw new Exception ("Service parameters are not set - please contact the Administrators");
|
||||
if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0)
|
||||
throw new Exception ("No file has been provided to the process");
|
||||
// if (mainPage==null || mainPage.trim().length()==0)
|
||||
// throw new Exception ("Please provide a valid main page");
|
||||
|
||||
File f = new File(fileAbsolutePath);
|
||||
String fileName = f.getName();
|
||||
|
||||
//unzip the file
|
||||
AnalysisLogger.getLogger().debug("Package is in file "+fileName);
|
||||
|
||||
boolean mkdir = folder.mkdir();
|
||||
AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir);
|
||||
AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath());
|
||||
ZipTools.unZip(f.getAbsolutePath(), folder.getAbsolutePath());
|
||||
// f.delete();
|
||||
AnalysisLogger.getLogger().debug("Package unzipped and original file deleted");
|
||||
|
||||
File[] webappfiles = folder.listFiles();
|
||||
//get all files for the upload
|
||||
String prefix = "/"+folder.getName()+"/";
|
||||
|
||||
Map<String,String> allfiles = getFilesPaths(webappfiles,prefix);
|
||||
|
||||
//discover the dataTransfer service with Apache
|
||||
getTransferInfo(config.getGcubeScope());
|
||||
String remoteFolder = "/var/www/html";
|
||||
//upload every file
|
||||
int nfiles = allfiles.size();
|
||||
float step = 80f/(float)nfiles;
|
||||
float initialStatus=status;
|
||||
int i=0;
|
||||
for (String key:allfiles.keySet()){
|
||||
status=(float)MathFunctions.roundDecimal(initialStatus+(float)i*step,2);
|
||||
String subpath = allfiles.get(key);
|
||||
subpath = subpath.substring(0,subpath.lastIndexOf("/"));
|
||||
String remotePath = remoteFolder+subpath+"/";
|
||||
AnalysisLogger.getLogger().debug("Uploading "+key+" -> "+remotePath);
|
||||
|
||||
boolean transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
|
||||
if (!transferout){
|
||||
throw new Exception("Error transferring files to the infrastructure ");
|
||||
}
|
||||
i++;
|
||||
}
|
||||
String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/";
|
||||
if (webappfiles.length==1 && webappfiles[0].isDirectory()){
|
||||
producedPage = producedPage + webappfiles[0].getName()+"/";
|
||||
}
|
||||
AnalysisLogger.getLogger().debug("Entry point of the page "+producedPage);
|
||||
|
||||
//get URL
|
||||
addOutputString("Generated Website - Main URL", producedPage);
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().debug(e);
|
||||
AnalysisLogger.getLogger().debug("An error occurred!");
|
||||
throw e;
|
||||
}finally{
|
||||
//clean everything
|
||||
if (folder.exists()){
|
||||
AnalysisLogger.getLogger().debug("Cleaning folder "+folder);
|
||||
FileUtils.cleanDirectory(folder);
|
||||
FileUtils.deleteDirectory(folder);
|
||||
}
|
||||
}
|
||||
AnalysisLogger.getLogger().debug("Process finished");
|
||||
status = 100;
|
||||
}
|
||||
|
||||
private Map<String,String> getFilesPaths(File[] toexplore, String prefix){
|
||||
Map<String,String> toreturn = new HashMap<String,String> ();
|
||||
for (File toex:toexplore){
|
||||
if (toex.isDirectory()){
|
||||
toreturn.putAll(getFilesPaths(toex.listFiles(),prefix+toex.getName()+"/"));
|
||||
}
|
||||
else
|
||||
toreturn.put(toex.getAbsolutePath(),prefix+toex.getName());
|
||||
}
|
||||
|
||||
return toreturn;
|
||||
}
|
||||
@Override
|
||||
protected void setInputParameters() {
|
||||
|
||||
inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site"));
|
||||
inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name"));
|
||||
// addStringInput(MainPageParam, "Main page of the website", "index.html");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
AnalysisLogger.getLogger().debug("WebApplicationPublisher - shutdown");
|
||||
}
|
||||
|
||||
|
||||
public void getTransferInfo(String scope) throws Exception{
|
||||
|
||||
List<String> apacheAddress = InfraRetrieval.retrieveServiceAddress("Application", "Apache Server", scope, "Transect");
|
||||
|
||||
if (apacheAddress.size()==0)
|
||||
throw new Exception("Apache Server resource is not available in scope "+scope);
|
||||
|
||||
String apacheServiceAddress = apacheAddress.get(0);
|
||||
apacheServiceAddress = apacheServiceAddress.substring(apacheServiceAddress.indexOf("http://")+7);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Found "+apacheAddress.size()+" services");
|
||||
AnalysisLogger.getLogger().debug("Apache address: "+apacheServiceAddress);
|
||||
List<String> dataTransferAddress = InfraRetrieval.retrieveService("agent-service", scope);
|
||||
|
||||
if (dataTransferAddress.size()==0)
|
||||
throw new Exception("Data Transfer services are not available in scope "+scope);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Found "+dataTransferAddress.size()+" transfer services");
|
||||
|
||||
int apacheDTPort = 9090;
|
||||
boolean found = false;
|
||||
for (String datatransferservice:dataTransferAddress){
|
||||
AnalysisLogger.getLogger().debug("Transfer service found");
|
||||
datatransferservice = datatransferservice.substring(datatransferservice.indexOf("http://")+7);
|
||||
String servicehost = datatransferservice.substring(0,datatransferservice.indexOf(":"));
|
||||
String serviceport = datatransferservice.substring(datatransferservice.indexOf(":")+1,datatransferservice.indexOf("/"));
|
||||
AnalysisLogger.getLogger().debug("Transfer service: "+servicehost+":"+serviceport);
|
||||
if (apacheServiceAddress.equals(servicehost)){
|
||||
apacheDTPort = Integer.parseInt(serviceport);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found)
|
||||
throw new Exception("Apache data transfer has not been found in the same scope of the catalog: "+scope);
|
||||
else
|
||||
AnalysisLogger.getLogger().debug("Transfer service found at address "+apacheServiceAddress+":"+apacheDTPort);
|
||||
|
||||
transferServiceAddress = apacheServiceAddress;
|
||||
transferServicePort = apacheDTPort;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package org.gcube.dataanalysis.executor.tests;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
|
||||
|
||||
public class TestWebAppPublisher {
|
||||
|
||||
static String cfg = "./cfg/";
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
|
||||
config.setAgent("WEB_APP_PUBLISHER");
|
||||
|
||||
config.setGcubeScope("/gcube/devsec/devVRE");
|
||||
|
||||
config.setParam("ServiceUserName","gianpaolo.coro");
|
||||
config.setParam("MainPage", "index.html");
|
||||
//config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\SitoUnirender2015.zip");
|
||||
//config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\gcube.zip");
|
||||
// config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/simplesite.zip");
|
||||
config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/verysimple.zip");
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Executing: " + config.getAgent());
|
||||
List<ComputationalAgent> trans = null;
|
||||
trans = TransducerersFactory.getTransducerers(config);
|
||||
trans.get(0).init();
|
||||
Regressor.process(trans.get(0));
|
||||
StatisticalType st = trans.get(0).getOutput();
|
||||
AnalysisLogger.getLogger().debug("ST:" + st);
|
||||
trans = null;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -4,11 +4,18 @@ import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent
|
|||
|
||||
import java.io.File;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.datatransfer.agent.library.AgentLibrary;
|
||||
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
|
||||
import org.gcube.datatransfer.common.agent.Types.storageType;
|
||||
|
|
@ -19,9 +26,12 @@ import org.gcube.datatransfer.common.outcome.TransferStatus;
|
|||
public class DataTransferer {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String scope = "/d4science.research-infrastructures.eu/gCubeApps";
|
||||
//String scope = "/d4science.research-infrastructures.eu/gCubeApps";
|
||||
|
||||
String scope = "/gcube/devsec/devVRE";
|
||||
ScopeProvider.instance.set(scope);
|
||||
String transferGHN = "dewn04.madgik.di.uoa.gr";
|
||||
//String transferGHN = "dewn04.madgik.di.uoa.gr";
|
||||
String transferGHN = "access.d4science.org";
|
||||
int transferPort = 8080;
|
||||
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
|
||||
|
||||
|
|
@ -34,7 +44,8 @@ public class DataTransferer {
|
|||
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||
*/
|
||||
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
|
||||
String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
|
||||
//String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
|
||||
String urlStorage ="http://goo.gl/Vq8QVY";
|
||||
|
||||
System.out.println("URL for storage: " + urlStorage);
|
||||
|
||||
|
|
@ -47,9 +58,14 @@ public class DataTransferer {
|
|||
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
|
||||
input.add(uri);
|
||||
|
||||
String outPath = "/tmp";
|
||||
|
||||
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort, "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg", outPath);
|
||||
//String outPath = "/tmp";
|
||||
String outPath = "/var/www/html/test/";
|
||||
String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
|
||||
//fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
|
||||
// fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png";
|
||||
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath);
|
||||
|
||||
|
||||
}
|
||||
|
||||
// returns the number of transferred bytes
|
||||
|
|
@ -66,7 +82,10 @@ public class DataTransferer {
|
|||
String localfolder = localFile.getParent();
|
||||
String file = localFile.getName();
|
||||
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
|
||||
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, file);
|
||||
ScopeProvider.instance.set(scope);
|
||||
AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
|
||||
|
||||
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);
|
||||
//urls for testing
|
||||
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
|
||||
|
|
@ -91,7 +110,8 @@ public class DataTransferer {
|
|||
|
||||
|
||||
ArrayList<URI> outputURI = new ArrayList<URI>();
|
||||
outputURI.add(new URI("file://"+remoteFolder+""+file));
|
||||
// outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_")));
|
||||
outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20")));
|
||||
|
||||
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
|
||||
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ public class StorageUtils {
|
|||
ScopeProvider.instance.set(scope);
|
||||
AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
|
||||
IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient();
|
||||
String remotef = remoteFolder+file;
|
||||
String remotef = remoteFolder+file.replace(" ","%20");
|
||||
client.put(true).LFile(new File(localFolder,file).getAbsolutePath()).RFile(remotef);
|
||||
String url = "";
|
||||
if (httplink)
|
||||
|
|
|
|||
Loading…
Reference in New Issue