WebApplicationPublisher

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@126658 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java

@@ -0,0 +1,196 @@
+package org.gcube.dataanalysis.executor.nodes.transducers;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.io.FileUtils;
+import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
+import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
+import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
+import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
+import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
+import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
+import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
+import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
+import org.gcube.dataanalysis.executor.util.DataTransferer;
+import org.gcube.dataanalysis.executor.util.InfraRetrieval;
+
+
+public class WebApplicationPublisher extends StandardLocalExternalAlgorithm{
+//	private static String MainPageParam = "MainPage";
+	private static String FileParam = "ZipFile";
+	private String transferServiceAddress = "";
+	private int transferServicePort = 0;
+	
+	@Override
+	public String getDescription() {
+		return "This algorithm publishes a zip file containing a Web site, based on html and javascript in the e-Infrastructure. It generates a public URL to the application that can be shared.";
+	}
+
+	@Override
+	public void init() throws Exception {
+	}
+
+	@Override
+	protected void process() throws Exception {
+		String uuid = "webpub_" + UUID.randomUUID();
+		File folder = new File(config.getConfigPath(), uuid);
+		try
+		{
+			status = 10;
+		String scope = config.getGcubeScope();
+		String username = config.getParam("ServiceUserName");
+		String fileAbsolutePath = config.getParam(FileParam);
+//		String mainPage = config.getParam(MainPageParam);
+		
+		AnalysisLogger.getLogger().debug("scope: "+scope);
+		AnalysisLogger.getLogger().debug("username: "+username);
+		AnalysisLogger.getLogger().debug("fileAbsolutePath: "+fileAbsolutePath);
+//		AnalysisLogger.getLogger().debug("layerTitle: "+mainPage);
+		
+		if (scope==null || username==null)
+			throw new Exception ("Service parameters are not set - please contact the Administrators");
+		if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0)
+			throw new Exception ("No file has been provided to the process");
+//		if (mainPage==null || mainPage.trim().length()==0)
+//			throw new Exception ("Please provide a valid main page");
+	 
+		File f = new File(fileAbsolutePath); 
+		String fileName = f.getName();
+       		
+		//unzip the file
+		AnalysisLogger.getLogger().debug("Package is in file "+fileName);
+		
+		boolean mkdir = folder.mkdir();
+		AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir);
+		AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath());
+		ZipTools.unZip(f.getAbsolutePath(), folder.getAbsolutePath());
+//		f.delete();
+		AnalysisLogger.getLogger().debug("Package unzipped and original file deleted");
+		
+		File[] webappfiles = folder.listFiles();
+		//get all files for the upload
+		String prefix = "/"+folder.getName()+"/";
+		
+		Map<String,String> allfiles = getFilesPaths(webappfiles,prefix);
+		
+		//discover the dataTransfer service with Apache
+		getTransferInfo(config.getGcubeScope());
+		String remoteFolder = "/var/www/html";
+	    //upload every file
+		int nfiles = allfiles.size();
+		float step = 80f/(float)nfiles;
+		float initialStatus=status;
+		int i=0;
+		for (String key:allfiles.keySet()){
+			status=(float)MathFunctions.roundDecimal(initialStatus+(float)i*step,2);
+			String subpath = allfiles.get(key);
+			subpath = subpath.substring(0,subpath.lastIndexOf("/"));
+			String remotePath = remoteFolder+subpath+"/";
+			AnalysisLogger.getLogger().debug("Uploading "+key+" -> "+remotePath);
+			
+			boolean transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
+			if (!transferout){
+				throw new Exception("Error transferring files to the infrastructure ");
+			}
+			i++;
+		}
+		String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/";
+		if (webappfiles.length==1 && webappfiles[0].isDirectory()){
+			producedPage = producedPage + webappfiles[0].getName()+"/";
+		}
+		AnalysisLogger.getLogger().debug("Entry point of the page "+producedPage);
+      
+      //get URL
+		addOutputString("Generated Website - Main URL", producedPage);
+	}catch(Exception e){
+			e.printStackTrace();
+			AnalysisLogger.getLogger().debug(e);
+			AnalysisLogger.getLogger().debug("An error occurred!");
+			throw e;
+		}finally{	
+		       //clean everything
+			if (folder.exists()){
+				AnalysisLogger.getLogger().debug("Cleaning folder "+folder);
+				FileUtils.cleanDirectory(folder);
+				FileUtils.deleteDirectory(folder);
+			}
+	}
+		AnalysisLogger.getLogger().debug("Process finished");
+		status = 100;
+	}
+
+	private Map<String,String> getFilesPaths(File[] toexplore, String prefix){
+		Map<String,String> toreturn = new HashMap<String,String> ();
+		for (File toex:toexplore){
+			if (toex.isDirectory()){
+				toreturn.putAll(getFilesPaths(toex.listFiles(),prefix+toex.getName()+"/"));
+			}
+			else
+				toreturn.put(toex.getAbsolutePath(),prefix+toex.getName());
+		}
+		
+		return toreturn;
+	}
+	@Override
+	protected void setInputParameters() {
+		
+			inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site"));
+			inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name"));
+//			addStringInput(MainPageParam, "Main page of the website", "index.html");
+		}
+
+	@Override
+	public void shutdown() {
+		AnalysisLogger.getLogger().debug("WebApplicationPublisher - shutdown");
+	}	
+
+	
+	public void getTransferInfo(String scope) throws Exception{
+		
+		List<String> apacheAddress = InfraRetrieval.retrieveServiceAddress("Application", "Apache Server", scope, "Transect");
+		
+		if (apacheAddress.size()==0)
+			throw new Exception("Apache Server resource is not available in scope "+scope);
+		
+		String apacheServiceAddress = apacheAddress.get(0);
+		apacheServiceAddress = apacheServiceAddress.substring(apacheServiceAddress.indexOf("http://")+7);
+		
+		AnalysisLogger.getLogger().debug("Found "+apacheAddress.size()+" services");
+		AnalysisLogger.getLogger().debug("Apache address: "+apacheServiceAddress);
+		List<String> dataTransferAddress = InfraRetrieval.retrieveService("agent-service", scope);
+		
+		if (dataTransferAddress.size()==0)
+			throw new Exception("Data Transfer services are not available in scope "+scope);
+		
+		AnalysisLogger.getLogger().debug("Found "+dataTransferAddress.size()+" transfer services");
+		
+		int apacheDTPort = 9090; 
+		boolean found = false;
+		for (String datatransferservice:dataTransferAddress){
+			AnalysisLogger.getLogger().debug("Transfer service found");
+			datatransferservice = datatransferservice.substring(datatransferservice.indexOf("http://")+7);
+			String servicehost = datatransferservice.substring(0,datatransferservice.indexOf(":"));
+			String serviceport = datatransferservice.substring(datatransferservice.indexOf(":")+1,datatransferservice.indexOf("/"));
+			AnalysisLogger.getLogger().debug("Transfer service: "+servicehost+":"+serviceport);
+			if (apacheServiceAddress.equals(servicehost)){
+				apacheDTPort = Integer.parseInt(serviceport);
+				found = true;
+				break;
+			}
+		}
+		
+		if (!found)
+			throw new Exception("Apache data transfer has not been found in the same scope of the catalog: "+scope);
+		else
+			AnalysisLogger.getLogger().debug("Transfer service found at address "+apacheServiceAddress+":"+apacheDTPort);
+		
+		transferServiceAddress = apacheServiceAddress;
+		transferServicePort = apacheDTPort;
+	}
+	
+}
+

src/main/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java

@@ -0,0 +1,45 @@
+package org.gcube.dataanalysis.executor.tests;
+
+import java.util.List;
+
+import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
+import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
+import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
+import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
+import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
+import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
+
+public class TestWebAppPublisher {
+
+	static String cfg = "./cfg/";
+
+	public static void main(String[] args) throws Exception {
+		AlgorithmConfiguration config = new AlgorithmConfiguration();
+
+		config.setConfigPath("./cfg/");
+		config.setPersistencePath("./");
+
+		config.setAgent("WEB_APP_PUBLISHER");
+
+		config.setGcubeScope("/gcube/devsec/devVRE");
+		
+		config.setParam("ServiceUserName","gianpaolo.coro");
+		config.setParam("MainPage", "index.html");
+		//config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\SitoUnirender2015.zip");
+		//config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\gcube.zip");
+//		config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/simplesite.zip");
+		config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/verysimple.zip");
+		AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
+
+		AnalysisLogger.getLogger().debug("Executing: " + config.getAgent());
+		List<ComputationalAgent> trans = null;
+		trans = TransducerersFactory.getTransducerers(config);
+		trans.get(0).init();
+		Regressor.process(trans.get(0));
+		StatisticalType st = trans.get(0).getOutput();
+		AnalysisLogger.getLogger().debug("ST:" + st);
+		trans = null;
+
+	}
+
+}

src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java

@@ -4,11 +4,18 @@ import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent
 
 import java.io.File;
 import java.net.URI;
+import java.net.URL;
+import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.concurrent.TimeUnit;
 
 import org.gcube.common.scope.api.ScopeProvider;
+import org.gcube.contentmanagement.blobstorage.service.IClient;
 import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
+import org.gcube.contentmanager.storageclient.wrapper.AccessType;
+import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
+import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
+import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
 import org.gcube.datatransfer.agent.library.AgentLibrary;
 import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
 import org.gcube.datatransfer.common.agent.Types.storageType;
@@ -19,9 +26,12 @@ import org.gcube.datatransfer.common.outcome.TransferStatus;
 public class DataTransferer {
 
 	public static void main(String[] args) throws Exception {
-		String scope = "/d4science.research-infrastructures.eu/gCubeApps";
+		//String scope = "/d4science.research-infrastructures.eu/gCubeApps";
+		
+		String scope = "/gcube/devsec/devVRE";
 		ScopeProvider.instance.set(scope);
-		String transferGHN = "dewn04.madgik.di.uoa.gr";
+		//String transferGHN = "dewn04.madgik.di.uoa.gr";
+		String transferGHN = "access.d4science.org";
 		int transferPort = 8080;
 		AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
 
@@ -34,7 +44,8 @@ public class DataTransferer {
 		 * String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
 		 */
 		// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
-		String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
+		//String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
+		String urlStorage ="http://goo.gl/Vq8QVY";
 
 		System.out.println("URL for storage: " + urlStorage);
 
@@ -47,9 +58,14 @@ public class DataTransferer {
 		// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
 		input.add(uri);
 
-		String outPath = "/tmp";
+		//String outPath = "/tmp";
-
+		String outPath = "/var/www/html/test/";
-		transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort, "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg", outPath);
+		String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
+		//fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
+//		fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png";
+		transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath);
+		
+		
 	}
 
 	// returns the number of transferred bytes
@@ -66,7 +82,10 @@ public class DataTransferer {
 		String localfolder = localFile.getParent();
 		String file = localFile.getName();
 		AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
-		String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, file);
+		ScopeProvider.instance.set(scope);
+		AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
+		
+		String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);
 		//urls for testing
 		//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
 		//	String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
@@ -91,7 +110,8 @@ public class DataTransferer {
 		
 		
 		ArrayList<URI> outputURI = new ArrayList<URI>();
-		outputURI.add(new URI("file://"+remoteFolder+""+file));
+//		outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_")));
+		outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20")));
 		
 		AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
 		

src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java

@@ -58,7 +58,7 @@ public class StorageUtils {
 			ScopeProvider.instance.set(scope);
 			AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
 			IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient();
-			String remotef = remoteFolder+file;
+			String remotef = remoteFolder+file.replace(" ","%20");
 			client.put(true).LFile(new File(localFolder,file).getAbsolutePath()).RFile(remotef);
 			String url = "";
 			if (httplink)