diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java new file mode 100644 index 0000000..dd915fe --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java @@ -0,0 +1,196 @@ +package org.gcube.dataanalysis.executor.nodes.transducers; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.commons.io.FileUtils; +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm; +import org.gcube.dataanalysis.ecoengine.utils.ZipTools; +import org.gcube.dataanalysis.executor.util.DataTransferer; +import org.gcube.dataanalysis.executor.util.InfraRetrieval; + + +public class WebApplicationPublisher extends StandardLocalExternalAlgorithm{ +// private static String MainPageParam = "MainPage"; + private static String FileParam = "ZipFile"; + private String transferServiceAddress = ""; + private int transferServicePort = 0; + + @Override + public String getDescription() { + return "This algorithm publishes a zip file containing a Web site, based on html and javascript in the e-Infrastructure. It generates a public URL to the application that can be shared."; + } + + @Override + public void init() throws Exception { + } + + @Override + protected void process() throws Exception { + String uuid = "webpub_" + UUID.randomUUID(); + File folder = new File(config.getConfigPath(), uuid); + try + { + status = 10; + String scope = config.getGcubeScope(); + String username = config.getParam("ServiceUserName"); + String fileAbsolutePath = config.getParam(FileParam); +// String mainPage = config.getParam(MainPageParam); + + AnalysisLogger.getLogger().debug("scope: "+scope); + AnalysisLogger.getLogger().debug("username: "+username); + AnalysisLogger.getLogger().debug("fileAbsolutePath: "+fileAbsolutePath); +// AnalysisLogger.getLogger().debug("layerTitle: "+mainPage); + + if (scope==null || username==null) + throw new Exception ("Service parameters are not set - please contact the Administrators"); + if (fileAbsolutePath==null || fileAbsolutePath.trim().length()==0) + throw new Exception ("No file has been provided to the process"); +// if (mainPage==null || mainPage.trim().length()==0) +// throw new Exception ("Please provide a valid main page"); + + File f = new File(fileAbsolutePath); + String fileName = f.getName(); + + //unzip the file + AnalysisLogger.getLogger().debug("Package is in file "+fileName); + + boolean mkdir = folder.mkdir(); + AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir); + AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath()); + ZipTools.unZip(f.getAbsolutePath(), folder.getAbsolutePath()); +// f.delete(); + AnalysisLogger.getLogger().debug("Package unzipped and original file deleted"); + + File[] webappfiles = folder.listFiles(); + //get all files for the upload + String prefix = "/"+folder.getName()+"/"; + + Map allfiles = getFilesPaths(webappfiles,prefix); + + //discover the dataTransfer service with Apache + getTransferInfo(config.getGcubeScope()); + String remoteFolder = "/var/www/html"; + //upload every file + int nfiles = allfiles.size(); + float step = 80f/(float)nfiles; + float initialStatus=status; + int i=0; + for (String key:allfiles.keySet()){ + status=(float)MathFunctions.roundDecimal(initialStatus+(float)i*step,2); + String subpath = allfiles.get(key); + subpath = subpath.substring(0,subpath.lastIndexOf("/")); + String remotePath = remoteFolder+subpath+"/"; + AnalysisLogger.getLogger().debug("Uploading "+key+" -> "+remotePath); + + boolean transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath); + if (!transferout){ + throw new Exception("Error transferring files to the infrastructure "); + } + i++; + } + String producedPage = "http://"+transferServiceAddress+"/"+uuid+"/"; + if (webappfiles.length==1 && webappfiles[0].isDirectory()){ + producedPage = producedPage + webappfiles[0].getName()+"/"; + } + AnalysisLogger.getLogger().debug("Entry point of the page "+producedPage); + + //get URL + addOutputString("Generated Website - Main URL", producedPage); + }catch(Exception e){ + e.printStackTrace(); + AnalysisLogger.getLogger().debug(e); + AnalysisLogger.getLogger().debug("An error occurred!"); + throw e; + }finally{ + //clean everything + if (folder.exists()){ + AnalysisLogger.getLogger().debug("Cleaning folder "+folder); + FileUtils.cleanDirectory(folder); + FileUtils.deleteDirectory(folder); + } + } + AnalysisLogger.getLogger().debug("Process finished"); + status = 100; + } + + private Map getFilesPaths(File[] toexplore, String prefix){ + Map toreturn = new HashMap (); + for (File toex:toexplore){ + if (toex.isDirectory()){ + toreturn.putAll(getFilesPaths(toex.listFiles(),prefix+toex.getName()+"/")); + } + else + toreturn.put(toex.getAbsolutePath(),prefix+toex.getName()); + } + + return toreturn; + } + @Override + protected void setInputParameters() { + + inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, FileParam, "Zip file containing the Web site")); + inputs.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name")); +// addStringInput(MainPageParam, "Main page of the website", "index.html"); + } + + @Override + public void shutdown() { + AnalysisLogger.getLogger().debug("WebApplicationPublisher - shutdown"); + } + + + public void getTransferInfo(String scope) throws Exception{ + + List apacheAddress = InfraRetrieval.retrieveServiceAddress("Application", "Apache Server", scope, "Transect"); + + if (apacheAddress.size()==0) + throw new Exception("Apache Server resource is not available in scope "+scope); + + String apacheServiceAddress = apacheAddress.get(0); + apacheServiceAddress = apacheServiceAddress.substring(apacheServiceAddress.indexOf("http://")+7); + + AnalysisLogger.getLogger().debug("Found "+apacheAddress.size()+" services"); + AnalysisLogger.getLogger().debug("Apache address: "+apacheServiceAddress); + List dataTransferAddress = InfraRetrieval.retrieveService("agent-service", scope); + + if (dataTransferAddress.size()==0) + throw new Exception("Data Transfer services are not available in scope "+scope); + + AnalysisLogger.getLogger().debug("Found "+dataTransferAddress.size()+" transfer services"); + + int apacheDTPort = 9090; + boolean found = false; + for (String datatransferservice:dataTransferAddress){ + AnalysisLogger.getLogger().debug("Transfer service found"); + datatransferservice = datatransferservice.substring(datatransferservice.indexOf("http://")+7); + String servicehost = datatransferservice.substring(0,datatransferservice.indexOf(":")); + String serviceport = datatransferservice.substring(datatransferservice.indexOf(":")+1,datatransferservice.indexOf("/")); + AnalysisLogger.getLogger().debug("Transfer service: "+servicehost+":"+serviceport); + if (apacheServiceAddress.equals(servicehost)){ + apacheDTPort = Integer.parseInt(serviceport); + found = true; + break; + } + } + + if (!found) + throw new Exception("Apache data transfer has not been found in the same scope of the catalog: "+scope); + else + AnalysisLogger.getLogger().debug("Transfer service found at address "+apacheServiceAddress+":"+apacheDTPort); + + transferServiceAddress = apacheServiceAddress; + transferServicePort = apacheDTPort; + } + +} + diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java new file mode 100644 index 0000000..d01d177 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java @@ -0,0 +1,45 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class TestWebAppPublisher { + + static String cfg = "./cfg/"; + + public static void main(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + + config.setAgent("WEB_APP_PUBLISHER"); + + config.setGcubeScope("/gcube/devsec/devVRE"); + + config.setParam("ServiceUserName","gianpaolo.coro"); + config.setParam("MainPage", "index.html"); + //config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\SitoUnirender2015.zip"); + //config.setParam("ZipFile", "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\WEB_APP_PUBLISHER\\gcube.zip"); +// config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/simplesite.zip"); + config.setParam("ZipFile", "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/verysimple.zip"); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + AnalysisLogger.getLogger().debug("Executing: " + config.getAgent()); + List trans = null; + trans = TransducerersFactory.getTransducerers(config); + trans.get(0).init(); + Regressor.process(trans.get(0)); + StatisticalType st = trans.get(0).getOutput(); + AnalysisLogger.getLogger().debug("ST:" + st); + trans = null; + + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java index 212caab..69235d7 100644 --- a/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java +++ b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java @@ -4,11 +4,18 @@ import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent import java.io.File; import java.net.URI; +import java.net.URL; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.datatransfer.agent.library.AgentLibrary; import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException; import org.gcube.datatransfer.common.agent.Types.storageType; @@ -19,9 +26,12 @@ import org.gcube.datatransfer.common.outcome.TransferStatus; public class DataTransferer { public static void main(String[] args) throws Exception { - String scope = "/d4science.research-infrastructures.eu/gCubeApps"; + //String scope = "/d4science.research-infrastructures.eu/gCubeApps"; + + String scope = "/gcube/devsec/devVRE"; ScopeProvider.instance.set(scope); - String transferGHN = "dewn04.madgik.di.uoa.gr"; + //String transferGHN = "dewn04.madgik.di.uoa.gr"; + String transferGHN = "access.d4science.org"; int transferPort = 8080; AgentLibrary library = transferAgent().at(transferGHN, transferPort).build(); @@ -34,7 +44,8 @@ public class DataTransferer { * String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file; */ // String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif"; - String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc="; + //String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc="; + String urlStorage ="http://goo.gl/Vq8QVY"; System.out.println("URL for storage: " + urlStorage); @@ -47,9 +58,14 @@ public class DataTransferer { // URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif"); input.add(uri); - String outPath = "/tmp"; - - transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort, "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg", outPath); + //String outPath = "/tmp"; + String outPath = "/var/www/html/test/"; + String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg"; + //fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg"; +// fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png"; + transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath); + + } // returns the number of transferred bytes @@ -66,7 +82,10 @@ public class DataTransferer { String localfolder = localFile.getParent(); String file = localFile.getName(); AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage"); - String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, file); + ScopeProvider.instance.set(scope); + AnalysisLogger.getLogger().info("Loading file on scope: " + scope); + + String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true); //urls for testing //storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file; // String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc="; @@ -91,7 +110,8 @@ public class DataTransferer { ArrayList outputURI = new ArrayList(); - outputURI.add(new URI("file://"+remoteFolder+""+file)); +// outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_"))); + outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20"))); AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0)); diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java b/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java index 632fc5a..7c2139c 100644 --- a/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java +++ b/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java @@ -58,7 +58,7 @@ public class StorageUtils { ScopeProvider.instance.set(scope); AnalysisLogger.getLogger().info("Loading file on scope: " + scope); IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); - String remotef = remoteFolder+file; + String remotef = remoteFolder+file.replace(" ","%20"); client.put(true).LFile(new File(localFolder,file).getAbsolutePath()).RFile(remotef); String url = ""; if (httplink)