From 27a8136cdd9a590518cd666a0bf601ed6c6b9a05 Mon Sep 17 00:00:00 2001
From: Lucio Lelii <lucio.lelii@isti.cnr.it>
Date: Tue, 19 Sep 2017 15:19:03 +0000
Subject: [PATCH] git-svn-id:
 https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/wps@153272
 82a268e6-3cf1-43bd-a215-b396298e98cf

---
 distro/web.xml                                |  16 ++-
 pom.xml                                       |   2 +-
 .../analysis/wps/GetCapabilitiesBuilder.java  |   6 +-
 .../wps/filter/ClassLoaderFilter.java         |  34 +++++
 .../wps/repository/AlgorithmUpdater.java      | 130 ++++++++++++++----
 .../repository/GcubeAlgorithmRepository.java  |   7 +-
 src/main/webapp/WEB-INF/README                |   2 +-
 src/main/webapp/WEB-INF/web.xml               |  16 ++-
 8 files changed, 177 insertions(+), 36 deletions(-)
 create mode 100644 src/main/java/org/gcube/data/analysis/wps/filter/ClassLoaderFilter.java

diff --git a/distro/web.xml b/distro/web.xml
index e7d0403..3f174c5 100644
--- a/distro/web.xml
+++ b/distro/web.xml
@@ -84,6 +84,15 @@
 	</context-param>
 
 	<filter>
+		<filter-name>ClassLoaderFilter</filter-name>
+		<filter-class>org.gcube.data.analysis.wps.filter.ClassLoaderFilter</filter-class>
+	</filter>
+
+	
+
+
+	<filter>
+
 		<filter-name>CORS</filter-name>
 		<filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
 		<init-param>
@@ -111,7 +120,12 @@
 		<filter-name>CORS</filter-name>
 		<url-pattern>/*</url-pattern>
 	</filter-mapping>
-
+	
+	<filter-mapping>
+		<filter-name>ClassLoaderFilter</filter-name>
+		<url-pattern>/*</url-pattern>
+	</filter-mapping>
+	
 	<!-- <filter> <filter-name>authn</filter-name> -->
 	<!-- display-name>Authentication Chain Filter</display-name -->
 	<!-- description> Delegates calls to AuthenticationChainFilter that is defined 
diff --git a/pom.xml b/pom.xml
index 621808a..36ccb8c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.gcube.data-analysis</groupId>
 	<artifactId>wps</artifactId>
-	<version>1.0.0-SNAPSHOT</version>
+	<version>1.1.0-SNAPSHOT</version>
 	<name>WPS</name>
 
 	<parent>
diff --git a/src/main/java/org/gcube/data/analysis/wps/GetCapabilitiesBuilder.java b/src/main/java/org/gcube/data/analysis/wps/GetCapabilitiesBuilder.java
index 8fb2875..5c6ee40 100644
--- a/src/main/java/org/gcube/data/analysis/wps/GetCapabilitiesBuilder.java
+++ b/src/main/java/org/gcube/data/analysis/wps/GetCapabilitiesBuilder.java
@@ -2,6 +2,7 @@ package org.gcube.data.analysis.wps;
 
 import java.io.InputStream;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -112,7 +113,10 @@ public class GetCapabilitiesBuilder {
 		LOGGER.debug("Found {}  algorithms in scope {} ",algorithmsInScope.size() ,ScopeProvider.instance.get());
 		StringBuffer capabilities = new StringBuffer();
 		
-		for (String algorithmInScope : algorithmsInScope) {
+		//TO eliminate duplicate coming from IS
+		Set<String> algorithmsSet = new HashSet<String>(algorithmsInScope);
+		
+		for (String algorithmInScope : algorithmsSet ) {
 			String classAlgorithm = allalgorithms.get(algorithmInScope);
 			if (classAlgorithm != null) {
 				LOGGER.debug("Approving " + classAlgorithm + " to capabilities ");
diff --git a/src/main/java/org/gcube/data/analysis/wps/filter/ClassLoaderFilter.java b/src/main/java/org/gcube/data/analysis/wps/filter/ClassLoaderFilter.java
new file mode 100644
index 0000000..6608c22
--- /dev/null
+++ b/src/main/java/org/gcube/data/analysis/wps/filter/ClassLoaderFilter.java
@@ -0,0 +1,34 @@
+package org.gcube.data.analysis.wps.filter;
+
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import org.gcube.data.analysis.wps.repository.GcubeAlgorithmRepository;
+
+public class ClassLoaderFilter implements Filter{
+
+
+		@Override
+		public void init(FilterConfig filterConfig) throws ServletException {
+
+		}
+
+		@Override
+		public void doFilter(ServletRequest request, ServletResponse response,
+				FilterChain chain) throws IOException, ServletException {
+			Thread.currentThread().setContextClassLoader(GcubeAlgorithmRepository.getUpdater().getLoader());
+			chain.doFilter(request, response);
+		}
+
+		@Override
+		public void destroy() {
+			// TODO Auto-generated method stub
+			
+		}
+	
+}
diff --git a/src/main/java/org/gcube/data/analysis/wps/repository/AlgorithmUpdater.java b/src/main/java/org/gcube/data/analysis/wps/repository/AlgorithmUpdater.java
index d48c2c8..d709168 100644
--- a/src/main/java/org/gcube/data/analysis/wps/repository/AlgorithmUpdater.java
+++ b/src/main/java/org/gcube/data/analysis/wps/repository/AlgorithmUpdater.java
@@ -1,19 +1,23 @@
 package org.gcube.data.analysis.wps.repository;
 
 import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
-import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
 import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
 
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileSystems;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.WatchEvent;
 import java.nio.file.WatchKey;
 import java.nio.file.WatchService;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,13 +25,13 @@ import org.slf4j.LoggerFactory;
 public final class AlgorithmUpdater {
 
 	private static final Logger log = LoggerFactory.getLogger(AlgorithmUpdater.class);
-	
+
 	private String algorithmDirectory;
-	
+
 	private boolean mustUpdate = false;
-		
+
 	private WatcherThread watcherThread = null;
-	
+
 	public AlgorithmUpdater(String algorithmDirectory) {
 		super();
 		this.algorithmDirectory = algorithmDirectory;
@@ -36,43 +40,62 @@ public final class AlgorithmUpdater {
 	protected synchronized boolean mustUpdate(){
 		return mustUpdate;
 	}
-	
+
 	protected synchronized void reset(){
 		mustUpdate = false;
 	}
-	
+
 	public boolean isStarted(){
 		return watcherThread!=null;
 	}
-	
-	protected void startWhatcher(){
+
+	protected void init(){
 		watcherThread = new WatcherThread(Thread.currentThread().getContextClassLoader(), algorithmDirectory);
+	}
+
+	protected void startWhatcher(){
 		watcherThread.start();
 	}
-	
-	protected void init(){
-		
+
+
+	public ClassLoader getLoader() {
+		return this.watcherThread.getLoader();
 	}
-	
+
+
 	private class WatcherThread extends Thread {
 
+		/*private Map<String, Long> justCreated = new WeakHashMap<String, Long>();
+		private static final long ENTRY_MAX_TIME = 300000;*/
 		private WatchService watcher;
 		private ClassLoader loader;
+		private ClassLoader parentLoader;
 		private Path dir;
-		
-		public WatcherThread(ClassLoader loader, String algorithmDirectory) {
+		private List<String> installedURLS;
+
+		public WatcherThread(ClassLoader parentLoader, String algorithmDirectory) {
 			super();
 			try {
+				log.debug("Watcher Thread created");
 				watcher = FileSystems.getDefault().newWatchService();
-				this.loader = loader;
-				//TODO: change with something from configuration
+				this.parentLoader = parentLoader;
+				log.debug("parent class loader is {}", parentLoader.getClass().getSimpleName());
 				dir = Paths.get(algorithmDirectory);
-				dir.register(watcher, ENTRY_CREATE, ENTRY_MODIFY);
+				installedURLS = updateClassLoader();
+				dir.register(watcher, ENTRY_CREATE);
 			} catch (IOException e) {
 				throw new RuntimeException(e);
 			}
 		}
 
+
+		protected ClassLoader getLoader() {
+			return loader;
+		}
+
+
+
+
 		@SuppressWarnings("unchecked")
 		public void run(){
 			for (;;) {
@@ -88,7 +111,7 @@ public final class AlgorithmUpdater {
 
 				for (WatchEvent<?> event: key.pollEvents()) {
 					WatchEvent.Kind<?> kind = event.kind();
-					log.trace("new event thrown for directory watcher");
+
 					// This key is registered only
 					// for ENTRY_CREATE events,
 					// but an OVERFLOW event can
@@ -102,21 +125,41 @@ public final class AlgorithmUpdater {
 					// context of the event.
 					WatchEvent<Path> ev = (WatchEvent<Path>)event;
 					Path filename = ev.context();
-					
-					if (filename.toString().endsWith(".jar")){
-						log.debug("found filename {} ",filename.toString());
+
+					log.trace("new event thrown for directory watcher with filename {} and kind {}", filename, kind);
+
+					if (filename.toString().endsWith("_interface.jar") ){
 						try{
-							final Class<URLClassLoader> sysclass = URLClassLoader.class;
-							// TODO some kind of a hack. Need to invent better solution.
-							final Method method = sysclass.getDeclaredMethod("addURL", new Class[] { URL.class });
-							method.setAccessible(true);
-							method.invoke(loader, new URL[] { dir.resolve(filename).toFile().toURI().toURL() });
-							log.info("filename added is {} in loader {}",filename, loader.getClass().getName());
+							if (installedURLS.contains(filename.getFileName().toString())){
+								log.debug("modifying an already installed algorithm");
+								installedURLS = updateClassLoader();
+							} else {
+								log.debug("installing new algorithm");
+								final Class<URLClassLoader> sysclass = URLClassLoader.class;
+								// TODO some kind of a hack. Need to invent better solution.
+								final Method method = sysclass.getDeclaredMethod("addURL", new Class[] { URL.class });
+								method.setAccessible(true);
+								URL realjarURL = null;
+								URL[] jarURLS ;
+								try{
+									realjarURL = dir.resolve(filename.toString().replaceFirst("_interface", "")).getFileName().toUri().toURL();
+									jarURLS = new URL[] {realjarURL, 
+											dir.resolve(filename).toFile().toURI().toURL() };
+									log.debug("found {} ",realjarURL);
+								}catch(Exception ipe){
+									jarURLS = new URL[] {dir.resolve(filename).toFile().toURI().toURL() };
+									log.warn("only {} have been found",filename);
+								}
+								method.invoke(loader, jarURLS );
+								log.info("filename added in loader {}",filename, loader.getClass().getName());
+								installedURLS.add(filename.getFileName().toString());
+							}
 							mustUpdate = true;
 						}catch(Exception e){
 							log.error("filename {} cannot be added to classpath",e,filename);
 						}
-					} else log.info("filename {} is not a jar",filename);	
+
+					} else log.info("filename {} is not an algorithm interface",filename);	
 
 				}
 
@@ -126,8 +169,35 @@ public final class AlgorithmUpdater {
 				}
 			}
 		}
+
+		private List<String> updateClassLoader(){
+			log.debug("getting the stream from directoy {}",dir.getFileName());
+			List<URL> urls = new ArrayList<URL>();
+			List<String> toReturn = new ArrayList<String>(urls.size());
+
+			DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
+				public boolean accept(Path file) throws IOException {
+					return (file.getFileName().toString().endsWith(".jar"));
+				}
+			}; 
+
+			try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, filter )){
+				for (Path file: stream) {
+					log.debug("loading url {}",file.getFileName());
+					urls.add(file.toUri().toURL());
+					toReturn.add(file.getFileName().toString());
+				}
+			} catch (IOException | DirectoryIteratorException x) {
+				log.error("error reading config dir",x);
+			}				
+			this.loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), parentLoader);
+			log.debug("loader object is {}", loader);
+			return toReturn;
+		}
 	}
-	
+
+
+
 	protected void shutdown(){
 		if (isStarted()){
 			//TODO : kill the watcherThread
diff --git a/src/main/java/org/gcube/data/analysis/wps/repository/GcubeAlgorithmRepository.java b/src/main/java/org/gcube/data/analysis/wps/repository/GcubeAlgorithmRepository.java
index a4460fa..4473c84 100644
--- a/src/main/java/org/gcube/data/analysis/wps/repository/GcubeAlgorithmRepository.java
+++ b/src/main/java/org/gcube/data/analysis/wps/repository/GcubeAlgorithmRepository.java
@@ -29,6 +29,7 @@ public class GcubeAlgorithmRepository implements IAlgorithmRepository {
 	public GcubeAlgorithmRepository(){
 		log.info("gcube algorithm repository started");
 		if (updater==null) throw new RuntimeException("GcubeAlgorithmRepository cannot be initialized: updater is null");
+		updater.init();
 		updateRepository();
 		updater.startWhatcher();
 	}
@@ -85,11 +86,12 @@ public class GcubeAlgorithmRepository implements IAlgorithmRepository {
 
 	private static synchronized void updateRepository(){
 		if (reflection==null || updater.mustUpdate()){
+			Thread.currentThread().setContextClassLoader(updater.getLoader());
 			log.info("updating repository ({}) ",updater.mustUpdate());
 			updater.reset();
 			ConfigurationBuilder confBuilder = new ConfigurationBuilder()
 			.filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(PACKAGE_TO_FIND)))
-			.setUrls(((URLClassLoader)Thread.currentThread().getContextClassLoader()).getURLs());
+			.setUrls(((URLClassLoader)updater.getLoader()).getURLs());
 			reflection = new Reflections(confBuilder);
 		}
 	}
@@ -117,4 +119,7 @@ public class GcubeAlgorithmRepository implements IAlgorithmRepository {
 		} else return false;
 	}
 	
+	public static AlgorithmUpdater getUpdater(){
+		return updater;
+	}
 }
diff --git a/src/main/webapp/WEB-INF/README b/src/main/webapp/WEB-INF/README
index 61fbb2e..2a4fbbe 100644
--- a/src/main/webapp/WEB-INF/README
+++ b/src/main/webapp/WEB-INF/README
@@ -25,7 +25,7 @@ no. 654119), SoBigData (grant no. 654024);
 Version
 --------------------------------------------------
  
-1.0.0-SNAPSHOT (${maven.build.timestamp})
+1.0.0-SNAPSHOT (2017-09-19)
  
 Please see the file named "changelog.xml" in this directory for the release notes.
  
diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml
index e7d0403..3f174c5 100644
--- a/src/main/webapp/WEB-INF/web.xml
+++ b/src/main/webapp/WEB-INF/web.xml
@@ -84,6 +84,15 @@
 	</context-param>
 
 	<filter>
+		<filter-name>ClassLoaderFilter</filter-name>
+		<filter-class>org.gcube.data.analysis.wps.filter.ClassLoaderFilter</filter-class>
+	</filter>
+
+	
+
+
+	<filter>
+
 		<filter-name>CORS</filter-name>
 		<filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
 		<init-param>
@@ -111,7 +120,12 @@
 		<filter-name>CORS</filter-name>
 		<url-pattern>/*</url-pattern>
 	</filter-mapping>
-
+	
+	<filter-mapping>
+		<filter-name>ClassLoaderFilter</filter-name>
+		<url-pattern>/*</url-pattern>
+	</filter-mapping>
+	
 	<!-- <filter> <filter-name>authn</filter-name> -->
 	<!-- display-name>Authentication Chain Filter</display-name -->
 	<!-- description> Delegates calls to AuthenticationChainFilter that is defined