git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@134306 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java

@@ -0,0 +1,107 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
+
+/**
+ * This class is responsible for the interface with ansible, retrieving log,
+ * etc. etc. It's not supposed to access templates and static stuff files. It
+ * does not know the service datamodel.
+ * 
+ * @author paolo
+ * 
+ */
+public class AnsibleWorker {
+  
+  /**
+   * The name of the inventory 
+   */
+  private static String INVENTORY_NAME = "inventory.yaml";
+  
+  /**
+   * The directory containing roles
+   */
+  private static String ROLES_DIR = "roles";
+
+  /**
+   * The name of the playbook
+   */
+  private static String PLAYBOOK_NAME = "playbook.yaml";
+
+  /**
+   * The root of the worker. This corresponds to a standard ansible working dir.
+   */
+  private File workerRoot;
+
+  public AnsibleWorker(File root) {
+    this.workerRoot = root;
+    this.ensureWorkStructure();
+  }
+
+  private File getWorkdir() {
+    return this.workerRoot;
+  }
+
+  private File getRolesDir() {
+    return new File(this.getWorkdir(), ROLES_DIR);
+  }
+
+  public String getWorkerId() {
+    return this.workerRoot.getName();
+  }
+
+  private void ensureWorkStructure() {
+    // generate root
+    this.getWorkdir().mkdirs();
+  }
+
+  private void removeWorkStructure() {
+    // remove the working dir
+//    this.getWorkdir().delete();
+  }
+
+  public File getPlaybookFile() {
+    return new File(this.getWorkdir(), PLAYBOOK_NAME);
+  }
+
+  public File getInventoryFile() {
+    return new File(this.getWorkdir(), INVENTORY_NAME);
+  }
+
+
+  public void setInventory(Inventory inventory) throws IOException {
+    // serialize the string to the 'inventory' file
+    AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile());
+  }
+
+  public void setPlaybook(Playbook playbook) throws IOException {
+    // serialize the string to the 'playbook' file
+    AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile());
+  }
+
+  public void addRole(Role r) throws IOException {
+    // Serialize role in the workdir
+    AnsibleSerializeHelper.serializeRole(r, this.getRolesDir());
+  }
+  
+  public void apply() {
+    // TODO execute the playbook and return output
+    System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
+  }
+  
+  /**
+   * Destroy the worker: 
+   * - remove the working dir
+   */
+  public void destroy() {
+    this.removeWorkStructure();
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java

@@ -0,0 +1,19 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+public class AnsibleHost {
+
+  private String name;
+
+  public AnsibleHost(String name) {
+    this.name = name;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java

@@ -0,0 +1,29 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class HostGroup {
+
+  private String name;
+
+  private Collection<AnsibleHost> hosts;
+
+  public HostGroup(String name) {
+    this.name = name;
+    this.hosts = new Vector<>();
+  }
+
+  public void addHost(AnsibleHost h) {
+    this.hosts.add(h);
+  }
+
+  public String getName() {
+    return this.name;
+  }
+
+  public Collection<AnsibleHost> getHosts() {
+    return new Vector<>(this.hosts);
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java

@@ -0,0 +1,37 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class Inventory {
+
+  private Collection<HostGroup> groups;
+
+  public Inventory() {
+    this.groups = new Vector<>();
+  }
+
+  public void addGroup(HostGroup group) {
+    this.groups.add(group);
+  }
+
+  public void addHost(AnsibleHost h, String groupName) {
+    this.getGroup(groupName).addHost(h);
+  }
+
+  private HostGroup getGroup(String groupName) {
+    for (HostGroup hg : this.groups) {
+      if (groupName.equals(hg.getName())) {
+        return hg;
+      }
+    }
+    HostGroup hg = new HostGroup(groupName);
+    this.groups.add(hg);
+    return hg;
+  }
+
+  public Collection<HostGroup> getHostGroups() {
+    return new Vector<>(this.groups);
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java

@@ -0,0 +1,32 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+import java.util.List;
+import java.util.Vector;
+
+public class Playbook {
+
+  private String hostGroupName;
+  
+  private List<String> roles;
+  
+  public Playbook() {
+    this.roles = new Vector<>();
+  }
+  
+  public void addRole(String role) {
+    roles.add(role);
+  }
+  
+  public void applyTo(String hostGroupName) {
+    this.hostGroupName = hostGroupName;
+  }
+
+  public String getHostGroupName() {
+    return hostGroupName;
+  }
+
+  public List<String> getRoles() {
+    return new Vector<>(roles);
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java

@@ -0,0 +1,51 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class Role {
+
+  /**
+   * The name of the role
+   */
+  private String name;
+  
+  private Collection<RoleFile> tasks;
+
+  private Collection<RoleFile> meta;
+  
+  public Role() {
+    this.tasks = new Vector<>();
+    this.meta = new Vector<>();
+  }
+
+  public Role(String name) {
+    this();
+    this.name = name;
+  }
+  
+  public void addTaskFile(RoleFile tf) {
+    this.tasks.add(tf);
+  }
+
+  public void addMeta(RoleFile tf) {
+    this.meta.add(tf);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+  
+  public Collection<RoleFile> getTaskFiles() {
+    return new Vector<>(this.tasks);
+  }
+  
+  public Collection<RoleFile> getMeta() {
+    return new Vector<>(this.meta);
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java

@@ -0,0 +1,54 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
+
+public class RoleFile {
+
+  /**
+   * The path to the file, starting from the role root
+   */
+  private String path;
+  
+  /**
+   * The name of the task file
+   */
+  private String name;
+  
+  /**
+   * The content of the task file
+   * @return
+   */
+  private String content;
+  
+  public RoleFile() {
+  }
+
+  public RoleFile(String name, String content) {
+    this();
+    this.setName(name);
+    this.setContent(content);
+  }
+  
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getContent() {
+    return content;
+  }
+
+  public void setContent(String content) {
+    this.content = content;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public void setPath(String path) {
+    this.path = path;
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java

@@ -0,0 +1,239 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.Vector;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.AlgorithmPackage;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CranDependencyPackage;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomDependencyPackage;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomRoleManager;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.DependencyPackage;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.OSDependencyPackage;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.StaticRoleManager;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.TemplateManager;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.AlgorithmComparator;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.DependencyComparator;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.HostComparator;
+
+public class AnsibleBridge {
+
+  /**
+   * The workdir for this service
+   */
+  private String dpmRoot;
+  
+  public AnsibleBridge() {
+    this("/home/paolo/tmp/dataminer-pool-manager");
+  }
+  
+  public AnsibleBridge(String root) {
+    this.dpmRoot = root;
+    this.ensureServiceRoot();
+  }
+  
+  private void ensureServiceRoot() {
+    // generate root
+    new File(dpmRoot).mkdirs();
+    // 'template' is for template roles
+    this.getTemplatesDir().mkdirs();
+    // 'static' is for custom roles
+    this.getCustomDir().mkdirs();
+    // 'work' is for temporary working directories
+    this.getWorkDir().mkdirs();    
+  }
+  
+  private File getWorkDir() {
+    return new File(this.dpmRoot, "work");
+  }
+  
+  private File getTemplatesDir() {
+    return new File(this.dpmRoot, "templates");
+  }
+  
+  private File getCustomDir() {
+    return new File(this.dpmRoot, "custom");
+  }
+
+  public AnsibleWorker createWorker() {
+    File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
+    AnsibleWorker worker = new AnsibleWorker(workerRoot);
+    return worker;
+  }
+
+  /**
+   * Groups hosts by domain and algorithm sets
+   * @param clusters
+   */
+  public void printInventoryByDomainAndSets(Collection<Cluster> clusters) {
+    Map<String, Set<Host>> inventory = new TreeMap<>();
+    for(Cluster cluster:clusters) {
+      for(AlgorithmSet as:cluster.getAlgorithmSets()) {
+        String asName = as.getName();
+        for(Host h:cluster.getHosts()) {
+          String domain = h.getDomain().getName();
+          String key = String.format("[%s@%s]", asName, domain);
+          Set<Host> hosts = inventory.get(key);
+          if(hosts==null) {
+            hosts = new TreeSet<>(new HostComparator());
+            inventory.put(key, hosts);
+          }
+          hosts.add(h);
+        }
+        
+      }
+    }
+    for(String key:inventory.keySet()) {
+      System.out.println(key);
+      Collection<Host> hosts = inventory.get(key);
+      for(Host h:hosts) {
+        System.out.println(h.getName()+"."+h.getDomain().getName());
+      }
+      System.out.println();
+    }
+  }
+
+  /**
+   * Groups hosts by algorithm sets only
+   * @param clusters
+   */
+  public void printInventoryBySets(Collection<Cluster> clusters) {
+    Map<String, Set<Host>> inventory = new TreeMap<>();
+    for (Cluster cluster : clusters) {
+      for (AlgorithmSet as : cluster.getAlgorithmSets()) {
+        String asName = as.getName();
+        for (Host h : cluster.getHosts()) {
+          String key = String.format("[%s]", asName);
+          Set<Host> hosts = inventory.get(key);
+          if (hosts == null) {
+            hosts = new TreeSet<>(new HostComparator());
+            inventory.put(key, hosts);
+          }
+          hosts.add(h);
+        }
+        
+      }
+    }
+    for (String key : inventory.keySet()) {
+      System.out.println(key);
+      Collection<Host> hosts = inventory.get(key);
+      for (Host h : hosts) {
+        System.out.println(h.getName()+"."+h.getDomain().getName());
+      }
+      System.out.println();
+    }
+  }
+  
+  public void applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException {
+    AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), UUID.randomUUID().toString()));
+
+    List<Role> algoRoles = new Vector<>();
+
+    // add algorithms and dependencies to the worker
+    for (Algorithm a : as.getAlgorithms()) {
+      for (Role r : this.generateRoles(a)) {
+        algoRoles.add(r);
+        worker.addRole(r);
+      }
+      for (Dependency d : a.getDependencies()) {
+        for (Role r : this.generateRoles(d)) {
+          worker.addRole(r);
+        }
+      }
+    }
+
+    // add static roles
+    for(Role r:this.getStaticRoleManager().getStaticRoles()) {
+      worker.addRole(r);
+    }
+    
+    // generate the inventory
+    Inventory inventory = new Inventory();
+    for (Host h : cluster.getHosts()) {
+      AnsibleHost ah = new AnsibleHost(h.getName());
+      inventory.addHost(ah, "universe");
+      inventory.addHost(ah, "d4science");
+    }
+    worker.setInventory(inventory);
+    
+    // generate the playbook
+    Playbook playbook = new Playbook();
+    playbook.applyTo("universe");
+    for(Role r:algoRoles) {
+      // add only 'add' roles
+      if(!r.getName().endsWith("remove")) {
+        playbook.addRole(r.getName());
+      }
+    }
+    
+    worker.setPlaybook(playbook);
+
+    // execute
+    worker.apply();
+    
+    // destroy the worker
+    worker.destroy();
+  
+  }
+  
+  private TemplateManager getTemplateManager() {
+    return new TemplateManager(this.dpmRoot+"/templates");
+  }
+
+  private CustomRoleManager getCustomRoleManager() {
+    return new CustomRoleManager(this.dpmRoot+"/custom");
+  }
+
+  private StaticRoleManager getStaticRoleManager() {
+    return new StaticRoleManager(this.dpmRoot+"/static");
+  }
+
+  /**
+   * Generate all roles for this dependency
+   * @param d
+   */
+  public Collection<Role> generateRoles(Dependency d) {
+    Collection<Role> roles = new Vector<>();
+    if("os".equals(d.getType())) {
+      OSDependencyPackage pkg = new OSDependencyPackage(d);
+      if(pkg!=null) {
+        roles.addAll(pkg.getRoles(this.getTemplateManager()));
+      }
+    } else if("custom".equals(d.getType())) {
+      CustomDependencyPackage pkg = new CustomDependencyPackage(d);
+      if(pkg!=null) {
+        roles.addAll(pkg.getRoles(this.getCustomRoleManager()));
+      }
+    } else if("cran".equals(d.getType())) {
+      CranDependencyPackage pkg = new CranDependencyPackage(d);
+      if(pkg!=null) {
+        roles.addAll(pkg.getRoles(this.getTemplateManager()));
+      }
+    }
+    return roles;
+  }
+
+  public Collection<Role> generateRoles(Algorithm a) {
+    AlgorithmPackage pkg = new AlgorithmPackage(a);
+    return pkg.getRoles(this.getTemplateManager());
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java

@@ -0,0 +1,117 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.commons.io.IOUtils;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.HostGroup;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+
+public class AnsibleSerializeHelper {
+  
+  public static void serialize(Inventory inventory, File inventoryFile) throws IOException {
+    String out = "";
+    for(HostGroup hg:inventory.getHostGroups()) {
+      out+=String.format("[%s]\n", hg.getName());
+      for(AnsibleHost h:hg.getHosts()) {
+        out+=h.getName()+"\n";
+      }
+      out+="\n";
+    }
+    out = out.trim();
+    serialize(out, inventoryFile);
+  }
+  
+  public static void serialize(Playbook playbook, File playbookFile) throws IOException {
+    String out = "- hosts: " + playbook.getHostGroupName() +  "\n";
+    out += "  remote_user: dpm\n";
+    out+="  roles:\n";
+    for(String r:playbook.getRoles()) {
+      out+="    - " + r+"\n";
+    }
+    out = out.trim();
+    serialize(out, playbookFile);
+  }
+
+  public static void serializeRole(Role r, File dir) throws IOException {
+    // create root
+    File root = new File(dir, r.getName());
+    root.mkdirs();
+    
+    // create tasks
+    if(r.getTaskFiles().size()>0) {
+      File tasks = new File(root, "tasks");
+      tasks.mkdirs();
+      for(RoleFile tf: r.getTaskFiles()) {
+        serializeTask(tf, tasks);
+      }
+    }
+    
+    // create meta
+    if(r.getMeta().size()>0) {
+      File meta = new File(root, "meta");
+      meta.mkdirs();
+      for(RoleFile tf: r.getMeta()) {
+        serializeTask(tf, meta);
+      }
+    }
+  }
+
+  public static void serializeTask(RoleFile tf, File dir) throws IOException {
+    File f = new File(dir, tf.getName());
+    serialize(tf.getContent().trim(), f);
+  }
+
+  public static void serialize(String s, File f) throws IOException {
+    PrintWriter out = new PrintWriter(f);
+    out.println(s);
+    out.close();
+  }
+  
+  public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException {
+    Role out = new Role();
+    out.setName(roleDir.getName());
+    
+    if(!roleDir.exists()) {
+      throw new FileNotFoundException();
+    }
+    
+    try {
+      File tasksDir = new File(roleDir, "tasks");
+      if(tasksDir.exists()) {
+        for(File main:tasksDir.listFiles()) {
+          String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
+          RoleFile tf = new RoleFile(main.getName(), content);
+          tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
+          out.addTaskFile(tf);
+        }
+      }
+    } catch(FileNotFoundException e) {
+      e.printStackTrace();
+    }
+    
+    try {
+      File metaDir = new File(roleDir, "meta");
+      if(metaDir.exists()) {
+        for(File main:metaDir.listFiles()) {
+          String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
+          RoleFile tf = new RoleFile(main.getName(), content);
+          tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
+          out.addMeta(tf);
+        }
+      }
+    } catch(FileNotFoundException e) {
+      e.printStackTrace();
+    }
+    
+    return out;
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java

@@ -0,0 +1,62 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Vector;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class AlgorithmPackage {
+
+  private Algorithm algorithm;
+
+  public AlgorithmPackage(Algorithm a) {
+    this.algorithm = a;
+  }
+
+  protected Map<String, String> getDictionary(Algorithm a) {
+    Map<String, String> out = new HashMap<String, String>();
+    out.put("name", a.getName());
+    String deps = "";
+    for(Dependency d:a.getDependencies()) {
+      deps+=String.format("- { role: %s }\n", d.getType()+"-"+d.getName());
+    }
+    deps = deps.trim();
+    out.put("dependencies", deps);
+    return out;
+  }
+  
+  protected Algorithm getAlgorithm() {
+    return this.algorithm;
+  }
+  
+  public Collection<Role> getRoles(TemplateManager tm) {
+    Collection<Role> out = new Vector<>();
+    for(String mode:new String[]{"add", "remove", "update"}) {
+      String roleName = "algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode);
+      try {
+        // find template
+        Role template = tm.getRoleTemplate("algorithm-" + mode);
+        // 
+        if(template!=null) {
+          Map<String, String> dictionary = this.getDictionary(this.getAlgorithm());
+          Role r = tm.fillRoleTemplate(template, dictionary);
+          r.setName(roleName);
+          out.add(r);
+        } else {
+          System.out.println("WARNING: template is null");
+        }
+      } catch (NoSuchElementException e) {
+//        e.printStackTrace();
+        System.out.println("WARNING: no template found for " + roleName);
+      }
+    }
+    return out;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java

@@ -0,0 +1,11 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class CranDependencyPackage extends DependencyPackage {
+
+  public CranDependencyPackage(Dependency d) {
+    super(d);
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java

@@ -0,0 +1,69 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Vector;
+
+import org.apache.commons.io.FileUtils;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class CustomDependencyPackage extends DependencyPackage {
+
+  public CustomDependencyPackage(Dependency d) {
+    super(d);
+  }
+
+  private String getCustomRepositoryLocation(String ansibleRoot) {
+    return ansibleRoot+"/custom";
+  }
+
+  /*
+  public void serializeTo(String ansibleRoot) {
+    for(String mode:new String[]{"add", "remove", "update"}) {
+      // look for roles in the 'custom' repository
+      try {
+        // role name
+        String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
+        // look for the custom role
+        File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName);
+        System.out.println("** CUSTOM ** " + src);
+        if(src.exists()) {
+          // do copy
+          System.out.println("copying CUSTOM role");
+          File dest = new File(ansibleRoot+"/work/"+roleName);
+          FileUtils.copyDirectory(src, dest);
+        }
+      } catch(IOException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+  */
+  
+  public Collection<Role> getRoles(CustomRoleManager crm) {
+    Collection<Role> out = new Vector<>();
+    for(String mode:new String[]{"add", "remove", "update"}) {
+      // role name
+      String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
+      try {
+        // look for custom role
+        Role role = crm.getRole(roleName);
+        if(role!=null) {
+          out.add(role);
+        }
+      } catch (NoSuchElementException e) {
+//        e.printStackTrace();
+        System.out.println("WARNING: no custom role found for " + roleName);
+      }
+    }
+    return out;
+  }
+
+  
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java

@@ -0,0 +1,37 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.NoSuchElementException;
+import java.util.Vector;
+
+import org.apache.commons.io.IOUtils;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
+
+public class CustomRoleManager {
+
+  private String root;
+
+  public CustomRoleManager(String root) {
+    this.root = root;
+  }
+
+  public String getRoot() {
+    return this.root;
+  }
+
+  public Role getRole(String roleName) throws NoSuchElementException {
+    File f = new File(this.getRoot(), roleName);
+    try {
+      return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
+    } catch (IOException e) {
+//      e.printStackTrace();
+      throw new NoSuchElementException("unable to find " + roleName);
+    }
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java

@@ -0,0 +1,56 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Vector;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class DependencyPackage {
+
+  private Dependency dependency;
+
+  public DependencyPackage(Dependency d) {
+    this.dependency = d;
+  }
+
+  protected Map<String, String> getDictionary(Dependency d) {
+    Map<String, String> out = new HashMap<String, String>();
+    out.put("name", d.getName());
+    out.put("type", d.getType());
+    return out;
+  }
+  
+  protected Dependency getDependency() {
+    return this.dependency;
+  }
+  
+  public Collection<Role> getRoles(TemplateManager tm) {
+    Collection<Role> out = new Vector<>();
+    for(String mode:new String[]{"add", "remove", "update"}) {
+      String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
+      try {
+        // find template
+        Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode);
+        // 
+        if(template!=null) {
+          Map<String, String> dictionary = this.getDictionary(this.getDependency());
+          Role r = tm.fillRoleTemplate(template, dictionary);
+          r.setName(roleName);
+          out.add(r);
+        } else {
+          System.out.println("WARNING: template is null");
+        }
+      } catch (NoSuchElementException e) {
+//        e.printStackTrace();
+        System.out.println("WARNING: no template found for " + roleName);
+      }
+    }
+    return out;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java

@@ -0,0 +1,11 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class OSDependencyPackage extends DependencyPackage {
+
+  public OSDependencyPackage(Dependency d) {
+    super(d);
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java

@@ -0,0 +1,37 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+  import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Vector;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
+
+  public class StaticRoleManager {
+
+    private String root;
+
+    public StaticRoleManager(String root) {
+      this.root = root;
+    }
+
+    public String getRoot() {
+      return this.root;
+    }
+
+    public Collection<Role> getStaticRoles() {
+      Collection<Role> out = new Vector<>();
+      for(File f: new File(this.getRoot()).listFiles()) {
+        try {
+          out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f));
+        } catch(IOException e) {
+          e.printStackTrace();
+        }
+      }
+      return out;
+    }
+    
+
+
+  }

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java

@@ -0,0 +1,99 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.commons.io.IOUtils;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+import org.stringtemplate.v4.ST;
+
+public class TemplateManager {
+
+  private String root;
+
+  public TemplateManager(String root) {
+    this.root = root;
+  }
+
+  public String getTemplateRoot() {
+    return this.root;
+  }
+
+  /**
+   * Read the given template
+   * @param templateName
+   * @return
+   * @throws IOException
+   */
+//  private String readTemplate(String templateName) throws IOException {
+//    File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml");
+//    System.out.println("looking for file " + templateFile.getName());
+//    String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8");
+//    return out;
+//  }
+
+  /**
+   * Return the content of the given template
+   * @param templateName
+   * @return
+   * @throws NoSuchElementException if no such template exists
+   */
+//  public String getTemplate(String templateName) throws NoSuchElementException {
+//    String template = null;
+//    try {
+//      template = this.readTemplate(templateName);
+//    } catch (IOException e) {
+//      throw new NoSuchElementException();
+//    }
+//    return template;
+//  }
+  
+  public Role fillRoleTemplate(Role template, Map<String, String> dictionary) {
+    Role out = new Role();
+    out.setName(template.getName());
+    for(RoleFile tf:template.getTaskFiles()) {
+      out.addTaskFile(this.fillTaskTemplate(tf, dictionary));
+    }
+    for(RoleFile tf:template.getMeta()) {
+      out.addMeta(this.fillTaskTemplate(tf, dictionary));
+    }
+    return out;
+  }
+
+  private RoleFile fillTaskTemplate(RoleFile template, Map<String, String> dictionary) {
+    RoleFile out = new RoleFile();
+    out.setName(template.getName());
+    out.setContent(this.fillTemplate(template.getContent(), dictionary));
+    return out;
+  }
+
+  private String fillTemplate(String template, Map<String, String> dictionary) {
+    if (template != null) {
+      ST t = new ST(template);
+      for (String key : dictionary.keySet()) {
+        t.add(key, dictionary.get(key));
+      }
+      String output = t.render();
+      return output;
+    }
+    return template;
+  }
+
+  public Role getRoleTemplate(String roleName) throws NoSuchElementException {
+    File f = new File(this.getTemplateRoot(), roleName);
+    try {
+      return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
+    } catch (IOException e) {
+//      e.printStackTrace();
+      throw new NoSuchElementException("unable to find " + roleName);
+    }
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java

@@ -0,0 +1,58 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.clients;
+
+import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
+import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Vector;
+
+import org.gcube.common.resources.gcore.ServiceEndpoint;
+import org.gcube.common.scope.api.ScopeProvider;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
+import org.gcube.resources.discovery.client.api.DiscoveryClient;
+import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
+
+public class ISClient {
+
+  /**
+   * Return the list of hosts (dataminers) in a given VRE
+   * 
+   * @param vreName
+   * @return
+   */
+  public Collection<Host> listDataminersInVRE(String scope) {
+
+    boolean remote = false;
+
+    if (!remote) {
+      Collection<Host> out = new Vector<>();
+      Host h = new Host();
+      h.setName("bb-dataminer.res.eng.it");
+      out.add(h);
+      return out;
+    } else {
+
+      ScopeProvider.instance.set(scope);
+
+      SimpleQuery query = queryFor(ServiceEndpoint.class);
+
+      query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
+          .addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
+
+      DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
+
+      List<ServiceEndpoint> resources = client.submit(query);
+
+      Collection<Host> out = new Vector<>();
+      for (ServiceEndpoint r : resources) {
+        Host h = new Host();
+        h.setName(r.profile().runtime().hostedOn());
+        out.add(h);
+      }
+      return out;
+    }
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java

@@ -0,0 +1,35 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+public class Action {
+
+  private String name;
+
+  private String description;
+
+  private String script;
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public String getScript() {
+    return script;
+  }
+
+  public void setScript(String script) {
+    this.script = script;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java

@@ -0,0 +1,71 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class Algorithm {
+
+  private String name;
+
+  private String description;
+
+  private String category;
+
+  private Collection<Action> actions;
+
+  private Collection<Dependency> dependencies;
+
+  public Algorithm() {
+    this.actions = new Vector<>();
+    this.dependencies = new Vector<>();
+    Dependency p = new Dependency();
+  }
+  
+  public void addDependency(Dependency dep) {
+    this.dependencies.add(dep);
+  }
+
+  public void addAction(Action action) {
+    this.actions.add(action);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public String getCategory() {
+    return category;
+  }
+
+  public void setCategory(String category) {
+    this.category = category;
+  }
+
+  public Collection<Action> getActions() {
+    return actions;
+  }
+
+  public Collection<Dependency> getDependencies() {
+    return dependencies;
+  }
+  
+  public String toString() {
+    String out = "Algorithm: " + this.getName()+"\n";
+    out+="  Description: " + this.getDescription()+"\n";
+    out+="  Dependencies: " + this.getDependencies()+"\n";
+    return out;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java

@@ -0,0 +1,49 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class AlgorithmSet {
+
+  private String name;
+
+  private Collection<Algorithm> algorithms;
+
+  public AlgorithmSet() {
+    this.algorithms = new Vector<>();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public Collection<Algorithm> getAlgorithms() {
+    return new Vector<>(algorithms);
+  }
+
+  public void addAlgorithm(Algorithm algoritm) {
+    this.algorithms.add(algoritm);
+  }
+
+  public Boolean hasAlgorithm(Algorithm algorithm) {
+    for (Algorithm a : this.algorithms) {
+      if (a.getName().equals(algorithm.getName())) {
+        return true;
+      }
+    }
+    return false;
+  }
+  
+  public String toString() {
+    String out = "ALGOSET: " + this.name + "\n";
+    for(Algorithm a:this.algorithms) {
+      out+=a+"\n";
+    }
+    return out;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java

@@ -0,0 +1,73 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+import java.util.Collection;
+import java.util.Vector;
+
+public class Cluster {
+
+  /**
+   * The set of hosts belonging to the cluster.
+   */
+  private Collection<Host> hosts;
+
+  /**
+   * A name for this cluster.
+   */
+  private String name;
+
+  /**
+   * A description of this cluster.
+   */
+  private String description;
+
+  /**
+   * The set of algorithms deployed on this cluster (i.e. on all its hosts)
+   */
+  private Collection<AlgorithmSet> algoSets;
+  
+  public Cluster() {
+    this.hosts = new Vector<>();
+    this.algoSets = new Vector<>();
+  }
+  
+  public void addAlgorithmSet(AlgorithmSet set) {
+    this.algoSets.add(set);
+  }
+
+  public void addHost(Host host) {
+    this.hosts.add(host);
+  }
+
+  public Collection<Host> getHosts() {
+    return hosts;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public Collection<AlgorithmSet> getAlgorithmSets() {
+    return algoSets;
+  }
+  
+  public String toString() {
+    String out = "Cluster: "+this.name+"\n";
+    for(Host h:this.getHosts()) {
+      out+="  "+h+"\n";
+    }
+    return out;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java

@@ -0,0 +1,29 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+public class Dependency {
+
+  private String name;
+
+  private String type;
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String toString() {
+    return this.type+":"+this.name;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java

@@ -0,0 +1,15 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+public class Domain {
+
+  private String name;
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+  
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java

@@ -0,0 +1,39 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
+
+public class Host {
+
+  private String name;
+  
+  private Domain domain;
+
+  public Host() {
+  }
+
+  public String getFullyQualifiedName() {
+    if(this.domain!=null && this.domain.getName()!=null)
+      return this.getName()+"."+this.getDomain().getName();
+    else
+      return this.getName();
+  }
+  
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public Domain getDomain() {
+    return domain;
+  }
+
+  public void setDomain(Domain domain) {
+    this.domain = domain;
+  }
+  
+  public String toString() {
+    return this.name + "@" + this.domain;
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java

@@ -0,0 +1,15 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
+
+import java.util.Comparator;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
+
+public class AlgorithmComparator implements Comparator<Algorithm> {
+
+  @Override
+  public int compare(Algorithm a1, Algorithm a2) {
+    return a1.getName().compareTo(a2.getName());
+  }
+  
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java

@@ -0,0 +1,19 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
+
+import java.util.Comparator;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+
+public class DependencyComparator implements Comparator<Dependency> {
+
+  @Override
+  public int compare(Dependency a1, Dependency a2) {
+    int out = a1.getType().compareTo(a2.getType());
+    if(out!=0)
+      return out;
+    return a1.getName().compareTo(a2.getName());
+  }
+  
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java

@@ -0,0 +1,17 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
+
+import java.util.Comparator;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
+
+public class HostComparator implements Comparator<Host> {
+
+  @Override
+  public int compare(Host h1, Host h2) {
+    int out = h1.getDomain().getName().compareTo(h2.getDomain().getName());
+    if(out!=0)
+      return out;
+    return h1.getName().compareTo(h2.getName());
+  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java

@@ -0,0 +1,121 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.service;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Vector;
+
+import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleBridge;
+import org.gcube.dataanalysys.dataminerpoolmanager.clients.ISClient;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
+import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
+
+public class DataminerPoolManager {
+
+//  static Collection<Algorithm> algorithms;
+//  
+//  static Collection<AlgorithmSet> sets;
+//
+//  static {
+//    algorithms = new Vector<>();
+//  }
+//
+//  public DataminerPoolManager() {
+//  }
+//
+//  /**
+//   * Add a new algorithm to the set of known ones. No further action is expected
+//   * on the pool.
+//   */
+//  public void publishAlgorithm(Algorithm algorithm) {
+//    algorithms.add(algorithm);
+//  }
+//
+//  /**
+//   * Re-deploy the given algorithm wherever it's installed
+//   * 
+//   * @param algorithm
+//   */
+//  /*
+//   * public void updateAlgorithm(Algorithm algorithm) { // TODO implement this }
+//   */
+//
+//  /**
+//   * Add the give algorithm to the given set
+//   * 
+//   * @param algorithmId
+//   * @param setId
+//   */
+//  public void addAlgorithmToSet(String algorithmName, String setName) {
+//    AlgorithmSet set = this.getAlgorithmSet(setName);
+//    Algorithm algorithm = this.getAlgorithm(algorithmName);
+//    if (set != null && algorithm != null) {
+//      set.addAlgorithm(algorithm);
+//      this.updateClusters();
+//    }
+//  }
+//
+//  /**
+//   * Apply the given set of algorithms to the given cluster
+//   * 
+//   * @param setId
+//   * @param clusterId
+//   */
+//  public void applyAlgorithmSetToCluster(String setName, String clusterName) {
+//    AlgorithmSet set = this.getAlgorithmSet(setName);
+//    Cluster cluster = new ISClient().getCluster(clusterName);
+//    if (set != null && cluster != null) {
+//      cluster.addAlgorithmSet(set);
+//      this.updateClusters();
+//    }
+//  }
+//
+//  private AlgorithmSet getAlgorithmSet(String name) {
+//    for (AlgorithmSet set : sets) {
+//      if (name.equals(set.getName())) {
+//        return set;
+//      }
+//    }
+//    return null;
+//  }
+//
+//  private Algorithm getAlgorithm(String name) {
+//    for (Algorithm a : algorithms) {
+//      if (name.equals(a.getName())) {
+//        return a;
+//      }
+//    }
+//    return null;
+//  }
+
+  /**
+   * Publish the given algorithm in the given VRE
+   * 
+   * @param algorithmName
+   * @param vre
+   */
+  public void addAlgorithmToVRE(Algorithm algorithm, String vre) throws IOException {
+    
+    // create a fake algorithm set
+    AlgorithmSet algoSet = new AlgorithmSet();
+    algoSet.setName("fake");
+    algoSet.addAlgorithm(algorithm);
+
+    // create the cluster (dataminers in the vre)
+    Cluster cluster = new Cluster();
+    for(Host h:new ISClient().listDataminersInVRE(vre)) {
+      cluster.addHost(h);
+    }
+
+    // apply the changes
+    new AnsibleBridge().applyAlgorithmSetToCluster(algoSet, cluster);
+
+  }
+
+//  private void updateClusters() {
+//    System.out.println("flushing changes to all clusters");
+//  }
+
+}

src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java

@@ -0,0 +1,145 @@
+package org.gcube.dataanalysys.dataminerpoolmanager.util;
+
+import java.io.IOException;
+import java.net.Authenticator;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
+import java.net.ProxySelector;
+import java.net.SocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
+
+interface NetworkConfiguration {
+
+  public String getProxyHost();
+
+  public String getProxyPort();
+
+  public String getProxyUser();
+
+  public String getProxyPassword();
+
+  public String getNonProxyHosts();
+
+}
+
+class FileBasedProxyConfiguration implements NetworkConfiguration {
+
+  private static PropertiesConfiguration configuration;
+
+  public FileBasedProxyConfiguration(String path) {
+    try {
+      // load the configuration
+      configuration = new PropertiesConfiguration(path);
+      // set the reloading strategy to enable hot-configuration
+      FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy();
+      configuration.setReloadingStrategy(fcrs);
+    } catch (ConfigurationException e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Override
+  public String getProxyHost() {
+    return configuration.getString("proxyHost");
+  }
+
+  @Override
+  public String getProxyPort() {
+    return configuration.getString("proxyPort");
+  }
+
+  @Override
+  public String getProxyUser() {
+    return configuration.getString("proxyUser");
+  }
+
+  @Override
+  public String getProxyPassword() {
+    return configuration.getString("proxyPassword");
+  }
+
+  @Override
+  public String getNonProxyHosts() {
+    return configuration.getString("nonProxyHosts");
+  }
+
+}
+
+public class PropertiesBasedProxySelector extends ProxySelector {
+
+  List<Proxy> proxies = null;
+
+  List<String> nonProxyHosts = null;
+
+  public PropertiesBasedProxySelector(String proxySettingsPath) {
+    this(new FileBasedProxyConfiguration(proxySettingsPath));
+  }
+
+  public PropertiesBasedProxySelector(NetworkConfiguration config) {
+    if (config == null || config.getProxyHost() == null) {
+      this.proxies = null;
+      return;
+    }
+
+    String host = config.getProxyHost();
+
+    int port = 80;
+
+    if (config.getProxyPort() != null) {
+      port = Integer.valueOf(config.getProxyPort());
+    }
+
+    if (config.getNonProxyHosts() != null) {
+      this.nonProxyHosts = Arrays
+          .asList(config.getNonProxyHosts().split("\\|"));
+    }
+
+    this.proxies = new ArrayList<Proxy>();
+    this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host,
+        port)));
+
+    if (config.getProxyUser() != null) {
+      final String username = config.getProxyUser();
+      final String password = config.getProxyPassword();
+
+      Authenticator.setDefault(new Authenticator() {
+        @Override
+        protected PasswordAuthentication getPasswordAuthentication() {
+          return new PasswordAuthentication(username, password.toCharArray());
+        }
+      });
+
+    }
+  }
+
+  @Override
+  public List<Proxy> select(URI uri) {
+    if (this.nonProxyHosts == null) {
+      return Arrays.asList(Proxy.NO_PROXY);
+    } else {
+      for (String entry : this.nonProxyHosts) {
+        entry = entry.trim();
+        if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) {
+          return Arrays.asList(Proxy.NO_PROXY);
+        }
+        if (uri.getHost().equals(entry)) {
+          return Arrays.asList(Proxy.NO_PROXY);
+        }
+      }
+      return this.proxies;
+    }
+  }
+
+  @Override
+  public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) {
+
+  }
+}