From 6d4a7d8b4e79ec8c856c7159f29ea2e9fa28c6dc Mon Sep 17 00:00:00 2001 From: Paolo Fabriani Date: Thu, 17 Nov 2016 11:34:53 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@134306 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../ansible/AnsibleWorker.java | 107 ++++++++ .../ansible/model/AnsibleHost.java | 19 ++ .../ansible/model/HostGroup.java | 29 +++ .../ansible/model/Inventory.java | 37 +++ .../ansible/model/Playbook.java | 32 +++ .../ansible/model/Role.java | 51 ++++ .../ansible/model/RoleFile.java | 54 ++++ .../ansiblebridge/AnsibleBridge.java | 239 ++++++++++++++++++ .../ansiblebridge/AnsibleSerializeHelper.java | 117 +++++++++ .../template/AlgorithmPackage.java | 62 +++++ .../template/CranDependencyPackage.java | 11 + .../template/CustomDependencyPackage.java | 69 +++++ .../template/CustomRoleManager.java | 37 +++ .../template/DependencyPackage.java | 56 ++++ .../template/OSDependencyPackage.java | 11 + .../template/StaticRoleManager.java | 37 +++ .../template/TemplateManager.java | 99 ++++++++ .../clients/ISClient.java | 58 +++++ .../datamodel/Action.java | 35 +++ .../datamodel/Algorithm.java | 71 ++++++ .../datamodel/AlgorithmSet.java | 49 ++++ .../datamodel/Cluster.java | 73 ++++++ .../datamodel/Dependency.java | 29 +++ .../datamodel/Domain.java | 15 ++ .../dataminerpoolmanager/datamodel/Host.java | 39 +++ .../comparator/AlgorithmComparator.java | 15 ++ .../comparator/DependencyComparator.java | 19 ++ .../datamodel/comparator/HostComparator.java | 17 ++ .../service/DataminerPoolManager.java | 121 +++++++++ .../util/PropertiesBasedProxySelector.java | 145 +++++++++++ 30 files changed, 1753 insertions(+) create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java new file mode 100644 index 0000000..4d67b29 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java @@ -0,0 +1,107 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible; + +import java.io.File; +***REMOVED*** +import java.util.UUID; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper; + +/** + * This class is responsible for the interface with ansible, retrieving log, + * etc. etc. It's not supposed to access templates and static stuff files. It + * does not know the service datamodel. + * + * @author paolo + * + */ +public class AnsibleWorker ***REMOVED*** + + /** + * The name of the inventory + */ + private static String INVENTORY_NAME = "inventory.yaml"; + + /** + * The directory containing roles + */ + private static String ROLES_DIR = "roles"; + + /** + * The name of the playbook + */ + private static String PLAYBOOK_NAME = "playbook.yaml"; + + /** + * The root of the worker. This corresponds to a standard ansible working dir. + */ + private File workerRoot; + + public AnsibleWorker(File root) ***REMOVED*** + this.workerRoot = root; + this.ensureWorkStructure(); + ***REMOVED*** + + private File getWorkdir() ***REMOVED*** + return this.workerRoot; + ***REMOVED*** + + private File getRolesDir() ***REMOVED*** + return new File(this.getWorkdir(), ROLES_DIR); + ***REMOVED*** + + public String getWorkerId() ***REMOVED*** + return this.workerRoot.getName(); + ***REMOVED*** + + private void ensureWorkStructure() ***REMOVED*** + ***REMOVED*** generate root + this.getWorkdir().mkdirs(); + ***REMOVED*** + + private void removeWorkStructure() ***REMOVED*** + ***REMOVED*** remove the working dir +***REMOVED*** this.getWorkdir().delete(); + ***REMOVED*** + + public File getPlaybookFile() ***REMOVED*** + return new File(this.getWorkdir(), PLAYBOOK_NAME); + ***REMOVED*** + + public File getInventoryFile() ***REMOVED*** + return new File(this.getWorkdir(), INVENTORY_NAME); + ***REMOVED*** + + + public void setInventory(Inventory inventory) throws IOException ***REMOVED*** + ***REMOVED*** serialize the string to the 'inventory' file + AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); + ***REMOVED*** + + public void setPlaybook(Playbook playbook) throws IOException ***REMOVED*** + ***REMOVED*** serialize the string to the 'playbook' file + AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); + ***REMOVED*** + + public void addRole(Role r) throws IOException ***REMOVED*** + ***REMOVED*** Serialize role in the workdir + AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); + ***REMOVED*** + + public void apply() ***REMOVED*** + ***REMOVED*** TODO execute the playbook and return output + System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); + ***REMOVED*** + + /** + * Destroy the worker: + * - remove the working dir + */ + public void destroy() ***REMOVED*** + this.removeWorkStructure(); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java new file mode 100644 index 0000000..b4e5dad --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +public class AnsibleHost ***REMOVED*** + + private String name; + + public AnsibleHost(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java new file mode 100644 index 0000000..c27b1f4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +import java.util.Collection; +import java.util.Vector; + +public class HostGroup ***REMOVED*** + + private String name; + + private Collection hosts; + + public HostGroup(String name) ***REMOVED*** + this.name = name; + this.hosts = new Vector<>(); + ***REMOVED*** + + public void addHost(AnsibleHost h) ***REMOVED*** + this.hosts.add(h); + ***REMOVED*** + + public String getName() ***REMOVED*** + return this.name; + ***REMOVED*** + + public Collection getHosts() ***REMOVED*** + return new Vector<>(this.hosts); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java new file mode 100644 index 0000000..922b1e0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java @@ -0,0 +1,37 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +import java.util.Collection; +import java.util.Vector; + +public class Inventory ***REMOVED*** + + private Collection groups; + + public Inventory() ***REMOVED*** + this.groups = new Vector<>(); + ***REMOVED*** + + public void addGroup(HostGroup group) ***REMOVED*** + this.groups.add(group); + ***REMOVED*** + + public void addHost(AnsibleHost h, String groupName) ***REMOVED*** + this.getGroup(groupName).addHost(h); + ***REMOVED*** + + private HostGroup getGroup(String groupName) ***REMOVED*** + for (HostGroup hg : this.groups) ***REMOVED*** + if (groupName.equals(hg.getName())) ***REMOVED*** + return hg; + ***REMOVED*** + ***REMOVED*** + HostGroup hg = new HostGroup(groupName); + this.groups.add(hg); + return hg; + ***REMOVED*** + + public Collection getHostGroups() ***REMOVED*** + return new Vector<>(this.groups); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java new file mode 100644 index 0000000..4d460da --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java @@ -0,0 +1,32 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +***REMOVED*** +import java.util.Vector; + +public class Playbook ***REMOVED*** + + private String hostGroupName; + + private List roles; + + public Playbook() ***REMOVED*** + this.roles = new Vector<>(); + ***REMOVED*** + + public void addRole(String role) ***REMOVED*** + roles.add(role); + ***REMOVED*** + + public void applyTo(String hostGroupName) ***REMOVED*** + this.hostGroupName = hostGroupName; + ***REMOVED*** + + public String getHostGroupName() ***REMOVED*** + return hostGroupName; + ***REMOVED*** + + public List getRoles() ***REMOVED*** + return new Vector<>(roles); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java new file mode 100644 index 0000000..e024de2 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java @@ -0,0 +1,51 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +import java.util.Collection; +import java.util.Vector; + +public class Role ***REMOVED*** + + /** + * The name of the role + */ + private String name; + + private Collection tasks; + + private Collection meta; + + public Role() ***REMOVED*** + this.tasks = new Vector<>(); + this.meta = new Vector<>(); + ***REMOVED*** + + public Role(String name) ***REMOVED*** + this(); + this.name = name; + ***REMOVED*** + + public void addTaskFile(RoleFile tf) ***REMOVED*** + this.tasks.add(tf); + ***REMOVED*** + + public void addMeta(RoleFile tf) ***REMOVED*** + this.meta.add(tf); + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public Collection getTaskFiles() ***REMOVED*** + return new Vector<>(this.tasks); + ***REMOVED*** + + public Collection getMeta() ***REMOVED*** + return new Vector<>(this.meta); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java new file mode 100644 index 0000000..ae25f7f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java @@ -0,0 +1,54 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; + +public class RoleFile ***REMOVED*** + + /** + * The path to the file, starting from the role root + */ + private String path; + + /** + * The name of the task file + */ + private String name; + + /** + * The content of the task file + * @return + */ + private String content; + + public RoleFile() ***REMOVED*** + ***REMOVED*** + + public RoleFile(String name, String content) ***REMOVED*** + this(); + this.setName(name); + this.setContent(content); + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getContent() ***REMOVED*** + return content; + ***REMOVED*** + + public void setContent(String content) ***REMOVED*** + this.content = content; + ***REMOVED*** + + public String getPath() ***REMOVED*** + return path; + ***REMOVED*** + + public void setPath(String path) ***REMOVED*** + this.path = path; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java new file mode 100644 index 0000000..1ba74c0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java @@ -0,0 +1,239 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge; + +import java.io.File; +***REMOVED*** +import java.util.Collection; +***REMOVED*** +import java.util.Map; +***REMOVED*** +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import java.util.Vector; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.AlgorithmPackage; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CranDependencyPackage; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomDependencyPackage; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomRoleManager; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.DependencyPackage; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.OSDependencyPackage; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.StaticRoleManager; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.TemplateManager; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.AlgorithmComparator; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.DependencyComparator; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.HostComparator; + +public class AnsibleBridge ***REMOVED*** + + /** + * The workdir for this service + */ + private String dpmRoot; + + public AnsibleBridge() ***REMOVED*** + this("/home/paolo/tmp/dataminer-pool-manager"); + ***REMOVED*** + + public AnsibleBridge(String root) ***REMOVED*** + this.dpmRoot = root; + this.ensureServiceRoot(); + ***REMOVED*** + + private void ensureServiceRoot() ***REMOVED*** + ***REMOVED*** generate root + new File(dpmRoot).mkdirs(); + ***REMOVED*** 'template' is for template roles + this.getTemplatesDir().mkdirs(); + ***REMOVED*** 'static' is for custom roles + this.getCustomDir().mkdirs(); + ***REMOVED*** 'work' is for temporary working directories + this.getWorkDir().mkdirs(); + ***REMOVED*** + + private File getWorkDir() ***REMOVED*** + return new File(this.dpmRoot, "work"); + ***REMOVED*** + + private File getTemplatesDir() ***REMOVED*** + return new File(this.dpmRoot, "templates"); + ***REMOVED*** + + private File getCustomDir() ***REMOVED*** + return new File(this.dpmRoot, "custom"); + ***REMOVED*** + + public AnsibleWorker createWorker() ***REMOVED*** + File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); + AnsibleWorker worker = new AnsibleWorker(workerRoot); + return worker; + ***REMOVED*** + + /** + * Groups hosts by domain and algorithm sets + * @param clusters + */ + public void printInventoryByDomainAndSets(Collection clusters) ***REMOVED*** + Map> inventory = new TreeMap<>(); + for(Cluster cluster:clusters) ***REMOVED*** + for(AlgorithmSet as:cluster.getAlgorithmSets()) ***REMOVED*** + String asName = as.getName(); + for(Host h:cluster.getHosts()) ***REMOVED*** + String domain = h.getDomain().getName(); + String key = String.format("[%s@%s]", asName, domain); + Set hosts = inventory.get(key); + if(hosts==null) ***REMOVED*** + hosts = new TreeSet<>(new HostComparator()); + inventory.put(key, hosts); + ***REMOVED*** + hosts.add(h); + ***REMOVED*** + + ***REMOVED*** + ***REMOVED*** + for(String key:inventory.keySet()) ***REMOVED*** + System.out.println(key); + Collection hosts = inventory.get(key); + for(Host h:hosts) ***REMOVED*** + System.out.println(h.getName()+"."+h.getDomain().getName()); + ***REMOVED*** + System.out.println(); + ***REMOVED*** + ***REMOVED*** + + /** + * Groups hosts by algorithm sets only + * @param clusters + */ + public void printInventoryBySets(Collection clusters) ***REMOVED*** + Map> inventory = new TreeMap<>(); + for (Cluster cluster : clusters) ***REMOVED*** + for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** + String asName = as.getName(); + for (Host h : cluster.getHosts()) ***REMOVED*** + String key = String.format("[%s]", asName); + Set hosts = inventory.get(key); + if (hosts == null) ***REMOVED*** + hosts = new TreeSet<>(new HostComparator()); + inventory.put(key, hosts); + ***REMOVED*** + hosts.add(h); + ***REMOVED*** + + ***REMOVED*** + ***REMOVED*** + for (String key : inventory.keySet()) ***REMOVED*** + System.out.println(key); + Collection hosts = inventory.get(key); + for (Host h : hosts) ***REMOVED*** + System.out.println(h.getName()+"."+h.getDomain().getName()); + ***REMOVED*** + System.out.println(); + ***REMOVED*** + ***REMOVED*** + + public void applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException ***REMOVED*** + AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), UUID.randomUUID().toString())); + + List algoRoles = new Vector<>(); + + ***REMOVED*** add algorithms and dependencies to the worker + for (Algorithm a : as.getAlgorithms()) ***REMOVED*** + for (Role r : this.generateRoles(a)) ***REMOVED*** + algoRoles.add(r); + worker.addRole(r); + ***REMOVED*** + for (Dependency d : a.getDependencies()) ***REMOVED*** + for (Role r : this.generateRoles(d)) ***REMOVED*** + worker.addRole(r); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + + ***REMOVED*** add static roles + for(Role r:this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** + worker.addRole(r); + ***REMOVED*** + + ***REMOVED*** generate the inventory + Inventory inventory = new Inventory(); + for (Host h : cluster.getHosts()) ***REMOVED*** + AnsibleHost ah = new AnsibleHost(h.getName()); + inventory.addHost(ah, "universe"); + inventory.addHost(ah, "d4science"); + ***REMOVED*** + worker.setInventory(inventory); + + ***REMOVED*** generate the playbook + Playbook playbook = new Playbook(); + playbook.applyTo("universe"); + for(Role r:algoRoles) ***REMOVED*** + ***REMOVED*** add only 'add' roles + if(!r.getName().endsWith("remove")) ***REMOVED*** + playbook.addRole(r.getName()); + ***REMOVED*** + ***REMOVED*** + + worker.setPlaybook(playbook); + + ***REMOVED*** execute + worker.apply(); + + ***REMOVED*** destroy the worker + worker.destroy(); + + ***REMOVED*** + + private TemplateManager getTemplateManager() ***REMOVED*** + return new TemplateManager(this.dpmRoot+"/templates"); + ***REMOVED*** + + private CustomRoleManager getCustomRoleManager() ***REMOVED*** + return new CustomRoleManager(this.dpmRoot+"/custom"); + ***REMOVED*** + + private StaticRoleManager getStaticRoleManager() ***REMOVED*** + return new StaticRoleManager(this.dpmRoot+"/static"); + ***REMOVED*** + + /** + * Generate all roles for this dependency + * @param d + */ + public Collection generateRoles(Dependency d) ***REMOVED*** + Collection roles = new Vector<>(); + if("os".equals(d.getType())) ***REMOVED*** + OSDependencyPackage pkg = new OSDependencyPackage(d); + if(pkg!=null) ***REMOVED*** + roles.addAll(pkg.getRoles(this.getTemplateManager())); + ***REMOVED*** + ***REMOVED*** else if("custom".equals(d.getType())) ***REMOVED*** + CustomDependencyPackage pkg = new CustomDependencyPackage(d); + if(pkg!=null) ***REMOVED*** + roles.addAll(pkg.getRoles(this.getCustomRoleManager())); + ***REMOVED*** + ***REMOVED*** else if("cran".equals(d.getType())) ***REMOVED*** + CranDependencyPackage pkg = new CranDependencyPackage(d); + if(pkg!=null) ***REMOVED*** + roles.addAll(pkg.getRoles(this.getTemplateManager())); + ***REMOVED*** + ***REMOVED*** + return roles; + ***REMOVED*** + + public Collection generateRoles(Algorithm a) ***REMOVED*** + AlgorithmPackage pkg = new AlgorithmPackage(a); + return pkg.getRoles(this.getTemplateManager()); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java new file mode 100644 index 0000000..b0eee20 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java @@ -0,0 +1,117 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +***REMOVED*** +import java.io.PrintWriter; + +import org.apache.commons.io.IOUtils; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.HostGroup; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; + +public class AnsibleSerializeHelper ***REMOVED*** + + public static void serialize(Inventory inventory, File inventoryFile) throws IOException ***REMOVED*** + String out = ""; + for(HostGroup hg:inventory.getHostGroups()) ***REMOVED*** + out+=String.format("[%s]\n", hg.getName()); + for(AnsibleHost h:hg.getHosts()) ***REMOVED*** + out+=h.getName()+"\n"; + ***REMOVED*** + out+="\n"; + ***REMOVED*** + out = out.trim(); + serialize(out, inventoryFile); + ***REMOVED*** + + public static void serialize(Playbook playbook, File playbookFile) throws IOException ***REMOVED*** + String out = "- hosts: " + playbook.getHostGroupName() + "\n"; + out += " remote_user: dpm\n"; + out+=" roles:\n"; + for(String r:playbook.getRoles()) ***REMOVED*** + out+=" - " + r+"\n"; + ***REMOVED*** + out = out.trim(); + serialize(out, playbookFile); + ***REMOVED*** + + public static void serializeRole(Role r, File dir) throws IOException ***REMOVED*** + ***REMOVED*** create root + File root = new File(dir, r.getName()); + root.mkdirs(); + + ***REMOVED*** create tasks + if(r.getTaskFiles().size()>0) ***REMOVED*** + File tasks = new File(root, "tasks"); + tasks.mkdirs(); + for(RoleFile tf: r.getTaskFiles()) ***REMOVED*** + serializeTask(tf, tasks); + ***REMOVED*** + ***REMOVED*** + + ***REMOVED*** create meta + if(r.getMeta().size()>0) ***REMOVED*** + File meta = new File(root, "meta"); + meta.mkdirs(); + for(RoleFile tf: r.getMeta()) ***REMOVED*** + serializeTask(tf, meta); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + + public static void serializeTask(RoleFile tf, File dir) throws IOException ***REMOVED*** + File f = new File(dir, tf.getName()); + serialize(tf.getContent().trim(), f); + ***REMOVED*** + + public static void serialize(String s, File f) throws IOException ***REMOVED*** + PrintWriter out = new PrintWriter(f); + out.println(s); + out.close(); + ***REMOVED*** + + public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException ***REMOVED*** + Role out = new Role(); + out.setName(roleDir.getName()); + + if(!roleDir.exists()) ***REMOVED*** + throw new FileNotFoundException(); + ***REMOVED*** + + try ***REMOVED*** + File tasksDir = new File(roleDir, "tasks"); + if(tasksDir.exists()) ***REMOVED*** + for(File main:tasksDir.listFiles()) ***REMOVED*** + String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); + RoleFile tf = new RoleFile(main.getName(), content); + tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); + out.addTaskFile(tf); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + + try ***REMOVED*** + File metaDir = new File(roleDir, "meta"); + if(metaDir.exists()) ***REMOVED*** + for(File main:metaDir.listFiles()) ***REMOVED*** + String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); + RoleFile tf = new RoleFile(main.getName(), content); + tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); + out.addMeta(tf); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java new file mode 100644 index 0000000..cb3bf49 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java @@ -0,0 +1,62 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Vector; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class AlgorithmPackage ***REMOVED*** + + private Algorithm algorithm; + + public AlgorithmPackage(Algorithm a) ***REMOVED*** + this.algorithm = a; + ***REMOVED*** + + protected Map getDictionary(Algorithm a) ***REMOVED*** + Map out = new HashMap(); + out.put("name", a.getName()); + String deps = ""; + for(Dependency d:a.getDependencies()) ***REMOVED*** + deps+=String.format("- ***REMOVED*** role: %s ***REMOVED***\n", d.getType()+"-"+d.getName()); + ***REMOVED*** + deps = deps.trim(); + out.put("dependencies", deps); + return out; + ***REMOVED*** + + protected Algorithm getAlgorithm() ***REMOVED*** + return this.algorithm; + ***REMOVED*** + + public Collection getRoles(TemplateManager tm) ***REMOVED*** + Collection out = new Vector<>(); + for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** + String roleName = "algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); + try ***REMOVED*** + ***REMOVED*** find template + Role template = tm.getRoleTemplate("algorithm-" + mode); + ***REMOVED*** + if(template!=null) ***REMOVED*** + Map dictionary = this.getDictionary(this.getAlgorithm()); + Role r = tm.fillRoleTemplate(template, dictionary); + r.setName(roleName); + out.add(r); + ***REMOVED*** else ***REMOVED*** + System.out.println("WARNING: template is null"); + ***REMOVED*** + ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** +***REMOVED*** e.printStackTrace(); + System.out.println("WARNING: no template found for " + roleName); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java new file mode 100644 index 0000000..588badc --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java @@ -0,0 +1,11 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class CranDependencyPackage extends DependencyPackage ***REMOVED*** + + public CranDependencyPackage(Dependency d) ***REMOVED*** + super(d); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java new file mode 100644 index 0000000..de64358 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java @@ -0,0 +1,69 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import java.io.File; +***REMOVED*** +import java.util.Collection; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Vector; + +import org.apache.commons.io.FileUtils; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class CustomDependencyPackage extends DependencyPackage ***REMOVED*** + + public CustomDependencyPackage(Dependency d) ***REMOVED*** + super(d); + ***REMOVED*** + + private String getCustomRepositoryLocation(String ansibleRoot) ***REMOVED*** + return ansibleRoot+"/custom"; + ***REMOVED*** + + /* + public void serializeTo(String ansibleRoot) ***REMOVED*** + for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** + ***REMOVED*** look for roles in the 'custom' repository + try ***REMOVED*** + ***REMOVED*** role name + String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); + ***REMOVED*** look for the custom role + File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); + System.out.println("** CUSTOM ** " + src); + if(src.exists()) ***REMOVED*** + ***REMOVED*** do copy + System.out.println("copying CUSTOM role"); + File dest = new File(ansibleRoot+"/work/"+roleName); + FileUtils.copyDirectory(src, dest); + ***REMOVED*** + ***REMOVED*** catch(IOException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + */ + + public Collection getRoles(CustomRoleManager crm) ***REMOVED*** + Collection out = new Vector<>(); + for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** + ***REMOVED*** role name + String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); + try ***REMOVED*** + ***REMOVED*** look for custom role + Role role = crm.getRole(roleName); + if(role!=null) ***REMOVED*** + out.add(role); + ***REMOVED*** + ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** +***REMOVED*** e.printStackTrace(); + System.out.println("WARNING: no custom role found for " + roleName); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java new file mode 100644 index 0000000..ff944c0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java @@ -0,0 +1,37 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import java.io.File; +import java.io.FileInputStream; +***REMOVED*** +import java.util.Collection; +import java.util.NoSuchElementException; +import java.util.Vector; + +import org.apache.commons.io.IOUtils; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper; + +public class CustomRoleManager ***REMOVED*** + + private String root; + + public CustomRoleManager(String root) ***REMOVED*** + this.root = root; + ***REMOVED*** + + public String getRoot() ***REMOVED*** + return this.root; + ***REMOVED*** + + public Role getRole(String roleName) throws NoSuchElementException ***REMOVED*** + File f = new File(this.getRoot(), roleName); + try ***REMOVED*** + return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); + ***REMOVED*** catch (IOException e) ***REMOVED*** +***REMOVED*** e.printStackTrace(); + throw new NoSuchElementException("unable to find " + roleName); + ***REMOVED*** + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java new file mode 100644 index 0000000..40b3c00 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java @@ -0,0 +1,56 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Vector; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class DependencyPackage ***REMOVED*** + + private Dependency dependency; + + public DependencyPackage(Dependency d) ***REMOVED*** + this.dependency = d; + ***REMOVED*** + + protected Map getDictionary(Dependency d) ***REMOVED*** + Map out = new HashMap(); + out.put("name", d.getName()); + out.put("type", d.getType()); + return out; + ***REMOVED*** + + protected Dependency getDependency() ***REMOVED*** + return this.dependency; + ***REMOVED*** + + public Collection getRoles(TemplateManager tm) ***REMOVED*** + Collection out = new Vector<>(); + for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** + String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); + try ***REMOVED*** + ***REMOVED*** find template + Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); + ***REMOVED*** + if(template!=null) ***REMOVED*** + Map dictionary = this.getDictionary(this.getDependency()); + Role r = tm.fillRoleTemplate(template, dictionary); + r.setName(roleName); + out.add(r); + ***REMOVED*** else ***REMOVED*** + System.out.println("WARNING: template is null"); + ***REMOVED*** + ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** +***REMOVED*** e.printStackTrace(); + System.out.println("WARNING: no template found for " + roleName); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java new file mode 100644 index 0000000..ecea9cf --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java @@ -0,0 +1,11 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class OSDependencyPackage extends DependencyPackage ***REMOVED*** + + public OSDependencyPackage(Dependency d) ***REMOVED*** + super(d); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java new file mode 100644 index 0000000..092b7f3 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java @@ -0,0 +1,37 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + + import java.io.File; +***REMOVED*** +import java.util.Collection; +import java.util.Vector; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper; + + public class StaticRoleManager ***REMOVED*** + + private String root; + + public StaticRoleManager(String root) ***REMOVED*** + this.root = root; + ***REMOVED*** + + public String getRoot() ***REMOVED*** + return this.root; + ***REMOVED*** + + public Collection getStaticRoles() ***REMOVED*** + Collection out = new Vector<>(); + for(File f: new File(this.getRoot()).listFiles()) ***REMOVED*** + try ***REMOVED*** + out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); + ***REMOVED*** catch(IOException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + + + + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java new file mode 100644 index 0000000..10eaee8 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java @@ -0,0 +1,99 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; + +import java.io.File; +import java.io.FileInputStream; +***REMOVED*** +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; + +import org.apache.commons.io.IOUtils; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; +import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; +import org.stringtemplate.v4.ST; + +public class TemplateManager ***REMOVED*** + + private String root; + + public TemplateManager(String root) ***REMOVED*** + this.root = root; + ***REMOVED*** + + public String getTemplateRoot() ***REMOVED*** + return this.root; + ***REMOVED*** + + /** + * Read the given template + * @param templateName + * @return + * @throws IOException + */ +***REMOVED*** private String readTemplate(String templateName) throws IOException ***REMOVED*** +***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); +***REMOVED*** System.out.println("looking for file " + templateFile.getName()); +***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); +***REMOVED*** return out; +***REMOVED*** ***REMOVED*** + + /** + * Return the content of the given template + * @param templateName + * @return + * @throws NoSuchElementException if no such template exists + */ +***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException ***REMOVED*** +***REMOVED*** String template = null; +***REMOVED*** try ***REMOVED*** +***REMOVED*** template = this.readTemplate(templateName); +***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED*** +***REMOVED*** throw new NoSuchElementException(); +***REMOVED*** ***REMOVED*** +***REMOVED*** return template; +***REMOVED*** ***REMOVED*** + + public Role fillRoleTemplate(Role template, Map dictionary) ***REMOVED*** + Role out = new Role(); + out.setName(template.getName()); + for(RoleFile tf:template.getTaskFiles()) ***REMOVED*** + out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); + ***REMOVED*** + for(RoleFile tf:template.getMeta()) ***REMOVED*** + out.addMeta(this.fillTaskTemplate(tf, dictionary)); + ***REMOVED*** + return out; + ***REMOVED*** + + private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) ***REMOVED*** + RoleFile out = new RoleFile(); + out.setName(template.getName()); + out.setContent(this.fillTemplate(template.getContent(), dictionary)); + return out; + ***REMOVED*** + + private String fillTemplate(String template, Map dictionary) ***REMOVED*** + if (template != null) ***REMOVED*** + ST t = new ST(template); + for (String key : dictionary.keySet()) ***REMOVED*** + t.add(key, dictionary.get(key)); + ***REMOVED*** + String output = t.render(); + return output; + ***REMOVED*** + return template; + ***REMOVED*** + + public Role getRoleTemplate(String roleName) throws NoSuchElementException ***REMOVED*** + File f = new File(this.getTemplateRoot(), roleName); + try ***REMOVED*** + return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); + ***REMOVED*** catch (IOException e) ***REMOVED*** +***REMOVED*** e.printStackTrace(); + throw new NoSuchElementException("unable to find " + roleName); + ***REMOVED*** + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java new file mode 100644 index 0000000..a980146 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java @@ -0,0 +1,58 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.clients; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.util.Collection; +***REMOVED*** +import java.util.Vector; + +import org.gcube.common.resources.gcore.ServiceEndpoint; +***REMOVED*** +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; + +public class ISClient ***REMOVED*** + + /** + * Return the list of hosts (dataminers) in a given VRE + * + * @param vreName + * @return + */ + public Collection listDataminersInVRE(String scope) ***REMOVED*** + + boolean remote = false; + + if (!remote) ***REMOVED*** + Collection out = new Vector<>(); + Host h = new Host(); + h.setName("bb-dataminer.res.eng.it"); + out.add(h); + return out; + ***REMOVED*** else ***REMOVED*** + + ScopeProvider.instance.set(scope); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + + query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") + .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + + List resources = client.submit(query); + + Collection out = new Vector<>(); + for (ServiceEndpoint r : resources) ***REMOVED*** + Host h = new Host(); + h.setName(r.profile().runtime().hostedOn()); + out.add(h); + ***REMOVED*** + return out; + ***REMOVED*** + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java new file mode 100644 index 0000000..dea7d4b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java @@ -0,0 +1,35 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +public class Action ***REMOVED*** + + private String name; + + private String description; + + private String script; + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getDescription() ***REMOVED*** + return description; + ***REMOVED*** + + public void setDescription(String description) ***REMOVED*** + this.description = description; + ***REMOVED*** + + public String getScript() ***REMOVED*** + return script; + ***REMOVED*** + + public void setScript(String script) ***REMOVED*** + this.script = script; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java new file mode 100644 index 0000000..ac180ae --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java @@ -0,0 +1,71 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +import java.util.Collection; +import java.util.Vector; + +public class Algorithm ***REMOVED*** + + private String name; + + private String description; + + private String category; + + private Collection actions; + + private Collection dependencies; + + public Algorithm() ***REMOVED*** + this.actions = new Vector<>(); + this.dependencies = new Vector<>(); + Dependency p = new Dependency(); + ***REMOVED*** + + public void addDependency(Dependency dep) ***REMOVED*** + this.dependencies.add(dep); + ***REMOVED*** + + public void addAction(Action action) ***REMOVED*** + this.actions.add(action); + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getDescription() ***REMOVED*** + return description; + ***REMOVED*** + + public void setDescription(String description) ***REMOVED*** + this.description = description; + ***REMOVED*** + + public String getCategory() ***REMOVED*** + return category; + ***REMOVED*** + + public void setCategory(String category) ***REMOVED*** + this.category = category; + ***REMOVED*** + + public Collection getActions() ***REMOVED*** + return actions; + ***REMOVED*** + + public Collection getDependencies() ***REMOVED*** + return dependencies; + ***REMOVED*** + + public String toString() ***REMOVED*** + String out = "Algorithm: " + this.getName()+"\n"; + out+=" Description: " + this.getDescription()+"\n"; + out+=" Dependencies: " + this.getDependencies()+"\n"; + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java new file mode 100644 index 0000000..b831375 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java @@ -0,0 +1,49 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +import java.util.Collection; +import java.util.Vector; + +public class AlgorithmSet ***REMOVED*** + + private String name; + + private Collection algorithms; + + public AlgorithmSet() ***REMOVED*** + this.algorithms = new Vector<>(); + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public Collection getAlgorithms() ***REMOVED*** + return new Vector<>(algorithms); + ***REMOVED*** + + public void addAlgorithm(Algorithm algoritm) ***REMOVED*** + this.algorithms.add(algoritm); + ***REMOVED*** + + public Boolean hasAlgorithm(Algorithm algorithm) ***REMOVED*** + for (Algorithm a : this.algorithms) ***REMOVED*** + if (a.getName().equals(algorithm.getName())) ***REMOVED*** + return true; + ***REMOVED*** + ***REMOVED*** + return false; + ***REMOVED*** + + public String toString() ***REMOVED*** + String out = "ALGOSET: " + this.name + "\n"; + for(Algorithm a:this.algorithms) ***REMOVED*** + out+=a+"\n"; + ***REMOVED*** + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java new file mode 100644 index 0000000..f53a0ea --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java @@ -0,0 +1,73 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +import java.util.Collection; +import java.util.Vector; + +public class Cluster ***REMOVED*** + + /** + * The set of hosts belonging to the cluster. + */ + private Collection hosts; + + /** + * A name for this cluster. + */ + private String name; + + /** + * A description of this cluster. + */ + private String description; + + /** + * The set of algorithms deployed on this cluster (i.e. on all its hosts) + */ + private Collection algoSets; + + public Cluster() ***REMOVED*** + this.hosts = new Vector<>(); + this.algoSets = new Vector<>(); + ***REMOVED*** + + public void addAlgorithmSet(AlgorithmSet set) ***REMOVED*** + this.algoSets.add(set); + ***REMOVED*** + + public void addHost(Host host) ***REMOVED*** + this.hosts.add(host); + ***REMOVED*** + + public Collection getHosts() ***REMOVED*** + return hosts; + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getDescription() ***REMOVED*** + return description; + ***REMOVED*** + + public void setDescription(String description) ***REMOVED*** + this.description = description; + ***REMOVED*** + + public Collection getAlgorithmSets() ***REMOVED*** + return algoSets; + ***REMOVED*** + + public String toString() ***REMOVED*** + String out = "Cluster: "+this.name+"\n"; + for(Host h:this.getHosts()) ***REMOVED*** + out+=" "+h+"\n"; + ***REMOVED*** + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java new file mode 100644 index 0000000..4947ffa --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +public class Dependency ***REMOVED*** + + private String name; + + private String type; + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public String getType() ***REMOVED*** + return type; + ***REMOVED*** + + public void setType(String type) ***REMOVED*** + this.type = type; + ***REMOVED*** + + public String toString() ***REMOVED*** + return this.type+":"+this.name; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java new file mode 100644 index 0000000..ec5898e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java @@ -0,0 +1,15 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +public class Domain ***REMOVED*** + + private String name; + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java new file mode 100644 index 0000000..f6720e8 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java @@ -0,0 +1,39 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; + +public class Host ***REMOVED*** + + private String name; + + private Domain domain; + + public Host() ***REMOVED*** + ***REMOVED*** + + public String getFullyQualifiedName() ***REMOVED*** + if(this.domain!=null && this.domain.getName()!=null) + return this.getName()+"."+this.getDomain().getName(); + else + return this.getName(); + ***REMOVED*** + + public String getName() ***REMOVED*** + return name; + ***REMOVED*** + + public void setName(String name) ***REMOVED*** + this.name = name; + ***REMOVED*** + + public Domain getDomain() ***REMOVED*** + return domain; + ***REMOVED*** + + public void setDomain(Domain domain) ***REMOVED*** + this.domain = domain; + ***REMOVED*** + + public String toString() ***REMOVED*** + return this.name + "@" + this.domain; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java new file mode 100644 index 0000000..a88335c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java @@ -0,0 +1,15 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; + +import java.util.Comparator; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; + +public class AlgorithmComparator implements Comparator ***REMOVED*** + + @Override + public int compare(Algorithm a1, Algorithm a2) ***REMOVED*** + return a1.getName().compareTo(a2.getName()); + ***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java new file mode 100644 index 0000000..1550b0a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; + +import java.util.Comparator; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class DependencyComparator implements Comparator ***REMOVED*** + + @Override + public int compare(Dependency a1, Dependency a2) ***REMOVED*** + int out = a1.getType().compareTo(a2.getType()); + if(out!=0) + return out; + return a1.getName().compareTo(a2.getName()); + ***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java new file mode 100644 index 0000000..906c6d2 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java @@ -0,0 +1,17 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; + +import java.util.Comparator; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; + +public class HostComparator implements Comparator ***REMOVED*** + + @Override + public int compare(Host h1, Host h2) ***REMOVED*** + int out = h1.getDomain().getName().compareTo(h2.getDomain().getName()); + if(out!=0) + return out; + return h1.getName().compareTo(h2.getName()); + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java new file mode 100644 index 0000000..1bd407e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java @@ -0,0 +1,121 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.service; + +***REMOVED*** +import java.util.Collection; +import java.util.Vector; + +import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleBridge; +import org.gcube.dataanalysys.dataminerpoolmanager.clients.ISClient; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; + +public class DataminerPoolManager ***REMOVED*** + +***REMOVED*** static Collection algorithms; +***REMOVED*** +***REMOVED*** static Collection sets; +***REMOVED*** +***REMOVED*** static ***REMOVED*** +***REMOVED*** algorithms = new Vector<>(); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** public DataminerPoolManager() ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** /** +***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected +***REMOVED*** * on the pool. +***REMOVED*** */ +***REMOVED*** public void publishAlgorithm(Algorithm algorithm) ***REMOVED*** +***REMOVED*** algorithms.add(algorithm); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** /** +***REMOVED*** * Re-deploy the given algorithm wherever it's installed +***REMOVED*** * +***REMOVED*** * @param algorithm +***REMOVED*** */ +***REMOVED*** /* +***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) ***REMOVED*** ***REMOVED*** TODO implement this ***REMOVED*** +***REMOVED*** */ +***REMOVED*** +***REMOVED*** /** +***REMOVED*** * Add the give algorithm to the given set +***REMOVED*** * +***REMOVED*** * @param algorithmId +***REMOVED*** * @param setId +***REMOVED*** */ +***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) ***REMOVED*** +***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); +***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); +***REMOVED*** if (set != null && algorithm != null) ***REMOVED*** +***REMOVED*** set.addAlgorithm(algorithm); +***REMOVED*** this.updateClusters(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** /** +***REMOVED*** * Apply the given set of algorithms to the given cluster +***REMOVED*** * +***REMOVED*** * @param setId +***REMOVED*** * @param clusterId +***REMOVED*** */ +***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) ***REMOVED*** +***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); +***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); +***REMOVED*** if (set != null && cluster != null) ***REMOVED*** +***REMOVED*** cluster.addAlgorithmSet(set); +***REMOVED*** this.updateClusters(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) ***REMOVED*** +***REMOVED*** for (AlgorithmSet set : sets) ***REMOVED*** +***REMOVED*** if (name.equals(set.getName())) ***REMOVED*** +***REMOVED*** return set; +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** return null; +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** private Algorithm getAlgorithm(String name) ***REMOVED*** +***REMOVED*** for (Algorithm a : algorithms) ***REMOVED*** +***REMOVED*** if (name.equals(a.getName())) ***REMOVED*** +***REMOVED*** return a; +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** return null; +***REMOVED*** ***REMOVED*** + + /** + * Publish the given algorithm in the given VRE + * + * @param algorithmName + * @param vre + */ + public void addAlgorithmToVRE(Algorithm algorithm, String vre) throws IOException ***REMOVED*** + + ***REMOVED*** create a fake algorithm set + AlgorithmSet algoSet = new AlgorithmSet(); + algoSet.setName("fake"); + algoSet.addAlgorithm(algorithm); + + ***REMOVED*** create the cluster (dataminers in the vre) + Cluster cluster = new Cluster(); + for(Host h:new ISClient().listDataminersInVRE(vre)) ***REMOVED*** + cluster.addHost(h); + ***REMOVED*** + + ***REMOVED*** apply the changes + new AnsibleBridge().applyAlgorithmSetToCluster(algoSet, cluster); + + ***REMOVED*** + +***REMOVED*** private void updateClusters() ***REMOVED*** +***REMOVED*** System.out.println("flushing changes to all clusters"); +***REMOVED*** ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java new file mode 100644 index 0000000..23a85bd --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java @@ -0,0 +1,145 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.util; + +***REMOVED*** +import java.net.Authenticator; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.SocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +***REMOVED*** + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; + +interface NetworkConfiguration ***REMOVED*** + + public String getProxyHost(); + + public String getProxyPort(); + + public String getProxyUser(); + + public String getProxyPassword(); + + public String getNonProxyHosts(); + +***REMOVED*** + +class FileBasedProxyConfiguration implements NetworkConfiguration ***REMOVED*** + + private static PropertiesConfiguration configuration; + + public FileBasedProxyConfiguration(String path) ***REMOVED*** + try ***REMOVED*** + ***REMOVED*** load the configuration + configuration = new PropertiesConfiguration(path); + ***REMOVED*** set the reloading strategy to enable hot-configuration + FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); + configuration.setReloadingStrategy(fcrs); + ***REMOVED*** catch (ConfigurationException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + + @Override + public String getProxyHost() ***REMOVED*** + return configuration.getString("proxyHost"); + ***REMOVED*** + + @Override + public String getProxyPort() ***REMOVED*** + return configuration.getString("proxyPort"); + ***REMOVED*** + + @Override + public String getProxyUser() ***REMOVED*** + return configuration.getString("proxyUser"); + ***REMOVED*** + + @Override + public String getProxyPassword() ***REMOVED*** + return configuration.getString("proxyPassword"); + ***REMOVED*** + + @Override + public String getNonProxyHosts() ***REMOVED*** + return configuration.getString("nonProxyHosts"); + ***REMOVED*** + +***REMOVED*** + +public class PropertiesBasedProxySelector extends ProxySelector ***REMOVED*** + + List proxies = null; + + List nonProxyHosts = null; + + public PropertiesBasedProxySelector(String proxySettingsPath) ***REMOVED*** + this(new FileBasedProxyConfiguration(proxySettingsPath)); + ***REMOVED*** + + public PropertiesBasedProxySelector(NetworkConfiguration config) ***REMOVED*** + if (config == null || config.getProxyHost() == null) ***REMOVED*** + this.proxies = null; + return; + ***REMOVED*** + + String host = config.getProxyHost(); + + int port = 80; + + if (config.getProxyPort() != null) ***REMOVED*** + port = Integer.valueOf(config.getProxyPort()); + ***REMOVED*** + + if (config.getNonProxyHosts() != null) ***REMOVED*** + this.nonProxyHosts = Arrays + .asList(config.getNonProxyHosts().split("\\|")); + ***REMOVED*** + + this.proxies = new ArrayList(); + this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, + port))); + + if (config.getProxyUser() != null) ***REMOVED*** + final String username = config.getProxyUser(); + final String password = config.getProxyPassword(); + + Authenticator.setDefault(new Authenticator() ***REMOVED*** + @Override + protected PasswordAuthentication getPasswordAuthentication() ***REMOVED*** + return new PasswordAuthentication(username, password.toCharArray()); + ***REMOVED*** + ***REMOVED***); + + ***REMOVED*** + ***REMOVED*** + + @Override + public List select(URI uri) ***REMOVED*** + if (this.nonProxyHosts == null) ***REMOVED*** + return Arrays.asList(Proxy.NO_PROXY); + ***REMOVED*** else ***REMOVED*** + for (String entry : this.nonProxyHosts) ***REMOVED*** + entry = entry.trim(); + if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) ***REMOVED*** + return Arrays.asList(Proxy.NO_PROXY); + ***REMOVED*** + if (uri.getHost().equals(entry)) ***REMOVED*** + return Arrays.asList(Proxy.NO_PROXY); + ***REMOVED*** + ***REMOVED*** + return this.proxies; + ***REMOVED*** + ***REMOVED*** + + @Override + public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) ***REMOVED*** + + ***REMOVED*** +***REMOVED***