package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; import java.io.FileOutputStream; ***REMOVED*** import java.io.PrintStream; import java.util.Collection; ***REMOVED*** import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; ***REMOVED*** public class AnsibleBridge { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); /** * The workdir for this service */ private String dpmRoot; public AnsibleBridge() { this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); ***REMOVED*** public AnsibleBridge(String root) { this.dpmRoot = root; this.ensureServiceRoot(); ***REMOVED*** private void ensureServiceRoot() { ***REMOVED*** generate root new File(dpmRoot).mkdirs(); ***REMOVED*** 'template' is for template roles this.getTemplatesDir().mkdirs(); ***REMOVED*** 'static' is for custom roles this.getCustomDir().mkdirs(); ***REMOVED*** 'work' is for temporary working directories this.getWorkDir().mkdirs(); ***REMOVED*** private File getWorkDir() { return new File(this.dpmRoot, "work"); ***REMOVED*** private File getTemplatesDir() { return new File(this.dpmRoot, "templates"); ***REMOVED*** private File getCustomDir() { return new File(this.dpmRoot, "custom"); ***REMOVED*** public AnsibleWorker createWorker() { File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); AnsibleWorker worker = new AnsibleWorker(workerRoot); return worker; ***REMOVED*** /** * Groups hosts by domain and algorithm sets * @param clusters */ public void printInventoryByDomainAndSets(Collection clusters) { Map> inventory = new TreeMap<>(); for(Cluster cluster:clusters) { for(AlgorithmSet as:cluster.getAlgorithmSets()) { String asName = as.getName(); for(Host h:cluster.getHosts()) { String domain = h.getDomain().getName(); String key = String.format("[%s@%s]", asName, domain); Set hosts = inventory.get(key); if(hosts==null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); ***REMOVED*** hosts.add(h); ***REMOVED*** ***REMOVED*** ***REMOVED*** for(String key:inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); for(Host h:hosts) { System.out.println(h.getName()+"."+h.getDomain().getName()); ***REMOVED*** System.out.println(); ***REMOVED*** ***REMOVED*** /** * Groups hosts by algorithm sets only * @param clusters */ public void printInventoryBySets(Collection clusters) { Map> inventory = new TreeMap<>(); for (Cluster cluster : clusters) { for (AlgorithmSet as : cluster.getAlgorithmSets()) { String asName = as.getName(); for (Host h : cluster.getHosts()) { String key = String.format("[%s]", asName); Set hosts = inventory.get(key); if (hosts == null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); ***REMOVED*** hosts.add(h); ***REMOVED*** ***REMOVED*** ***REMOVED*** for (String key : inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); for (Host h : hosts) { System.out.println(h.getName()+"."+h.getDomain().getName()); ***REMOVED*** System.out.println(); ***REMOVED*** ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException { return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString()); ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException { AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); List algoRoles = new Vector<>(); ***REMOVED*** add algorithms and dependencies to the worker for (Algorithm a : as.getAlgorithms()) { for (Role r : this.generateRoles(a)) { algoRoles.add(r); worker.addRole(r); ***REMOVED*** for (Dependency d : a.getDependencies()) { for (Role r : this.generateRoles(d)) { worker.addRole(r); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** add static roles for(Role r:this.getStaticRoleManager().getStaticRoles()) { worker.addRole(r); ***REMOVED*** ***REMOVED*** generate the inventory Inventory inventory = new Inventory(); for (Host h : cluster.getHosts()) { AnsibleHost ah = new AnsibleHost(h.getName()); inventory.addHost(ah, "universe"); inventory.addHost(ah, "d4science"); ***REMOVED*** worker.setInventory(inventory); ***REMOVED*** generate the playbook Playbook playbook = new Playbook(); playbook.setRemote_user("root"); playbook.applyTo("universe"); for(Role r:algoRoles) { ***REMOVED*** add only 'add' roles if(!r.getName().endsWith("remove")) { playbook.addRole(r.getName()); ***REMOVED*** ***REMOVED*** worker.setPlaybook(playbook); ***REMOVED*** execute and save log locally PrintStream console = System.out; File path = new File(worker.getWorkdir() + File.separator + "logs"); path.mkdirs(); File n = new File(path + File.separator + worker.getWorkerId()); FileOutputStream fos = new FileOutputStream(n); PrintStream ps = new PrintStream(fos); System.setOut(ps); System.setErr(ps); ***REMOVED***System.setErr(console); worker.apply(); ***REMOVED***System.setOut(console); ***REMOVED***worker.apply(); System.out.println("Log stored to to " + n.getAbsolutePath()); ***REMOVED*** destroy the worker worker.destroy(); return worker; ***REMOVED*** private TemplateManager getTemplateManager() { return new TemplateManager(this.dpmRoot+"/templates"); ***REMOVED*** private CustomRoleManager getCustomRoleManager() { return new CustomRoleManager(this.dpmRoot+"/custom"); ***REMOVED*** private StaticRoleManager getStaticRoleManager() { return new StaticRoleManager(this.dpmRoot+"/static"); ***REMOVED*** /** * Generate all roles for this dependency * @param d */ public Collection generateRoles(Dependency d) { Collection roles = new Vector<>(); if("os".equalsIgnoreCase(d.getType())) { OSDependencyPackage pkg = new OSDependencyPackage(d); if(pkg!=null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); ***REMOVED*** ***REMOVED*** else if("custom".equalsIgnoreCase(d.getType())) { CustomDependencyPackage pkg = new CustomDependencyPackage(d); if(pkg!=null) { roles.addAll(pkg.getRoles(this.getCustomRoleManager())); ***REMOVED*** ***REMOVED*** else if("cran".equalsIgnoreCase(d.getType())) { CranDependencyPackage pkg = new CranDependencyPackage(d); if(pkg!=null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); ***REMOVED*** ***REMOVED*** return roles; ***REMOVED*** public Collection generateRoles(Algorithm a) { AlgorithmPackage pkg = new AlgorithmPackage(a); return pkg.getRoles(this.getTemplateManager()); ***REMOVED*** ***REMOVED***