package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; ***REMOVED*** import java.util.Collection; ***REMOVED*** import java.util.Map; ***REMOVED*** import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; public class AnsibleBridge ***REMOVED*** ***REMOVED*** private static final org.slf4j.Logger LOGGER = ***REMOVED*** LoggerFactory.getLogger(AnsibleBridge.class); private String dpmRoot; ***REMOVED*** public AnsibleBridge() ***REMOVED*** ***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); ***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); ***REMOVED*** ***REMOVED*** ***REMOVED*** public AnsibleBridge(String root) ***REMOVED*** this.dpmRoot = root; this.ensureServiceRoot(); ***REMOVED*** private void ensureServiceRoot() ***REMOVED*** ***REMOVED*** generate root new File(dpmRoot).mkdirs(); ***REMOVED*** 'template' is for template roles ***REMOVED*** this.getTemplatesDir().mkdirs(); ***REMOVED*** 'static' is for custom roles ***REMOVED*** this.getCustomDir().mkdirs(); ***REMOVED*** 'work' is for temporary working directories this.getWorkDir().mkdirs(); ***REMOVED*** private File getWorkDir() ***REMOVED*** return new File(this.dpmRoot, "work"); ***REMOVED*** ***REMOVED*** private String getTemplatesDir() ***REMOVED*** ***REMOVED*** String input = null; ***REMOVED*** input = ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); ***REMOVED*** return input; ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** private String getCustomDir() ***REMOVED*** ***REMOVED*** String input = null; ***REMOVED*** input = ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); ***REMOVED*** return input; ***REMOVED*** ***REMOVED*** public AnsibleWorker createWorker(Algorithm algorithm, Cluster dataminerCluster, boolean includeAlgorithmDependencies, String user) throws IOException ***REMOVED*** File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); AnsibleWorker worker = new AnsibleWorker(workerRoot); List algoRoles = new Vector<>(); ***REMOVED*** add algorithms and dependencies to the worker for (Role r : this.generateRoles(algorithm, includeAlgorithmDependencies)) ***REMOVED*** algoRoles.add(r); worker.addRole(r); ***REMOVED*** ***REMOVED*** to comment the for in case of just install algo if (includeAlgorithmDependencies) ***REMOVED*** for (Dependency d : algorithm.getDependencies()) ***REMOVED*** for (Role r : this.generateRoles(d)) ***REMOVED*** worker.addRole(r); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** add static roles for (Role r : this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** worker.addRole(r); ***REMOVED*** ***REMOVED*** generate the inventory Inventory inventory = new Inventory(); for (Host h : dataminerCluster.getHosts()) ***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName()); inventory.addHost(ah, "universe"); inventory.addHost(ah, "d4science"); ***REMOVED*** worker.setInventory(inventory); ***REMOVED*** generate the playbook Playbook playbook = new Playbook(); playbook.setRemote_user(user); playbook.applyTo("universe"); for (Role r : algoRoles) ***REMOVED*** ***REMOVED*** add only 'add' roles if (!r.getName().endsWith("remove")) ***REMOVED*** playbook.addRole(r.getName()); ***REMOVED*** ***REMOVED*** worker.setPlaybook(playbook); return worker; ***REMOVED*** public void printInventoryByDomainAndSets(Collection clusters) ***REMOVED*** Map> inventory = new TreeMap<>(); for (Cluster cluster : clusters) ***REMOVED*** for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** String asName = as.getName(); for (Host h : cluster.getHosts()) ***REMOVED*** String domain = h.getDomain().getName(); String key = String.format("[%s@%s]", asName, domain); Set hosts = inventory.get(key); if (hosts == null) ***REMOVED*** hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); ***REMOVED*** hosts.add(h); ***REMOVED*** ***REMOVED*** ***REMOVED*** for (String key : inventory.keySet()) ***REMOVED*** System.out.println(key); Collection hosts = inventory.get(key); for (Host h : hosts) ***REMOVED*** System.out.println(h.getName() + "." + h.getDomain().getName()); ***REMOVED*** System.out.println(); ***REMOVED*** ***REMOVED*** public void printInventoryBySets(Collection clusters) ***REMOVED*** Map> inventory = new TreeMap<>(); for (Cluster cluster : clusters) ***REMOVED*** for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** String asName = as.getName(); for (Host h : cluster.getHosts()) ***REMOVED*** String key = String.format("[%s]", asName); Set hosts = inventory.get(key); if (hosts == null) ***REMOVED*** hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); ***REMOVED*** hosts.add(h); ***REMOVED*** ***REMOVED*** ***REMOVED*** for (String key : inventory.keySet()) ***REMOVED*** System.out.println(key); Collection hosts = inventory.get(key); for (Host h : hosts) ***REMOVED*** System.out.println(h.getName() + "." + h.getDomain().getName()); ***REMOVED*** System.out.println(); ***REMOVED*** ***REMOVED*** ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster ***REMOVED*** cluster, /*boolean updateSVN,*/ boolean test) throws IOException, ***REMOVED*** InterruptedException, SVNException ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** return applyAlgorithmSetToCluster ***REMOVED*** (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); ***REMOVED*** ***REMOVED*** ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster ***REMOVED*** cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws ***REMOVED*** IOException, InterruptedException, SVNException ***REMOVED*** ***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), ***REMOVED*** uuid)); ***REMOVED*** ***REMOVED*** ***REMOVED*** List algoRoles = new Vector<>(); ***REMOVED*** ***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker ***REMOVED*** for (Algorithm a : as.getAlgorithms()) ***REMOVED*** ***REMOVED*** for (Role r : this.generateRoles(a)) ***REMOVED*** ***REMOVED*** algoRoles.add(r); ***REMOVED*** worker.addRole(r); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***to comment the for in case of just install algo ***REMOVED*** if(test)***REMOVED*** ***REMOVED*** for (Dependency d : a.getDependencies()) ***REMOVED*** ***REMOVED*** for (Role r : this.generateRoles(d)) ***REMOVED*** ***REMOVED*** worker.addRole(r); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** add static roles ***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** ***REMOVED*** worker.addRole(r); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** generate the inventory ***REMOVED*** Inventory inventory = new Inventory(); ***REMOVED*** for (Host h : cluster.getHosts()) ***REMOVED*** ***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName()); ***REMOVED*** inventory.addHost(ah, "universe"); ***REMOVED*** inventory.addHost(ah, "d4science"); ***REMOVED*** ***REMOVED*** ***REMOVED*** worker.setInventory(inventory); ***REMOVED*** ***REMOVED*** ***REMOVED*** generate the playbook ***REMOVED*** Playbook playbook = new Playbook(); ***REMOVED*** if(test)***REMOVED*** ***REMOVED*** playbook.setRemote_user("root");***REMOVED*** ***REMOVED*** playbook.setRemote_user("gcube"); ***REMOVED*** playbook.applyTo("universe"); ***REMOVED*** for(Role r:algoRoles) ***REMOVED*** ***REMOVED*** ***REMOVED*** add only 'add' roles ***REMOVED*** if(!r.getName().endsWith("remove")) ***REMOVED*** ***REMOVED*** playbook.addRole(r.getName()); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** worker.setPlaybook(playbook); ***REMOVED*** ***REMOVED*** ***REMOVED*** execute and save log locally ***REMOVED*** ***REMOVED***PrintStream console = System.out; ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); ***REMOVED*** path.mkdirs(); ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); ***REMOVED*** FileOutputStream fos = new FileOutputStream(n); ***REMOVED*** PrintStream ps = new PrintStream(fos); ***REMOVED*** ***REMOVED*** ***REMOVED***System.setErr(console); ***REMOVED*** ***REMOVED*** worker.apply(as,ps,test); ***REMOVED*** ***REMOVED***System.setOut(console); ***REMOVED*** ***REMOVED***worker.apply(); ***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); ***REMOVED*** ***REMOVED*** ***REMOVED*** destroy the worker ***REMOVED*** worker.destroy(); ***REMOVED*** return worker; ***REMOVED*** ***REMOVED*** private TemplateManager getTemplateManager() ***REMOVED*** return new TemplateManager(); ***REMOVED*** private CustomRoleManager getCustomRoleManager() ***REMOVED*** return new CustomRoleManager(); ***REMOVED*** private StaticRoleManager getStaticRoleManager() ***REMOVED*** return new StaticRoleManager(); ***REMOVED*** ***REMOVED**** * Generate all roles for this dependency * * @param dep * Dependency * @return Collection of Roles ***REMOVED*** public Collection generateRoles(Dependency dep) ***REMOVED*** Collection roles = new Vector<>(); if ("os".equalsIgnoreCase(dep.getType())) ***REMOVED*** OSDependencyPackage pkg = new OSDependencyPackage(dep); if (pkg != null) ***REMOVED*** roles.addAll(pkg.getRoles(this.getTemplateManager())); ***REMOVED*** ***REMOVED*** else if ("custom".equalsIgnoreCase(dep.getType())) ***REMOVED*** CustomDependencyPackage pkg = new CustomDependencyPackage(dep); if (pkg != null) ***REMOVED*** roles.addAll(pkg.getRoles(this.getCustomRoleManager())); ***REMOVED*** ***REMOVED*** else if ("github".equalsIgnoreCase(dep.getType())) ***REMOVED*** CranDependencyPackage pkg = new CranDependencyPackage(dep); if (pkg != null) ***REMOVED*** roles.addAll(pkg.getRoles(this.getTemplateManager())); ***REMOVED*** ***REMOVED*** else if ("cran".equalsIgnoreCase(dep.getType())) ***REMOVED*** CranDependencyPackage pkg = new CranDependencyPackage(dep); if (pkg != null) ***REMOVED*** roles.addAll(pkg.getRoles(this.getTemplateManager())); ***REMOVED*** ***REMOVED*** return roles; ***REMOVED*** public Collection generateRoles(Algorithm a, boolean includeAlgorithmDependencies) ***REMOVED*** AlgorithmPackage pkg = new AlgorithmPackage(a, includeAlgorithmDependencies); return pkg.getRoles(this.getTemplateManager()); ***REMOVED*** ***REMOVED***