This commit is contained in:
Paolo Fabriani 2016-11-17 11:34:53 +00:00
parent 705dd23458
commit de9f4fa4f3
30 changed files with 1753 additions and 0 deletions

View File

@ -0,0 +1,107 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
/**
* This class is responsible for the interface with ansible, retrieving log,
* etc. etc. It's not supposed to access templates and static stuff files. It
* does not know the service datamodel.
*
* @author paolo
*
*/
public class AnsibleWorker {
/**
* The name of the inventory
*/
private static String INVENTORY_NAME = "inventory.yaml";
/**
* The directory containing roles
*/
private static String ROLES_DIR = "roles";
/**
* The name of the playbook
*/
private static String PLAYBOOK_NAME = "playbook.yaml";
/**
* The root of the worker. This corresponds to a standard ansible working dir.
*/
private File workerRoot;
public AnsibleWorker(File root) {
this.workerRoot = root;
this.ensureWorkStructure();
}
private File getWorkdir() {
return this.workerRoot;
}
private File getRolesDir() {
return new File(this.getWorkdir(), ROLES_DIR);
}
public String getWorkerId() {
return this.workerRoot.getName();
}
private void ensureWorkStructure() {
// generate root
this.getWorkdir().mkdirs();
}
private void removeWorkStructure() {
// remove the working dir
// this.getWorkdir().delete();
}
public File getPlaybookFile() {
return new File(this.getWorkdir(), PLAYBOOK_NAME);
}
public File getInventoryFile() {
return new File(this.getWorkdir(), INVENTORY_NAME);
}
public void setInventory(Inventory inventory) throws IOException {
// serialize the string to the 'inventory' file
AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile());
}
public void setPlaybook(Playbook playbook) throws IOException {
// serialize the string to the 'playbook' file
AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile());
}
public void addRole(Role r) throws IOException {
// Serialize role in the workdir
AnsibleSerializeHelper.serializeRole(r, this.getRolesDir());
}
public void apply() {
// TODO execute the playbook and return output
System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
}
/**
* Destroy the worker:
* - remove the working dir
*/
public void destroy() {
this.removeWorkStructure();
}
}

View File

@ -0,0 +1,19 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
public class AnsibleHost {
private String name;
public AnsibleHost(String name) {
this.name = name;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

View File

@ -0,0 +1,29 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class HostGroup {
private String name;
private Collection<AnsibleHost> hosts;
public HostGroup(String name) {
this.name = name;
this.hosts = new Vector<>();
}
public void addHost(AnsibleHost h) {
this.hosts.add(h);
}
public String getName() {
return this.name;
}
public Collection<AnsibleHost> getHosts() {
return new Vector<>(this.hosts);
}
}

View File

@ -0,0 +1,37 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class Inventory {
private Collection<HostGroup> groups;
public Inventory() {
this.groups = new Vector<>();
}
public void addGroup(HostGroup group) {
this.groups.add(group);
}
public void addHost(AnsibleHost h, String groupName) {
this.getGroup(groupName).addHost(h);
}
private HostGroup getGroup(String groupName) {
for (HostGroup hg : this.groups) {
if (groupName.equals(hg.getName())) {
return hg;
}
}
HostGroup hg = new HostGroup(groupName);
this.groups.add(hg);
return hg;
}
public Collection<HostGroup> getHostGroups() {
return new Vector<>(this.groups);
}
}

View File

@ -0,0 +1,32 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
import java.util.List;
import java.util.Vector;
public class Playbook {
private String hostGroupName;
private List<String> roles;
public Playbook() {
this.roles = new Vector<>();
}
public void addRole(String role) {
roles.add(role);
}
public void applyTo(String hostGroupName) {
this.hostGroupName = hostGroupName;
}
public String getHostGroupName() {
return hostGroupName;
}
public List<String> getRoles() {
return new Vector<>(roles);
}
}

View File

@ -0,0 +1,51 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class Role {
/**
* The name of the role
*/
private String name;
private Collection<RoleFile> tasks;
private Collection<RoleFile> meta;
public Role() {
this.tasks = new Vector<>();
this.meta = new Vector<>();
}
public Role(String name) {
this();
this.name = name;
}
public void addTaskFile(RoleFile tf) {
this.tasks.add(tf);
}
public void addMeta(RoleFile tf) {
this.meta.add(tf);
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Collection<RoleFile> getTaskFiles() {
return new Vector<>(this.tasks);
}
public Collection<RoleFile> getMeta() {
return new Vector<>(this.meta);
}
}

View File

@ -0,0 +1,54 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model;
public class RoleFile {
/**
* The path to the file, starting from the role root
*/
private String path;
/**
* The name of the task file
*/
private String name;
/**
* The content of the task file
* @return
*/
private String content;
public RoleFile() {
}
public RoleFile(String name, String content) {
this();
this.setName(name);
this.setContent(content);
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
}

View File

@ -0,0 +1,239 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.AlgorithmPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CranDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomRoleManager;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.DependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.OSDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.StaticRoleManager;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.TemplateManager;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.AlgorithmComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.DependencyComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.HostComparator;
public class AnsibleBridge {
/**
* The workdir for this service
*/
private String dpmRoot;
public AnsibleBridge() {
this("/home/paolo/tmp/dataminer-pool-manager");
}
public AnsibleBridge(String root) {
this.dpmRoot = root;
this.ensureServiceRoot();
}
private void ensureServiceRoot() {
// generate root
new File(dpmRoot).mkdirs();
// 'template' is for template roles
this.getTemplatesDir().mkdirs();
// 'static' is for custom roles
this.getCustomDir().mkdirs();
// 'work' is for temporary working directories
this.getWorkDir().mkdirs();
}
private File getWorkDir() {
return new File(this.dpmRoot, "work");
}
private File getTemplatesDir() {
return new File(this.dpmRoot, "templates");
}
private File getCustomDir() {
return new File(this.dpmRoot, "custom");
}
public AnsibleWorker createWorker() {
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
AnsibleWorker worker = new AnsibleWorker(workerRoot);
return worker;
}
/**
* Groups hosts by domain and algorithm sets
* @param clusters
*/
public void printInventoryByDomainAndSets(Collection<Cluster> clusters) {
Map<String, Set<Host>> inventory = new TreeMap<>();
for(Cluster cluster:clusters) {
for(AlgorithmSet as:cluster.getAlgorithmSets()) {
String asName = as.getName();
for(Host h:cluster.getHosts()) {
String domain = h.getDomain().getName();
String key = String.format("[%s@%s]", asName, domain);
Set<Host> hosts = inventory.get(key);
if(hosts==null) {
hosts = new TreeSet<>(new HostComparator());
inventory.put(key, hosts);
}
hosts.add(h);
}
}
}
for(String key:inventory.keySet()) {
System.out.println(key);
Collection<Host> hosts = inventory.get(key);
for(Host h:hosts) {
System.out.println(h.getName()+"."+h.getDomain().getName());
}
System.out.println();
}
}
/**
* Groups hosts by algorithm sets only
* @param clusters
*/
public void printInventoryBySets(Collection<Cluster> clusters) {
Map<String, Set<Host>> inventory = new TreeMap<>();
for (Cluster cluster : clusters) {
for (AlgorithmSet as : cluster.getAlgorithmSets()) {
String asName = as.getName();
for (Host h : cluster.getHosts()) {
String key = String.format("[%s]", asName);
Set<Host> hosts = inventory.get(key);
if (hosts == null) {
hosts = new TreeSet<>(new HostComparator());
inventory.put(key, hosts);
}
hosts.add(h);
}
}
}
for (String key : inventory.keySet()) {
System.out.println(key);
Collection<Host> hosts = inventory.get(key);
for (Host h : hosts) {
System.out.println(h.getName()+"."+h.getDomain().getName());
}
System.out.println();
}
}
public void applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException {
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), UUID.randomUUID().toString()));
List<Role> algoRoles = new Vector<>();
// add algorithms and dependencies to the worker
for (Algorithm a : as.getAlgorithms()) {
for (Role r : this.generateRoles(a)) {
algoRoles.add(r);
worker.addRole(r);
}
for (Dependency d : a.getDependencies()) {
for (Role r : this.generateRoles(d)) {
worker.addRole(r);
}
}
}
// add static roles
for(Role r:this.getStaticRoleManager().getStaticRoles()) {
worker.addRole(r);
}
// generate the inventory
Inventory inventory = new Inventory();
for (Host h : cluster.getHosts()) {
AnsibleHost ah = new AnsibleHost(h.getName());
inventory.addHost(ah, "universe");
inventory.addHost(ah, "d4science");
}
worker.setInventory(inventory);
// generate the playbook
Playbook playbook = new Playbook();
playbook.applyTo("universe");
for(Role r:algoRoles) {
// add only 'add' roles
if(!r.getName().endsWith("remove")) {
playbook.addRole(r.getName());
}
}
worker.setPlaybook(playbook);
// execute
worker.apply();
// destroy the worker
worker.destroy();
}
private TemplateManager getTemplateManager() {
return new TemplateManager(this.dpmRoot+"/templates");
}
private CustomRoleManager getCustomRoleManager() {
return new CustomRoleManager(this.dpmRoot+"/custom");
}
private StaticRoleManager getStaticRoleManager() {
return new StaticRoleManager(this.dpmRoot+"/static");
}
/**
* Generate all roles for this dependency
* @param d
*/
public Collection<Role> generateRoles(Dependency d) {
Collection<Role> roles = new Vector<>();
if("os".equals(d.getType())) {
OSDependencyPackage pkg = new OSDependencyPackage(d);
if(pkg!=null) {
roles.addAll(pkg.getRoles(this.getTemplateManager()));
}
} else if("custom".equals(d.getType())) {
CustomDependencyPackage pkg = new CustomDependencyPackage(d);
if(pkg!=null) {
roles.addAll(pkg.getRoles(this.getCustomRoleManager()));
}
} else if("cran".equals(d.getType())) {
CranDependencyPackage pkg = new CranDependencyPackage(d);
if(pkg!=null) {
roles.addAll(pkg.getRoles(this.getTemplateManager()));
}
}
return roles;
}
public Collection<Role> generateRoles(Algorithm a) {
AlgorithmPackage pkg = new AlgorithmPackage(a);
return pkg.getRoles(this.getTemplateManager());
}
}

View File

@ -0,0 +1,117 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import org.apache.commons.io.IOUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.HostGroup;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
public class AnsibleSerializeHelper {
public static void serialize(Inventory inventory, File inventoryFile) throws IOException {
String out = "";
for(HostGroup hg:inventory.getHostGroups()) {
out+=String.format("[%s]\n", hg.getName());
for(AnsibleHost h:hg.getHosts()) {
out+=h.getName()+"\n";
}
out+="\n";
}
out = out.trim();
serialize(out, inventoryFile);
}
public static void serialize(Playbook playbook, File playbookFile) throws IOException {
String out = "- hosts: " + playbook.getHostGroupName() + "\n";
out += " remote_user: dpm\n";
out+=" roles:\n";
for(String r:playbook.getRoles()) {
out+=" - " + r+"\n";
}
out = out.trim();
serialize(out, playbookFile);
}
public static void serializeRole(Role r, File dir) throws IOException {
// create root
File root = new File(dir, r.getName());
root.mkdirs();
// create tasks
if(r.getTaskFiles().size()>0) {
File tasks = new File(root, "tasks");
tasks.mkdirs();
for(RoleFile tf: r.getTaskFiles()) {
serializeTask(tf, tasks);
}
}
// create meta
if(r.getMeta().size()>0) {
File meta = new File(root, "meta");
meta.mkdirs();
for(RoleFile tf: r.getMeta()) {
serializeTask(tf, meta);
}
}
}
public static void serializeTask(RoleFile tf, File dir) throws IOException {
File f = new File(dir, tf.getName());
serialize(tf.getContent().trim(), f);
}
public static void serialize(String s, File f) throws IOException {
PrintWriter out = new PrintWriter(f);
out.println(s);
out.close();
}
public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException {
Role out = new Role();
out.setName(roleDir.getName());
if(!roleDir.exists()) {
throw new FileNotFoundException();
}
try {
File tasksDir = new File(roleDir, "tasks");
if(tasksDir.exists()) {
for(File main:tasksDir.listFiles()) {
String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
RoleFile tf = new RoleFile(main.getName(), content);
tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
out.addTaskFile(tf);
}
}
} catch(FileNotFoundException e) {
e.printStackTrace();
}
try {
File metaDir = new File(roleDir, "meta");
if(metaDir.exists()) {
for(File main:metaDir.listFiles()) {
String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
RoleFile tf = new RoleFile(main.getName(), content);
tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
out.addMeta(tf);
}
}
} catch(FileNotFoundException e) {
e.printStackTrace();
}
return out;
}
}

View File

@ -0,0 +1,62 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class AlgorithmPackage {
private Algorithm algorithm;
public AlgorithmPackage(Algorithm a) {
this.algorithm = a;
}
protected Map<String, String> getDictionary(Algorithm a) {
Map<String, String> out = new HashMap<String, String>();
out.put("name", a.getName());
String deps = "";
for(Dependency d:a.getDependencies()) {
deps+=String.format("- { role: %s }\n", d.getType()+"-"+d.getName());
}
deps = deps.trim();
out.put("dependencies", deps);
return out;
}
protected Algorithm getAlgorithm() {
return this.algorithm;
}
public Collection<Role> getRoles(TemplateManager tm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"}) {
String roleName = "algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode);
try {
// find template
Role template = tm.getRoleTemplate("algorithm-" + mode);
//
if(template!=null) {
Map<String, String> dictionary = this.getDictionary(this.getAlgorithm());
Role r = tm.fillRoleTemplate(template, dictionary);
r.setName(roleName);
out.add(r);
} else {
System.out.println("WARNING: template is null");
}
} catch (NoSuchElementException e) {
// e.printStackTrace();
System.out.println("WARNING: no template found for " + roleName);
}
}
return out;
}
}

View File

@ -0,0 +1,11 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class CranDependencyPackage extends DependencyPackage {
public CranDependencyPackage(Dependency d) {
super(d);
}
}

View File

@ -0,0 +1,69 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.apache.commons.io.FileUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class CustomDependencyPackage extends DependencyPackage {
public CustomDependencyPackage(Dependency d) {
super(d);
}
private String getCustomRepositoryLocation(String ansibleRoot) {
return ansibleRoot+"/custom";
}
/*
public void serializeTo(String ansibleRoot) {
for(String mode:new String[]{"add", "remove", "update"}) {
// look for roles in the 'custom' repository
try {
// role name
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
// look for the custom role
File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName);
System.out.println("** CUSTOM ** " + src);
if(src.exists()) {
// do copy
System.out.println("copying CUSTOM role");
File dest = new File(ansibleRoot+"/work/"+roleName);
FileUtils.copyDirectory(src, dest);
}
} catch(IOException e) {
e.printStackTrace();
}
}
}
*/
public Collection<Role> getRoles(CustomRoleManager crm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"}) {
// role name
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
try {
// look for custom role
Role role = crm.getRole(roleName);
if(role!=null) {
out.add(role);
}
} catch (NoSuchElementException e) {
// e.printStackTrace();
System.out.println("WARNING: no custom role found for " + roleName);
}
}
return out;
}
}

View File

@ -0,0 +1,37 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.apache.commons.io.IOUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
public class CustomRoleManager {
private String root;
public CustomRoleManager(String root) {
this.root = root;
}
public String getRoot() {
return this.root;
}
public Role getRole(String roleName) throws NoSuchElementException {
File f = new File(this.getRoot(), roleName);
try {
return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
} catch (IOException e) {
// e.printStackTrace();
throw new NoSuchElementException("unable to find " + roleName);
}
}
}

View File

@ -0,0 +1,56 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class DependencyPackage {
private Dependency dependency;
public DependencyPackage(Dependency d) {
this.dependency = d;
}
protected Map<String, String> getDictionary(Dependency d) {
Map<String, String> out = new HashMap<String, String>();
out.put("name", d.getName());
out.put("type", d.getType());
return out;
}
protected Dependency getDependency() {
return this.dependency;
}
public Collection<Role> getRoles(TemplateManager tm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"}) {
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
try {
// find template
Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode);
//
if(template!=null) {
Map<String, String> dictionary = this.getDictionary(this.getDependency());
Role r = tm.fillRoleTemplate(template, dictionary);
r.setName(roleName);
out.add(r);
} else {
System.out.println("WARNING: template is null");
}
} catch (NoSuchElementException e) {
// e.printStackTrace();
System.out.println("WARNING: no template found for " + roleName);
}
}
return out;
}
}

View File

@ -0,0 +1,11 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class OSDependencyPackage extends DependencyPackage {
public OSDependencyPackage(Dependency d) {
super(d);
}
}

View File

@ -0,0 +1,37 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
public class StaticRoleManager {
private String root;
public StaticRoleManager(String root) {
this.root = root;
}
public String getRoot() {
return this.root;
}
public Collection<Role> getStaticRoles() {
Collection<Role> out = new Vector<>();
for(File f: new File(this.getRoot()).listFiles()) {
try {
out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f));
} catch(IOException e) {
e.printStackTrace();
}
}
return out;
}
}

View File

@ -0,0 +1,99 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.commons.io.IOUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.stringtemplate.v4.ST;
public class TemplateManager {
private String root;
public TemplateManager(String root) {
this.root = root;
}
public String getTemplateRoot() {
return this.root;
}
/**
* Read the given template
* @param templateName
* @return
* @throws IOException
*/
// private String readTemplate(String templateName) throws IOException {
// File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml");
// System.out.println("looking for file " + templateFile.getName());
// String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8");
// return out;
// }
/**
* Return the content of the given template
* @param templateName
* @return
* @throws NoSuchElementException if no such template exists
*/
// public String getTemplate(String templateName) throws NoSuchElementException {
// String template = null;
// try {
// template = this.readTemplate(templateName);
// } catch (IOException e) {
// throw new NoSuchElementException();
// }
// return template;
// }
public Role fillRoleTemplate(Role template, Map<String, String> dictionary) {
Role out = new Role();
out.setName(template.getName());
for(RoleFile tf:template.getTaskFiles()) {
out.addTaskFile(this.fillTaskTemplate(tf, dictionary));
}
for(RoleFile tf:template.getMeta()) {
out.addMeta(this.fillTaskTemplate(tf, dictionary));
}
return out;
}
private RoleFile fillTaskTemplate(RoleFile template, Map<String, String> dictionary) {
RoleFile out = new RoleFile();
out.setName(template.getName());
out.setContent(this.fillTemplate(template.getContent(), dictionary));
return out;
}
private String fillTemplate(String template, Map<String, String> dictionary) {
if (template != null) {
ST t = new ST(template);
for (String key : dictionary.keySet()) {
t.add(key, dictionary.get(key));
}
String output = t.render();
return output;
}
return template;
}
public Role getRoleTemplate(String roleName) throws NoSuchElementException {
File f = new File(this.getTemplateRoot(), roleName);
try {
return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
} catch (IOException e) {
// e.printStackTrace();
throw new NoSuchElementException("unable to find " + roleName);
}
}
}

View File

@ -0,0 +1,58 @@
package org.gcube.dataanalysys.dataminerpoolmanager.clients;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Collection;
import java.util.List;
import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
public class ISClient {
/**
* Return the list of hosts (dataminers) in a given VRE
*
* @param vreName
* @return
*/
public Collection<Host> listDataminersInVRE(String scope) {
boolean remote = false;
if (!remote) {
Collection<Host> out = new Vector<>();
Host h = new Host();
h.setName("bb-dataminer.res.eng.it");
out.add(h);
return out;
} else {
ScopeProvider.instance.set(scope);
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
Collection<Host> out = new Vector<>();
for (ServiceEndpoint r : resources) {
Host h = new Host();
h.setName(r.profile().runtime().hostedOn());
out.add(h);
}
return out;
}
}
}

View File

@ -0,0 +1,35 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
public class Action {
private String name;
private String description;
private String script;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getScript() {
return script;
}
public void setScript(String script) {
this.script = script;
}
}

View File

@ -0,0 +1,71 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class Algorithm {
private String name;
private String description;
private String category;
private Collection<Action> actions;
private Collection<Dependency> dependencies;
public Algorithm() {
this.actions = new Vector<>();
this.dependencies = new Vector<>();
Dependency p = new Dependency();
}
public void addDependency(Dependency dep) {
this.dependencies.add(dep);
}
public void addAction(Action action) {
this.actions.add(action);
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
public Collection<Action> getActions() {
return actions;
}
public Collection<Dependency> getDependencies() {
return dependencies;
}
public String toString() {
String out = "Algorithm: " + this.getName()+"\n";
out+=" Description: " + this.getDescription()+"\n";
out+=" Dependencies: " + this.getDependencies()+"\n";
return out;
}
}

View File

@ -0,0 +1,49 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class AlgorithmSet {
private String name;
private Collection<Algorithm> algorithms;
public AlgorithmSet() {
this.algorithms = new Vector<>();
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Collection<Algorithm> getAlgorithms() {
return new Vector<>(algorithms);
}
public void addAlgorithm(Algorithm algoritm) {
this.algorithms.add(algoritm);
}
public Boolean hasAlgorithm(Algorithm algorithm) {
for (Algorithm a : this.algorithms) {
if (a.getName().equals(algorithm.getName())) {
return true;
}
}
return false;
}
public String toString() {
String out = "ALGOSET: " + this.name + "\n";
for(Algorithm a:this.algorithms) {
out+=a+"\n";
}
return out;
}
}

View File

@ -0,0 +1,73 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class Cluster {
/**
* The set of hosts belonging to the cluster.
*/
private Collection<Host> hosts;
/**
* A name for this cluster.
*/
private String name;
/**
* A description of this cluster.
*/
private String description;
/**
* The set of algorithms deployed on this cluster (i.e. on all its hosts)
*/
private Collection<AlgorithmSet> algoSets;
public Cluster() {
this.hosts = new Vector<>();
this.algoSets = new Vector<>();
}
public void addAlgorithmSet(AlgorithmSet set) {
this.algoSets.add(set);
}
public void addHost(Host host) {
this.hosts.add(host);
}
public Collection<Host> getHosts() {
return hosts;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public Collection<AlgorithmSet> getAlgorithmSets() {
return algoSets;
}
public String toString() {
String out = "Cluster: "+this.name+"\n";
for(Host h:this.getHosts()) {
out+=" "+h+"\n";
}
return out;
}
}

View File

@ -0,0 +1,29 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
public class Dependency {
private String name;
private String type;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String toString() {
return this.type+":"+this.name;
}
}

View File

@ -0,0 +1,15 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
public class Domain {
private String name;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

View File

@ -0,0 +1,39 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel;
public class Host {
private String name;
private Domain domain;
public Host() {
}
public String getFullyQualifiedName() {
if(this.domain!=null && this.domain.getName()!=null)
return this.getName()+"."+this.getDomain().getName();
else
return this.getName();
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Domain getDomain() {
return domain;
}
public void setDomain(Domain domain) {
this.domain = domain;
}
public String toString() {
return this.name + "@" + this.domain;
}
}

View File

@ -0,0 +1,15 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
public class AlgorithmComparator implements Comparator<Algorithm> {
@Override
public int compare(Algorithm a1, Algorithm a2) {
return a1.getName().compareTo(a2.getName());
}
}

View File

@ -0,0 +1,19 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class DependencyComparator implements Comparator<Dependency> {
@Override
public int compare(Dependency a1, Dependency a2) {
int out = a1.getType().compareTo(a2.getType());
if(out!=0)
return out;
return a1.getName().compareTo(a2.getName());
}
}

View File

@ -0,0 +1,17 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator;
import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
public class HostComparator implements Comparator<Host> {
@Override
public int compare(Host h1, Host h2) {
int out = h1.getDomain().getName().compareTo(h2.getDomain().getName());
if(out!=0)
return out;
return h1.getName().compareTo(h2.getName());
}
}

View File

@ -0,0 +1,121 @@
package org.gcube.dataanalysys.dataminerpoolmanager.service;
import java.io.IOException;
import java.util.Collection;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysys.dataminerpoolmanager.clients.ISClient;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
public class DataminerPoolManager {
// static Collection<Algorithm> algorithms;
//
// static Collection<AlgorithmSet> sets;
//
// static {
// algorithms = new Vector<>();
// }
//
// public DataminerPoolManager() {
// }
//
// /**
// * Add a new algorithm to the set of known ones. No further action is expected
// * on the pool.
// */
// public void publishAlgorithm(Algorithm algorithm) {
// algorithms.add(algorithm);
// }
//
// /**
// * Re-deploy the given algorithm wherever it's installed
// *
// * @param algorithm
// */
// /*
// * public void updateAlgorithm(Algorithm algorithm) { // TODO implement this }
// */
//
// /**
// * Add the give algorithm to the given set
// *
// * @param algorithmId
// * @param setId
// */
// public void addAlgorithmToSet(String algorithmName, String setName) {
// AlgorithmSet set = this.getAlgorithmSet(setName);
// Algorithm algorithm = this.getAlgorithm(algorithmName);
// if (set != null && algorithm != null) {
// set.addAlgorithm(algorithm);
// this.updateClusters();
// }
// }
//
// /**
// * Apply the given set of algorithms to the given cluster
// *
// * @param setId
// * @param clusterId
// */
// public void applyAlgorithmSetToCluster(String setName, String clusterName) {
// AlgorithmSet set = this.getAlgorithmSet(setName);
// Cluster cluster = new ISClient().getCluster(clusterName);
// if (set != null && cluster != null) {
// cluster.addAlgorithmSet(set);
// this.updateClusters();
// }
// }
//
// private AlgorithmSet getAlgorithmSet(String name) {
// for (AlgorithmSet set : sets) {
// if (name.equals(set.getName())) {
// return set;
// }
// }
// return null;
// }
//
// private Algorithm getAlgorithm(String name) {
// for (Algorithm a : algorithms) {
// if (name.equals(a.getName())) {
// return a;
// }
// }
// return null;
// }
/**
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*/
public void addAlgorithmToVRE(Algorithm algorithm, String vre) throws IOException {
// create a fake algorithm set
AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
// create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for(Host h:new ISClient().listDataminersInVRE(vre)) {
cluster.addHost(h);
}
// apply the changes
new AnsibleBridge().applyAlgorithmSetToCluster(algoSet, cluster);
}
// private void updateClusters() {
// System.out.println("flushing changes to all clusters");
// }
}

View File

@ -0,0 +1,145 @@
package org.gcube.dataanalysys.dataminerpoolmanager.util;
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
interface NetworkConfiguration {
public String getProxyHost();
public String getProxyPort();
public String getProxyUser();
public String getProxyPassword();
public String getNonProxyHosts();
}
class FileBasedProxyConfiguration implements NetworkConfiguration {
private static PropertiesConfiguration configuration;
public FileBasedProxyConfiguration(String path) {
try {
// load the configuration
configuration = new PropertiesConfiguration(path);
// set the reloading strategy to enable hot-configuration
FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy();
configuration.setReloadingStrategy(fcrs);
} catch (ConfigurationException e) {
e.printStackTrace();
}
}
@Override
public String getProxyHost() {
return configuration.getString("proxyHost");
}
@Override
public String getProxyPort() {
return configuration.getString("proxyPort");
}
@Override
public String getProxyUser() {
return configuration.getString("proxyUser");
}
@Override
public String getProxyPassword() {
return configuration.getString("proxyPassword");
}
@Override
public String getNonProxyHosts() {
return configuration.getString("nonProxyHosts");
}
}
public class PropertiesBasedProxySelector extends ProxySelector {
List<Proxy> proxies = null;
List<String> nonProxyHosts = null;
public PropertiesBasedProxySelector(String proxySettingsPath) {
this(new FileBasedProxyConfiguration(proxySettingsPath));
}
public PropertiesBasedProxySelector(NetworkConfiguration config) {
if (config == null || config.getProxyHost() == null) {
this.proxies = null;
return;
}
String host = config.getProxyHost();
int port = 80;
if (config.getProxyPort() != null) {
port = Integer.valueOf(config.getProxyPort());
}
if (config.getNonProxyHosts() != null) {
this.nonProxyHosts = Arrays
.asList(config.getNonProxyHosts().split("\\|"));
}
this.proxies = new ArrayList<Proxy>();
this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host,
port)));
if (config.getProxyUser() != null) {
final String username = config.getProxyUser();
final String password = config.getProxyPassword();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password.toCharArray());
}
});
}
}
@Override
public List<Proxy> select(URI uri) {
if (this.nonProxyHosts == null) {
return Arrays.asList(Proxy.NO_PROXY);
} else {
for (String entry : this.nonProxyHosts) {
entry = entry.trim();
if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) {
return Arrays.asList(Proxy.NO_PROXY);
}
if (uri.getHost().equals(entry)) {
return Arrays.asList(Proxy.NO_PROXY);
}
}
return this.proxies;
}
}
@Override
public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) {
}
}