gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@147247 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-28 13:26:47 +00:00
parent 24c7b210a6
commit 3035a8ee2f
3 changed files with 52 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java
View File

@ -62,14 +62,16 @@ public class HAProxy {
cl.setName("genericworkers_proto");
}
//dev HAProxy
if (HProxy.equals("dataminer-d-workers.d4science.org")) {
if (HProxy.equals("dataminer-d-workers.d4science.org")||(HProxy.equals("dataminer-d-d4s.d4science.org"))) {
cl.setName("devnext_backend");
}
}
//preProd HAProxy
// if (HProxy.equals("dataminer1-pre.d4science.org")) {
// cl.setName("dataminer1-pre.d4science.org");
// }
return cl;
}
public List<Host> listDataMinersByCluster() throws IOException {
@ -77,26 +79,40 @@ public class HAProxy {
// Cluster cluster = this.getClusterByHProxy();
Cluster cluster = this.MapCluster();
List<Host> out = new LinkedList<Host>();
// prod
//URL stockURL = new URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
URL stockURL = new URL("http://"+ ISClient.getHProxy() +":8880/;csv");
System.out.println(stockURL);
// dev
// URL stockURL = new
// URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in, ',');
String[] nextLine;
while ((nextLine = reader.readNext()) != null) {
if (nextLine[1].equals("BACKEND") || (nextLine[1].equals("FRONTEND"))) {
continue;
}
Host a = new Host();
if (nextLine[0].equals(cluster.getName())) {
Host a = new Host();
a.setName(nextLine[1]);
out.add(a);
System.out.println(a.getFullyQualifiedName());
//no proxy dataminer (preprod)
if (cluster.getName() == null){
a.setName(ISClient.getHProxy());
out.add(a);
}
// if preprod, just one dm available
// if (cluster.getName().equals("dataminer1-pre.d4science.org")) {
// a.setName("dataminer1-pre.d4science.org");
// out.add(a);
//}
else {
// prod
// URL stockURL = new
// URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
URL stockURL = new URL("http://" + ISClient.getHProxy() + ":8880/;csv");
System.out.println(stockURL);
// dev
// URL stockURL = new
// URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in, ',');
String[] nextLine;
while ((nextLine = reader.readNext()) != null) {
if (nextLine[1].equals("BACKEND") || (nextLine[1].equals("FRONTEND"))) {
continue;
}
if (nextLine[0].equals(cluster.getName())) {
a.setName(nextLine[1]);
out.add(a);
System.out.println(a.getFullyQualifiedName());
}
}
}
return out;

52
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -197,33 +197,10 @@ public class RestPoolManager implements PoolManager {
@DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN)
throws IOException, InterruptedException, SVNException {
Algorithm algo = this.getAlgorithm(algorithm, /*vre*/null, null, name, description, category, algorithmType, skipJava);
// publish algo
if (publish) {
service.addAlgToIs(algo);
}
// update svn
// if (updateSVN){
// for (Dependency d : algo.getDependencies()) {
//
// if (d.getType().equals("os")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_deb_pkgs.txt", ls);
// }
// if (d.getType().equals("cran")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_cran_pkgs.txt", ls);
// }
// if (d.getType().equals("github")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_github_pkgs.txt", ls);
// }
// }
//}
return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),updateSVN);
}
@ -246,32 +223,9 @@ public class RestPoolManager implements PoolManager {
Algorithm algo = this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType,
skipJava);
// publish algo
if (publish) {
service.addAlgToIs(algo);
}
// update svn
// if (updateSVN){
// for (Dependency d : algo.getDependencies()) {
//
// if (d.getType().equals("os")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_deb_pkgs.txt", ls);
// }
// if (d.getType().equals("cran")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_cran_pkgs.txt", ls);
// }
// if (d.getType().equals("github")) {
// List<String> ls = new LinkedList<String>();
// ls.add(d.getName());
// service.updateSVN("r_github_pkgs.txt", ls);
// }
// }
//}
return service.addAlgorithmToHost(algo, hostname,updateSVN);
}
@ -329,7 +283,7 @@ public class RestPoolManager implements PoolManager {
// ProxySelector.setDefault(new
// PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ScopeProvider.instance.set("/gcube/preprod/preVRE");
// PoolManager aa = new DataminerPoolManager();
// System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3"));
@ -349,9 +303,9 @@ public class RestPoolManager implements PoolManager {
a.addAlgorithmToHost(
"http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0",
"dataminer1-devnext.d4science.org",
"dataminer1-pre.d4science.org",
"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers",
"N",false, true);
"N",false, false);

23
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -566,30 +566,27 @@ public class DataminerPoolManager implements PoolManager {
}
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException {
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN)
throws IOException, InterruptedException, SVNException {
// create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
cluster.addHost(new ISClient().getDataminer(hostname));
// for(Host h:new ISClient().listDataminersInVRE()) {
// cluster.addHost(h);
for (Host h : new HAProxy().listDataMinersByCluster()) {
if (h.getName().equals(hostname)) {
cluster.addHost(h);
}
}
// if(ISClient.getHProxy().equals(hostname)){
// cluster.addHost(new ISClient().getDataminer(hostname));
// }
// apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid,updateSVN ).getWorkerId();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId();
}
public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url);
}