gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148412 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-05-09 15:10:03 +00:00
parent 7d286d5326
commit 3d6a1273ff
6 changed files with 154 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -24,6 +24,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSeriali
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@ -118,7 +119,7 @@ public class AnsibleWorker {
public void apply(AlgorithmSet as, PrintStream ps, boolean updateSVN) public void apply(AlgorithmSet as, PrintStream ps, boolean test)
throws IOException, InterruptedException, SVNException { throws IOException, InterruptedException, SVNException {
***REMOVED*** TODO execute the playbook and return output ***REMOVED*** TODO execute the playbook and return output
System.out.println(this.getWorkdir()); System.out.println(this.getWorkdir());
@ -128,43 +129,70 @@ public class AnsibleWorker {
inheritIO(p.getInputStream(), ps); inheritIO(p.getInputStream(), ps);
inheritIO(p.getErrorStream(), ps); inheritIO(p.getErrorStream(), ps);
if (updateSVN) {
int exitValue = p.waitFor(); int exitValue = p.waitFor();
if (exitValue == 0) { if (exitValue == 0) {
if (test) {
for (Algorithm algo : as.getAlgorithms()) { for (Algorithm algo : as.getAlgorithms()) {
for (Dependency d : algo.getDependencies()) { for (Dependency d : algo.getDependencies()) {
if (d.getType().equals("os")) { if (d.getType().equals("os")) {
List<String> ls = new LinkedList<String>(); List<String> ls = new LinkedList<String>();
ls.add(d.getName()); ls.add(d.getName());
this.updateSVN("r_deb_pkgs.txt", ls); this.updateSVN("test_r_deb_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
if (d.getType().equals("cran")) { if (d.getType().equals("cran")) {
List<String> ls = new LinkedList<String>(); List<String> ls = new LinkedList<String>();
ls.add(d.getName()); ls.add(d.getName());
this.updateSVN("r_cran_pkgs.txt", ls); this.updateSVN("test_r_cran_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
if (d.getType().equals("github")) { if (d.getType().equals("github")) {
List<String> ls = new LinkedList<String>(); List<String> ls = new LinkedList<String>();
ls.add(d.getName()); ls.add(d.getName());
this.updateSVN("r_github_pkgs.txt", ls); this.updateSVN("test_r_github_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
else if(!test){
for (Algorithm algo : as.getAlgorithms()) {
for (Dependency d : algo.getDependencies()) {
if (d.getType().equals("os")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
for (Algorithm algo : as.getAlgorithms()) {
DataminerPoolManager a = new DataminerPoolManager();
a.addAlgToIs(algo);
***REMOVED***
***REMOVED***
***REMOVED*** catch (IOException e) { ***REMOVED*** catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** System.out.println("TODO: execute: ansible-playbook -v -i " +
***REMOVED*** this.getInventoryFile().getName() + " " +
***REMOVED*** this.getPlaybookFile().getName());
***REMOVED*** ***REMOVED***

15
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -158,13 +158,13 @@ public class AnsibleBridge {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, boolean updateSVN) throws IOException, InterruptedException, SVNException { public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),updateSVN); return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
***REMOVED*** ***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
@ -176,11 +176,14 @@ public class AnsibleBridge {
algoRoles.add(r); algoRoles.add(r);
worker.addRole(r); worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED***to comment the for in case of just install algo
if(test){
for (Dependency d : a.getDependencies()) { for (Dependency d : a.getDependencies()) {
for (Role r : this.generateRoles(d)) { for (Role r : this.generateRoles(d)) {
worker.addRole(r); worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** add static roles ***REMOVED*** add static roles
@ -199,7 +202,9 @@ public class AnsibleBridge {
***REMOVED*** generate the playbook ***REMOVED*** generate the playbook
Playbook playbook = new Playbook(); Playbook playbook = new Playbook();
playbook.setRemote_user("root"); if(test){
playbook.setRemote_user("root");***REMOVED***
playbook.setRemote_user("gcube");
playbook.applyTo("universe"); playbook.applyTo("universe");
for(Role r:algoRoles) { for(Role r:algoRoles) {
***REMOVED*** add only 'add' roles ***REMOVED*** add only 'add' roles
@ -220,7 +225,7 @@ public class AnsibleBridge {
***REMOVED***System.setErr(console); ***REMOVED***System.setErr(console);
worker.apply(as,ps,updateSVN); worker.apply(as,ps,test);
***REMOVED***System.setOut(console); ***REMOVED***System.setOut(console);
***REMOVED***worker.apply(); ***REMOVED***worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath()); System.out.println("Log stored to to " + n.getAbsolutePath());

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -72,6 +72,7 @@ public class ISClient {
***REMOVED*** ***REMOVED***
return h; return h;
***REMOVED*** ***REMOVED***
***REMOVED*** return the HProxy hostname in the VRE ***REMOVED*** return the HProxy hostname in the VRE
public static String getHProxy() { public static String getHProxy() {

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
View File

@ -35,12 +35,13 @@ import java.net.UnknownHostException;
import java.util.Set; import java.util.Set;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
public interface PoolManager { public interface PoolManager {
String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn ) throws IOException, InterruptedException; String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException;
String addAlgorithmToHost(Algorithm algo, String host, boolean svn) throws IOException, InterruptedException; String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException;
Algorithm extractAlgorithm(String url) throws IOException; Algorithm extractAlgorithm(String url) throws IOException;
@ -57,5 +58,5 @@ public interface PoolManager {
Set<Algorithm> getAlgoFromIs(); Set<Algorithm> getAlgoFromIs();
List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException; List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException;
***REMOVED*** ***REMOVED***

98
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -162,8 +162,10 @@ import javax.ws.rs.QueryParam;
import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.lang.math.RandomUtils;
***REMOVED*** ***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
@ -171,6 +173,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySel
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
import ch.qos.logback.core.subst.Token;
@Path("/") @Path("/")
public class RestPoolManager implements PoolManager { public class RestPoolManager implements PoolManager {
@ -201,37 +205,46 @@ public class RestPoolManager implements PoolManager {
if (publish) { if (publish) {
service.addAlgToIs(algo); service.addAlgToIs(algo);
***REMOVED*** ***REMOVED***
return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),updateSVN); return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),/*updateSVN,*/false);
***REMOVED*** ***REMOVED***
***REMOVED***Test Phase
***REMOVED***The DMPM contacts the Staging Dataminer and installs the algorithm and the dependencies
@GET @GET
@Path("/hosts/add") @Path("/algo/add")
@Produces("text/plain") @Produces("text/plain")
public String addAlgorithmToHost( public String addAlgorithmToStagingHost(
@QueryParam("algorithm") String algorithm, @QueryParam("algorithm") String algorithm,
@QueryParam("hostname") String hostname, @DefaultValue("insert hostname here") @QueryParam("hostname") String hostname,
@QueryParam("name") String name, @QueryParam("name") String name,
@QueryParam("description") String description, @QueryParam("description") String description,
@QueryParam("category") String category, @QueryParam("category") String category,
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava, @DefaultValue("N") @QueryParam("skipJava") String skipJava,
@DefaultValue("false") @QueryParam("publish") boolean publish, @DefaultValue("prodtoken") @QueryParam("prodtoken") String prodtoken,
@DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN) @DefaultValue("true") @QueryParam("test") boolean test)
throws IOException, InterruptedException, SVNException { throws IOException, InterruptedException, SVNException {
Algorithm algo = this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, String id="";
skipJava); Algorithm algo = this.getAlgorithm(algorithm, ScopeProvider.instance.get(), hostname, name, description, category, algorithmType,skipJava);
***REMOVED*** publish algo
if (publish) { ***REMOVED***test phase
service.addAlgToIs(algo); if(test){
service.addAlgorithmToHost(algo, hostname,true);
id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),true);
***REMOVED*** ***REMOVED***
return service.addAlgorithmToHost(algo, hostname,updateSVN); ***REMOVED***release phase
if(!test){
SecurityTokenProvider.instance.set(prodtoken);
id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),false);
***REMOVED***
return id;
***REMOVED*** ***REMOVED***
private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description, private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description,
String category, String algorithmType, String skipJava) throws IOException, InterruptedException { String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + ScopeProvider.instance.get());
Algorithm algo = service.extractAlgorithm(algorithm); Algorithm algo = service.extractAlgorithm(algorithm);
if (algo.getCategory() == null) { if (algo.getCategory() == null) {
@ -283,8 +296,10 @@ public class RestPoolManager implements PoolManager {
***REMOVED*** ProxySelector.setDefault(new ***REMOVED*** ProxySelector.setDefault(new
***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext"); ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
***REMOVED*** PoolManager aa = new DataminerPoolManager(); ***REMOVED*** PoolManager aa = new DataminerPoolManager();
***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3")); ***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3"));
***REMOVED*** System.out.println(aa.getAlgoFromIs()); ***REMOVED*** System.out.println(aa.getAlgoFromIs());
@ -306,9 +321,17 @@ public class RestPoolManager implements PoolManager {
***REMOVED*** "dataminer1-pre.d4science.org", ***REMOVED*** "dataminer1-pre.d4science.org",
***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", ***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers",
***REMOVED*** "N",false, false); ***REMOVED*** "N",false, false);
RestPoolManager b = new RestPoolManager();
b.addAlgorithmToStagingHost(
"http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0",
"dataminer1-devnext.d4science.org",
null,
null,
"N",
"transducerers",
false,
false
***REMOVED*** PoolManager aa = new DataminerPoolManager(); ***REMOVED*** PoolManager aa = new DataminerPoolManager();
***REMOVED*** List<String> ls = new LinkedList<String>(); ***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** String afa = "test"; ***REMOVED*** String afa = "test";
@ -357,21 +380,32 @@ public class RestPoolManager implements PoolManager {
***REMOVED*** ***REMOVED***
@Override @Override
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn) throws IOException, InterruptedException { public String addAlgorithmToHost(Algorithm algo, String host, boolean test)
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public String addAlgorithmToHost(Algorithm algo, String host, boolean svn)
throws IOException, InterruptedException { throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
return null; return null;
***REMOVED*** ***REMOVED***
@Override
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test)
throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED*** ***REMOVED***

431
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -1,265 +1,3 @@
***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.service;
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***import java.io.File;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***import java.net.MalformedURLException;
***REMOVED******REMOVED***
***REMOVED***import java.net.URLConnection;
***REMOVED***import java.net.UnknownHostException;
***REMOVED***import java.util.UUID;
***REMOVED***
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***public class DataminerPoolManager implements PoolManager {
***REMOVED***
***REMOVED***
***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED***
***REMOVED***
***REMOVED******REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED******REMOVED***
***REMOVED******REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED******REMOVED***
***REMOVED******REMOVED*** static {
***REMOVED******REMOVED*** algorithms = new Vector<>();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** public DataminerPoolManager() {
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
***REMOVED******REMOVED*** * on the pool.
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED******REMOVED*** algorithms.add(algorithm);
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param algorithm
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** /*
***REMOVED******REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED***
***REMOVED******REMOVED*** */
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Add the give algorithm to the given set
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param algorithmId
***REMOVED******REMOVED*** * @param setId
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED******REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED******REMOVED*** if (set != null && algorithm != null) {
***REMOVED******REMOVED*** set.addAlgorithm(algorithm);
***REMOVED******REMOVED*** this.updateClusters();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param setId
***REMOVED******REMOVED*** * @param clusterId
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) {
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED******REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED******REMOVED*** if (set != null && cluster != null) {
***REMOVED******REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED******REMOVED*** this.updateClusters();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED******REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED******REMOVED*** if (name.equals(set.getName())) {
***REMOVED******REMOVED*** return set;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED******REMOVED*** for (Algorithm a : algorithms) {
***REMOVED******REMOVED*** if (name.equals(a.getName())) {
***REMOVED******REMOVED*** return a;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
***REMOVED******REMOVED*** new Thread() {
***REMOVED******REMOVED*** public void run() {
***REMOVED******REMOVED*** while (true) {
***REMOVED******REMOVED*** try {
***REMOVED******REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED******REMOVED*** ***REMOVED***log here
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***.start();
***REMOVED******REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED******REMOVED*** public String getLogId(){
***REMOVED******REMOVED*** PrintStream console = System.out;
***REMOVED******REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED******REMOVED*** path.mkdirs();
***REMOVED******REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED******REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED******REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED******REMOVED*** System.setOut(ps);
***REMOVED******REMOVED*** worker.apply();
***REMOVED******REMOVED*** System.setOut(console);
***REMOVED******REMOVED*** worker.apply();
***REMOVED******REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED******REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED*** public String getLogById(String id) throws IOException {
***REMOVED******REMOVED*** String strLine = null;
***REMOVED******REMOVED*** try{
***REMOVED******REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED******REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED******REMOVED*** /* read log line by line */
***REMOVED******REMOVED*** while ((strLine = br.readLine()) != null) {
***REMOVED******REMOVED*** /* parse strLine to obtain what you want */
***REMOVED******REMOVED*** System.out.println (strLine);
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** br.close();
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED******REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return strLine;
***REMOVED******REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getScriptFromURL(URL url) throws IOException {
***REMOVED*** if (url == null) {
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED*** URLConnection yc = url.openConnection();
***REMOVED*** BufferedReader input = new BufferedReader(new InputStreamReader(
***REMOVED*** yc.getInputStream()));
***REMOVED*** String line;
***REMOVED*** StringBuffer buffer = new StringBuffer();
***REMOVED*** while ((line = input.readLine()) != null) {
***REMOVED*** buffer.append(line + "\n");
***REMOVED*** ***REMOVED***
***REMOVED*** String bufferScript = buffer.substring(0, buffer.length());
***REMOVED*** input.close();
***REMOVED*** return bufferScript;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Publish the given algorithm in the given VRE
***REMOVED*** *
***REMOVED*** * @param algorithmName
***REMOVED*** * @param vre
***REMOVED*** *
***REMOVED*** */
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
***REMOVED*** ***REMOVED*** create a fake algorithm set
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
***REMOVED*** algoSet.setName("fake");
***REMOVED*** algoSet.addAlgorithm(algorithm);
***REMOVED*** final String uuid = UUID.randomUUID().toString();
***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid);
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***).start();
***REMOVED*** ***REMOVED***this line will execute immediately, not waiting for your task to complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
***REMOVED***
***REMOVED*** File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator +a);
***REMOVED*** ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress();
***REMOVED***
***REMOVED*** return new File(n.getPath()).toURI().toURL();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
***REMOVED***
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException {
***REMOVED*** return new AlgorithmPackageParser().parsePackage(url);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***@Override
***REMOVED***public void getLogId(Algorithm algo, String vre) {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***@Override
***REMOVED***public String getLogById(String logId) throws IOException {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** return null;
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED***
package org.gcube.dataanalysis.dataminer.poolmanager.service; package org.gcube.dataanalysis.dataminer.poolmanager.service;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
@ -325,134 +63,7 @@ public class DataminerPoolManager implements PoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED***
***REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED***
***REMOVED*** static {
***REMOVED*** algorithms = new Vector<>();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public DataminerPoolManager() {
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is
***REMOVED*** expected
***REMOVED*** * on the pool.
***REMOVED*** */
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED*** algorithms.add(algorithm);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED*** *
***REMOVED*** * @param algorithm
***REMOVED*** */
***REMOVED*** /*
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement
***REMOVED*** this ***REMOVED***
***REMOVED*** */
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add the give algorithm to the given set
***REMOVED*** *
***REMOVED*** * @param algorithmId
***REMOVED*** * @param setId
***REMOVED*** */
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED*** if (set != null && algorithm != null) {
***REMOVED*** set.addAlgorithm(algorithm);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED*** *
***REMOVED*** * @param setId
***REMOVED*** * @param clusterId
***REMOVED*** */
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String
***REMOVED*** clusterName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED*** if (set != null && cluster != null) {
***REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED*** if (name.equals(set.getName())) {
***REMOVED*** return set;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED*** for (Algorithm a : algorithms) {
***REMOVED*** if (name.equals(a.getName())) {
***REMOVED*** return a;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
***REMOVED*** new Thread() {
***REMOVED*** public void run() {
***REMOVED*** while (true) {
***REMOVED*** try {
***REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** ***REMOVED***log here
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***.start();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getLogId(){
***REMOVED*** PrintStream console = System.out;
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED*** System.setOut(ps);
***REMOVED*** worker.apply();
***REMOVED*** System.setOut(console);
***REMOVED*** worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** ***REMOVED***
***REMOVED*** public String getLogById(String id) throws IOException {
***REMOVED*** String strLine = null;
***REMOVED*** try{
***REMOVED*** FileInputStream fstream = new
***REMOVED*** FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED*** /* read log line by line */
***REMOVED*** while ((strLine = br.readLine()) != null) {
***REMOVED*** /* parse strLine to obtain what you want */
***REMOVED*** System.out.println (strLine);
***REMOVED*** ***REMOVED***
***REMOVED*** br.close();
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED*** ***REMOVED***
***REMOVED*** return strLine;
***REMOVED*** ***REMOVED***
public String getScriptFromURL(URL url) throws IOException { public String getScriptFromURL(URL url) throws IOException {
if (url == null) { if (url == null) {
return null; return null;
@ -476,7 +87,7 @@ public class DataminerPoolManager implements PoolManager {
* @param vre * @param vre
* *
*/ */
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, final boolean updateSVN) throws IOException { public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set ***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet(); final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake"); algoSet.setName("fake");
@ -489,7 +100,7 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
try { try {
try { try {
addAlgorithmsToVRE(algoSet, vre, uuid, updateSVN); addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
***REMOVED*** catch (SVNException e) { ***REMOVED*** catch (SVNException e) {
***REMOVED*** TODO Auto-generated catch block ***REMOVED*** TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
@ -503,13 +114,13 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***).start(); ***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for your task to ***REMOVED*** this line will execute immediately, not waiting for task to
***REMOVED*** complete ***REMOVED*** complete
System.out.println(uuid); System.out.println(uuid);
return uuid; return uuid;
***REMOVED*** ***REMOVED***
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, final boolean updateSVN) throws IOException { public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set ***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet(); final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake"); algoSet.setName("fake");
@ -521,7 +132,8 @@ public class DataminerPoolManager implements PoolManager {
public void run() { public void run() {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
try { try {
addAlgorithmsToHost(algoSet, hostname, uuid, updateSVN); if(test){
addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
***REMOVED*** catch (IOException e) { ***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block ***REMOVED*** TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
@ -551,7 +163,7 @@ public class DataminerPoolManager implements PoolManager {
return new File(n.getPath()).toURI().toURL(); return new File(n.getPath()).toURI().toURL();
***REMOVED*** ***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
@ -562,11 +174,11 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** apply the changes ***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge(); AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId(); return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
***REMOVED*** ***REMOVED***
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN) public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
throws IOException, InterruptedException, SVNException { throws IOException, InterruptedException, SVNException {
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** create the cluster (dataminers in the vre)
@ -581,11 +193,23 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** apply the changes ***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge(); AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId(); return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED*** ***REMOVED***
public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
throws IOException, InterruptedException, SVNException {
Cluster cluster = new Cluster();
Host h = new Host();
h.setName(hostname);
cluster.addHost(h);
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException { public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url); return new AlgorithmPackageParser().parsePackage(url);
@ -726,4 +350,13 @@ public class DataminerPoolManager implements PoolManager {
return null; return null;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***