git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148412 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
7d286d5326
commit
3d6a1273ff
|
|
@ -24,6 +24,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSeriali
|
|||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
|
|
@ -118,7 +119,7 @@ public class AnsibleWorker {
|
|||
|
||||
|
||||
|
||||
public void apply(AlgorithmSet as, PrintStream ps, boolean updateSVN)
|
||||
public void apply(AlgorithmSet as, PrintStream ps, boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
***REMOVED*** TODO execute the playbook and return output
|
||||
System.out.println(this.getWorkdir());
|
||||
|
|
@ -128,43 +129,70 @@ public class AnsibleWorker {
|
|||
|
||||
inheritIO(p.getInputStream(), ps);
|
||||
inheritIO(p.getErrorStream(), ps);
|
||||
|
||||
if (updateSVN) {
|
||||
int exitValue = p.waitFor();
|
||||
if (exitValue == 0) {
|
||||
|
||||
|
||||
|
||||
int exitValue = p.waitFor();
|
||||
if (exitValue == 0) {
|
||||
if (test) {
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
|
||||
|
||||
for (Dependency d : algo.getDependencies()) {
|
||||
|
||||
if (d.getType().equals("os")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_deb_pkgs.txt", ls);
|
||||
this.updateSVN("test_r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_cran_pkgs.txt", ls);
|
||||
this.updateSVN("test_r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_github_pkgs.txt", ls);
|
||||
this.updateSVN("test_r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
else if(!test){
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
|
||||
for (Dependency d : algo.getDependencies()) {
|
||||
|
||||
if (d.getType().equals("os")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
DataminerPoolManager a = new DataminerPoolManager();
|
||||
a.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED*** catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** System.out.println("TODO: execute: ansible-playbook -v -i " +
|
||||
***REMOVED*** this.getInventoryFile().getName() + " " +
|
||||
***REMOVED*** this.getPlaybookFile().getName());
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -158,13 +158,13 @@ public class AnsibleBridge {
|
|||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, boolean updateSVN) throws IOException, InterruptedException, SVNException {
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
|
||||
|
||||
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),updateSVN);
|
||||
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
|
||||
***REMOVED***
|
||||
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException {
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
|
||||
|
||||
|
||||
|
|
@ -176,11 +176,14 @@ public class AnsibleBridge {
|
|||
algoRoles.add(r);
|
||||
worker.addRole(r);
|
||||
***REMOVED***
|
||||
***REMOVED***to comment the for in case of just install algo
|
||||
if(test){
|
||||
for (Dependency d : a.getDependencies()) {
|
||||
for (Role r : this.generateRoles(d)) {
|
||||
worker.addRole(r);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED*** add static roles
|
||||
|
|
@ -199,7 +202,9 @@ public class AnsibleBridge {
|
|||
|
||||
***REMOVED*** generate the playbook
|
||||
Playbook playbook = new Playbook();
|
||||
playbook.setRemote_user("root");
|
||||
if(test){
|
||||
playbook.setRemote_user("root");***REMOVED***
|
||||
playbook.setRemote_user("gcube");
|
||||
playbook.applyTo("universe");
|
||||
for(Role r:algoRoles) {
|
||||
***REMOVED*** add only 'add' roles
|
||||
|
|
@ -220,7 +225,7 @@ public class AnsibleBridge {
|
|||
|
||||
***REMOVED***System.setErr(console);
|
||||
|
||||
worker.apply(as,ps,updateSVN);
|
||||
worker.apply(as,ps,test);
|
||||
***REMOVED***System.setOut(console);
|
||||
***REMOVED***worker.apply();
|
||||
System.out.println("Log stored to to " + n.getAbsolutePath());
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ public class ISClient {
|
|||
***REMOVED***
|
||||
return h;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED*** return the HProxy hostname in the VRE
|
||||
public static String getHProxy() {
|
||||
|
|
|
|||
|
|
@ -35,12 +35,13 @@ import java.net.UnknownHostException;
|
|||
import java.util.Set;
|
||||
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
|
||||
public interface PoolManager {
|
||||
|
||||
String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn ) throws IOException, InterruptedException;
|
||||
String addAlgorithmToHost(Algorithm algo, String host, boolean svn) throws IOException, InterruptedException;
|
||||
String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException;
|
||||
String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException;
|
||||
|
||||
Algorithm extractAlgorithm(String url) throws IOException;
|
||||
|
||||
|
|
@ -57,5 +58,5 @@ public interface PoolManager {
|
|||
Set<Algorithm> getAlgoFromIs();
|
||||
List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException;
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -162,8 +162,10 @@ import javax.ws.rs.QueryParam;
|
|||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
|
||||
|
|
@ -171,6 +173,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySel
|
|||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
|
||||
import ch.qos.logback.core.subst.Token;
|
||||
|
||||
@Path("/")
|
||||
public class RestPoolManager implements PoolManager {
|
||||
|
||||
|
|
@ -201,37 +205,46 @@ public class RestPoolManager implements PoolManager {
|
|||
if (publish) {
|
||||
service.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),updateSVN);
|
||||
return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),/*updateSVN,*/false);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***Test Phase
|
||||
***REMOVED***The DMPM contacts the Staging Dataminer and installs the algorithm and the dependencies
|
||||
@GET
|
||||
@Path("/hosts/add")
|
||||
@Path("/algo/add")
|
||||
@Produces("text/plain")
|
||||
public String addAlgorithmToHost(
|
||||
@QueryParam("algorithm") String algorithm,
|
||||
@QueryParam("hostname") String hostname,
|
||||
public String addAlgorithmToStagingHost(
|
||||
@QueryParam("algorithm") String algorithm,
|
||||
@DefaultValue("insert hostname here") @QueryParam("hostname") String hostname,
|
||||
@QueryParam("name") String name,
|
||||
@QueryParam("description") String description,
|
||||
@QueryParam("category") String category,
|
||||
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
|
||||
@DefaultValue("false") @QueryParam("publish") boolean publish,
|
||||
@DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN)
|
||||
@DefaultValue("prodtoken") @QueryParam("prodtoken") String prodtoken,
|
||||
@DefaultValue("true") @QueryParam("test") boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
Algorithm algo = this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType,
|
||||
skipJava);
|
||||
***REMOVED*** publish algo
|
||||
if (publish) {
|
||||
service.addAlgToIs(algo);
|
||||
String id="";
|
||||
Algorithm algo = this.getAlgorithm(algorithm, ScopeProvider.instance.get(), hostname, name, description, category, algorithmType,skipJava);
|
||||
|
||||
***REMOVED***test phase
|
||||
if(test){
|
||||
service.addAlgorithmToHost(algo, hostname,true);
|
||||
id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),true);
|
||||
***REMOVED***
|
||||
return service.addAlgorithmToHost(algo, hostname,updateSVN);
|
||||
***REMOVED***release phase
|
||||
if(!test){
|
||||
SecurityTokenProvider.instance.set(prodtoken);
|
||||
id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),false);
|
||||
***REMOVED***
|
||||
|
||||
return id;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description,
|
||||
String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
|
||||
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + ScopeProvider.instance.get());
|
||||
Algorithm algo = service.extractAlgorithm(algorithm);
|
||||
|
||||
if (algo.getCategory() == null) {
|
||||
|
|
@ -283,8 +296,10 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED*** ProxySelector.setDefault(new
|
||||
***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
|
||||
|
||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||
|
||||
***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
|
||||
|
||||
|
||||
***REMOVED*** PoolManager aa = new DataminerPoolManager();
|
||||
***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3"));
|
||||
***REMOVED*** System.out.println(aa.getAlgoFromIs());
|
||||
|
|
@ -306,9 +321,17 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED*** "dataminer1-pre.d4science.org",
|
||||
***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers",
|
||||
***REMOVED*** "N",false, false);
|
||||
|
||||
|
||||
RestPoolManager b = new RestPoolManager();
|
||||
|
||||
b.addAlgorithmToStagingHost(
|
||||
"http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0",
|
||||
"dataminer1-devnext.d4science.org",
|
||||
null,
|
||||
null,
|
||||
"N",
|
||||
"transducerers",
|
||||
false,
|
||||
false
|
||||
***REMOVED*** PoolManager aa = new DataminerPoolManager();
|
||||
***REMOVED*** List<String> ls = new LinkedList<String>();
|
||||
***REMOVED*** String afa = "test";
|
||||
|
|
@ -357,21 +380,32 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn) throws IOException, InterruptedException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String addAlgorithmToHost(Algorithm algo, String host, boolean svn)
|
||||
public String addAlgorithmToHost(Algorithm algo, String host, boolean test)
|
||||
throws IOException, InterruptedException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
@Override
|
||||
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test)
|
||||
throws IOException, InterruptedException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -1,265 +1,3 @@
|
|||
***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***import java.io.File;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***import java.net.MalformedURLException;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***import java.net.URLConnection;
|
||||
***REMOVED***import java.net.UnknownHostException;
|
||||
***REMOVED***import java.util.UUID;
|
||||
***REMOVED***
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
|
||||
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***public class DataminerPoolManager implements PoolManager {
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED*** static Collection<Algorithm> algorithms;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** static Collection<AlgorithmSet> sets;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** static {
|
||||
***REMOVED******REMOVED*** algorithms = new Vector<>();
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** public DataminerPoolManager() {
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** /**
|
||||
***REMOVED******REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
|
||||
***REMOVED******REMOVED*** * on the pool.
|
||||
***REMOVED******REMOVED*** */
|
||||
***REMOVED******REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
|
||||
***REMOVED******REMOVED*** algorithms.add(algorithm);
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** /**
|
||||
***REMOVED******REMOVED*** * Re-deploy the given algorithm wherever it's installed
|
||||
***REMOVED******REMOVED*** *
|
||||
***REMOVED******REMOVED*** * @param algorithm
|
||||
***REMOVED******REMOVED*** */
|
||||
***REMOVED******REMOVED*** /*
|
||||
***REMOVED******REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED***
|
||||
***REMOVED******REMOVED*** */
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** /**
|
||||
***REMOVED******REMOVED*** * Add the give algorithm to the given set
|
||||
***REMOVED******REMOVED*** *
|
||||
***REMOVED******REMOVED*** * @param algorithmId
|
||||
***REMOVED******REMOVED*** * @param setId
|
||||
***REMOVED******REMOVED*** */
|
||||
***REMOVED******REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
|
||||
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
|
||||
***REMOVED******REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
|
||||
***REMOVED******REMOVED*** if (set != null && algorithm != null) {
|
||||
***REMOVED******REMOVED*** set.addAlgorithm(algorithm);
|
||||
***REMOVED******REMOVED*** this.updateClusters();
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** /**
|
||||
***REMOVED******REMOVED*** * Apply the given set of algorithms to the given cluster
|
||||
***REMOVED******REMOVED*** *
|
||||
***REMOVED******REMOVED*** * @param setId
|
||||
***REMOVED******REMOVED*** * @param clusterId
|
||||
***REMOVED******REMOVED*** */
|
||||
***REMOVED******REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) {
|
||||
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
|
||||
***REMOVED******REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
|
||||
***REMOVED******REMOVED*** if (set != null && cluster != null) {
|
||||
***REMOVED******REMOVED*** cluster.addAlgorithmSet(set);
|
||||
***REMOVED******REMOVED*** this.updateClusters();
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
|
||||
***REMOVED******REMOVED*** for (AlgorithmSet set : sets) {
|
||||
***REMOVED******REMOVED*** if (name.equals(set.getName())) {
|
||||
***REMOVED******REMOVED*** return set;
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** return null;
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** private Algorithm getAlgorithm(String name) {
|
||||
***REMOVED******REMOVED*** for (Algorithm a : algorithms) {
|
||||
***REMOVED******REMOVED*** if (name.equals(a.getName())) {
|
||||
***REMOVED******REMOVED*** return a;
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** return null;
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
|
||||
***REMOVED******REMOVED*** new Thread() {
|
||||
***REMOVED******REMOVED*** public void run() {
|
||||
***REMOVED******REMOVED*** while (true) {
|
||||
***REMOVED******REMOVED*** try {
|
||||
***REMOVED******REMOVED*** addAlgorithmToVRE(algorithm, vre);
|
||||
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
|
||||
***REMOVED******REMOVED*** ***REMOVED***log here
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** ***REMOVED***.start();
|
||||
***REMOVED******REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED*** public String getLogId(){
|
||||
***REMOVED******REMOVED*** PrintStream console = System.out;
|
||||
***REMOVED******REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||||
***REMOVED******REMOVED*** path.mkdirs();
|
||||
***REMOVED******REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
|
||||
***REMOVED******REMOVED*** FileOutputStream fos = new FileOutputStream(n);
|
||||
***REMOVED******REMOVED*** PrintStream ps = new PrintStream(fos);
|
||||
***REMOVED******REMOVED*** System.setOut(ps);
|
||||
***REMOVED******REMOVED*** worker.apply();
|
||||
***REMOVED******REMOVED*** System.setOut(console);
|
||||
***REMOVED******REMOVED*** worker.apply();
|
||||
***REMOVED******REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
|
||||
***REMOVED******REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED*** public String getLogById(String id) throws IOException {
|
||||
***REMOVED******REMOVED*** String strLine = null;
|
||||
***REMOVED******REMOVED*** try{
|
||||
***REMOVED******REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
|
||||
***REMOVED******REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
|
||||
***REMOVED******REMOVED*** /* read log line by line */
|
||||
***REMOVED******REMOVED*** while ((strLine = br.readLine()) != null) {
|
||||
***REMOVED******REMOVED*** /* parse strLine to obtain what you want */
|
||||
***REMOVED******REMOVED*** System.out.println (strLine);
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** br.close();
|
||||
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
|
||||
***REMOVED******REMOVED*** System.err.println("Error: " + e.getMessage());
|
||||
***REMOVED******REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED*** return strLine;
|
||||
***REMOVED******REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public String getScriptFromURL(URL url) throws IOException {
|
||||
***REMOVED*** if (url == null) {
|
||||
***REMOVED*** return null;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** URLConnection yc = url.openConnection();
|
||||
***REMOVED*** BufferedReader input = new BufferedReader(new InputStreamReader(
|
||||
***REMOVED*** yc.getInputStream()));
|
||||
***REMOVED*** String line;
|
||||
***REMOVED*** StringBuffer buffer = new StringBuffer();
|
||||
***REMOVED*** while ((line = input.readLine()) != null) {
|
||||
***REMOVED*** buffer.append(line + "\n");
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** String bufferScript = buffer.substring(0, buffer.length());
|
||||
***REMOVED*** input.close();
|
||||
***REMOVED*** return bufferScript;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** /**
|
||||
***REMOVED*** * Publish the given algorithm in the given VRE
|
||||
***REMOVED*** *
|
||||
***REMOVED*** * @param algorithmName
|
||||
***REMOVED*** * @param vre
|
||||
***REMOVED*** *
|
||||
***REMOVED*** */
|
||||
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
|
||||
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
||||
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
***REMOVED*** algoSet.setName("fake");
|
||||
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
||||
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
||||
***REMOVED***
|
||||
***REMOVED*** new Thread(new Runnable() {
|
||||
***REMOVED*** @Override
|
||||
***REMOVED*** public void run() {
|
||||
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
||||
***REMOVED*** try {
|
||||
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid);
|
||||
***REMOVED*** ***REMOVED*** catch (IOException e) {
|
||||
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
||||
***REMOVED*** e.printStackTrace();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***).start();
|
||||
***REMOVED*** ***REMOVED***this line will execute immediately, not waiting for your task to complete
|
||||
***REMOVED*** System.out.println(uuid);
|
||||
***REMOVED*** return uuid;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
|
||||
***REMOVED***
|
||||
***REMOVED*** File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
|
||||
***REMOVED*** path.mkdirs();
|
||||
***REMOVED*** File n = new File(path + File.separator +a);
|
||||
***REMOVED*** ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress();
|
||||
***REMOVED***
|
||||
***REMOVED*** return new File(n.getPath()).toURI().toURL();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
||||
***REMOVED*** Cluster cluster = new Cluster();
|
||||
***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) {
|
||||
***REMOVED*** cluster.addHost(h);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** apply the changes
|
||||
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
||||
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException {
|
||||
***REMOVED*** return new AlgorithmPackageParser().parsePackage(url);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***@Override
|
||||
***REMOVED***public void getLogId(Algorithm algo, String vre) {
|
||||
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***@Override
|
||||
***REMOVED***public String getLogById(String logId) throws IOException {
|
||||
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
||||
***REMOVED*** return null;
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||||
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
|
|
@ -325,134 +63,7 @@ public class DataminerPoolManager implements PoolManager {
|
|||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||||
|
||||
***REMOVED*** static Collection<Algorithm> algorithms;
|
||||
***REMOVED***
|
||||
***REMOVED*** static Collection<AlgorithmSet> sets;
|
||||
***REMOVED***
|
||||
***REMOVED*** static {
|
||||
***REMOVED*** algorithms = new Vector<>();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public DataminerPoolManager() {
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** /**
|
||||
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is
|
||||
***REMOVED*** expected
|
||||
***REMOVED*** * on the pool.
|
||||
***REMOVED*** */
|
||||
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
|
||||
***REMOVED*** algorithms.add(algorithm);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** /**
|
||||
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
|
||||
***REMOVED*** *
|
||||
***REMOVED*** * @param algorithm
|
||||
***REMOVED*** */
|
||||
***REMOVED*** /*
|
||||
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement
|
||||
***REMOVED*** this ***REMOVED***
|
||||
***REMOVED*** */
|
||||
***REMOVED***
|
||||
***REMOVED*** /**
|
||||
***REMOVED*** * Add the give algorithm to the given set
|
||||
***REMOVED*** *
|
||||
***REMOVED*** * @param algorithmId
|
||||
***REMOVED*** * @param setId
|
||||
***REMOVED*** */
|
||||
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
|
||||
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
|
||||
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
|
||||
***REMOVED*** if (set != null && algorithm != null) {
|
||||
***REMOVED*** set.addAlgorithm(algorithm);
|
||||
***REMOVED*** this.updateClusters();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** /**
|
||||
***REMOVED*** * Apply the given set of algorithms to the given cluster
|
||||
***REMOVED*** *
|
||||
***REMOVED*** * @param setId
|
||||
***REMOVED*** * @param clusterId
|
||||
***REMOVED*** */
|
||||
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String
|
||||
***REMOVED*** clusterName) {
|
||||
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
|
||||
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
|
||||
***REMOVED*** if (set != null && cluster != null) {
|
||||
***REMOVED*** cluster.addAlgorithmSet(set);
|
||||
***REMOVED*** this.updateClusters();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
|
||||
***REMOVED*** for (AlgorithmSet set : sets) {
|
||||
***REMOVED*** if (name.equals(set.getName())) {
|
||||
***REMOVED*** return set;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** return null;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** private Algorithm getAlgorithm(String name) {
|
||||
***REMOVED*** for (Algorithm a : algorithms) {
|
||||
***REMOVED*** if (name.equals(a.getName())) {
|
||||
***REMOVED*** return a;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** return null;
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
|
||||
***REMOVED*** new Thread() {
|
||||
***REMOVED*** public void run() {
|
||||
***REMOVED*** while (true) {
|
||||
***REMOVED*** try {
|
||||
***REMOVED*** addAlgorithmToVRE(algorithm, vre);
|
||||
***REMOVED*** ***REMOVED*** catch (Exception e) {
|
||||
***REMOVED*** ***REMOVED***log here
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***.start();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED*** public String getLogId(){
|
||||
***REMOVED*** PrintStream console = System.out;
|
||||
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||||
***REMOVED*** path.mkdirs();
|
||||
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
|
||||
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
|
||||
***REMOVED*** PrintStream ps = new PrintStream(fos);
|
||||
***REMOVED*** System.setOut(ps);
|
||||
***REMOVED*** worker.apply();
|
||||
***REMOVED*** System.setOut(console);
|
||||
***REMOVED*** worker.apply();
|
||||
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
***REMOVED*** public String getLogById(String id) throws IOException {
|
||||
***REMOVED*** String strLine = null;
|
||||
***REMOVED*** try{
|
||||
***REMOVED*** FileInputStream fstream = new
|
||||
***REMOVED*** FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
|
||||
***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
|
||||
***REMOVED*** /* read log line by line */
|
||||
***REMOVED*** while ((strLine = br.readLine()) != null) {
|
||||
***REMOVED*** /* parse strLine to obtain what you want */
|
||||
***REMOVED*** System.out.println (strLine);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** br.close();
|
||||
***REMOVED*** ***REMOVED*** catch (Exception e) {
|
||||
***REMOVED*** System.err.println("Error: " + e.getMessage());
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** return strLine;
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
|
||||
public String getScriptFromURL(URL url) throws IOException {
|
||||
if (url == null) {
|
||||
return null;
|
||||
|
|
@ -476,7 +87,7 @@ public class DataminerPoolManager implements PoolManager {
|
|||
* @param vre
|
||||
*
|
||||
*/
|
||||
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, final boolean updateSVN) throws IOException {
|
||||
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
***REMOVED*** create a fake algorithm set
|
||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
algoSet.setName("fake");
|
||||
|
|
@ -489,7 +100,7 @@ public class DataminerPoolManager implements PoolManager {
|
|||
***REMOVED*** TODO Auto-generated method stub
|
||||
try {
|
||||
try {
|
||||
addAlgorithmsToVRE(algoSet, vre, uuid, updateSVN);
|
||||
addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
||||
***REMOVED*** catch (SVNException e) {
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
|
|
@ -503,13 +114,13 @@ public class DataminerPoolManager implements PoolManager {
|
|||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***).start();
|
||||
***REMOVED*** this line will execute immediately, not waiting for your task to
|
||||
***REMOVED*** this line will execute immediately, not waiting for task to
|
||||
***REMOVED*** complete
|
||||
System.out.println(uuid);
|
||||
return uuid;
|
||||
***REMOVED***
|
||||
|
||||
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, final boolean updateSVN) throws IOException {
|
||||
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
***REMOVED*** create a fake algorithm set
|
||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
algoSet.setName("fake");
|
||||
|
|
@ -521,7 +132,8 @@ public class DataminerPoolManager implements PoolManager {
|
|||
public void run() {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
try {
|
||||
addAlgorithmsToHost(algoSet, hostname, uuid, updateSVN);
|
||||
if(test){
|
||||
addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
|
||||
***REMOVED*** catch (IOException e) {
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
|
|
@ -551,7 +163,7 @@ public class DataminerPoolManager implements PoolManager {
|
|||
return new File(n.getPath()).toURI().toURL();
|
||||
***REMOVED***
|
||||
|
||||
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException {
|
||||
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
|
||||
***REMOVED*** create the cluster (dataminers in the vre)
|
||||
Cluster cluster = new Cluster();
|
||||
|
|
@ -562,11 +174,11 @@ public class DataminerPoolManager implements PoolManager {
|
|||
|
||||
***REMOVED*** apply the changes
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN)
|
||||
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
|
||||
***REMOVED*** create the cluster (dataminers in the vre)
|
||||
|
|
@ -581,11 +193,23 @@ public class DataminerPoolManager implements PoolManager {
|
|||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** apply the changes
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
Cluster cluster = new Cluster();
|
||||
Host h = new Host();
|
||||
h.setName(hostname);
|
||||
cluster.addHost(h);
|
||||
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public Algorithm extractAlgorithm(String url) throws IOException {
|
||||
return new AlgorithmPackageParser().parsePackage(url);
|
||||
|
|
@ -726,4 +350,13 @@ public class DataminerPoolManager implements PoolManager {
|
|||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
Loading…
Reference in New Issue