gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity
dataminer-pool-manager/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java

781 lines
31 KiB
Java
Raw Blame History

***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.service;
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***import java.io.File;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***import java.net.MalformedURLException;
***REMOVED******REMOVED***
***REMOVED***import java.net.URLConnection;
***REMOVED***import java.net.UnknownHostException;
***REMOVED***import java.util.UUID;
***REMOVED***
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***public class DataminerPoolManager implements PoolManager {
***REMOVED***
***REMOVED***
***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED***
***REMOVED***
***REMOVED******REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED******REMOVED***
***REMOVED******REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED******REMOVED***
***REMOVED******REMOVED*** static {
***REMOVED******REMOVED*** algorithms = new Vector<>();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** public DataminerPoolManager() {
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
***REMOVED******REMOVED*** * on the pool.
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED******REMOVED*** algorithms.add(algorithm);
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param algorithm
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** /*
***REMOVED******REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED***
***REMOVED******REMOVED*** */
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Add the give algorithm to the given set
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param algorithmId
***REMOVED******REMOVED*** * @param setId
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED******REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED******REMOVED*** if (set != null && algorithm != null) {
***REMOVED******REMOVED*** set.addAlgorithm(algorithm);
***REMOVED******REMOVED*** this.updateClusters();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** /**
***REMOVED******REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED******REMOVED*** *
***REMOVED******REMOVED*** * @param setId
***REMOVED******REMOVED*** * @param clusterId
***REMOVED******REMOVED*** */
***REMOVED******REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) {
***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED******REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED******REMOVED*** if (set != null && cluster != null) {
***REMOVED******REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED******REMOVED*** this.updateClusters();
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED******REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED******REMOVED*** if (name.equals(set.getName())) {
***REMOVED******REMOVED*** return set;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED******REMOVED*** for (Algorithm a : algorithms) {
***REMOVED******REMOVED*** if (name.equals(a.getName())) {
***REMOVED******REMOVED*** return a;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
***REMOVED******REMOVED*** new Thread() {
***REMOVED******REMOVED*** public void run() {
***REMOVED******REMOVED*** while (true) {
***REMOVED******REMOVED*** try {
***REMOVED******REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED******REMOVED*** ***REMOVED***log here
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED***.start();
***REMOVED******REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED******REMOVED*** public String getLogId(){
***REMOVED******REMOVED*** PrintStream console = System.out;
***REMOVED******REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED******REMOVED*** path.mkdirs();
***REMOVED******REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED******REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED******REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED******REMOVED*** System.setOut(ps);
***REMOVED******REMOVED*** worker.apply();
***REMOVED******REMOVED*** System.setOut(console);
***REMOVED******REMOVED*** worker.apply();
***REMOVED******REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED******REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED*** public String getLogById(String id) throws IOException {
***REMOVED******REMOVED*** String strLine = null;
***REMOVED******REMOVED*** try{
***REMOVED******REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED******REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED******REMOVED*** /* read log line by line */
***REMOVED******REMOVED*** while ((strLine = br.readLine()) != null) {
***REMOVED******REMOVED*** /* parse strLine to obtain what you want */
***REMOVED******REMOVED*** System.out.println (strLine);
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** br.close();
***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED******REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED******REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** return strLine;
***REMOVED******REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getScriptFromURL(URL url) throws IOException {
***REMOVED*** if (url == null) {
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED*** URLConnection yc = url.openConnection();
***REMOVED*** BufferedReader input = new BufferedReader(new InputStreamReader(
***REMOVED*** yc.getInputStream()));
***REMOVED*** String line;
***REMOVED*** StringBuffer buffer = new StringBuffer();
***REMOVED*** while ((line = input.readLine()) != null) {
***REMOVED*** buffer.append(line + "\n");
***REMOVED*** ***REMOVED***
***REMOVED*** String bufferScript = buffer.substring(0, buffer.length());
***REMOVED*** input.close();
***REMOVED*** return bufferScript;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Publish the given algorithm in the given VRE
***REMOVED*** *
***REMOVED*** * @param algorithmName
***REMOVED*** * @param vre
***REMOVED*** *
***REMOVED*** */
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
***REMOVED*** ***REMOVED*** create a fake algorithm set
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
***REMOVED*** algoSet.setName("fake");
***REMOVED*** algoSet.addAlgorithm(algorithm);
***REMOVED*** final String uuid = UUID.randomUUID().toString();
***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid);
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***).start();
***REMOVED*** ***REMOVED***this line will execute immediately, not waiting for your task to complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
***REMOVED***
***REMOVED*** File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator +a);
***REMOVED*** ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress();
***REMOVED***
***REMOVED*** return new File(n.getPath()).toURI().toURL();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
***REMOVED***
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException {
***REMOVED*** return new AlgorithmPackageParser().parsePackage(url);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***@Override
***REMOVED***public void getLogId(Algorithm algo, String vre) {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***@Override
***REMOVED***public String getLogById(String logId) throws IOException {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** return null;
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED******REMOVED***
package org.gcube.dataanalysis.dataminer.poolmanager.service;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
***REMOVED***
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
***REMOVED***
import java.io.StringWriter;
import java.net.MalformedURLException;
***REMOVED***
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
***REMOVED***
import java.util.Set;
import java.util.SortedSet;
import java.util.UUID;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.HostingNode;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.common.resources.gcore.Software.Profile.Dependency;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.informationsystem.publisher.ScopedPublisher;
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNCommitInfo;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class DataminerPoolManager implements PoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED***
***REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED***
***REMOVED*** static {
***REMOVED*** algorithms = new Vector<>();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public DataminerPoolManager() {
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is
***REMOVED*** expected
***REMOVED*** * on the pool.
***REMOVED*** */
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED*** algorithms.add(algorithm);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED*** *
***REMOVED*** * @param algorithm
***REMOVED*** */
***REMOVED*** /*
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement
***REMOVED*** this ***REMOVED***
***REMOVED*** */
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add the give algorithm to the given set
***REMOVED*** *
***REMOVED*** * @param algorithmId
***REMOVED*** * @param setId
***REMOVED*** */
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED*** if (set != null && algorithm != null) {
***REMOVED*** set.addAlgorithm(algorithm);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED*** *
***REMOVED*** * @param setId
***REMOVED*** * @param clusterId
***REMOVED*** */
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String
***REMOVED*** clusterName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED*** if (set != null && cluster != null) {
***REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED*** if (name.equals(set.getName())) {
***REMOVED*** return set;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED*** for (Algorithm a : algorithms) {
***REMOVED*** if (name.equals(a.getName())) {
***REMOVED*** return a;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
***REMOVED*** new Thread() {
***REMOVED*** public void run() {
***REMOVED*** while (true) {
***REMOVED*** try {
***REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** ***REMOVED***log here
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***.start();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getLogId(){
***REMOVED*** PrintStream console = System.out;
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED*** System.setOut(ps);
***REMOVED*** worker.apply();
***REMOVED*** System.setOut(console);
***REMOVED*** worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** ***REMOVED***
***REMOVED*** public String getLogById(String id) throws IOException {
***REMOVED*** String strLine = null;
***REMOVED*** try{
***REMOVED*** FileInputStream fstream = new
***REMOVED*** FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED*** /* read log line by line */
***REMOVED*** while ((strLine = br.readLine()) != null) {
***REMOVED*** /* parse strLine to obtain what you want */
***REMOVED*** System.out.println (strLine);
***REMOVED*** ***REMOVED***
***REMOVED*** br.close();
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED*** ***REMOVED***
***REMOVED*** return strLine;
***REMOVED*** ***REMOVED***
public String getScriptFromURL(URL url) throws IOException {
if (url == null) {
return null;
***REMOVED***
URLConnection yc = url.openConnection();
BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String line;
StringBuffer buffer = new StringBuffer();
while ((line = input.readLine()) != null) {
buffer.append(line + "\n");
***REMOVED***
String bufferScript = buffer.substring(0, buffer.length());
input.close();
return bufferScript;
***REMOVED***
/**
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*
*/
public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
addAlgorithmsToVRE(algoSet, vre, uuid);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED***
public String addAlgorithmToHost(Algorithm algorithm, final String hostname) throws IOException {
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
addAlgorithmsToHost(algoSet, hostname, uuid);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
File path = new File(System.getProperty("user.home") + File.separator + "/gcube/dataminer-pool-manager/work/"
+ a + File.separator + "logs");
path.mkdirs();
File n = new File(path + File.separator + a);
***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress();
return new File(n.getPath()).toURI().toURL();
***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
***REMOVED*** for (Host h : new ISClient().listDataMinersByCluster()) {
for (Host h : new ISClient().listDataminersInVRE()) {
cluster.addHost(h);
***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId();
***REMOVED***
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid) throws IOException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
cluster.addHost(new ISClient().getDataminer(hostname));
***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId();
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException {
String usr = "nunzioandrea.galante";
String pwd = "Newuser1";
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(usr, pwd);
repository.setAuthenticationManager(authManager);
System.out.println(repository.getLocation());
return repository;
***REMOVED***
@Override
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
final SVNRepository svnRepository = this.getSvnRepository(
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
try {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
System.out.println("aaaa"+byteArrayOutputStream);
byte[] originalContents = byteArrayOutputStream.toByteArray();
System.out.println(byteArrayOutputStream.toByteArray());
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) {
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
System.out.println("bbbb"+baos);
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
return aa;
***REMOVED*** finally {
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public static String md5(byte[] contents) {
final byte[] tmp = new byte[1024];
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
try {
while (checksumStream.read(tmp) > 0) {
***REMOVED***
***REMOVED***
return checksumStream.getDigest();
***REMOVED*** catch (IOException e) {
***REMOVED***never happens
e.printStackTrace();
return null;
***REMOVED*** finally {
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) {
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
return new ArrayList<>(ss);
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***
@Override
public void getLogId(Algorithm algo, String vre) {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public String getLogById(String logId) throws IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED*** 2017 March 29
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
String id = resource.id();
LOGGER.debug("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id,
ScopeProvider.instance.get());
***REMOVED*** scopedPublisher.remove(resource, scopes);
advancedScopedPublisher.forceRemove(resource);
LOGGER.debug("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id);
***REMOVED***
public void publishScopedResource(GenericResource a, List<String> scopes)
throws RegistryNotFoundException, Exception {
StringWriter stringWriter = new StringWriter();
Resources.marshal(a, stringWriter);
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
try {
System.out.println(scopes);
System.out.println(stringWriter);
scopedPublisher.create(a, scopes);
***REMOVED*** catch (RegistryNotFoundException e) {
System.out.println(e);
throw e;
***REMOVED***
***REMOVED***
@Override
public void addAlgToIs(Algorithm algo) {
GenericResource a = new GenericResource();
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
a.profile().newBody(this.getAlgoBody(algo));
try {
publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***));
***REMOVED*** catch (Exception e) {
e.printStackTrace();
***REMOVED***
***REMOVED***
private String getAlgoBody(Algorithm algo) {
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
***REMOVED***
public void updateAlg(Algorithm algo) {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
if (ds.isEmpty()) {
return;
***REMOVED***
GenericResource a = ds.get(0);
a.profile().newBody(this.getAlgoBody(algo));
try {
scopedPublisher.update(a);
***REMOVED*** catch (RegistryNotFoundException e) {
e.printStackTrace();
***REMOVED***
***REMOVED***
private Algorithm convertAlgo(GenericResource a) {
Algorithm out = new Algorithm();
***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
out.setName(a.profile().name());
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
out.setDescription(a.profile().description());
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) {
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
deps.add(d1);
***REMOVED***
out.setDependencies(deps);
***REMOVED***
***REMOVED***
***REMOVED*** public Algorithm getAlgoById(String id) {
***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) {
***REMOVED*** if (aa.getId().equals(id)) {
***REMOVED*** return aa;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
@Override
public Set<Algorithm> getAlgoFromIs() {
***REMOVED*** TODO Auto-generated method stub
Set<Algorithm> out = new HashSet<Algorithm>();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
.setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) {
out.add(this.convertAlgo(a));
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
View Git Blame Copy Permalink