Updated project configuration
This commit is contained in:
Giancarlo Panichi
parent
2c476927c6
commit
bd981214c6
|
|
@ -17,6 +17,7 @@
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
</attributes>
|
</attributes>
|
||||||
</classpathentry>
|
</classpathentry>
|
||||||
|
<classpathentry kind="src" output="target/test-classes" path="src/test/resources"/>
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||||
<attributes>
|
<attributes>
|
||||||
<attribute name="maven.pomderived" value="true"/>
|
<attribute name="maven.pomderived" value="true"/>
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
|
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
|
||||||
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
|
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
|
||||||
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
|
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
|
||||||
|
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/resources"/>
|
||||||
<property name="context-root" value="dataminer-pool-manager"/>
|
<property name="context-root" value="dataminer-pool-manager"/>
|
||||||
<property name="java-output-path" value="/dataminer-pool-manager/target/classes"/>
|
<property name="java-output-path" value="/dataminer-pool-manager/target/classes"/>
|
||||||
</wb-module>
|
</wb-module>
|
||||||
|
|
|
||||||
|
|
@ -1,117 +0,0 @@
|
||||||
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Vector;
|
|
||||||
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
|
|
||||||
|
|
||||||
public class AlgorithmPackageParserTest {
|
|
||||||
|
|
||||||
private static int BUFFER_SIZE = 2048;
|
|
||||||
|
|
||||||
public void extractAllAlgorithms() throws IOException {
|
|
||||||
String url = "http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms";
|
|
||||||
List<String> commands = this.extractAddAlgorithmCommands(url);
|
|
||||||
AlgorithmSet algorithms = new AlgorithmSet();
|
|
||||||
for (String cmd : commands) {
|
|
||||||
System.out.println("-----------------------------------------");
|
|
||||||
System.out.println(cmd);
|
|
||||||
AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
|
|
||||||
System.out.println(aac);
|
|
||||||
|
|
||||||
// start creating the algo from the command
|
|
||||||
Algorithm algo = new Algorithm();
|
|
||||||
algo.setAlgorithmType(aac.getAlgorithmType());
|
|
||||||
algo.setCategory(aac.getCategory());
|
|
||||||
algo.setClazz(aac.getClazz());
|
|
||||||
algo.setDescription(aac.getDescription());
|
|
||||||
algo.setName(aac.getName());
|
|
||||||
algo.setPackageURL(aac.getUrl());
|
|
||||||
algo.setSkipJava(aac.getSkipjava());
|
|
||||||
|
|
||||||
// then override with info from the package
|
|
||||||
if (aac.getUrl().length() > 4) {
|
|
||||||
Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl());
|
|
||||||
if (packagedAlgo != null) {
|
|
||||||
algo.setDependencies(packagedAlgo.getDependencies());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
algorithms.addAlgorithm(algo);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
//to uncomment
|
|
||||||
// new DataminerPoolManager().addAlgorithmsToVRE(algorithms,
|
|
||||||
// "/gcube/devNext/NextNext");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract 'addAlgorithm' commands from a file containing wiki-table-style
|
|
||||||
* entries for algorithm.
|
|
||||||
*
|
|
||||||
* @return
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
private List<String> extractAddAlgorithmCommands(String listUrl)
|
|
||||||
throws IOException {
|
|
||||||
URL url = new URL(listUrl);
|
|
||||||
InputStream is = url.openStream();
|
|
||||||
|
|
||||||
StringBuilder s = new StringBuilder();
|
|
||||||
byte[] buffer = new byte[BUFFER_SIZE];
|
|
||||||
int read = 0;
|
|
||||||
while ((read = is.read(buffer)) != -1) {
|
|
||||||
s.append(new String(buffer, 0, read));
|
|
||||||
}
|
|
||||||
List<String> out = new Vector<>();
|
|
||||||
String[] lines = s.toString().split("\n");
|
|
||||||
for (String line : lines) {
|
|
||||||
System.out.println("--------------------");
|
|
||||||
if (!line.isEmpty()) {
|
|
||||||
String[] parts = line.split("\\|");
|
|
||||||
int c = 1;
|
|
||||||
for (String part : parts) {
|
|
||||||
if (part == null || part.trim().isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
System.out.println(c + ". " + part);
|
|
||||||
c++;
|
|
||||||
if (part.contains("addAlgorithm.sh")) {
|
|
||||||
String cmd = part.trim();
|
|
||||||
cmd = cmd.replaceAll("<notextile>", "");
|
|
||||||
cmd = cmd.replaceAll("</notextile>", "");
|
|
||||||
System.out.println(cmd);
|
|
||||||
// AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
|
|
||||||
// System.out.println(aac);
|
|
||||||
out.add(cmd);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create an Algorithm starting from the algorithm jar.
|
|
||||||
*
|
|
||||||
* @param url
|
|
||||||
* @return
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
private Algorithm extractAlgorithm(String url) throws IOException {
|
|
||||||
return new AlgorithmPackageParser().parsePackage(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
// ProxySelector.setDefault(new PropertiesBasedProxySelector(
|
|
||||||
// "/home/ngalante/.proxy-settings"));
|
|
||||||
|
|
||||||
new AlgorithmPackageParserTest().extractAllAlgorithms();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
|
|
||||||
|
|
||||||
public class AnsibleWorkerTest {
|
|
||||||
|
|
||||||
public static void main(String[] args) throws IOException {
|
|
||||||
AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString()));
|
|
||||||
|
|
||||||
System.out.println("created worker named " + worker.getWorkerId());
|
|
||||||
|
|
||||||
worker.setInventory(new Inventory());
|
|
||||||
worker.setPlaybook(new Playbook());
|
|
||||||
|
|
||||||
Role r = new Role();
|
|
||||||
r.setName("latex");
|
|
||||||
|
|
||||||
RoleFile tf = new RoleFile("main", "do something special for " + r.getName());
|
|
||||||
r.addTaskFile(tf);
|
|
||||||
worker.addRole(r);
|
|
||||||
|
|
||||||
//worker.apply();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,225 +0,0 @@
|
||||||
|
|
||||||
|
|
||||||
import java.net.ProxySelector;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
|
|
||||||
|
|
||||||
public class DataminerPoolManagerTest {
|
|
||||||
|
|
||||||
/*
|
|
||||||
private static Map<String, Domain> domains = new HashMap<>();
|
|
||||||
private static Map<String, Cluster> clusters = new HashMap<>();
|
|
||||||
|
|
||||||
private static Dependency createDependency(String depName) {
|
|
||||||
String[] parts = depName.split(":");
|
|
||||||
Dependency out = new Dependency();
|
|
||||||
if(parts.length>1) {
|
|
||||||
out.setType(parts[0]);
|
|
||||||
out.setName(parts[1]);
|
|
||||||
} else {
|
|
||||||
out.setType("os");
|
|
||||||
out.setName(depName);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Algorithm createAlgorithm(String name, String ... deps) {
|
|
||||||
Algorithm a = new Algorithm();
|
|
||||||
a.setName(name);
|
|
||||||
for(String dep:deps) {
|
|
||||||
a.addDependency(createDependency(dep));
|
|
||||||
}
|
|
||||||
return a;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) {
|
|
||||||
AlgorithmSet out = new AlgorithmSet();
|
|
||||||
out.setName(name);
|
|
||||||
for(Algorithm a:algs) {
|
|
||||||
out.addAlgorithm(a);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Domain getDomain(String name) {
|
|
||||||
if(domains.get(name)==null) {
|
|
||||||
Domain d = new Domain();
|
|
||||||
d.setName(name);
|
|
||||||
domains.put(name, d);
|
|
||||||
return d;
|
|
||||||
} else {
|
|
||||||
return domains.get(name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Host createHost(String hostname, String domainName) {
|
|
||||||
Host out = new Host();
|
|
||||||
out.setName(hostname);
|
|
||||||
Domain d = getDomain(domainName);
|
|
||||||
out.setDomain(d);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Cluster getCluster(String name) {
|
|
||||||
if(clusters.get(name)==null) {
|
|
||||||
Cluster d = new Cluster();
|
|
||||||
d.setName(name);
|
|
||||||
clusters.put(name, d);
|
|
||||||
return d;
|
|
||||||
} else {
|
|
||||||
return clusters.get(name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Collection<Dependency> extractDependencies() {
|
|
||||||
Collection<Dependency> out = new TreeSet<>(new DependencyComparator());
|
|
||||||
for(Cluster c:clusters.values()) {
|
|
||||||
for(AlgorithmSet as:c.getAlgorithmSets()) {
|
|
||||||
for(Algorithm a:as.getAlgorithms()) {
|
|
||||||
for(Dependency d:a.getDependencies()) {
|
|
||||||
out.add(d);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Collection<Algorithm> extractAlgorithms() {
|
|
||||||
Collection<Algorithm> out = new TreeSet<>(new AlgorithmComparator());
|
|
||||||
for(Cluster c:clusters.values()) {
|
|
||||||
for(AlgorithmSet as:c.getAlgorithmSets()) {
|
|
||||||
for(Algorithm a:as.getAlgorithms()) {
|
|
||||||
out.add(a);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
static {
|
|
||||||
|
|
||||||
Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package");
|
|
||||||
Algorithm ensemble = createAlgorithm("ensemble", "python");
|
|
||||||
Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah");
|
|
||||||
|
|
||||||
AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe);
|
|
||||||
AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble);
|
|
||||||
AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe);
|
|
||||||
|
|
||||||
Cluster cluster1 = getCluster("cluster-1");
|
|
||||||
cluster1.addHost(createHost("host1", "domain1"));
|
|
||||||
cluster1.addHost(createHost("host2", "domain1"));
|
|
||||||
cluster1.addHost(createHost("host3", "domain1"));
|
|
||||||
cluster1.addHost(createHost("host1", "domain2"));
|
|
||||||
cluster1.addHost(createHost("host2", "domain2"));
|
|
||||||
|
|
||||||
Cluster cluster2 = getCluster("cluster-2");
|
|
||||||
cluster2.addHost(createHost("host4", "domain1"));
|
|
||||||
cluster2.addHost(createHost("host5", "domain1"));
|
|
||||||
cluster2.addHost(createHost("host6", "domain1"));
|
|
||||||
cluster2.addHost(createHost("host3", "domain2"));
|
|
||||||
cluster2.addHost(createHost("host4", "domain2"));
|
|
||||||
cluster2.addHost(createHost("host5", "domain2"));
|
|
||||||
|
|
||||||
cluster1.addAlgorithmSet(as1);
|
|
||||||
cluster1.addAlgorithmSet(as2);
|
|
||||||
|
|
||||||
cluster2.addAlgorithmSet(as1);
|
|
||||||
cluster2.addAlgorithmSet(as3);
|
|
||||||
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
/*
|
|
||||||
AnsibleBridge ab = new AnsibleBridge();
|
|
||||||
ab.printInventoryByDomainAndSets(clusters.values());
|
|
||||||
System.out.println("-----------");
|
|
||||||
ab.printInventoryBySets(clusters.values());
|
|
||||||
|
|
||||||
AnsibleWorker worker = ab.createWorker();
|
|
||||||
|
|
||||||
for(Algorithm a:extractAlgorithms()) {
|
|
||||||
for(Role r:ab.generateRoles(a)) {
|
|
||||||
worker.addRole(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for(Dependency d:extractDependencies()) {
|
|
||||||
for(Role r:ab.generateRoles(d)) {
|
|
||||||
worker.addRole(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
|
||||||
|
|
||||||
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
|
|
||||||
//
|
|
||||||
// // create the algorithm (download it, etc etc)
|
|
||||||
// Algorithm algorithm = new Algorithm();
|
|
||||||
// algorithm.setName("ichtyop");
|
|
||||||
// algorithm.setClazz("org.gcube...");
|
|
||||||
// algorithm.setDescription("some description");
|
|
||||||
//
|
|
||||||
// Dependency d = new Dependency();
|
|
||||||
// d.setName("libpng");
|
|
||||||
// d.setType("os");
|
|
||||||
// algorithm.addDependency(d);
|
|
||||||
//
|
|
||||||
// d = new Dependency();
|
|
||||||
// d.setName("some-r-package");
|
|
||||||
// d.setType("cran");
|
|
||||||
// algorithm.addDependency(d);
|
|
||||||
//
|
|
||||||
// d = new Dependency();
|
|
||||||
// d.setName("some-other-r-package");
|
|
||||||
// d.setType("cran");
|
|
||||||
// algorithm.addDependency(d);
|
|
||||||
//
|
|
||||||
// d = new Dependency();
|
|
||||||
// d.setName("voodoo");
|
|
||||||
// d.setType("custom");
|
|
||||||
// algorithm.addDependency(d);
|
|
||||||
//
|
|
||||||
// // create the algorithm (download it, etc etc)
|
|
||||||
// Algorithm ewe = new Algorithm();
|
|
||||||
// ewe.setName("ewe");
|
|
||||||
//
|
|
||||||
// d = new Dependency();
|
|
||||||
// d.setName("voodoo");
|
|
||||||
// d.setType("custom");
|
|
||||||
// ewe.addDependency(d);
|
|
||||||
|
|
||||||
AlgorithmSet algorithms = new AlgorithmSet();
|
|
||||||
algorithms.setName("dummy-set");
|
|
||||||
|
|
||||||
|
|
||||||
// algorithms.addAlgorithm(algorithm);
|
|
||||||
// algorithms.addAlgorithm(ewe);
|
|
||||||
|
|
||||||
Algorithm ensemble = new Algorithm();
|
|
||||||
ensemble.setName("ewe");
|
|
||||||
ensemble.setCategory("a");
|
|
||||||
ensemble.setAlgorithmType("transducerers");
|
|
||||||
ensemble.setPackageURL("http://data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0");
|
|
||||||
ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone");
|
|
||||||
ensemble.setSkipJava("N");
|
|
||||||
ensemble.setDescription("test");
|
|
||||||
|
|
||||||
Dependency d = new Dependency();
|
|
||||||
d.setName("libpng3");
|
|
||||||
d.setType("os");
|
|
||||||
ensemble.addDependency(d);
|
|
||||||
algorithms.addAlgorithm(ensemble);
|
|
||||||
|
|
||||||
//new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
|
|
||||||
|
|
||||||
import java.net.ProxySelector;
|
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
|
|
||||||
|
|
||||||
public class ISClientTest {
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
|
|
||||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
|
||||||
System.out.println(new ISClient().listDataminersInVRE());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue