From a5667a7c57c264b8d0ceb356916149825e978158 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Wed, 18 Jan 2023 17:56:48 +0100 Subject: [PATCH] Removed sensitive data --- .settings/org.eclipse.core.resources.prefs | 8 +- CHANGELOG.md | 2 +- FUNDING.md | 38 +- README.md | 14 +- descriptor.xml | 10 +- gcube-app.xml | 8 +- pom.xml | 18 +- profile.xml | 18 +- .../poolmanager/ansible/AnsibleWorker.java | 98 ++-- .../ansible/model/AnsibleHost.java | 16 +- .../poolmanager/ansible/model/HostGroup.java | 20 +- .../poolmanager/ansible/model/Inventory.java | 32 +- .../poolmanager/ansible/model/Playbook.java | 42 +- .../poolmanager/ansible/model/Role.java | 36 +- .../poolmanager/ansible/model/RoleFile.java | 36 +- .../ansiblebridge/AnsibleBridge.java | 394 ++++++++-------- .../ansiblebridge/AnsibleSerializeHelper.java | 96 ++-- .../template/AlgorithmPackage.java | 62 +-- .../template/CranDependencyPackage.java | 8 +- .../template/CustomDependencyPackage.java | 68 +-- .../template/CustomRoleManager.java | 22 +- .../template/DependencyPackage.java | 46 +- .../template/OSDependencyPackage.java | 8 +- .../template/StaticRoleManager.java | 28 +- .../template/TemplateManager.java | 84 ++-- .../poolmanager/clients/HAProxy.java | 204 ++++----- .../poolmanager/clients/ISClient.java | 246 +++++----- .../poolmanager/clients/ScopedCacheMap.java | 8 +- .../ClientConfigurationCache.java | 56 +-- .../clients/configuration/Configuration.java | 4 +- .../configuration/ConfigurationImpl.java | 152 +++---- .../DMPMClientConfiguratorManager.java | 74 +-- .../clients/configuration/SVNRepository.java | 36 +- .../configurations/AbstractConfiguration.java | 158 +++---- .../clients/configurations/Prod.java | 16 +- .../clients/configurations/Stage.java | 16 +- .../poolmanager/datamodel/Action.java | 28 +- .../poolmanager/datamodel/AlgoResource.java | 20 +- .../poolmanager/datamodel/Algorithm.java | 166 +++---- .../poolmanager/datamodel/AlgorithmSet.java | 44 +- .../poolmanager/datamodel/Cluster.java | 48 +- .../poolmanager/datamodel/Dependency.java | 24 +- .../poolmanager/datamodel/Domain.java | 12 +- .../dataminer/poolmanager/datamodel/Host.java | 42 +- .../comparator/AlgorithmComparator.java | 10 +- .../comparator/DependencyComparator.java | 8 +- .../datamodel/comparator/HostComparator.java | 8 +- .../process/AddAlgorithmCommand.java | 60 +-- .../process/AlgorithmPackageParser.java | 320 ++++++------- .../poolmanager/rest/PoolManager.java | 70 +-- .../poolmanager/rest/RestPoolManager.java | 245 ++++++++++ .../poolmanager/service/DMPMJob.java | 146 +++--- .../service/DataminerPoolManager.java | 52 +-- .../service/ProductionPublishingJob.java | 26 +- .../poolmanager/service/StagingJob.java | 20 +- .../service/exceptions/AnsibleException.java | 18 +- .../UndefinedDependenciesException.java | 22 +- .../poolmanager/util/AlgorithmBuilder.java | 40 +- .../poolmanager/util/CheckMethod.java | 144 +++--- .../poolmanager/util/CheckPermission.java | 18 +- .../poolmanager/util/ClusterBuilder.java | 34 +- .../poolmanager/util/NotificationHelper.java | 92 ++-- .../util/PropertiesBasedProxySelector.java | 108 ++--- .../util/SVNRepositoryManager.java | 30 +- .../poolmanager/util/SVNUpdater.java | 422 +++++++++--------- .../dataminer/poolmanager/util/SendMail.java | 178 ++++---- .../util/exception/AlgorithmException.java | 22 +- .../util/exception/DMPMException.java | 16 +- .../util/exception/EMailException.java | 24 +- .../util/exception/GenericException.java | 18 +- .../util/exception/SVNCommitException.java | 26 +- .../util/impl/CheckMethodProduction.java | 110 ++--- .../util/impl/CheckMethodStaging.java | 108 ++--- .../util/impl/ClusterBuilderProduction.java | 8 +- .../util/impl/ClusterBuilderStaging.java | 8 +- .../impl/NotificationHelperProduction.java | 12 +- .../util/impl/NotificationHelperStaging.java | 12 +- .../util/impl/SVNUpdaterProduction.java | 10 +- .../util/impl/SVNUpdaterStaging.java | 10 +- .../custom/custom-flr-Flash/meta/main.yaml | 2 +- src/main/resources/service.properties | 4 +- .../gcube-algorithm-installer/meta/main.yaml | 6 +- .../gcube-algorithm-installer/tasks/main.yaml | 6 +- .../static/gcube-dataminer/meta/main.yaml | 2 +- .../static/os-java-1.7.0/tasks/main.yaml | 2 +- .../resources/static/os-unzip/tasks/main.yaml | 2 +- .../templates/cran-package-add/meta/main.yaml | 2 +- .../cran-package-add/tasks/main.yaml | 2 +- .../gcube-algorithm-add/meta/main.yaml | 2 +- .../github-package-add/meta/main.yaml | 2 +- .../templates/os-package-add/tasks/main.yaml | 2 +- src/main/webapp/WEB-INF/web.xml | 6 +- .../AlgorithmPackageParserTest.java | 90 ++-- .../AnsibleWorkerTest.java | 12 +- .../ApplicationTokenTest.java | 42 +- .../DataminerPoolManagerTest.java | 196 ++++---- .../dataminerpoolmanager/ISClientTest.java | 10 +- .../dataminerpoolmanager/JobTest.java | 38 +- .../dataminerpoolmanager/SVNTests.java | 30 +- .../dataminerpoolmanager/SendMailTest.java | 44 +- 100 files changed, 2884 insertions(+), 2639 deletions(-) create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs index a931bf8..29abf99 100755 --- a/.settings/org.eclipse.core.resources.prefs +++ b/.settings/org.eclipse.core.resources.prefs @@ -1,6 +1,6 @@ eclipse.preferences.version=1 -encoding***REMOVED***src/main/java=UTF-8 -encoding***REMOVED***src/main/resources=UTF-8 -encoding***REMOVED***src/test/java=UTF-8 -encoding***REMOVED***src/test/resources=UTF-8 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 encoding/=UTF-8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 232e134..ea06cf1 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,4 +55,4 @@ - First Release -This project adheres to [Semantic Versioning](https:***REMOVED***semver.org/spec/v2.0.0.html). +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/FUNDING.md b/FUNDING.md index 3805322..6fa9eac 100755 --- a/FUNDING.md +++ b/FUNDING.md @@ -3,24 +3,24 @@ The projects leading to this software have received funding from a series of European Union programmes including: - the Sixth Framework Programme for Research and Technological Development - - [DILIGENT](https:***REMOVED***cordis.europa.eu/project/id/004260) (grant no. 004260). + - [DILIGENT](https://cordis.europa.eu/project/id/004260) (grant no. 004260). - the Seventh Framework Programme for research, technological development and demonstration - - [D4Science](https:***REMOVED***cordis.europa.eu/project/id/212488) (grant no. 212488); - - [D4Science-II](https:***REMOVED***cordis.europa.eu/project/id/239019) (grant no.239019); - - [ENVRI](https:***REMOVED***cordis.europa.eu/project/id/283465) (grant no. 283465); - - [iMarine](https:***REMOVED***cordis.europa.eu/project/id/283644) (grant no. 283644); - - [EUBrazilOpenBio](https:***REMOVED***cordis.europa.eu/project/id/288754) (grant no. 288754). + - [D4Science](https://cordis.europa.eu/project/id/212488) (grant no. 212488); + - [D4Science-II](https://cordis.europa.eu/project/id/239019) (grant no.239019); + - [ENVRI](https://cordis.europa.eu/project/id/283465) (grant no. 283465); + - [iMarine](https://cordis.europa.eu/project/id/283644) (grant no. 283644); + - [EUBrazilOpenBio](https://cordis.europa.eu/project/id/288754) (grant no. 288754). - the H2020 research and innovation programme - - [SoBigData](https:***REMOVED***cordis.europa.eu/project/id/654024) (grant no. 654024); - - [PARTHENOS](https:***REMOVED***cordis.europa.eu/project/id/654119) (grant no. 654119); - - [EGI-Engage](https:***REMOVED***cordis.europa.eu/project/id/654142) (grant no. 654142); - - [ENVRI PLUS](https:***REMOVED***cordis.europa.eu/project/id/654182) (grant no. 654182); - - [BlueBRIDGE](https:***REMOVED***cordis.europa.eu/project/id/675680) (grant no. 675680); - - [PerformFISH](https:***REMOVED***cordis.europa.eu/project/id/727610) (grant no. 727610); - - [AGINFRA PLUS](https:***REMOVED***cordis.europa.eu/project/id/731001) (grant no. 731001); - - [DESIRA](https:***REMOVED***cordis.europa.eu/project/id/818194) (grant no. 818194); - - [ARIADNEplus](https:***REMOVED***cordis.europa.eu/project/id/823914) (grant no. 823914); - - [RISIS 2](https:***REMOVED***cordis.europa.eu/project/id/824091) (grant no. 824091); - - [EOSC-Pillar](https:***REMOVED***cordis.europa.eu/project/id/857650) (grant no. 857650); - - [Blue Cloud](https:***REMOVED***cordis.europa.eu/project/id/862409) (grant no. 862409); - - [SoBigData-PlusPlus](https:***REMOVED***cordis.europa.eu/project/id/871042) (grant no. 871042); \ No newline at end of file + - [SoBigData](https://cordis.europa.eu/project/id/654024) (grant no. 654024); + - [PARTHENOS](https://cordis.europa.eu/project/id/654119) (grant no. 654119); + - [EGI-Engage](https://cordis.europa.eu/project/id/654142) (grant no. 654142); + - [ENVRI PLUS](https://cordis.europa.eu/project/id/654182) (grant no. 654182); + - [BlueBRIDGE](https://cordis.europa.eu/project/id/675680) (grant no. 675680); + - [PerformFISH](https://cordis.europa.eu/project/id/727610) (grant no. 727610); + - [AGINFRA PLUS](https://cordis.europa.eu/project/id/731001) (grant no. 731001); + - [DESIRA](https://cordis.europa.eu/project/id/818194) (grant no. 818194); + - [ARIADNEplus](https://cordis.europa.eu/project/id/823914) (grant no. 823914); + - [RISIS 2](https://cordis.europa.eu/project/id/824091) (grant no. 824091); + - [EOSC-Pillar](https://cordis.europa.eu/project/id/857650) (grant no. 857650); + - [Blue Cloud](https://cordis.europa.eu/project/id/862409) (grant no. 862409); + - [SoBigData-PlusPlus](https://cordis.europa.eu/project/id/871042) (grant no. 871042); \ No newline at end of file diff --git a/README.md b/README.md index f9e9ce2..3be2c89 100755 --- a/README.md +++ b/README.md @@ -8,12 +8,12 @@ DataMiner Pool Manager is a service to support the integration of algorithms in ## Built With -* [OpenJDK](https:***REMOVED***openjdk.java.net/) - The JDK used -* [Maven](https:***REMOVED***maven.apache.org/) - Dependency Management +* [OpenJDK](https://openjdk.java.net/) - The JDK used +* [Maven](https://maven.apache.org/) - Dependency Management ## Documentation -* Use of this widget is is described on [Wiki](https:***REMOVED***wiki.gcube-system.org/gcube/How_to_use_the_DataMiner_Pool_Manager). +* Use of this widget is is described on [Wiki](https://wiki.gcube-system.org/gcube/How_to_use_the_DataMiner_Pool_Manager). ## Change log @@ -21,9 +21,9 @@ See [CHANGELOG.md](CHANGELOG.md). ## Authors -* **Paolo Fabriani** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) -* **Nunzio Andrea Galante** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) -* **Ciro Formisano** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) +* **Paolo Fabriani** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) +* **Nunzio Andrea Galante** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) +* **Ciro Formisano** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) ## License @@ -32,7 +32,7 @@ This project is licensed under the EUPL V.1.1 License - see the [LICENSE.md](LIC ## About the gCube Framework -This software is part of the [gCubeFramework](https:***REMOVED***www.gcube-system.org/ "gCubeFramework"): an +This software is part of the [gCubeFramework](https://www.gcube-system.org/ "gCubeFramework"): an open-source software toolkit used for building and operating Hybrid Data Infrastructures enabling the dynamic deployment of Virtual Research Environments by favouring the realisation of reuse oriented policies. diff --git a/descriptor.xml b/descriptor.xml index 49931da..8af47b2 100755 --- a/descriptor.xml +++ b/descriptor.xml @@ -1,7 +1,7 @@ + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> servicearchive tar.gz @@ -24,8 +24,8 @@ - target/$***REMOVED***build.finalName***REMOVED***.$***REMOVED***project.packaging***REMOVED*** - /$***REMOVED***artifactId***REMOVED*** + target/${build.finalName}.${project.packaging} + /${artifactId} \ No newline at end of file diff --git a/gcube-app.xml b/gcube-app.xml index 6e6ed0f..4979186 100755 --- a/gcube-app.xml +++ b/gcube-app.xml @@ -1,9 +1,9 @@ - $***REMOVED***serviceName***REMOVED*** - $***REMOVED***serviceClass***REMOVED*** - $***REMOVED***version***REMOVED*** - $***REMOVED***description***REMOVED*** + ${serviceName} + ${serviceClass} + ${version} + ${description} /api/swagger.* \ No newline at end of file diff --git a/pom.xml b/pom.xml index dd49ddb..708390d 100755 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ - + 4.0.0 @@ -18,9 +18,9 @@ DataMiner Pool Manager is a service to support the integration of algorithms in D4Science Infrastructure - scm:git:https:***REMOVED***code-repo.d4science.org/gCubeSystem/$***REMOVED***project.artifactId***REMOVED***.git - scm:git:https:***REMOVED***code-repo.d4science.org/gCubeSystem/$***REMOVED***project.artifactId***REMOVED***.git - https:***REMOVED***code-repo.d4science.org/gCubeSystem/$***REMOVED***project.artifactId***REMOVED*** + scm:git:https://code-repo.d4science.org/gCubeSystem/${project.artifactId}.git + scm:git:https://code-repo.d4science.org/gCubeSystem/${project.artifactId}.git + https://code-repo.d4science.org/gCubeSystem/${project.artifactId} @@ -28,7 +28,7 @@ dataminer-pool-manager DataAnalysis - $***REMOVED***project.basedir***REMOVED***/src/main/webapp/WEB-INF + ${project.basedir}/src/main/webapp/WEB-INF distro UTF-8 1.5.13 @@ -103,7 +103,7 @@ 1.10 - + com.jcraft jsch @@ -172,10 +172,10 @@ process-resources - $***REMOVED***webappDirectory***REMOVED*** + ${webappDirectory} - $***REMOVED***project.basedir***REMOVED*** + ${project.basedir} true src diff --git a/profile.xml b/profile.xml index 3a026c9..270b2eb 100755 --- a/profile.xml +++ b/profile.xml @@ -1,23 +1,23 @@ - + Service - $***REMOVED***description***REMOVED*** + ${description} DataminerPoolManager - $***REMOVED***artifactId***REMOVED*** + ${artifactId} 1.0.0 - $***REMOVED***artifactId***REMOVED*** - $***REMOVED***version***REMOVED*** + ${artifactId} + ${version} - $***REMOVED***groupId***REMOVED*** - $***REMOVED***artifactId***REMOVED*** - $***REMOVED***version***REMOVED*** + ${groupId} + ${artifactId} + ${version} - $***REMOVED***build.finalName***REMOVED***.jar + ${build.finalName}.jar diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 3b802ce..b840ec6 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.util.Scanner; @@ -10,7 +10,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; /** * This class is responsible for the interface with ansible, retrieving log, @@ -20,7 +20,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSeriali * @author paolo * */ -public class AnsibleWorker ***REMOVED*** +public class AnsibleWorker { /** * The name of the inventory @@ -42,99 +42,99 @@ public class AnsibleWorker ***REMOVED*** */ private File workerRoot; - public AnsibleWorker(File root) ***REMOVED*** + public AnsibleWorker(File root) { this.workerRoot = root; this.ensureWorkStructure(); - ***REMOVED*** + } -***REMOVED*** public File getWorkdir() ***REMOVED*** -***REMOVED*** return this.workerRoot; -***REMOVED*** ***REMOVED*** +// public File getWorkdir() { +// return this.workerRoot; +// } - public File getRolesDir() ***REMOVED*** + public File getRolesDir() { return new File(this.workerRoot, ROLES_DIR); - ***REMOVED*** + } - public String getWorkerId() ***REMOVED*** + public String getWorkerId() { return this.workerRoot.getName(); - ***REMOVED*** + } - public void ensureWorkStructure() ***REMOVED*** - ***REMOVED*** generate root + public void ensureWorkStructure() { + // generate root this.workerRoot.mkdirs(); - ***REMOVED*** + } - public void removeWorkStructure() ***REMOVED*** - ***REMOVED*** remove the working dir + public void removeWorkStructure() { + // remove the working dir this.workerRoot.delete(); - ***REMOVED*** + } - public File getPlaybookFile() ***REMOVED*** + public File getPlaybookFile() { return new File(this.workerRoot, PLAYBOOK_NAME); - ***REMOVED*** + } - public File getInventoryFile() ***REMOVED*** + public File getInventoryFile() { return new File(this.workerRoot, INVENTORY_NAME); - ***REMOVED*** + } - public void setInventory(Inventory inventory) throws IOException ***REMOVED*** - ***REMOVED*** serialize the string to the 'inventory' file + public void setInventory(Inventory inventory) throws IOException { + // serialize the string to the 'inventory' file AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); - ***REMOVED*** + } - public void setPlaybook(Playbook playbook) throws IOException ***REMOVED*** - ***REMOVED*** serialize the string to the 'playbook' file + public void setPlaybook(Playbook playbook) throws IOException { + // serialize the string to the 'playbook' file AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); - ***REMOVED*** + } - public void addRole(Role r) throws IOException ***REMOVED*** - ***REMOVED*** Serialize role in the workdir + public void addRole(Role r) throws IOException { + // Serialize role in the workdir AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); - ***REMOVED*** + } public int execute(PrintStream ps) - throws IOException, InterruptedException, SVNException ***REMOVED*** + throws IOException, InterruptedException, SVNException { System.out.println(this.workerRoot); - try ***REMOVED*** + try { Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); inheritIO(p.getInputStream(), ps); inheritIO(p.getErrorStream(), ps); -***REMOVED*** writer.println(this.getStatus(p.waitFor())); -***REMOVED*** writer.close(); +// writer.println(this.getStatus(p.waitFor())); +// writer.close(); return p.waitFor(); - ***REMOVED*** catch (IOException e) ***REMOVED*** + } catch (IOException e) { e.printStackTrace(); - ***REMOVED*** + } return -1; -***REMOVED*** + } - private static void inheritIO(final InputStream src, final PrintStream dest) ***REMOVED*** - new Thread(new Runnable() ***REMOVED*** - public void run() ***REMOVED*** + private static void inheritIO(final InputStream src, final PrintStream dest) { + new Thread(new Runnable() { + public void run() { Scanner sc = new Scanner(src); - while (sc.hasNextLine()) ***REMOVED*** + while (sc.hasNextLine()) { dest.println(sc.nextLine()); - ***REMOVED*** + } sc.close(); - ***REMOVED*** - ***REMOVED***).start(); -***REMOVED*** + } + }).start(); + } /** * Destroy the worker: * - remove the working dir */ - public void destroy() ***REMOVED*** + public void destroy() { this.removeWorkStructure(); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java index 9e6a8d7..9ed34b3 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java @@ -1,19 +1,19 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; -public class AnsibleHost ***REMOVED*** +public class AnsibleHost { private String name; - public AnsibleHost(String name) ***REMOVED*** + public AnsibleHost(String name) { this.name = name; - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java index 1b786e4..077e1a1 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java @@ -3,27 +3,27 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; import java.util.Collection; import java.util.Vector; -public class HostGroup ***REMOVED*** +public class HostGroup { private String name; private Collection hosts; - public HostGroup(String name) ***REMOVED*** + public HostGroup(String name) { this.name = name; this.hosts = new Vector<>(); - ***REMOVED*** + } - public void addHost(AnsibleHost h) ***REMOVED*** + public void addHost(AnsibleHost h) { this.hosts.add(h); - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return this.name; - ***REMOVED*** + } - public Collection getHosts() ***REMOVED*** + public Collection getHosts() { return new Vector<>(this.hosts); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java index a53e151..4d6400d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java @@ -3,35 +3,35 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; import java.util.Collection; import java.util.Vector; -public class Inventory ***REMOVED*** +public class Inventory { private Collection groups; - public Inventory() ***REMOVED*** + public Inventory() { this.groups = new Vector<>(); - ***REMOVED*** + } - public void addGroup(HostGroup group) ***REMOVED*** + public void addGroup(HostGroup group) { this.groups.add(group); - ***REMOVED*** + } - public void addHost(AnsibleHost h, String groupName) ***REMOVED*** + public void addHost(AnsibleHost h, String groupName) { this.getGroup(groupName).addHost(h); - ***REMOVED*** + } - private HostGroup getGroup(String groupName) ***REMOVED*** - for (HostGroup hg : this.groups) ***REMOVED*** - if (groupName.equals(hg.getName())) ***REMOVED*** + private HostGroup getGroup(String groupName) { + for (HostGroup hg : this.groups) { + if (groupName.equals(hg.getName())) { return hg; - ***REMOVED*** - ***REMOVED*** + } + } HostGroup hg = new HostGroup(groupName); this.groups.add(hg); return hg; - ***REMOVED*** + } - public Collection getHostGroups() ***REMOVED*** + public Collection getHostGroups() { return new Vector<>(this.groups); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java index 9a7f045..c08c7e2 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java @@ -1,9 +1,9 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; -***REMOVED*** +import java.util.List; import java.util.Vector; -public class Playbook ***REMOVED*** +public class Playbook { private String hostGroupName; @@ -11,40 +11,40 @@ public class Playbook ***REMOVED*** private String remote_user; - public Playbook() ***REMOVED*** + public Playbook() { this.roles = new Vector<>(); - ***REMOVED*** + } - public void addRole(String role) ***REMOVED*** + public void addRole(String role) { roles.add(role); - ***REMOVED*** + } - public void applyTo(String hostGroupName) ***REMOVED*** + public void applyTo(String hostGroupName) { this.hostGroupName = hostGroupName; - ***REMOVED*** + } - public String getHostGroupName() ***REMOVED*** + public String getHostGroupName() { return hostGroupName; - ***REMOVED*** + } - public List getRoles() ***REMOVED*** + public List getRoles() { return new Vector<>(roles); - ***REMOVED*** + } -public String getRemote_user() ***REMOVED*** +public String getRemote_user() { return remote_user; -***REMOVED*** +} -public void setRemote_user(String remote_user) ***REMOVED*** +public void setRemote_user(String remote_user) { this.remote_user = remote_user; -***REMOVED*** +} -public void setHostGroupName(String hostGroupName) ***REMOVED*** +public void setHostGroupName(String hostGroupName) { this.hostGroupName = hostGroupName; -***REMOVED*** +} -public void setRoles(List roles) ***REMOVED*** +public void setRoles(List roles) { this.roles = roles; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java index 1876ea7..3f3aa33 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; import java.util.Collection; import java.util.Vector; -public class Role ***REMOVED*** +public class Role { /** * The name of the role @@ -14,38 +14,38 @@ public class Role ***REMOVED*** private Collection meta; - public Role() ***REMOVED*** + public Role() { this.tasks = new Vector<>(); this.meta = new Vector<>(); - ***REMOVED*** + } - public Role(String name) ***REMOVED*** + public Role(String name) { this(); this.name = name; - ***REMOVED*** + } - public void addTaskFile(RoleFile tf) ***REMOVED*** + public void addTaskFile(RoleFile tf) { this.tasks.add(tf); - ***REMOVED*** + } - public void addMeta(RoleFile tf) ***REMOVED*** + public void addMeta(RoleFile tf) { this.meta.add(tf); - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public Collection getTaskFiles() ***REMOVED*** + public Collection getTaskFiles() { return new Vector<>(this.tasks); - ***REMOVED*** + } - public Collection getMeta() ***REMOVED*** + public Collection getMeta() { return new Vector<>(this.meta); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java index ba903a9..3cff918 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; -public class RoleFile ***REMOVED*** +public class RoleFile { /** * The path to the file, starting from the role root @@ -18,37 +18,37 @@ public class RoleFile ***REMOVED*** */ private String content; - public RoleFile() ***REMOVED*** - ***REMOVED*** + public RoleFile() { + } - public RoleFile(String name, String content) ***REMOVED*** + public RoleFile(String name, String content) { this(); this.setName(name); this.setContent(content); - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public String getContent() ***REMOVED*** + public String getContent() { return content; - ***REMOVED*** + } - public void setContent(String content) ***REMOVED*** + public void setContent(String content) { this.content = content; - ***REMOVED*** + } - public String getPath() ***REMOVED*** + public String getPath() { return path; - ***REMOVED*** + } - public void setPath(String path) ***REMOVED*** + public void setPath(String path) { this.path = path; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 32b533e..e2b5d23 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -1,11 +1,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Collection; -***REMOVED*** +import java.util.List; import java.util.Map; -***REMOVED*** +import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; @@ -23,302 +23,302 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.Custo import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; -public class AnsibleBridge ***REMOVED*** - ***REMOVED*** private static final org.slf4j.Logger LOGGER = - ***REMOVED*** LoggerFactory.getLogger(AnsibleBridge.class); +public class AnsibleBridge { + // private static final org.slf4j.Logger LOGGER = + // LoggerFactory.getLogger(AnsibleBridge.class); private String dpmRoot; - ***REMOVED*** public AnsibleBridge() ***REMOVED*** - ***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); - ***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** + // public AnsibleBridge() { + // this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); + // //this(System.getProperty("/home/gcube/dataminer-pool-manager")); + // + // } - public AnsibleBridge(String root) ***REMOVED*** + public AnsibleBridge(String root) { this.dpmRoot = root; this.ensureServiceRoot(); -***REMOVED*** + } - private void ensureServiceRoot() ***REMOVED*** - ***REMOVED*** generate root + private void ensureServiceRoot() { + // generate root new File(dpmRoot).mkdirs(); - ***REMOVED*** 'template' is for template roles - ***REMOVED*** this.getTemplatesDir().mkdirs(); - ***REMOVED*** 'static' is for custom roles - ***REMOVED*** this.getCustomDir().mkdirs(); - ***REMOVED*** 'work' is for temporary working directories + // 'template' is for template roles + // this.getTemplatesDir().mkdirs(); + // 'static' is for custom roles + // this.getCustomDir().mkdirs(); + // 'work' is for temporary working directories this.getWorkDir().mkdirs(); -***REMOVED*** + } - private File getWorkDir() ***REMOVED*** + private File getWorkDir() { return new File(this.dpmRoot, "work"); -***REMOVED*** + } - ***REMOVED*** private String getTemplatesDir() ***REMOVED*** - ***REMOVED*** String input = null; - ***REMOVED*** input = - ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); - ***REMOVED*** return input; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** private String getCustomDir() ***REMOVED*** - ***REMOVED*** String input = null; - ***REMOVED*** input = - ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); - ***REMOVED*** return input; - ***REMOVED*** ***REMOVED*** + // private String getTemplatesDir() { + // String input = null; + // input = + // AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); + // return input; + // } + // + // + // private String getCustomDir() { + // String input = null; + // input = + // AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); + // return input; + // } public AnsibleWorker createWorker(Algorithm algorithm, Cluster dataminerCluster, - boolean includeAlgorithmDependencies, String user) throws IOException ***REMOVED*** + boolean includeAlgorithmDependencies, String user) throws IOException { File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); AnsibleWorker worker = new AnsibleWorker(workerRoot); List algoRoles = new Vector<>(); - ***REMOVED*** add algorithms and dependencies to the worker - for (Role r : this.generateRoles(algorithm, includeAlgorithmDependencies)) ***REMOVED*** + // add algorithms and dependencies to the worker + for (Role r : this.generateRoles(algorithm, includeAlgorithmDependencies)) { algoRoles.add(r); worker.addRole(r); - ***REMOVED*** + } - ***REMOVED*** to comment the for in case of just install algo - if (includeAlgorithmDependencies) ***REMOVED*** - for (Dependency d : algorithm.getDependencies()) ***REMOVED*** - for (Role r : this.generateRoles(d)) ***REMOVED*** + // to comment the for in case of just install algo + if (includeAlgorithmDependencies) { + for (Dependency d : algorithm.getDependencies()) { + for (Role r : this.generateRoles(d)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } - ***REMOVED*** add static roles - for (Role r : this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** + // add static roles + for (Role r : this.getStaticRoleManager().getStaticRoles()) { worker.addRole(r); - ***REMOVED*** + } - ***REMOVED*** generate the inventory + // generate the inventory Inventory inventory = new Inventory(); - for (Host h : dataminerCluster.getHosts()) ***REMOVED*** + for (Host h : dataminerCluster.getHosts()) { AnsibleHost ah = new AnsibleHost(h.getName()); inventory.addHost(ah, "universe"); inventory.addHost(ah, "d4science"); - ***REMOVED*** + } worker.setInventory(inventory); - ***REMOVED*** generate the playbook + // generate the playbook Playbook playbook = new Playbook(); playbook.setRemote_user(user); playbook.applyTo("universe"); - for (Role r : algoRoles) ***REMOVED*** - ***REMOVED*** add only 'add' roles - if (!r.getName().endsWith("remove")) ***REMOVED*** + for (Role r : algoRoles) { + // add only 'add' roles + if (!r.getName().endsWith("remove")) { playbook.addRole(r.getName()); - ***REMOVED*** - ***REMOVED*** + } + } worker.setPlaybook(playbook); return worker; -***REMOVED*** + } - public void printInventoryByDomainAndSets(Collection clusters) ***REMOVED*** + public void printInventoryByDomainAndSets(Collection clusters) { Map> inventory = new TreeMap<>(); - for (Cluster cluster : clusters) ***REMOVED*** - for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** + for (Cluster cluster : clusters) { + for (AlgorithmSet as : cluster.getAlgorithmSets()) { String asName = as.getName(); - for (Host h : cluster.getHosts()) ***REMOVED*** + for (Host h : cluster.getHosts()) { String domain = h.getDomain().getName(); String key = String.format("[%s@%s]", asName, domain); Set hosts = inventory.get(key); - if (hosts == null) ***REMOVED*** + if (hosts == null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); - ***REMOVED*** + } hosts.add(h); - ***REMOVED*** + } - ***REMOVED*** - ***REMOVED*** - for (String key : inventory.keySet()) ***REMOVED*** + } + } + for (String key : inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); - for (Host h : hosts) ***REMOVED*** + for (Host h : hosts) { System.out.println(h.getName() + "." + h.getDomain().getName()); - ***REMOVED*** + } System.out.println(); - ***REMOVED*** -***REMOVED*** + } + } - public void printInventoryBySets(Collection clusters) ***REMOVED*** + public void printInventoryBySets(Collection clusters) { Map> inventory = new TreeMap<>(); - for (Cluster cluster : clusters) ***REMOVED*** - for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** + for (Cluster cluster : clusters) { + for (AlgorithmSet as : cluster.getAlgorithmSets()) { String asName = as.getName(); - for (Host h : cluster.getHosts()) ***REMOVED*** + for (Host h : cluster.getHosts()) { String key = String.format("[%s]", asName); Set hosts = inventory.get(key); - if (hosts == null) ***REMOVED*** + if (hosts == null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); - ***REMOVED*** + } hosts.add(h); - ***REMOVED*** + } - ***REMOVED*** - ***REMOVED*** - for (String key : inventory.keySet()) ***REMOVED*** + } + } + for (String key : inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); - for (Host h : hosts) ***REMOVED*** + for (Host h : hosts) { System.out.println(h.getName() + "." + h.getDomain().getName()); - ***REMOVED*** + } System.out.println(); - ***REMOVED*** -***REMOVED*** + } + } - ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster - ***REMOVED*** cluster, /*boolean updateSVN,*/ boolean test) throws IOException, - ***REMOVED*** InterruptedException, SVNException ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** return applyAlgorithmSetToCluster - ***REMOVED*** (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); - ***REMOVED*** ***REMOVED*** + // public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster + // cluster, /*boolean updateSVN,*/ boolean test) throws IOException, + // InterruptedException, SVNException { + // + // + // return applyAlgorithmSetToCluster + // (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); + // } - ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster - ***REMOVED*** cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws - ***REMOVED*** IOException, InterruptedException, SVNException ***REMOVED*** - ***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), - ***REMOVED*** uuid)); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** List algoRoles = new Vector<>(); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker - ***REMOVED*** for (Algorithm a : as.getAlgorithms()) ***REMOVED*** - ***REMOVED*** for (Role r : this.generateRoles(a)) ***REMOVED*** - ***REMOVED*** algoRoles.add(r); - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED***to comment the for in case of just install algo - ***REMOVED*** if(test)***REMOVED*** - ***REMOVED*** for (Dependency d : a.getDependencies()) ***REMOVED*** - ***REMOVED*** for (Role r : this.generateRoles(d)) ***REMOVED*** - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** add static roles - ***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** generate the inventory - ***REMOVED*** Inventory inventory = new Inventory(); - ***REMOVED*** for (Host h : cluster.getHosts()) ***REMOVED*** - ***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName()); - ***REMOVED*** inventory.addHost(ah, "universe"); - ***REMOVED*** inventory.addHost(ah, "d4science"); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** worker.setInventory(inventory); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** generate the playbook - ***REMOVED*** Playbook playbook = new Playbook(); - ***REMOVED*** if(test)***REMOVED*** - ***REMOVED*** playbook.setRemote_user("root");***REMOVED*** - ***REMOVED*** playbook.setRemote_user("gcube"); - ***REMOVED*** playbook.applyTo("universe"); - ***REMOVED*** for(Role r:algoRoles) ***REMOVED*** - ***REMOVED*** ***REMOVED*** add only 'add' roles - ***REMOVED*** if(!r.getName().endsWith("remove")) ***REMOVED*** - ***REMOVED*** playbook.addRole(r.getName()); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** worker.setPlaybook(playbook); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** execute and save log locally - ***REMOVED*** ***REMOVED***PrintStream console = System.out; - ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); - ***REMOVED*** path.mkdirs(); - ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); - ***REMOVED*** FileOutputStream fos = new FileOutputStream(n); - ***REMOVED*** PrintStream ps = new PrintStream(fos); - ***REMOVED*** - ***REMOVED*** ***REMOVED***System.setErr(console); - ***REMOVED*** - ***REMOVED*** worker.apply(as,ps,test); - ***REMOVED*** ***REMOVED***System.setOut(console); - ***REMOVED*** ***REMOVED***worker.apply(); - ***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** destroy the worker - ***REMOVED*** worker.destroy(); - ***REMOVED*** return worker; - ***REMOVED*** ***REMOVED*** + // public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster + // cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws + // IOException, InterruptedException, SVNException { + // AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), + // uuid)); + // + // + // List algoRoles = new Vector<>(); + // + // // add algorithms and dependencies to the worker + // for (Algorithm a : as.getAlgorithms()) { + // for (Role r : this.generateRoles(a)) { + // algoRoles.add(r); + // worker.addRole(r); + // } + // //to comment the for in case of just install algo + // if(test){ + // for (Dependency d : a.getDependencies()) { + // for (Role r : this.generateRoles(d)) { + // worker.addRole(r); + // } + // } + // } + // } + // + // // add static roles + // for(Role r:this.getStaticRoleManager().getStaticRoles()) { + // worker.addRole(r); + // } + // + // // generate the inventory + // Inventory inventory = new Inventory(); + // for (Host h : cluster.getHosts()) { + // AnsibleHost ah = new AnsibleHost(h.getName()); + // inventory.addHost(ah, "universe"); + // inventory.addHost(ah, "d4science"); + // } + // worker.setInventory(inventory); + // + // // generate the playbook + // Playbook playbook = new Playbook(); + // if(test){ + // playbook.setRemote_user("root");} + // playbook.setRemote_user("gcube"); + // playbook.applyTo("universe"); + // for(Role r:algoRoles) { + // // add only 'add' roles + // if(!r.getName().endsWith("remove")) { + // playbook.addRole(r.getName()); + // } + // } + // + // worker.setPlaybook(playbook); + // + // // execute and save log locally + // //PrintStream console = System.out; + // File path = new File(worker.getWorkdir() + File.separator + "logs"); + // path.mkdirs(); + // File n = new File(path + File.separator + worker.getWorkerId()); + // FileOutputStream fos = new FileOutputStream(n); + // PrintStream ps = new PrintStream(fos); + // + // //System.setErr(console); + // + // worker.apply(as,ps,test); + // //System.setOut(console); + // //worker.apply(); + // System.out.println("Log stored to to " + n.getAbsolutePath()); + // + // // destroy the worker + // worker.destroy(); + // return worker; + // } - private TemplateManager getTemplateManager() ***REMOVED*** + private TemplateManager getTemplateManager() { return new TemplateManager(); -***REMOVED*** + } - private CustomRoleManager getCustomRoleManager() ***REMOVED*** + private CustomRoleManager getCustomRoleManager() { return new CustomRoleManager(); -***REMOVED*** + } - private StaticRoleManager getStaticRoleManager() ***REMOVED*** + private StaticRoleManager getStaticRoleManager() { return new StaticRoleManager(); -***REMOVED*** + } -***REMOVED**** + /** * Generate all roles for this dependency * * @param dep * Dependency * @return Collection of Roles -***REMOVED*** - public Collection generateRoles(Dependency dep) ***REMOVED*** + */ + public Collection generateRoles(Dependency dep) { Collection roles = new Vector<>(); - if ("os".equalsIgnoreCase(dep.getType())) ***REMOVED*** + if ("os".equalsIgnoreCase(dep.getType())) { OSDependencyPackage pkg = new OSDependencyPackage(dep); - if (pkg != null) ***REMOVED*** + if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** + } - ***REMOVED*** else if ("custom".equalsIgnoreCase(dep.getType())) ***REMOVED*** + } else if ("custom".equalsIgnoreCase(dep.getType())) { CustomDependencyPackage pkg = new CustomDependencyPackage(dep); - if (pkg != null) ***REMOVED*** + if (pkg != null) { roles.addAll(pkg.getRoles(this.getCustomRoleManager())); - ***REMOVED*** - ***REMOVED*** else if ("github".equalsIgnoreCase(dep.getType())) ***REMOVED*** + } + } else if ("github".equalsIgnoreCase(dep.getType())) { CranDependencyPackage pkg = new CranDependencyPackage(dep); - if (pkg != null) ***REMOVED*** + if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - ***REMOVED*** else if ("cran".equalsIgnoreCase(dep.getType())) ***REMOVED*** + } + } else if ("cran".equalsIgnoreCase(dep.getType())) { CranDependencyPackage pkg = new CranDependencyPackage(dep); - if (pkg != null) ***REMOVED*** + if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** + } - ***REMOVED*** + } return roles; -***REMOVED*** + } - public Collection generateRoles(Algorithm a, boolean includeAlgorithmDependencies) ***REMOVED*** + public Collection generateRoles(Algorithm a, boolean includeAlgorithmDependencies) { AlgorithmPackage pkg = new AlgorithmPackage(a, includeAlgorithmDependencies); return pkg.getRoles(this.getTemplateManager()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java index eda7460..fa0496c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.io.PrintWriter; import org.apache.commons.io.IOUtils; @@ -14,106 +14,106 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; -public class AnsibleSerializeHelper ***REMOVED*** +public class AnsibleSerializeHelper { - public static void serialize(Inventory inventory, File inventoryFile) throws IOException ***REMOVED*** + public static void serialize(Inventory inventory, File inventoryFile) throws IOException { String out = ""; - for(HostGroup hg:inventory.getHostGroups()) ***REMOVED*** + for(HostGroup hg:inventory.getHostGroups()) { out+=String.format("[%s]\n", hg.getName()); - for(AnsibleHost h:hg.getHosts()) ***REMOVED*** + for(AnsibleHost h:hg.getHosts()) { out+=h.getName()+"\n"; - ***REMOVED*** + } out+="\n"; - ***REMOVED*** + } out = out.trim(); serialize(out, inventoryFile); - ***REMOVED*** + } - public static void serialize(Playbook playbook, File playbookFile) throws IOException ***REMOVED*** + public static void serialize(Playbook playbook, File playbookFile) throws IOException { String out = "- hosts: " + playbook.getHostGroupName() + "\n"; out += " remote_user: "+playbook.getRemote_user()+"\n"; out+=" roles:\n"; - for(String r:playbook.getRoles()) ***REMOVED*** + for(String r:playbook.getRoles()) { out+=" - " + r+"\n"; - ***REMOVED*** + } out+=" vars:\n"; out+=" os_package_state: present\n"; out = out.trim(); serialize(out, playbookFile); - ***REMOVED*** + } - public static void serializeRole(Role r, File dir) throws IOException ***REMOVED*** - ***REMOVED*** create root + public static void serializeRole(Role r, File dir) throws IOException { + // create root File root = new File(dir, r.getName()); root.mkdirs(); - ***REMOVED*** create tasks - if(r.getTaskFiles().size()>0) ***REMOVED*** + // create tasks + if(r.getTaskFiles().size()>0) { File tasks = new File(root, "tasks"); tasks.mkdirs(); - for(RoleFile tf: r.getTaskFiles()) ***REMOVED*** + for(RoleFile tf: r.getTaskFiles()) { serializeTask(tf, tasks); - ***REMOVED*** - ***REMOVED*** + } + } - ***REMOVED*** create meta - if(r.getMeta().size()>0) ***REMOVED*** + // create meta + if(r.getMeta().size()>0) { File meta = new File(root, "meta"); meta.mkdirs(); - for(RoleFile tf: r.getMeta()) ***REMOVED*** + for(RoleFile tf: r.getMeta()) { serializeTask(tf, meta); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } - public static void serializeTask(RoleFile tf, File dir) throws IOException ***REMOVED*** + public static void serializeTask(RoleFile tf, File dir) throws IOException { File f = new File(dir, tf.getName()); serialize(tf.getContent().trim(), f); - ***REMOVED*** + } - public static void serialize(String s, File f) throws IOException ***REMOVED*** + public static void serialize(String s, File f) throws IOException { PrintWriter out = new PrintWriter(f); out.println(s); out.close(); - ***REMOVED*** + } - public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException ***REMOVED*** + public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException { Role out = new Role(); out.setName(roleDir.getName()); - if(!roleDir.exists()) ***REMOVED*** + if(!roleDir.exists()) { throw new FileNotFoundException(); - ***REMOVED*** + } - try ***REMOVED*** + try { File tasksDir = new File(roleDir, "tasks"); - if(tasksDir.exists()) ***REMOVED*** - for(File main:tasksDir.listFiles()) ***REMOVED*** + if(tasksDir.exists()) { + for(File main:tasksDir.listFiles()) { String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); RoleFile tf = new RoleFile(main.getName(), content); tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); out.addTaskFile(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** + } + } + } catch(FileNotFoundException e) { e.printStackTrace(); - ***REMOVED*** + } - try ***REMOVED*** + try { File metaDir = new File(roleDir, "meta"); - if(metaDir.exists()) ***REMOVED*** - for(File main:metaDir.listFiles()) ***REMOVED*** + if(metaDir.exists()) { + for(File main:metaDir.listFiles()) { String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); RoleFile tf = new RoleFile(main.getName(), content); tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); out.addMeta(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** + } + } + } catch(FileNotFoundException e) { e.printStackTrace(); - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java index d4a43e2..35c5575 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java @@ -6,29 +6,29 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Vector; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class AlgorithmPackage ***REMOVED*** +public class AlgorithmPackage { private Algorithm algorithm; private boolean includeAlgorithmDependencies; private Logger logger; public AlgorithmPackage(Algorithm a,boolean includeAlgorithmDependencies) - ***REMOVED*** + { this.logger = LoggerFactory.getLogger(AlgorithmPackage.class); this.algorithm = a; this.includeAlgorithmDependencies = includeAlgorithmDependencies; - ***REMOVED*** + } - protected Map getDictionary(Algorithm a) ***REMOVED*** + protected Map getDictionary(Algorithm a) { Map out = new HashMap(); out.put("name", a.getName()); out.put("category", a.getCategory()); @@ -36,47 +36,47 @@ public class AlgorithmPackage ***REMOVED*** out.put("atype", a.getAlgorithmType()); out.put("skipjava", a.getSkipJava()); out.put("vre", ScopeProvider.instance.get()); - ***REMOVED***out.put("vre", "FAKE_VRE"); + //out.put("vre", "FAKE_VRE"); out.put("packageurl", a.getPackageURL()); out.put("description", a.getDescription()); String deps = ""; - if(includeAlgorithmDependencies)***REMOVED*** - for(Dependency d:a.getDependencies()) ***REMOVED*** - deps+=String.format("- ***REMOVED*** role: %s ***REMOVED***\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); - ***REMOVED******REMOVED*** + if(includeAlgorithmDependencies){ + for(Dependency d:a.getDependencies()) { + deps+=String.format("- { role: %s }\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); + }} deps = deps.trim(); out.put("dependencies", deps); return out; - ***REMOVED*** + } - protected Algorithm getAlgorithm() ***REMOVED*** + protected Algorithm getAlgorithm() { return this.algorithm; - ***REMOVED*** + } - public Collection getRoles(TemplateManager tm) ***REMOVED*** + public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" + for(String mode:new String[]{"add"}) { // "remove", "update" String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** find template + try { + // find template Role template = tm.getRoleTemplate("gcube-algorithm-" + mode); - ***REMOVED*** - if(template!=null) ***REMOVED*** + // + if(template!=null) { Map dictionary = this.getDictionary(this.getAlgorithm()); Role r = tm.fillRoleTemplate(template, dictionary); r.setName(roleName); out.add(r); - ***REMOVED*** else - ***REMOVED*** + } else + { this.logger.warn("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java index a040404..baa4910 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java @@ -2,10 +2,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -public class CranDependencyPackage extends DependencyPackage ***REMOVED*** +public class CranDependencyPackage extends DependencyPackage { - public CranDependencyPackage(Dependency d) ***REMOVED*** + public CranDependencyPackage(Dependency d) { super(d); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java index 10f34e5..ae51255 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java @@ -6,64 +6,64 @@ import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** +import org.slf4j.Logger; -public class CustomDependencyPackage extends DependencyPackage ***REMOVED*** +public class CustomDependencyPackage extends DependencyPackage { private Logger logger; public CustomDependencyPackage(Dependency dependency) - ***REMOVED*** + { super(dependency); - ***REMOVED*** + } -***REMOVED*** private String getCustomRepositoryLocation(String ansibleRoot) ***REMOVED*** -***REMOVED*** return ansibleRoot+"/custom"; -***REMOVED*** ***REMOVED*** +// private String getCustomRepositoryLocation(String ansibleRoot) { +// return ansibleRoot+"/custom"; +// } /* - public void serializeTo(String ansibleRoot) ***REMOVED*** - for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** - ***REMOVED*** look for roles in the 'custom' repository - try ***REMOVED*** - ***REMOVED*** role name + public void serializeTo(String ansibleRoot) { + for(String mode:new String[]{"add", "remove", "update"}) { + // look for roles in the 'custom' repository + try { + // role name String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - ***REMOVED*** look for the custom role + // look for the custom role File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); System.out.println("** CUSTOM ** " + src); - if(src.exists()) ***REMOVED*** - ***REMOVED*** do copy + if(src.exists()) { + // do copy System.out.println("copying CUSTOM role"); File dest = new File(ansibleRoot+"/work/"+roleName); FileUtils.copyDirectory(src, dest); - ***REMOVED*** - ***REMOVED*** catch(IOException e) ***REMOVED*** + } + } catch(IOException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } */ - public Collection getRoles(CustomRoleManager crm) ***REMOVED*** + public Collection getRoles(CustomRoleManager crm) { Collection out = new Vector<>(); -***REMOVED*** for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" - ***REMOVED*** role name +// for(String mode:new String[]{"add", "remove", "update"}) { + for(String mode:new String[]{"add"}) { // "remove", "update" + // role name String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** look for custom role + try { + // look for custom role Role role = crm.getRole(roleName); - if(role!=null) ***REMOVED*** + if(role!=null) { out.add(role); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no custom role found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java index 3d59036..b49851c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java @@ -1,31 +1,31 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.NoSuchElementException; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; -public class CustomRoleManager ***REMOVED*** +public class CustomRoleManager { - public String getRoot() ***REMOVED*** + public String getRoot() { String input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); return input; - ***REMOVED*** + } - public Role getRole(String roleName) throws NoSuchElementException ***REMOVED*** + public Role getRole(String roleName) throws NoSuchElementException { File f = new File(this.getRoot(), roleName); - try ***REMOVED*** + try { return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java index bb06412..b5b44fd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java @@ -8,53 +8,53 @@ import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class DependencyPackage ***REMOVED*** +public class DependencyPackage { private Logger logger; private Dependency dependency; - public DependencyPackage(Dependency dependency) ***REMOVED*** + public DependencyPackage(Dependency dependency) { this.dependency = dependency; this.logger = LoggerFactory.getLogger(DependencyPackage.class); - ***REMOVED*** + } - protected Map getDictionary(Dependency d) ***REMOVED*** + protected Map getDictionary(Dependency d) { Map out = new HashMap(); out.put("name", d.getName()); out.put("type", d.getType()); return out; - ***REMOVED*** + } - protected Dependency getDependency() ***REMOVED*** + protected Dependency getDependency() { return this.dependency; - ***REMOVED*** + } - public Collection getRoles(TemplateManager tm) ***REMOVED*** + public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" + for(String mode:new String[]{"add"}) { // "remove", "update" String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** find template + try { + // find template Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); - ***REMOVED*** - if(template!=null) ***REMOVED*** + // + if(template!=null) { Map dictionary = this.getDictionary(this.getDependency()); Role r = tm.fillRoleTemplate(template, dictionary); r.setName(roleName); out.add(r); - ***REMOVED*** else ***REMOVED*** + } else { this.logger.warn("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java index 5540411..61648f0 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java @@ -2,10 +2,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -public class OSDependencyPackage extends DependencyPackage ***REMOVED*** +public class OSDependencyPackage extends DependencyPackage { - public OSDependencyPackage(Dependency dependency) ***REMOVED*** + public OSDependencyPackage(Dependency dependency) { super(dependency); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java index a11b6d4..98f377e 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Collection; import java.util.Vector; @@ -9,30 +9,30 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - public class StaticRoleManager ***REMOVED*** + public class StaticRoleManager { - public StaticRoleManager() ***REMOVED*** + public StaticRoleManager() { - ***REMOVED*** + } - public String getRoot() ***REMOVED*** + public String getRoot() { String input = AnsibleBridge.class.getClassLoader().getResource("static").getPath(); return input; - ***REMOVED*** + } - public Collection getStaticRoles() ***REMOVED*** + public Collection getStaticRoles() { Collection out = new Vector<>(); - for(File f: new File(this.getRoot()).listFiles()) ***REMOVED*** - try ***REMOVED*** + for(File f: new File(this.getRoot()).listFiles()) { + try { out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); - ***REMOVED*** catch(IOException e) ***REMOVED*** + } catch(IOException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } - ***REMOVED*** + } diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java index ed2202f..855c83c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Map; import java.util.NoSuchElementException; @@ -11,75 +11,75 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; import org.stringtemplate.v4.ST; -public class TemplateManager ***REMOVED*** +public class TemplateManager { - public TemplateManager() ***REMOVED*** + public TemplateManager() { - ***REMOVED*** + } - public String getTemplateRoot() ***REMOVED*** + public String getTemplateRoot() { String input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); return input; - ***REMOVED*** + } -***REMOVED*** private String readTemplate(String templateName) throws IOException ***REMOVED*** -***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); -***REMOVED*** System.out.println("looking for file " + templateFile.getName()); -***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** +// private String readTemplate(String templateName) throws IOException { +// File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); +// System.out.println("looking for file " + templateFile.getName()); +// String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); +// return out; +// } -***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException ***REMOVED*** -***REMOVED*** String template = null; -***REMOVED*** try ***REMOVED*** -***REMOVED*** template = this.readTemplate(templateName); -***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** throw new NoSuchElementException(); -***REMOVED*** ***REMOVED*** -***REMOVED*** return template; -***REMOVED*** ***REMOVED*** +// public String getTemplate(String templateName) throws NoSuchElementException { +// String template = null; +// try { +// template = this.readTemplate(templateName); +// } catch (IOException e) { +// throw new NoSuchElementException(); +// } +// return template; +// } - public Role fillRoleTemplate(Role template, Map dictionary) ***REMOVED*** + public Role fillRoleTemplate(Role template, Map dictionary) { Role out = new Role(); out.setName(template.getName()); - for(RoleFile tf:template.getTaskFiles()) ***REMOVED*** + for(RoleFile tf:template.getTaskFiles()) { out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** - for(RoleFile tf:template.getMeta()) ***REMOVED*** + } + for(RoleFile tf:template.getMeta()) { out.addMeta(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** + } return out; - ***REMOVED*** + } - private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) ***REMOVED*** + private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) { RoleFile out = new RoleFile(); out.setName(template.getName()); out.setContent(this.fillTemplate(template.getContent(), dictionary)); return out; - ***REMOVED*** + } - private String fillTemplate(String template, Map dictionary) ***REMOVED*** - if (template != null) ***REMOVED*** + private String fillTemplate(String template, Map dictionary) { + if (template != null) { ST t = new ST(template); - for (String key : dictionary.keySet()) ***REMOVED*** + for (String key : dictionary.keySet()) { t.add(key, dictionary.get(key)); - ***REMOVED*** + } String output = t.render(); return output; - ***REMOVED*** + } return template; - ***REMOVED*** + } - public Role getRoleTemplate(String roleName) throws NoSuchElementException ***REMOVED*** + public Role getRoleTemplate(String roleName) throws NoSuchElementException { File f = new File(this.getTemplateRoot(), roleName); - try ***REMOVED*** + try { return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java index 84a47f3..160ff51 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java @@ -1,167 +1,167 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients; import java.io.BufferedReader; -***REMOVED*** +import java.io.IOException; import java.io.InputStreamReader; -***REMOVED*** +import java.net.URL; import java.util.LinkedList; -***REMOVED*** +import java.util.List; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckPermission; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import au.com.bytecode.opencsv.CSVReader; -public class HAProxy ***REMOVED*** +public class HAProxy { private Logger logger; private CSVReader reader; - public HAProxy() ***REMOVED*** + public HAProxy() { this.logger = LoggerFactory.getLogger(HAProxy.class); -***REMOVED*** + } -***REMOVED*** public Cluster getClusterByHProxy() throws IOException ***REMOVED*** -***REMOVED*** Cluster cl = new Cluster(); -***REMOVED*** String HProxy = ISClient.getHProxy(); -***REMOVED*** URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); -***REMOVED*** BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); -***REMOVED*** reader = new CSVReader(in); -***REMOVED*** String[] nextLine; -***REMOVED*** while ((nextLine = reader.readNext()) != null) ***REMOVED*** -***REMOVED*** ***REMOVED*** rules to add -***REMOVED*** if (HProxy.contains(nextLine[0])) ***REMOVED*** -***REMOVED*** cl.setName(nextLine[0]); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return cl; -***REMOVED*** -***REMOVED******REMOVED*** +// public Cluster getClusterByHProxy() throws IOException { +// Cluster cl = new Cluster(); +// String HProxy = ISClient.getHProxy(); +// URL stockURL = new URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); +// BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); +// reader = new CSVReader(in); +// String[] nextLine; +// while ((nextLine = reader.readNext()) != null) { +// // rules to add +// if (HProxy.contains(nextLine[0])) { +// cl.setName(nextLine[0]); +// } +// } +// return cl; +// +// } - public Cluster MapCluster() throws IOException ***REMOVED*** + public Cluster MapCluster() throws IOException { Cluster cl = new Cluster(); String HProxy = ISClient.getHProxy(); - ***REMOVED***Prod HAproxy - if (HProxy.equals("dataminer-cluster1.d4science.org")) ***REMOVED*** + //Prod HAproxy + if (HProxy.equals("dataminer-cluster1.d4science.org")) { cl.setName("dataminer_cluster1"); - ***REMOVED*** - if (HProxy.equals("dataminer-bigdata.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-bigdata.d4science.org")) { cl.setName("bigdata"); - ***REMOVED*** - if (HProxy.equals("dataminer-cluster1.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-cluster1.d4science.org")) { cl.setName("dataminer_cluster1"); - ***REMOVED*** - if (HProxy.equals("dataminer-cloud1.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-cloud1.d4science.org")) { cl.setName("dataminer_cloud1"); - ***REMOVED*** - if (HProxy.equals("dataminer-prototypes.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-prototypes.d4science.org")) { cl.setName("prototypes"); - ***REMOVED*** - if (HProxy.equals("dataminer.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer.d4science.org")) { cl.setName("gcubeapps"); - ***REMOVED*** - if (HProxy.equals("dataminer-genericworkers.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-genericworkers.d4science.org")) { cl.setName("genericworkers"); - ***REMOVED*** - if (HProxy.equals("dataminer-genericworkers-proto.d4science.org")) ***REMOVED*** + } + if (HProxy.equals("dataminer-genericworkers-proto.d4science.org")) { cl.setName("genericworkers_proto"); - ***REMOVED*** - ***REMOVED***dev HAProxy - if (HProxy.equals("dataminer-d-workers.d4science.org")||(HProxy.equals("dataminer-d-d4s.d4science.org"))) ***REMOVED*** + } + //dev HAProxy + if (HProxy.equals("dataminer-d-workers.d4science.org")||(HProxy.equals("dataminer-d-d4s.d4science.org"))) { cl.setName("devnext_backend"); - ***REMOVED*** - ***REMOVED***preProd HAProxy -***REMOVED*** if (HProxy.equals("dataminer1-pre.d4science.org")) ***REMOVED*** -***REMOVED*** cl.setName("dataminer1-pre.d4science.org"); -***REMOVED*** ***REMOVED*** + } + //preProd HAProxy +// if (HProxy.equals("dataminer1-pre.d4science.org")) { +// cl.setName("dataminer1-pre.d4science.org"); +// } return cl; -***REMOVED*** + } - public List listDataMinersByCluster(String targetVREToken,String targetVRE) throws IOException ***REMOVED*** + public List listDataMinersByCluster(String targetVREToken,String targetVRE) throws IOException { SecurityTokenProvider.instance.set(targetVREToken); ScopeProvider.instance.set(targetVRE); - ***REMOVED*** next op to use when Cluster info available in the IS - ***REMOVED*** Cluster cluster = this.getClusterByHProxy(); + // next op to use when Cluster info available in the IS + // Cluster cluster = this.getClusterByHProxy(); Cluster cluster = this.MapCluster(); List out = new LinkedList(); Host a = new Host(); - ***REMOVED***no proxy dataminer (preprod) - if (cluster.getName() == null)***REMOVED*** + //no proxy dataminer (preprod) + if (cluster.getName() == null){ a.setName(ISClient.getHProxy()); out.add(a); - ***REMOVED*** + } - ***REMOVED*** if preprod, just one dm available -***REMOVED*** if (cluster.getName().equals("dataminer1-pre.d4science.org")) ***REMOVED*** -***REMOVED*** a.setName("dataminer1-pre.d4science.org"); -***REMOVED*** out.add(a); - ***REMOVED******REMOVED*** - else ***REMOVED*** - ***REMOVED*** prod - ***REMOVED***URL stockURL = new - ***REMOVED***URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); - URL stockURL = new URL("http:***REMOVED***"+ ISClient.getHProxy() +":8880/;csv"); - ***REMOVED***URL stockURL = new URL("http:***REMOVED***data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); - ***REMOVED***System.out.println(stockURL); - ***REMOVED*** dev - ***REMOVED***URL stockURL = new - ***REMOVED***URL("http:***REMOVED***data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); + // if preprod, just one dm available +// if (cluster.getName().equals("dataminer1-pre.d4science.org")) { +// a.setName("dataminer1-pre.d4science.org"); +// out.add(a); + //} + else { + // prod + //URL stockURL = new + //URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); + URL stockURL = new URL("http://"+ ISClient.getHProxy() +":8880/;csv"); + //URL stockURL = new URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); + //System.out.println(stockURL); + // dev + //URL stockURL = new + //URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); reader = new CSVReader(in, ','); String[] nextLine; - while ((nextLine = reader.readNext()) != null) ***REMOVED*** - if (nextLine[1].equals("BACKEND") || (nextLine[1].equals("FRONTEND"))) ***REMOVED*** + while ((nextLine = reader.readNext()) != null) { + if (nextLine[1].equals("BACKEND") || (nextLine[1].equals("FRONTEND"))) { continue; - ***REMOVED*** - if (nextLine[0].equals(cluster.getName())) ***REMOVED*** + } + if (nextLine[0].equals(cluster.getName())) { Host b = new Host(); b.setName(nextLine[1]); out.add(b); this.logger.info(b.getFullyQualifiedName()); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } this.logger.info(out.toString()); return out; -***REMOVED*** + } - public static void main(String[] args) throws ObjectNotFound, Exception ***REMOVED*** + public static void main(String[] args) throws ObjectNotFound, Exception { HAProxy a = new HAProxy(); - ***REMOVED***ScopeProvider.instance.set("/gcube/devNext/NextNext"); - ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - SecurityTokenProvider.instance.set("***REMOVED***"); + //ScopeProvider.instance.set("/gcube/devNext/NextNext"); + //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); CheckPermission test = new CheckPermission(); - CheckPermission.apply("***REMOVED***", "/gcube/devNext/NextNext"); + CheckPermission.apply("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548", "/gcube/devNext/NextNext"); - ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - ***REMOVED*** System.out.println(a.getHProxy()); - ***REMOVED*** System.out.println(a.MapCluster()); - ***REMOVED***System.out.println(a.listDataMinersByCluster("***REMOVED***","/gcube/devNext/NextNext")); - ***REMOVED*** System.out.println(a.listDataMinersByCluster()); + //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + // System.out.println(a.getHProxy()); + // System.out.println(a.MapCluster()); + //System.out.println(a.listDataMinersByCluster("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext")); + // System.out.println(a.listDataMinersByCluster()); - ***REMOVED*** List list = new LinkedList(); - ***REMOVED*** Dependency aa = new Dependency(); - ***REMOVED*** aa.setName("testnunzio"); - ***REMOVED*** aa.setType("cran:"); - ***REMOVED*** list.add(aa); + // List list = new LinkedList(); + // Dependency aa = new Dependency(); + // aa.setName("testnunzio"); + // aa.setType("cran:"); + // list.add(aa); - ***REMOVED*** a.checkSVNdep(); - ***REMOVED*** System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); - ***REMOVED*** System.out.println(a.listDataminersInVRE()); -***REMOVED*** -***REMOVED*** + // a.checkSVNdep(); + // System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); + // System.out.println(a.listDataminersInVRE()); + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 69a4010..a299693 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -3,19 +3,19 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -***REMOVED*** +import java.io.IOException; import java.io.StringWriter; import java.util.Arrays; import java.util.Collection; -***REMOVED*** +import java.util.List; import java.util.Vector; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.Resources; import org.gcube.common.resources.gcore.ServiceEndpoint; -***REMOVED*** -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; @@ -23,40 +23,40 @@ import org.gcube.informationsystem.publisher.ScopedPublisher; import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; -public class ISClient ***REMOVED*** +public class ISClient { private Logger logger; - public ISClient() ***REMOVED*** + public ISClient() { this.logger = LoggerFactory.getLogger(ISClient.class); -***REMOVED*** + } - public Host getDataminer(String hostname) ***REMOVED*** + public Host getDataminer(String hostname) { Host h = new Host(); boolean remote2 = true; - if (!remote2) ***REMOVED*** + if (!remote2) { h.setName("dataminer1-devnext.d4science.org"); return h; - ***REMOVED*** else ***REMOVED*** - ***REMOVED***SimpleQuery query = queryFor(ServiceEndpoint.class); - ***REMOVED***query.addCondition("$resource/Profile/RunTime/HostedOn/text() eq '" + hostname + "'"); - ***REMOVED***DiscoveryClient client = clientFor(ServiceEndpoint.class); - ***REMOVED***List resources = client.submit(query); - ***REMOVED***ServiceEndpoint a = resources.get(0); - ***REMOVED***h.setName(a.profile().runtime().hostedOn()); + } else { + //SimpleQuery query = queryFor(ServiceEndpoint.class); + //query.addCondition("$resource/Profile/RunTime/HostedOn/text() eq '" + hostname + "'"); + //DiscoveryClient client = clientFor(ServiceEndpoint.class); + //List resources = client.submit(query); + //ServiceEndpoint a = resources.get(0); + //h.setName(a.profile().runtime().hostedOn()); h.setName(hostname); - ***REMOVED*** + } return h; -***REMOVED*** + } - ***REMOVED*** return the HProxy hostname in the VRE - public static String getHProxy() ***REMOVED*** + // return the HProxy hostname in the VRE + public static String getHProxy() { Host h = new Host(); SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); @@ -64,29 +64,29 @@ public class ISClient ***REMOVED*** List resources = client.submit(query); h.setName(resources.get(0).profile().runtime().hostedOn()); return h.getName(); -***REMOVED*** + } - public Collection listDataminersInVRE() ***REMOVED*** + public Collection listDataminersInVRE() { boolean remote = false; - if (!remote) ***REMOVED*** + if (!remote) { Collection out = new Vector<>(); Host h = new Host(); - ***REMOVED*** h.setName("bb-dataminer.res.eng.it"); - ***REMOVED*** h.setName("vm101.ui.savba.sk"); + // h.setName("bb-dataminer.res.eng.it"); + // h.setName("vm101.ui.savba.sk"); h.setName("dataminer1-devnext.d4science.org"); out.add(h); return out; - ***REMOVED*** else ***REMOVED*** + } else { SimpleQuery query = queryFor(ServiceEndpoint.class); - ***REMOVED*** old version - ***REMOVED*** query.addCondition("$resource/Profile/Category/text() eq - ***REMOVED*** 'DataAnalysis'") - ***REMOVED*** .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); + // old version + // query.addCondition("$resource/Profile/Category/text() eq + // 'DataAnalysis'") + // .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); @@ -95,17 +95,17 @@ public class ISClient ***REMOVED*** List resources = client.submit(query); Collection out = new Vector<>(); - for (ServiceEndpoint r : resources) ***REMOVED*** + for (ServiceEndpoint r : resources) { Host h = new Host(); h.setName(r.profile().runtime().hostedOn()); out.add(h); - ***REMOVED*** + } return out; - ***REMOVED*** -***REMOVED*** + } + } - public void updateAlg(Algorithm algo) ***REMOVED*** + public void updateAlg(Algorithm algo) { ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); @@ -113,137 +113,137 @@ public class ISClient ***REMOVED*** query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); DiscoveryClient client = clientFor(GenericResource.class); List ds = client.submit(query); - if (ds.isEmpty()) ***REMOVED*** + if (ds.isEmpty()) { return; - ***REMOVED*** + } GenericResource a = ds.get(0); a.profile().newBody(this.getAlgoBody(algo)); - try ***REMOVED*** + try { scopedPublisher.update(a); - ***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED*** + } catch (RegistryNotFoundException e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } - private String getAlgoBody(Algorithm algo) ***REMOVED*** + private String getAlgoBody(Algorithm algo) { return "" + algo.getCategory() + "" + "\n" + "" + algo.getClazz() + "" + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" + "\n" + "" + algo.getDependencies() + ""; -***REMOVED*** + } -***REMOVED*** public void addAlgToIs(Algorithm algo) ***REMOVED*** -***REMOVED*** GenericResource a = new GenericResource(); -***REMOVED*** a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); -***REMOVED*** a.profile().newBody(this.getAlgoBody(algo)); -***REMOVED*** try ***REMOVED*** -***REMOVED*** publishScopedResource(a, Arrays.asList(new String[] ***REMOVED*** ScopeProvider.instance.get() ***REMOVED***)); -***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** +// public void addAlgToIs(Algorithm algo) { +// GenericResource a = new GenericResource(); +// a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); +// a.profile().newBody(this.getAlgoBody(algo)); +// try { +// publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() })); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } - public void addAlgToIs(Algorithm algo, String token) ***REMOVED*** + public void addAlgToIs(Algorithm algo, String token) { GenericResource a = new GenericResource(); a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); a.profile().newBody(this.getAlgoBody(algo)); - try ***REMOVED*** + try { SecurityTokenProvider.instance.set(token); - publishScopedResource(a, Arrays.asList(new String[] ***REMOVED*** SecurityTokenProvider.instance.get() ***REMOVED***)); - ***REMOVED*** catch (Exception e) ***REMOVED*** + publishScopedResource(a, Arrays.asList(new String[] { SecurityTokenProvider.instance.get() })); + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } - public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception ***REMOVED*** + public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); String id = resource.id(); - this.logger.info("Trying to remove ***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** from ***REMOVED******REMOVED***", resource.getClass().getSimpleName(), id, + this.logger.info("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id, ScopeProvider.instance.get()); - ***REMOVED*** scopedPublisher.remove(resource, scopes); + // scopedPublisher.remove(resource, scopes); advancedScopedPublisher.forceRemove(resource); - this.logger.info("***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** removed successfully", resource.getClass().getSimpleName(), id); -***REMOVED*** + this.logger.info("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id); + } public void publishScopedResource(GenericResource a, List scopes) - throws RegistryNotFoundException, Exception ***REMOVED*** + throws RegistryNotFoundException, Exception { StringWriter stringWriter = new StringWriter(); Resources.marshal(a, stringWriter); ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - try ***REMOVED*** + try { this.logger.debug(scopes.toString()); this.logger.debug(stringWriter.toString()); scopedPublisher.create(a, scopes); - ***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED*** + } catch (RegistryNotFoundException e) { this.logger.error("Registry not found",e); throw e; - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED*** public Set getAlgoFromIs() ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** Set out = new HashSet(); -***REMOVED*** SimpleQuery query = queryFor(GenericResource.class); -***REMOVED*** query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") -***REMOVED*** .setResult("$resource"); -***REMOVED*** DiscoveryClient client = clientFor(GenericResource.class); -***REMOVED*** List ds = client.submit(query); -***REMOVED*** for (GenericResource a : ds) ***REMOVED*** -***REMOVED*** out.add(this.convertAlgo(a)); -***REMOVED*** ***REMOVED*** -***REMOVED*** return out; -***REMOVED******REMOVED*** +// public Set getAlgoFromIs() { +// // TODO Auto-generated method stub +// +// Set out = new HashSet(); +// SimpleQuery query = queryFor(GenericResource.class); +// query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") +// .setResult("$resource"); +// DiscoveryClient client = clientFor(GenericResource.class); +// List ds = client.submit(query); +// for (GenericResource a : ds) { +// out.add(this.convertAlgo(a)); +// } +// return out; +// } -***REMOVED*** private Algorithm convertAlgo(GenericResource a) ***REMOVED*** -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** -***REMOVED*** ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); -***REMOVED*** out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); -***REMOVED*** out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); -***REMOVED*** out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); -***REMOVED*** out.setName(a.profile().name()); -***REMOVED*** out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); -***REMOVED*** out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); -***REMOVED*** out.setDescription(a.profile().description()); -***REMOVED*** -***REMOVED*** Set deps = new HashSet(); -***REMOVED*** for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) ***REMOVED*** -***REMOVED*** org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); -***REMOVED*** d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); -***REMOVED*** deps.add(d1); -***REMOVED*** ***REMOVED*** -***REMOVED*** out.setDependencies(deps); -***REMOVED*** return out; -***REMOVED******REMOVED*** +// private Algorithm convertAlgo(GenericResource a) { +// Algorithm out = new Algorithm(); +// +// // out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); +// out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); +// out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); +// out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); +// out.setName(a.profile().name()); +// out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); +// out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); +// out.setDescription(a.profile().description()); +// +// Set deps = new HashSet(); +// for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { +// org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); +// d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); +// deps.add(d1); +// } +// out.setDependencies(deps); +// return out; +// } - public static void main(String[] args) throws IOException, SVNException ***REMOVED*** + public static void main(String[] args) throws IOException, SVNException { ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - ***REMOVED*** System.out.println(a.getHProxy()); - ***REMOVED*** System.out.println(a.MapCluster()); - ***REMOVED*** System.out.println(a.listDataMinersByCluster()); - ***REMOVED*** System.out.println(a.listDataMinersByCluster()); + // System.out.println(a.getHProxy()); + // System.out.println(a.MapCluster()); + // System.out.println(a.listDataMinersByCluster()); + // System.out.println(a.listDataMinersByCluster()); - ***REMOVED*** List list = new LinkedList(); - ***REMOVED*** Dependency aa = new Dependency(); - ***REMOVED*** aa.setName("testnunzio"); - ***REMOVED*** aa.setType("cran:"); - ***REMOVED*** list.add(aa); + // List list = new LinkedList(); + // Dependency aa = new Dependency(); + // aa.setName("testnunzio"); + // aa.setType("cran:"); + // list.add(aa); - ***REMOVED*** a.checkSVNdep(); - ***REMOVED***System.out.println(a.getDataminer("dataminer1-d-d4s.d4science.org").getDomain()); - ***REMOVED*** System.out.println(a.listDataminersInVRE()); -***REMOVED*** -***REMOVED*** + // a.checkSVNdep(); + //System.out.println(a.getDataminer("dataminer1-d-d4s.d4science.org").getDomain()); + // System.out.println(a.listDataminersInVRE()); + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java index 43260f9..941c63f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java @@ -4,15 +4,15 @@ import java.util.HashMap; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ClientConfigurationCache; -public class ScopedCacheMap extends HashMap ***REMOVED*** +public class ScopedCacheMap extends HashMap { -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 1L; -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java index af9fdbb..774a33d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java @@ -6,7 +6,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.util.Date; import java.util.HashMap; import java.util.Iterator; -***REMOVED*** +import java.util.List; import java.util.Map; import org.gcube.common.resources.gcore.GenericResource; @@ -14,11 +14,11 @@ import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Config import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.AbstractConfiguration; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientConfigurationCache -***REMOVED*** +{ private Logger logger; private SVNRepository svnRepository; @@ -26,48 +26,48 @@ public class ClientConfigurationCache private long svnRepositoryTimeout; private Map configurationsTimeouts; - private final long duration = 120000; ***REMOVED***2 minutes + private final long duration = 120000; //2 minutes ClientConfigurationCache () - ***REMOVED*** + { this.logger = LoggerFactory.getLogger(ClientConfigurationCache.class); this.svnRepository = null; this.configurations = new HashMap<>(); this.svnRepositoryTimeout = 0; this.configurationsTimeouts = new HashMap<>(); -***REMOVED*** + } public AbstractConfiguration getConfiguration (CONFIGURATIONS configuration) - ***REMOVED*** + { Long time = this.configurationsTimeouts.get(configuration.toString()); long currentTime = new Date().getTime(); if (time == null || currentTime > time+this.duration) - ***REMOVED*** + { this.configurations.put(configuration.toString(), getConfiguration (configuration.getType())); this.configurationsTimeouts.put(configuration.toString(), currentTime); - ***REMOVED*** + } return this.configurations.get(configuration.toString()); -***REMOVED*** + } public SVNRepository getSVNRepository () - ***REMOVED*** + { long currentTime = new Date().getTime(); if (this.svnRepositoryTimeout == 0 || currentTime > this.svnRepositoryTimeout+this.duration) - ***REMOVED*** + { this.svnRepository = queryForRepository(); this.svnRepositoryTimeout = currentTime; - ***REMOVED*** + } return this.svnRepository; -***REMOVED*** + } private SVNRepository queryForRepository() - ***REMOVED*** + { SimpleQuery query = queryFor(GenericResource.class); query.addCondition("$resource/Profile/SecondaryType/text() eq 'DMPMConfigurator'").setResult("$resource"); DiscoveryClient client = clientFor(GenericResource.class); @@ -76,12 +76,12 @@ public class ClientConfigurationCache SVNRepository response = null; while (resourcesIterator.hasNext() && response == null) - ***REMOVED*** + { GenericResource resource = resourcesIterator.next(); String repositoryURL = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_URL).item(0).getTextContent(); if (repositoryURL != null) - ***REMOVED*** + { String repositoryPath = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PATH).item(0).getTextContent(); String repositoryUsername = null; @@ -89,7 +89,7 @@ public class ClientConfigurationCache try - ***REMOVED*** + { repositoryUsername = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_USERNAME).item(0).getTextContent(); repositoryPassword = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PASSWORD).item(0).getTextContent(); @@ -100,31 +100,31 @@ public class ClientConfigurationCache this.logger.debug("Repository username "+repositoryUsername); this.logger.debug("Repository password "+repositoryPassword); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { this.logger.debug("SVN Username and password not present"); - ***REMOVED*** + } this.logger.debug("SVN Repository URL: "+repositoryURL); this.logger.debug("SVN Repository path: "+repositoryPath); response = new SVNRepository(repositoryURL, repositoryPath,repositoryUsername, repositoryPassword); - ***REMOVED*** - ***REMOVED*** + } + } return response; -***REMOVED*** + } private AbstractConfiguration getConfiguration (AbstractConfiguration type) - ***REMOVED*** + { SimpleQuery query = queryFor(GenericResource.class); query.addCondition("$resource/Profile/SecondaryType/text() eq 'DMPMConfigurator'").setResult(type.getXMLModel()); DiscoveryClient client = clientFor(type.getClass()); List configurations = client.submit(query); if (configurations != null && !configurations.isEmpty()) return configurations.get(0); else return null; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java index 7fc719c..94357af 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; -public interface Configuration ***REMOVED*** +public interface Configuration { public String getHost(); @@ -35,4 +35,4 @@ public interface Configuration ***REMOVED*** public String getGhostAlgoDirectory(); -***REMOVED*** \ No newline at end of file +} \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java index f3353ef..363ad1d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java @@ -4,155 +4,155 @@ import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.Abstr import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.Prod; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.Stage; -public class ConfigurationImpl implements Configuration ***REMOVED*** +public class ConfigurationImpl implements Configuration { - enum CONFIGURATIONS ***REMOVED*** + enum CONFIGURATIONS { STAGE (new Stage ()), PROD (new Prod ()); private AbstractConfiguration type; - private CONFIGURATIONS(AbstractConfiguration type) ***REMOVED*** + private CONFIGURATIONS(AbstractConfiguration type) { this.type = type; - ***REMOVED*** + } -***REMOVED*** - public String toString() ***REMOVED*** + @Override + public String toString() { return this.type.getType(); - ***REMOVED*** + } public AbstractConfiguration getType () - ***REMOVED*** + { return this.type; - ***REMOVED*** + } -***REMOVED*** + } -***REMOVED*** enum REPOSITORIES ***REMOVED*** -***REMOVED*** REPO ("svn.repository"), -***REMOVED*** MAIN_ALGO ("svn.algo.main.repo"); -***REMOVED*** -***REMOVED*** private String type; -***REMOVED*** -***REMOVED*** private REPOSITORIES(String type) ***REMOVED*** -***REMOVED*** this.type = type; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** public String toString() ***REMOVED*** -***REMOVED*** return this.type; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** +// enum REPOSITORIES { +// REPO ("svn.repository"), +// MAIN_ALGO ("svn.algo.main.repo"); +// +// private String type; +// +// private REPOSITORIES(String type) { +// this.type = type; +// } +// +// @Override +// public String toString() { +// return this.type; +// } +// } private CONFIGURATIONS type; private ClientConfigurationCache cache; - public ConfigurationImpl(CONFIGURATIONS type,ClientConfigurationCache cache) ***REMOVED*** + public ConfigurationImpl(CONFIGURATIONS type,ClientConfigurationCache cache) { this.type = type; this.cache = cache; -***REMOVED*** + } -***REMOVED*** - public String getHost() ***REMOVED*** + @Override + public String getHost() { return this.cache.getConfiguration(this.type).getHost (); -***REMOVED*** + } -***REMOVED*** - public String getSVNAlgorithmsList() ***REMOVED*** + @Override + public String getSVNAlgorithmsList() { return this.cache.getConfiguration(this.type).getAlgorithmsList(); -***REMOVED*** + } -***REMOVED*** - public String getRepository() ***REMOVED*** + @Override + public String getRepository() { return this.cache.getConfiguration(this.type).getSoftwareRepo(); -***REMOVED*** + } -***REMOVED*** + @Override public String getSVNLinuxCompiledDepsList() - ***REMOVED*** + { return this.cache.getConfiguration(this.type).getDepsLinuxCompiled(); -***REMOVED*** + } -***REMOVED*** - public String getSVNPreInstalledDepsList() ***REMOVED*** + @Override + public String getSVNPreInstalledDepsList() { return this.cache.getConfiguration(this.type).getDepsPreInstalled(); -***REMOVED*** + } -***REMOVED*** + @Override public String getSVNRBDepsList() - ***REMOVED*** + { return this.cache.getConfiguration(this.type).getDepsRBlackbox(); -***REMOVED*** + } -***REMOVED*** - public String getSVNCRANDepsList() ***REMOVED*** + @Override + public String getSVNCRANDepsList() { return this.cache.getConfiguration(this.type).getDepsR(); -***REMOVED*** + } -***REMOVED*** - public String getSVNJavaDepsList() ***REMOVED*** + @Override + public String getSVNJavaDepsList() { return this.cache.getConfiguration(this.type).getDepsJava(); -***REMOVED*** + } -***REMOVED*** - public String getSVNKWDepsList() ***REMOVED*** + @Override + public String getSVNKWDepsList() { return this.cache.getConfiguration(this.type).getDepsKnimeWorkflow(); -***REMOVED*** + } -***REMOVED*** - public String getSVNKW4_1DepsList() ***REMOVED*** + @Override + public String getSVNKW4_1DepsList() { return this.cache.getConfiguration(this.type).getDepsKnimeWorkflow4_1(); -***REMOVED*** + } -***REMOVED*** - public String getSVNOctaveDepsList() ***REMOVED*** + @Override + public String getSVNOctaveDepsList() { return this.cache.getConfiguration(this.type).getDepsOctave(); -***REMOVED*** + } -***REMOVED*** - public String getSVNPythonDepsList() ***REMOVED*** + @Override + public String getSVNPythonDepsList() { return this.cache.getConfiguration(this.type).getDepsPython(); -***REMOVED*** + } -***REMOVED*** - public String getSVNPython3_6DepsList() ***REMOVED*** + @Override + public String getSVNPython3_6DepsList() { return this.cache.getConfiguration(this.type).getDepsPython3_6(); -***REMOVED*** + } -***REMOVED*** - public String getSVNWCDepsList() ***REMOVED*** + @Override + public String getSVNWCDepsList() { return this.cache.getConfiguration(this.type).getDepsWindowsCompiled(); -***REMOVED*** + } -***REMOVED*** + @Override public SVNRepository getSVNRepository() - ***REMOVED*** + { return this.cache.getSVNRepository(); -***REMOVED*** + } -***REMOVED*** - public String getGhostAlgoDirectory() ***REMOVED*** + @Override + public String getGhostAlgoDirectory() { return this.cache.getConfiguration(this.type).getGhostRepo(); -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java index 7e4d076..25f8f3f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java @@ -1,102 +1,102 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; -***REMOVED*** +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; -***REMOVED*** +import java.util.List; import java.util.Properties; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ScopedCacheMap; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.CONFIGURATIONS; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; -public class DMPMClientConfiguratorManager ***REMOVED*** -***REMOVED*** +public class DMPMClientConfiguratorManager { + private final Logger logger; private Properties defaultAdmins; static DMPMClientConfiguratorManager instance; private ScopedCacheMap cacheMap; - private DMPMClientConfiguratorManager() ***REMOVED*** + private DMPMClientConfiguratorManager() { this.cacheMap = new ScopedCacheMap(); this.logger = LoggerFactory.getLogger(DMPMClientConfiguratorManager.class); this.defaultAdmins = new Properties(); - try ***REMOVED*** + try { this.defaultAdmins.load(this.getClass().getResourceAsStream("/default.admins")); this.logger.debug("Default users successfully loaded"); - ***REMOVED*** catch (Exception e) ***REMOVED*** + } catch (Exception e) { this.logger.error("Unable to get default users", e); - ***REMOVED*** -***REMOVED*** + } + } - private ClientConfigurationCache getCurrentCache() ***REMOVED*** + private ClientConfigurationCache getCurrentCache() { String currentScope = ScopeProvider.instance.get(); this.logger.debug("Current scope = " + currentScope); this.logger.debug("Getting current configuration cache"); ClientConfigurationCache cache = this.cacheMap.get(currentScope); - if (cache == null) ***REMOVED*** + if (cache == null) { this.logger.debug("Cache not created yet, creating..."); cache = new ClientConfigurationCache(); this.cacheMap.put(currentScope, cache); - ***REMOVED*** + } return cache; -***REMOVED*** + } - public static DMPMClientConfiguratorManager getInstance() ***REMOVED*** + public static DMPMClientConfiguratorManager getInstance() { if (instance == null) instance = new DMPMClientConfiguratorManager(); return instance; -***REMOVED*** + } - public Configuration getProductionConfiguration() ***REMOVED*** + public Configuration getProductionConfiguration() { return new ConfigurationImpl(CONFIGURATIONS.PROD, getCurrentCache()); -***REMOVED*** + } - public Configuration getStagingConfiguration() ***REMOVED*** + public Configuration getStagingConfiguration() { return new ConfigurationImpl(CONFIGURATIONS.STAGE, getCurrentCache()); -***REMOVED*** + } - public List getDefaultAdmins() ***REMOVED*** + public List getDefaultAdmins() { List admins = new ArrayList(); - if (defaultAdmins == null || defaultAdmins.isEmpty()) ***REMOVED*** + if (defaultAdmins == null || defaultAdmins.isEmpty()) { admins.add("statistical.manager"); - ***REMOVED*** else ***REMOVED*** + } else { Iterator keys = this.defaultAdmins.keySet().iterator(); - while (keys.hasNext()) ***REMOVED*** + while (keys.hasNext()) { String key = (String) keys.next(); admins.add(defaultAdmins.getProperty(key)); - ***REMOVED*** + } - ***REMOVED*** + } this.logger.debug("Default admins list: " + admins); return admins; -***REMOVED*** + } - public static void main(String[] args) throws IOException, SVNException ***REMOVED*** + public static void main(String[] args) throws IOException, SVNException { DMPMClientConfiguratorManager a = new DMPMClientConfiguratorManager(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - ***REMOVED*** SecurityTokenProvider.instance.set("***REMOVED***"); + // SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); System.out.println("RESULT 1" + a.getStagingConfiguration().getSVNCRANDepsList()); System.out.println("RESULT 2" + a.getProductionConfiguration().getRepository()); System.out.println("RESULT 3" + a.getStagingConfiguration().getSVNRepository().getPath()); - ***REMOVED*** System.out.println(a.getRepo()); - ***REMOVED*** System.out.println(a.getAlgoRepo()); - ***REMOVED*** System.out.println(a.getSVNRepo()); + // System.out.println(a.getRepo()); + // System.out.println(a.getAlgoRepo()); + // System.out.println(a.getSVNRepo()); -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java index 2316218..8323706 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; public class SVNRepository -***REMOVED*** +{ static final String REPOSITORY_URL = "svn.repository", REPOSITORY_PATH = "svn.algo.main.repo", @@ -13,41 +13,41 @@ public class SVNRepository username, password; - SVNRepository(String baseUrl, String path, String username, String password) ***REMOVED*** + SVNRepository(String baseUrl, String path, String username, String password) { this.baseUrl = baseUrl; this.path = path; this.username = username; this.password = password; -***REMOVED*** + } - SVNRepository(String baseUrl, String path) ***REMOVED*** + SVNRepository(String baseUrl, String path) { this (baseUrl, path, null, null); -***REMOVED*** + } - public String getUsername() ***REMOVED*** + public String getUsername() { return username; -***REMOVED*** + } - public void setUsername(String username) ***REMOVED*** + public void setUsername(String username) { this.username = username; -***REMOVED*** + } - public String getPassword() ***REMOVED*** + public String getPassword() { return password; -***REMOVED*** + } - public void setPassword(String password) ***REMOVED*** + public void setPassword(String password) { this.password = password; -***REMOVED*** + } - public String getBaseUrl() ***REMOVED*** + public String getBaseUrl() { return baseUrl; -***REMOVED*** + } - public String getPath() ***REMOVED*** + public String getPath() { return path; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java index 55db9e3..dac6a94 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java @@ -4,7 +4,7 @@ import javax.xml.bind.annotation.XmlElement; public abstract class AbstractConfiguration -***REMOVED*** +{ private String host; private String algorithmsList; @@ -24,154 +24,154 @@ public abstract class AbstractConfiguration @XmlElement (name="host") - public String getHost() ***REMOVED*** + public String getHost() { return host; -***REMOVED*** - public void setHost(String host) ***REMOVED*** + } + public void setHost(String host) { this.host = host; -***REMOVED*** + } @XmlElement (name="algorithms-list") - public String getAlgorithmsList() ***REMOVED*** + public String getAlgorithmsList() { return algorithmsList; -***REMOVED*** - public void setAlgorithmsList(String algorithmsList) ***REMOVED*** + } + public void setAlgorithmsList(String algorithmsList) { this.algorithmsList = algorithmsList; -***REMOVED*** + } @XmlElement (name="software-repo") - public String getSoftwareRepo() ***REMOVED*** + public String getSoftwareRepo() { return softwareRepo; -***REMOVED*** - public void setSoftwareRepo(String softwareRepo) ***REMOVED*** + } + public void setSoftwareRepo(String softwareRepo) { this.softwareRepo = softwareRepo; -***REMOVED*** + } @XmlElement (name="ghost-repo") - public String getGhostRepo() ***REMOVED*** + public String getGhostRepo() { return ghostRepo; -***REMOVED*** + } - public void setGhostRepo(String ghostRepo) ***REMOVED*** + public void setGhostRepo(String ghostRepo) { this.ghostRepo = ghostRepo; -***REMOVED*** + } @XmlElement (name="deps-linux-compiled") - public String getDepsLinuxCompiled() ***REMOVED*** + public String getDepsLinuxCompiled() { return depsLinuxCompiled; -***REMOVED*** + } - public void setDepsLinuxCompiled(String depsLinuxCompiled) ***REMOVED*** + public void setDepsLinuxCompiled(String depsLinuxCompiled) { this.depsLinuxCompiled = depsLinuxCompiled; -***REMOVED*** + } @XmlElement (name="deps-pre-installed") - public String getDepsPreInstalled() ***REMOVED*** + public String getDepsPreInstalled() { return depsPreInstalled; -***REMOVED*** - public void setDepsPreInstalled(String depsPreInstalled) ***REMOVED*** + } + public void setDepsPreInstalled(String depsPreInstalled) { this.depsPreInstalled = depsPreInstalled; -***REMOVED*** + } @XmlElement (name="deps-r-blackbox") - public String getDepsRBlackbox() ***REMOVED*** + public String getDepsRBlackbox() { return depsRBlackbox; -***REMOVED*** - public void setDepsRBlackbox(String depsRBlackbox) ***REMOVED*** + } + public void setDepsRBlackbox(String depsRBlackbox) { this.depsRBlackbox = depsRBlackbox; -***REMOVED*** + } @XmlElement (name="deps-r") - public String getDepsR() ***REMOVED*** + public String getDepsR() { return depsR; -***REMOVED*** - public void setDepsR(String depsR) ***REMOVED*** + } + public void setDepsR(String depsR) { this.depsR = depsR; -***REMOVED*** + } @XmlElement (name="deps-java") - public String getDepsJava() ***REMOVED*** + public String getDepsJava() { return depsJava; -***REMOVED*** - public void setDepsJava(String depsJava) ***REMOVED*** + } + public void setDepsJava(String depsJava) { this.depsJava = depsJava; -***REMOVED*** + } @XmlElement (name="deps-knime-workflow") - public String getDepsKnimeWorkflow() ***REMOVED*** + public String getDepsKnimeWorkflow() { return depsKnimeWorkflow; -***REMOVED*** - public void setDepsKnimeWorkflow(String depsKnimeWorkflow) ***REMOVED*** + } + public void setDepsKnimeWorkflow(String depsKnimeWorkflow) { this.depsKnimeWorkflow = depsKnimeWorkflow; -***REMOVED*** + } @XmlElement (name="deps-knime-workflow4_1") - public String getDepsKnimeWorkflow4_1() ***REMOVED*** + public String getDepsKnimeWorkflow4_1() { return depsKnimeWorkflow4_1; -***REMOVED*** - public void setDepsKnimeWorkflow4_1(String depsKnimeWorkflow4_1) ***REMOVED*** + } + public void setDepsKnimeWorkflow4_1(String depsKnimeWorkflow4_1) { this.depsKnimeWorkflow4_1 = depsKnimeWorkflow4_1; -***REMOVED*** + } @XmlElement (name="deps-octave") - public String getDepsOctave() ***REMOVED*** + public String getDepsOctave() { return depsOctave; -***REMOVED*** - public void setDepsOctave(String depsOctave) ***REMOVED*** + } + public void setDepsOctave(String depsOctave) { this.depsOctave = depsOctave; -***REMOVED*** + } @XmlElement (name="deps-python") - public String getDepsPython() ***REMOVED*** + public String getDepsPython() { return depsPython; -***REMOVED*** - public void setDepsPython(String depsPython) ***REMOVED*** + } + public void setDepsPython(String depsPython) { this.depsPython = depsPython; -***REMOVED*** + } @XmlElement (name="deps-python3_6") - public String getDepsPython3_6() ***REMOVED*** + public String getDepsPython3_6() { return depsPython3_6; -***REMOVED*** - public void setDepsPython3_6(String depsPython3_6) ***REMOVED*** + } + public void setDepsPython3_6(String depsPython3_6) { this.depsPython3_6 = depsPython3_6; -***REMOVED*** + } @XmlElement (name="deps-windows-compiled") - public String getDepsWindowsCompiled() ***REMOVED*** + public String getDepsWindowsCompiled() { return depsWindowsCompiled; -***REMOVED*** - public void setDepsWindowsCompiled(String depsWindowsCompiled) ***REMOVED*** + } + public void setDepsWindowsCompiled(String depsWindowsCompiled) { this.depsWindowsCompiled = depsWindowsCompiled; -***REMOVED*** + } protected String getXML (String type) - ***REMOVED*** - return "<"+type+">***REMOVED***$resource/Profile/Body/"+type+"/ghost/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/algorithms-list/text()***REMOVED***"+ - " ***REMOVED***$resource/Profile/Body/"+type+"/software.repo/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/algo.ghost.repo/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-linux-compiled/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-pre-installed/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-r-blackbox/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-r/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-java/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-knime-workflow/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-knime-workflow4_1/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-octave/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-python/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-python3_6/text()***REMOVED***"+ - "***REMOVED***$resource/Profile/Body/"+type+"/deps-windows-compiled/text()***REMOVED***"; -***REMOVED*** + { + return "<"+type+">{$resource/Profile/Body/"+type+"/ghost/text()}"+ + "{$resource/Profile/Body/"+type+"/algorithms-list/text()}"+ + " {$resource/Profile/Body/"+type+"/software.repo/text()}"+ + "{$resource/Profile/Body/"+type+"/algo.ghost.repo/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-linux-compiled/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-pre-installed/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-r-blackbox/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-r/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-java/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-knime-workflow/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-knime-workflow4_1/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-octave/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-python/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-python3_6/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-windows-compiled/text()}"; + } abstract public String getXMLModel (); abstract public String getType (); -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java index 02c003b..b01fd05 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java @@ -3,21 +3,21 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations; import javax.xml.bind.annotation.XmlRootElement; @XmlRootElement(name="prod") -public class Prod extends AbstractConfiguration ***REMOVED*** +public class Prod extends AbstractConfiguration { private final String TYPE = "prod"; -***REMOVED*** + @Override public String getXMLModel () - ***REMOVED*** + { return super.getXML(TYPE); -***REMOVED*** + } -***REMOVED*** - public String getType() ***REMOVED*** + @Override + public String getType() { return TYPE; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java index b3e311b..2b02b74 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java @@ -3,21 +3,21 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations; import javax.xml.bind.annotation.XmlRootElement; @XmlRootElement(name="stage") -public class Stage extends AbstractConfiguration ***REMOVED*** +public class Stage extends AbstractConfiguration { private final String TYPE = "stage"; -***REMOVED*** + @Override public String getXMLModel () - ***REMOVED*** + { return super.getXML(TYPE); -***REMOVED*** + } -***REMOVED*** - public String getType() ***REMOVED*** + @Override + public String getType() { return TYPE; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java index dadf22e..989cec4 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java @@ -1,33 +1,33 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; -public class Action ***REMOVED*** +public class Action { private String name; private String description; private String script; - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public String getDescription() ***REMOVED*** + public String getDescription() { return description; - ***REMOVED*** + } - public void setDescription(String description) ***REMOVED*** + public void setDescription(String description) { this.description = description; - ***REMOVED*** + } - public String getScript() ***REMOVED*** + public String getScript() { return script; - ***REMOVED*** + } - public void setScript(String script) ***REMOVED*** + public void setScript(String script) { this.script = script; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java index 976733a..5144963 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java @@ -1,22 +1,22 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; -public class AlgoResource ***REMOVED*** +public class AlgoResource { protected String id; - public AlgoResource() ***REMOVED*** -***REMOVED*** + public AlgoResource() { + } - public AlgoResource(String id) ***REMOVED*** + public AlgoResource(String id) { this.id = id; -***REMOVED*** + } - public String getId() ***REMOVED*** + public String getId() { return this.id; -***REMOVED*** + } - public void setId(String id) ***REMOVED*** + public void setId(String id) { this.id = id; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index 2218106..48e564b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -2,11 +2,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; import java.util.Collection; import java.util.HashSet; -***REMOVED*** +import java.util.Set; import java.util.Vector; -public class Algorithm ***REMOVED*** +public class Algorithm { private String username; @@ -23,166 +23,166 @@ public class Algorithm ***REMOVED*** private Collection actions; private Collection dependencies; - public Algorithm() ***REMOVED*** + public Algorithm() { this.actions = new Vector<>(); this.dependencies = new Vector<>(); - ***REMOVED*** Dependency p = new Dependency(); + // Dependency p = new Dependency(); - ***REMOVED***init with default values + //init with default values this.skipJava = "N"; this.algorithmType = "transducerers"; - ***REMOVED*** + } - public void addDependency(Dependency dep) ***REMOVED*** + public void addDependency(Dependency dep) { this.dependencies.add(dep); - ***REMOVED*** + } - public void addAction(Action action) ***REMOVED*** + public void addAction(Action action) { this.actions.add(action); - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public String getDescription() ***REMOVED*** + public String getDescription() { return description; - ***REMOVED*** + } - public void setDescription(String description) ***REMOVED*** + public void setDescription(String description) { this.description = description.replace(": ", " "); - ***REMOVED*** + } - public String getCategory() ***REMOVED*** + public String getCategory() { return category; - ***REMOVED*** + } - public void setCategory(String category) ***REMOVED*** + public void setCategory(String category) { this.category = category; - ***REMOVED*** + } - public Collection getActions() ***REMOVED*** + public Collection getActions() { return actions; - ***REMOVED*** + } - public Collection getDependencies() ***REMOVED*** + public Collection getDependencies() { return dependencies; - ***REMOVED*** + } - public Collection getOSDependencies()***REMOVED*** + public Collection getOSDependencies(){ Set deps = new HashSet<>(); - for(Dependency d: this.getDependencies())***REMOVED*** - if(d.getType().equals("os"))***REMOVED*** + for(Dependency d: this.getDependencies()){ + if(d.getType().equals("os")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } - public Collection getCranDependencies()***REMOVED*** + public Collection getCranDependencies(){ Set deps = new HashSet<>(); - for(Dependency d: this.getDependencies())***REMOVED*** - if(d.getType().equals("cran"))***REMOVED*** + for(Dependency d: this.getDependencies()){ + if(d.getType().equals("cran")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } - public Collection getGitHubDependencies()***REMOVED*** + public Collection getGitHubDependencies(){ Set deps = new HashSet<>(); - for(Dependency d: this.getDependencies())***REMOVED*** - if(d.getType().equals("github"))***REMOVED*** + for(Dependency d: this.getDependencies()){ + if(d.getType().equals("github")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } - public void setDependencies(Collection deps) ***REMOVED*** + public void setDependencies(Collection deps) { this.dependencies = deps; - ***REMOVED*** + } - public String toString() ***REMOVED*** + public String toString() { String out = "Algorithm: " + this.getName()+"\n"; out+=" Class Name: " + this.getClazz()+"\n"; out+=" Description: " + this.getDescription()+"\n"; out+=" Dependencies: " + this.getDependencies()+"\n"; return out; - ***REMOVED*** + } - public String getClazz() ***REMOVED*** + public String getClazz() { return clazz; - ***REMOVED*** + } - public void setClazz(String clazz) ***REMOVED*** + public void setClazz(String clazz) { this.clazz = clazz; - ***REMOVED*** + } - public String getPackageURL() ***REMOVED*** + public String getPackageURL() { return packageURL; - ***REMOVED*** + } - public void setPackageURL(String packageURL) ***REMOVED*** + public void setPackageURL(String packageURL) { this.packageURL = packageURL; - ***REMOVED*** + } - public String getAlgorithmType() ***REMOVED*** + public String getAlgorithmType() { return algorithmType; - ***REMOVED*** + } - public void setAlgorithmType(String algorithmType) ***REMOVED*** + public void setAlgorithmType(String algorithmType) { this.algorithmType = algorithmType; - ***REMOVED*** + } - public String getSkipJava() ***REMOVED*** + public String getSkipJava() { return skipJava; - ***REMOVED*** + } - public void setSkipJava(String skipJava) ***REMOVED*** + public void setSkipJava(String skipJava) { this.skipJava = skipJava; - ***REMOVED*** + } -public String getUsername() ***REMOVED*** +public String getUsername() { return username; -***REMOVED*** +} -public void setUsername(String username) ***REMOVED*** +public void setUsername(String username) { this.username = username; -***REMOVED*** +} -public String getFullname() ***REMOVED*** +public String getFullname() { return fullname; -***REMOVED*** +} -public void setFullname(String fullname) ***REMOVED*** +public void setFullname(String fullname) { this.fullname = fullname; -***REMOVED*** +} -public String getEmail() ***REMOVED*** +public String getEmail() { return email; -***REMOVED*** +} -public void setEmail(String email) ***REMOVED*** +public void setEmail(String email) { this.email = email; -***REMOVED*** +} -public String getLanguage() ***REMOVED*** +public String getLanguage() { return language; -***REMOVED*** +} -public void setLanguage(String language) ***REMOVED*** +public void setLanguage(String language) { this.language = language; -***REMOVED*** +} -public void setActions(Collection actions) ***REMOVED*** +public void setActions(Collection actions) { this.actions = actions; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java index 7f9c9ed..7144103 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java @@ -3,54 +3,54 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; import java.util.Collection; import java.util.Vector; -public class AlgorithmSet ***REMOVED*** +public class AlgorithmSet { private String name; private Collection algorithms; public AlgorithmSet() - ***REMOVED*** + { this.algorithms = new Vector<>(); - ***REMOVED*** + } public String getName() - ***REMOVED*** + { return name; - ***REMOVED*** + } public void setName(String name) - ***REMOVED*** + { this.name = name; - ***REMOVED*** + } public Collection getAlgorithms() - ***REMOVED*** + { return new Vector<>(algorithms); - ***REMOVED*** + } public void addAlgorithm(Algorithm algoritm) - ***REMOVED*** + { this.algorithms.add(algoritm); - ***REMOVED*** + } public Boolean hasAlgorithm(Algorithm algorithm) - ***REMOVED*** - for (Algorithm a : this.algorithms) ***REMOVED*** + { + for (Algorithm a : this.algorithms) { if (a.getName().equals(algorithm.getName())) - ***REMOVED*** + { return true; - ***REMOVED*** - ***REMOVED*** + } + } return false; - ***REMOVED*** + } public String toString() - ***REMOVED*** + { String out = "ALGOSET: " + this.name + "\n"; - for(Algorithm a:this.algorithms) ***REMOVED*** + for(Algorithm a:this.algorithms) { out+=a+"\n"; - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java index 914a688..11f5771 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; import java.util.Collection; import java.util.Vector; -public class Cluster ***REMOVED*** +public class Cluster { /** * The set of hosts belonging to the cluster. @@ -26,57 +26,57 @@ public class Cluster ***REMOVED*** private Collection algoSets; public Cluster() - ***REMOVED*** + { this.hosts = new Vector<>(); this.algoSets = new Vector<>(); - ***REMOVED*** + } public void addAlgorithmSet(AlgorithmSet set) - ***REMOVED*** + { this.algoSets.add(set); - ***REMOVED*** + } public void addHost(Host host) - ***REMOVED*** + { this.hosts.add(host); - ***REMOVED*** + } public Collection getHosts() - ***REMOVED*** + { return hosts; - ***REMOVED*** + } public String getName() - ***REMOVED*** + { return name; - ***REMOVED*** + } public void setName(String name) - ***REMOVED*** + { this.name = name; - ***REMOVED*** + } public String getDescription() - ***REMOVED*** + { return description; - ***REMOVED*** + } public void setDescription(String description) - ***REMOVED*** + { this.description = description; - ***REMOVED*** + } public Collection getAlgorithmSets() - ***REMOVED*** + { return algoSets; - ***REMOVED*** + } - public String toString() ***REMOVED*** + public String toString() { String out = "Cluster: "+this.name+"\n"; - for(Host h:this.getHosts()) ***REMOVED*** + for(Host h:this.getHosts()) { out+=" "+h+"\n"; - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java index cb7ea89..bd4a65b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java @@ -1,29 +1,29 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; -public class Dependency ***REMOVED*** +public class Dependency { private String name; private String type; public String getName() - ***REMOVED*** + { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public String getType() ***REMOVED*** + public String getType() { return type; - ***REMOVED*** + } - public void setType(String type) ***REMOVED*** + public void setType(String type) { this.type = type; - ***REMOVED*** + } - public String toString() ***REMOVED*** + public String toString() { return this.type+":"+this.name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java index a7fc3ee..49f463b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java @@ -1,15 +1,15 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; -public class Domain ***REMOVED*** +public class Domain { private String name; - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java index e150476..f37dc80 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java @@ -1,48 +1,48 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; -public class Host ***REMOVED*** +public class Host { private String name; private Domain domain; - public Host(String hostname) ***REMOVED*** + public Host(String hostname) { this.setName(hostname); - ***REMOVED*** + } - public Host() ***REMOVED*** + public Host() { -***REMOVED*** +} -public String getFullyQualifiedName() ***REMOVED*** +public String getFullyQualifiedName() { if(this.domain!=null && this.domain.getName()!=null) return this.getName()+"."+this.getDomain().getName(); else return this.getName(); - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public void setName(String name) ***REMOVED*** + public void setName(String name) { this.name = name; - ***REMOVED*** + } - public Domain getDomain() ***REMOVED*** + public Domain getDomain() { return domain; - ***REMOVED*** + } - public void setDomain(Domain domain) ***REMOVED*** + public void setDomain(Domain domain) { this.domain = domain; - ***REMOVED*** + } -***REMOVED*** public String toString() ***REMOVED*** -***REMOVED*** return this.name + "@" + this.domain; -***REMOVED*** ***REMOVED*** +// public String toString() { +// return this.name + "@" + this.domain; +// } -public String toString() ***REMOVED*** +public String toString() { return this.name; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java index c3f3894..024c2d5 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java @@ -2,14 +2,14 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; import java.util.Comparator; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -public class AlgorithmComparator implements Comparator ***REMOVED*** +public class AlgorithmComparator implements Comparator { @Override - public int compare(Algorithm a1, Algorithm a2) ***REMOVED*** + public int compare(Algorithm a1, Algorithm a2) { return a1.getName().compareTo(a2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java index 77c27c5..abbfbbd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java @@ -4,15 +4,15 @@ import java.util.Comparator; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -public class DependencyComparator implements Comparator ***REMOVED*** +public class DependencyComparator implements Comparator { @Override - public int compare(Dependency a1, Dependency a2) ***REMOVED*** + public int compare(Dependency a1, Dependency a2) { int out = a1.getType().compareTo(a2.getType()); if(out!=0) return out; return a1.getName().compareTo(a2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java index c57f9ca..1e159b8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java @@ -4,14 +4,14 @@ import java.util.Comparator; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -public class HostComparator implements Comparator ***REMOVED*** +public class HostComparator implements Comparator { @Override - public int compare(Host h1, Host h2) ***REMOVED*** + public int compare(Host h1, Host h2) { int out = h1.getDomain().getName().compareTo(h2.getDomain().getName()); if(out!=0) return out; return h1.getName().compareTo(h2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java index 6ee5ada..b6b3c27 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java @@ -2,7 +2,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.process; import java.util.StringTokenizer; -public class AddAlgorithmCommand ***REMOVED*** +public class AddAlgorithmCommand { private String command; private String name; @@ -14,7 +14,7 @@ public class AddAlgorithmCommand ***REMOVED*** private String url; private String description; - public AddAlgorithmCommand(String cmd) ***REMOVED*** + public AddAlgorithmCommand(String cmd) { StringTokenizer st = new StringTokenizer(cmd, " "); if (st.hasMoreElements()) command = st.nextToken(); @@ -38,55 +38,55 @@ public class AddAlgorithmCommand ***REMOVED*** d = d + st.nextToken() + " "; this.setDescription(d); - ***REMOVED*** + } - public void setDescription(String d) ***REMOVED*** - if(d!=null) ***REMOVED*** + public void setDescription(String d) { + if(d!=null) { d = d.trim(); - if(d.startsWith("\"") && d.endsWith("\"")) ***REMOVED*** + if(d.startsWith("\"") && d.endsWith("\"")) { d = d.substring(1, d.length()-1).trim(); - ***REMOVED*** - ***REMOVED*** + } + } this.description = d; - ***REMOVED*** + } - public String getCommand() ***REMOVED*** + public String getCommand() { return command; - ***REMOVED*** + } - public String getName() ***REMOVED*** + public String getName() { return name; - ***REMOVED*** + } - public String getCategory() ***REMOVED*** + public String getCategory() { return category; - ***REMOVED*** + } - public String getClazz() ***REMOVED*** + public String getClazz() { return clazz; - ***REMOVED*** + } - public String getVRE() ***REMOVED*** + public String getVRE() { return scope; - ***REMOVED*** + } - public String getAlgorithmType() ***REMOVED*** + public String getAlgorithmType() { return algorithmType; - ***REMOVED*** + } - public String getSkipjava() ***REMOVED*** + public String getSkipjava() { return skipJava; - ***REMOVED*** + } - public String getUrl() ***REMOVED*** + public String getUrl() { return url; - ***REMOVED*** + } - public String getDescription() ***REMOVED*** + public String getDescription() { return description; - ***REMOVED*** + } - public String toString() ***REMOVED*** + public String toString() { String out = ""; out += String.format("%-12s: %s\n", "command", command); out += String.format("%-12s: %s\n", "algo name", name); @@ -98,6 +98,6 @@ public class AddAlgorithmCommand ***REMOVED*** out += String.format("%-12s: %s\n", "url", url); out += String.format("%-12s: %s\n", "description", this.description); return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index 21ca2c2..e5e186b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -1,21 +1,21 @@ package org.gcube.dataanalysis.dataminer.poolmanager.process; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; -***REMOVED*** +import java.net.URL; import java.util.HashMap; -***REMOVED*** +import java.util.List; import java.util.Map; import java.util.Vector; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class AlgorithmPackageParser ***REMOVED*** +public class AlgorithmPackageParser { /** * The name of the file containing algorithm metadata. Expected in the root @@ -31,7 +31,7 @@ public class AlgorithmPackageParser ***REMOVED*** METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description", METADATA_CLASS_NAME = "Class Name", - ***REMOVED*** private static final String METADATA_PACKAGES = "Packages"; + // private static final String METADATA_PACKAGES = "Packages"; METADATA_KEY_VALUE_SEPARATOR = ":"; @@ -39,116 +39,116 @@ public class AlgorithmPackageParser ***REMOVED*** private Logger logger; - public AlgorithmPackageParser() ***REMOVED*** + public AlgorithmPackageParser() { this.logger = LoggerFactory.getLogger(AlgorithmPackageParser.class); - ***REMOVED*** + } - public Algorithm parsePackage(String url) throws IOException ***REMOVED*** + public Algorithm parsePackage(String url) throws IOException { String packageMetadata = this.getPackageMetadata(url); - if (packageMetadata == null) ***REMOVED*** + if (packageMetadata == null) { this.logger.warn("WARNING: No metadata found for " + url); return null; - ***REMOVED*** else ***REMOVED*** + } else { Map> parsedMetadata = this.parseMetadata(packageMetadata); Algorithm a = this.createAlgorithm(parsedMetadata); a.setPackageURL(url); return a; - ***REMOVED*** - ***REMOVED*** + } + } - private String getPackageMetadata(String url) throws IOException ***REMOVED*** + private String getPackageMetadata(String url) throws IOException { InputStream is = new URL(url).openStream(); ZipInputStream zipIs = new ZipInputStream(is); ZipEntry entry = zipIs.getNextEntry(); String out = null; - while (entry != null) ***REMOVED*** - if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) ***REMOVED*** + while (entry != null) { + if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) { out = this.getEntryContent(zipIs); break; - ***REMOVED*** + } entry = zipIs.getNextEntry(); - ***REMOVED*** + } is.close(); zipIs.close(); return out; - ***REMOVED*** + } - private String getEntryContent(ZipInputStream zipIn) throws IOException ***REMOVED*** + private String getEntryContent(ZipInputStream zipIn) throws IOException { StringBuilder s = new StringBuilder(); byte[] buffer = new byte[BUFFER_SIZE]; int read = 0; - while ((read = zipIn.read(buffer)) != -1) ***REMOVED*** + while ((read = zipIn.read(buffer)) != -1) { s.append(new String(buffer, 0, read)); - ***REMOVED*** + } return s.toString(); - ***REMOVED*** + } - private Map> parseMetadata(String metadata) ***REMOVED*** + private Map> parseMetadata(String metadata) { Map> out = new HashMap>(); String[] lines = metadata.split("\n"); String key = null; String value = null; - for (String line : lines) ***REMOVED*** - ***REMOVED*** skip empty lines - if (line.trim().isEmpty()) ***REMOVED*** + for (String line : lines) { + // skip empty lines + if (line.trim().isEmpty()) { continue; - ***REMOVED*** - ***REMOVED*** scan lines one by one, looking for key and values + } + // scan lines one by one, looking for key and values String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); - if (parts.length > 1) ***REMOVED*** - ***REMOVED*** key and value on the same line + if (parts.length > 1) { + // key and value on the same line key = parts[0].trim(); value = line.substring(parts[0].length() + 1).trim(); - ***REMOVED*** else if (parts.length == 1) ***REMOVED*** - ***REMOVED*** either a key or a value - if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) ***REMOVED*** - ***REMOVED*** key + } else if (parts.length == 1) { + // either a key or a value + if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) { + // key key = parts[0].trim(); value = null; - ***REMOVED*** else ***REMOVED*** - ***REMOVED*** value + } else { + // value value = line.trim(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** add key+value to the map - if (key != null && value != null) ***REMOVED*** + } + } + // add key+value to the map + if (key != null && value != null) { List values = out.get(key); - if (values == null) ***REMOVED*** + if (values == null) { values = new Vector<>(); out.put(key, values); - ***REMOVED*** + } values.add(value); this.logger.debug(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** private Algorithm createAlgorithm(Map> metadata) ***REMOVED*** -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); -***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); -***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); -***REMOVED*** List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); -***REMOVED*** if (dependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : dependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** +// private Algorithm createAlgorithm(Map> metadata) { +// Algorithm out = new Algorithm(); +// out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); +// out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); +// out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); +// List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); +// if (dependencies != null) { +// for (String pkg : dependencies) { +// Dependency dep = new Dependency(); +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// return out; +// } - private Algorithm createAlgorithm(Map> metadata) ***REMOVED*** + private Algorithm createAlgorithm(Map> metadata) { Algorithm out = new Algorithm(); out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); @@ -161,13 +161,13 @@ public class AlgorithmPackageParser ***REMOVED*** List dependencies = extractMultipleValues(metadata, "Package Name"); - if (dependencies != null) ***REMOVED*** - for (String pkg : dependencies) ***REMOVED*** + if (dependencies != null) { + for (String pkg : dependencies) { Dependency dep = new Dependency(); dep.setName(pkg); out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** + } + } @@ -176,119 +176,119 @@ public class AlgorithmPackageParser ***REMOVED*** -***REMOVED*** List rdependencies = extractMultipleValues(metadata, "cran"); -***REMOVED*** if (rdependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : rdependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List defdependencies = extractMultipleValues(metadata, "Packages"); -***REMOVED*** if (defdependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : defdependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** List osdependencies = extractMultipleValues(metadata, "os"); -***REMOVED*** if (osdependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : osdependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List gitdependencies = extractMultipleValues(metadata, "github"); -***REMOVED*** if (gitdependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : gitdependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("github"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List cdependencies = extractMultipleValues(metadata, "custom"); -***REMOVED*** if (cdependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : cdependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** +// List rdependencies = extractMultipleValues(metadata, "cran"); +// if (rdependencies != null) { +// for (String pkg : rdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("cran"); +// out.addDependency(dep); +// } +// } +// +// +// List defdependencies = extractMultipleValues(metadata, "Packages"); +// if (defdependencies != null) { +// for (String pkg : defdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// +// List osdependencies = extractMultipleValues(metadata, "os"); +// if (osdependencies != null) { +// for (String pkg : osdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// +// +// +// List gitdependencies = extractMultipleValues(metadata, "github"); +// if (gitdependencies != null) { +// for (String pkg : gitdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("github"); +// out.addDependency(dep); +// } +// } +// +// +// +// List cdependencies = extractMultipleValues(metadata, "custom"); +// if (cdependencies != null) { +// for (String pkg : cdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("custom"); +// out.addDependency(dep); +// } +// } -***REMOVED*** if (pkg.startsWith("r:"))***REMOVED*** -***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (pkg.startsWith("custom:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** ***REMOVED*** +// if (pkg.startsWith("r:")){ +// //String results = StringEscapeUtils.escapeJava(pkg); +// dep.setName(pkg); +// dep.setType("cran"); +// } +// if (pkg.startsWith("custom:")){ +// dep.setName(pkg); +// dep.setType("custom"); +// } +// if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ +// dep.setName(pkg); +// dep.setType("os"); +// } return out; - ***REMOVED*** + } private static String extractSingleValue(Map> metadata, - String key) ***REMOVED*** + String key) { List l = metadata.get(key); - if (l != null && l.size() == 1) ***REMOVED*** + if (l != null && l.size() == 1) { return l.get(0); - ***REMOVED*** else ***REMOVED*** + } else { return null; - ***REMOVED*** - ***REMOVED*** + } + } private static List extractMultipleValues( - Map> metadata, String key) ***REMOVED*** + Map> metadata, String key) { List l = metadata.get(key); - if (l != null) ***REMOVED*** + if (l != null) { return new Vector<>(l); - ***REMOVED*** else ***REMOVED*** + } else { return null; - ***REMOVED*** - ***REMOVED*** + } + } - public static void main(String[] args) ***REMOVED*** + public static void main(String[] args) { AlgorithmPackageParser ap = new AlgorithmPackageParser(); String txt = "Username: giancarlo.panichi\n"+ @@ -312,6 +312,6 @@ public class AlgorithmPackageParser ***REMOVED*** ap.parseMetadata(txt); -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 68c7833..653597f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -1,44 +1,44 @@ -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED***public interface PoolManager ***REMOVED*** -***REMOVED*** -***REMOVED*** String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; -***REMOVED*** -***REMOVED*** Algorithm extractAlgorithm(String url) throws IOException; -***REMOVED*** -***REMOVED*** String getLogById(String logId) throws IOException; -***REMOVED*** -***REMOVED*** void getLogId(Algorithm algo, String vre); -***REMOVED*** -***REMOVED*** String getScriptFromURL(URL logId) throws IOException; -***REMOVED*** -***REMOVED*** URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; -***REMOVED*** -***REMOVED******REMOVED*** +//package org.gcube.dataanalysis.dataminer.poolmanager.rest; +// +//import java.io.IOException; +//import java.net.MalformedURLException; +//import java.net.URL; +//import java.net.UnknownHostException; +// +//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +// +//public interface PoolManager { +// +// String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; +// +// Algorithm extractAlgorithm(String url) throws IOException; +// +// String getLogById(String logId) throws IOException; +// +// void getLogId(Algorithm algo, String vre); +// +// String getScriptFromURL(URL logId) throws IOException; +// +// URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; +// +//} -***REMOVED*** +package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Set; -***REMOVED*** -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.tmatesoft.svn.core.SVNException; public interface PoolManager -***REMOVED*** +{ String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException; String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; @@ -64,4 +64,4 @@ public interface PoolManager List updateSVN(String file, List ldep) throws SVNException, IOException; -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java new file mode 100755 index 0000000..550f4f9 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -0,0 +1,245 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.rest; + + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Set; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; + +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; + +@Path("/") +public class RestPoolManager implements PoolManager +{ + //@Context ServletContext context; + + private final Logger logger; + private DataminerPoolManager service; + + //@Context + //private ApplicationContext context = ContextProvider.get(); + + public RestPoolManager() + { + this.logger = LoggerFactory.getLogger(RestPoolManager.class); + this.service = new DataminerPoolManager(); + + } + + + @GET + @Path("/algorithm/stage") + @Produces("text/plain") + public String stageAlgorithm( + @QueryParam("algorithmPackageURL") String algorithmPackageURL, + @QueryParam("targetVRE") String targetVRE, + @QueryParam("category") String category, + @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { + this.logger.debug("Stage algorithm method called"); + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); + //String env = context.application().getInitParameter("Environment"); + return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type/*,env*/); + } + + + @GET + @Path("/algorithm/add") + @Produces("text/plain") + public String publishAlgorithm( + @QueryParam("algorithmPackageURL") String algorithmPackageURL, + //@QueryParam("targetVREToken") String targetVREToken, + @QueryParam("targetVRE") String targetVRE, + @QueryParam("category") String category, + @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { + this.logger.debug("Publish algorithm method called"); + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); + //String env = context.application().getInitParameter("Environment"); + return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type/*,env*/); + } + + /* + * /scopes/ POST // add an algorithm to all dataminers in the scope + * /hosts/ POST // add an algorithm to the given host + */ + + @GET + @Path("/log") + @Produces("text/plain") + public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { + // TODO Auto-generated method stub + this.logger.debug("Get log by id method called"); + this.logger.debug("Returning Log =" + logUrl); + return service.getLogById(logUrl); + } + + + @GET + @Path("/monitor") + @Produces("text/plain") + public String getMonitorById(@QueryParam("logUrl") String logUrl) throws IOException { + // TODO Auto-generated method stub + this.logger.debug("Get monitor by id method called"); + this.logger.debug("Returning Log =" + logUrl); + return service.getMonitorById(logUrl); + } + + + + + + + @Override + public Algorithm extractAlgorithm(String url) throws IOException { + // TODO Auto-generated method stub + return null; + } + + public static void main(String[] args) throws ObjectNotFound, Exception { + + + // System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); +// // ProxySelector.setDefault(new +// // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); +// +// ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); +// SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx"); + + ScopeProvider.instance.set("/gcube/devNext"); + SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx"); + + + +// AuthorizationEntry entry = authorizationService().get("xxx-xxx-xxx-xxx"); +// System.out.println(entry.getContext()); + + + RestPoolManager a = new RestPoolManager(); + + a.stageAlgorithm("http://data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0"); +// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxx-xxx","/gcube/devNext/NextNext"); +// // PoolManager aa = new DataminerPoolManager(); +// + + } + + + + //Production Testing + /* + stageAlgorithm(Rproto caller token,pacchetto, category) + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL + + publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre) + node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxx-xxx-xxx&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab + + getLogById(Rproto caller token, logid) + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxxx-xxxx-xxx-xxx&logUrl= + */ + + + //dev Testing + /* + stageAlgorithm(dev_caller_vre_token,pacchetto, category) + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxxx-xxxx-xxx-xxxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL + + publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token, target prod vre) + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=450bb7f9-9e38-4bde-8f4d-f3296f95deba + + getLogById(dev_caller_vre_token, logid) + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-c90929c32c27 */ + + + @Override + public void getLogId(Algorithm algo, String vre) { + // TODO Auto-generated method stub + + } + + @Override + public String getScriptFromURL(URL logId) throws IOException { + // TODO Auto-generated method stub + return null; + } + + + + @Override + public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void addAlgToIs(Algorithm algo) { + // TODO Auto-generated method stub + + } + + @Override + public Set getAlgoFromIs() { + // TODO Auto-generated method stub + return null; + } + + + @Override + public List updateSVN(String file, List ldep) throws SVNException { + // TODO Auto-generated method stub + return null; + } + + + @Override + public String addAlgorithmToHost(Algorithm algo, String host, boolean test) + throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } + + + @Override + public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) + throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } + + + @Override + public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } + + + @Override + public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) + throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } + + + + + + + + + +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index b1b8531..8179c72 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -2,18 +2,18 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import java.io.File; import java.io.FileOutputStream; -***REMOVED*** +import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.util.Collection; import java.util.UUID; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.AnsibleException; import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.UndefinedDependenciesException; @@ -24,11 +24,11 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class DMPMJob ***REMOVED*** +public abstract class DMPMJob { private Configuration configuration; private String dmpmHomeDirectory; @@ -43,7 +43,7 @@ public abstract class DMPMJob ***REMOVED*** private Logger logger; private enum STATUS - ***REMOVED*** + { PROGRESS ("IN PROGRESS"), COMPLETED ("COMPLETED"), FAILED ("FAILED"); @@ -51,13 +51,13 @@ public abstract class DMPMJob ***REMOVED*** private String status; STATUS (String status) - ***REMOVED*** + { this.status = status; - ***REMOVED*** -***REMOVED*** + } + } public DMPMJob(SVNUpdater svnUpdater,Configuration configuration,Algorithm algorithm, Cluster cluster,String vREName, - String category, String algorithm_type)***REMOVED*** + String category, String algorithm_type){ this.logger = LoggerFactory.getLogger(DMPMJob.class); this.configuration = configuration; @@ -71,82 +71,82 @@ public abstract class DMPMJob ***REMOVED*** this.dmpmHomeDirectory = new String (System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); this.id = UUID.randomUUID().toString(); - ***REMOVED***TODO: dmpm work directory should be loaded from configuration file + //TODO: dmpm work directory should be loaded from configuration file this.jobLogs = new File(this.dmpmHomeDirectory+File.separator+"jobs"); this.jobLogs.mkdirs(); -***REMOVED*** + } public String start() - ***REMOVED*** + { setStatusInformation(STATUS.PROGRESS); - new Thread(new Runnable() ***REMOVED*** - ***REMOVED*** - public void run() ***REMOVED*** - try ***REMOVED*** + new Thread(new Runnable() { + @Override + public void run() { + try { execute(); - ***REMOVED*** catch (Exception e) ***REMOVED*** + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); + } + } + }).start(); return this.id; -***REMOVED*** + } protected AnsibleWorker createWorker(Algorithm algo, Cluster dataminerCluster, boolean includeAlgorithmDependencies, - String user)***REMOVED*** + String user){ AnsibleBridge ansibleBridge = new AnsibleBridge(this.dmpmHomeDirectory); - try ***REMOVED*** + try { return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); - ***REMOVED*** catch (IOException e) ***REMOVED*** + } catch (IOException e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + return null; -***REMOVED*** + } - public void setStatusInformation(STATUS exitStatus) ***REMOVED*** + public void setStatusInformation(STATUS exitStatus) { try - ***REMOVED*** + { File statusFile = new File (this.jobLogs,this.id + "_exitStatus"); - ***REMOVED***File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus"); + //File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus"); PrintWriter writer = new PrintWriter(statusFile, "UTF-8"); writer.println(exitStatus.status); writer.close(); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { this.logger.error ("Unable to update exit status file with status "+exitStatus.status,e); - ***REMOVED*** + } -***REMOVED*** + } private void updateLogFile (File logFile, String message) - ***REMOVED*** + { try - ***REMOVED*** + { PrintWriter writer = new PrintWriter(logFile,"UTF-8"); writer.print(message); writer.close(); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { this.logger.error("Unable to log the error message: "+message,e); - ***REMOVED*** + } -***REMOVED*** + } protected abstract void execute (); private void preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws GenericException, EMailException,UndefinedDependenciesException - ***REMOVED*** + { this.logger.debug("Checking dependencies..."); Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( @@ -154,16 +154,16 @@ public abstract class DMPMJob ***REMOVED*** this.algorithm.getDependencies()); if (!undefinedDependencies.isEmpty()) - ***REMOVED*** + { this.logger.debug("Some dependencies are not defined"); throw new UndefinedDependenciesException(undefinedDependencies); - ***REMOVED*** + } -***REMOVED*** + } private String installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws DMPMException - ***REMOVED*** + { this.logger.debug("Installation process started"); methodChecker.deleteFiles(this.algorithm/*, env*/); int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.cluster, false, "root"),logFile); @@ -172,9 +172,9 @@ public abstract class DMPMJob ***REMOVED*** if (ret != 0) throw new AnsibleException(ret); else - ***REMOVED*** + { this.logger.debug("Operation completed"); - ***REMOVED***this.setStatusInformation(STATUS.PROGRESS); + //this.setStatusInformation(STATUS.PROGRESS); this.logger.debug("Checking the method..."); methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get()); methodChecker.copyAlgorithms(this.algorithm); @@ -185,20 +185,20 @@ public abstract class DMPMJob ***REMOVED*** this.algorithm.getFullname()); this.setStatusInformation(STATUS.COMPLETED); return algorithmListResult ?"":"\nWARNING: algorithm list could not be updated on SVN"; - ***REMOVED*** -***REMOVED*** + } + } protected void execute(NotificationHelper nh, CheckMethod methodChecker) - ***REMOVED*** + { SendMail sm = new SendMail(); File logFile = new File(this.jobLogs,this.id); try - ***REMOVED*** + { - try ***REMOVED*** + try { this.logger.debug("Pre installation operations"); preInstallation(sm, nh, logFile); @@ -213,8 +213,8 @@ public abstract class DMPMJob ***REMOVED*** - ***REMOVED*** catch (DMPMException dmpme) - ***REMOVED*** + } catch (DMPMException dmpme) + { this.logger.error("Operation failed: "+dmpme.getMessage()); this.logger.error("Exception: ",dmpme); this.setStatusInformation(STATUS.FAILED); @@ -222,41 +222,41 @@ public abstract class DMPMJob ***REMOVED*** this.updateLogFile(logFile, errorMessage); sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage); - ***REMOVED*** + } - ***REMOVED*** catch (EMailException eme) - ***REMOVED*** + } catch (EMailException eme) + { this.logger.error("Unable to send notification email",eme); - ***REMOVED*** + } -***REMOVED*** + } protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws GenericException - ***REMOVED*** + { try - ***REMOVED*** + { FileOutputStream fos = new FileOutputStream(logFile, true); PrintStream ps = new PrintStream(fos); -***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); -***REMOVED*** PrintWriter fos2 = new PrintWriter(m, "UTF-8"); +// File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); +// PrintWriter fos2 = new PrintWriter(m, "UTF-8"); return worker.execute(ps); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } - public String buildInfo() ***REMOVED*** + public String buildInfo() { return "\n"+ @@ -266,8 +266,8 @@ public abstract class DMPMJob ***REMOVED*** "Staging DataMiner Host: "+ this.configuration.getHost()+"\n"+ "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ "Target VRE: "+this.vREName+"\n"; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 523ee6d..5ee25a8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -2,55 +2,55 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import java.io.File; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.util.Scanner; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderStaging; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; -public class DataminerPoolManager ***REMOVED*** +public class DataminerPoolManager { private Logger logger; private SVNUpdaterStaging svnUpdaterStaging; private SVNUpdaterProduction svnUpdaterProduction; - public DataminerPoolManager() ***REMOVED*** + public DataminerPoolManager() { this.logger = LoggerFactory.getLogger(this.getClass()); - try ***REMOVED*** - ***REMOVED***TODO: read this from configuration + try { + //TODO: read this from configuration this.svnUpdaterStaging = new SVNUpdaterStaging(); this.svnUpdaterProduction = new SVNUpdaterProduction(); - ***REMOVED*** catch (SVNException e) ***REMOVED*** + } catch (SVNException e) { this.logger.error("SVN Exception",e); - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED***Algorithm algo,String targetVRE, String category, String algorithm_type/*,String env*/) throws IOException, InterruptedException - ***REMOVED*** + public String stageAlgorithm(Algorithm algo,String targetVRE, String category, String algorithm_type/*,String env*/) throws IOException, InterruptedException + { this.logger.debug("Stage algorithm"); this.logger.debug("Algo "+algo); this.logger.debug("Category "+category); this.logger.debug("Algo type "+algorithm_type); ClusterBuilder stagingClusterBuilder = new ClusterBuilderStaging(); Cluster stagingCluster = stagingClusterBuilder.getDataminerCluster(); - ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + //Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); DMPMJob job = new StagingJob(this.svnUpdaterStaging, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; -***REMOVED*** + } -***REMOVED***Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException - ***REMOVED*** + public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException + { this.logger.debug("publish algorithm"); this.logger.debug("Algo "+algo); this.logger.debug("Category "+category); @@ -60,12 +60,12 @@ public class DataminerPoolManager ***REMOVED*** DMPMJob job = new ProductionPublishingJob(this.svnUpdaterProduction, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; -***REMOVED*** + } - public String getLogById(String id) throws FileNotFoundException***REMOVED*** + public String getLogById(String id) throws FileNotFoundException{ - ***REMOVED***TODO: load dir from configuration file + //TODO: load dir from configuration file this.logger.debug("Getting log by id "+id); File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + id); @@ -75,13 +75,13 @@ public class DataminerPoolManager ***REMOVED*** this.logger.debug("Response "+response); scanner.close(); return response; -***REMOVED*** + } - public String getMonitorById(String id) throws FileNotFoundException***REMOVED*** + public String getMonitorById(String id) throws FileNotFoundException{ this.logger.debug("Getting monitor by id "+id); - ***REMOVED***TODO: load dir from configuration file + //TODO: load dir from configuration file File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + id + "_exitStatus"); Scanner scanner = new Scanner(path); @@ -89,9 +89,9 @@ public class DataminerPoolManager ***REMOVED*** this.logger.debug("Response "+response); scanner.close(); return response; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index 515ec43..071dbdb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -4,34 +4,34 @@ import java.io.FileNotFoundException; import java.io.UnsupportedEncodingException; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperProduction; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class ProductionPublishingJob extends DMPMJob ***REMOVED*** +public class ProductionPublishingJob extends DMPMJob { - ***REMOVED***private String targetVREToken; - ***REMOVED***private String env; + //private String targetVREToken; + //private String env; private Logger logger; public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, - Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** + Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),algorithm,prodCluster,targetVREName,category,algorithm_type); - this.logger = LoggerFactory.getLogger(StagingJob.class);***REMOVED*** this.jobLogs = new File( + this.logger = LoggerFactory.getLogger(StagingJob.class);// this.jobLogs = new File( -***REMOVED*** + } -***REMOVED*** - protected void execute() ***REMOVED*** + @Override + protected void execute() { this.logger.debug("Executing staging job..."); super.execute(new NotificationHelperProduction(), new CheckMethodProduction()); -***REMOVED*** + } @@ -42,4 +42,4 @@ public class ProductionPublishingJob extends DMPMJob ***REMOVED*** -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index ae76189..b7e761f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -4,15 +4,15 @@ import java.io.FileNotFoundException; import java.io.UnsupportedEncodingException; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodStaging; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperStaging; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class StagingJob extends DMPMJob ***REMOVED*** +public class StagingJob extends DMPMJob { @@ -20,18 +20,18 @@ public class StagingJob extends DMPMJob ***REMOVED*** public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, /* Cluster rProtoCluster, */ - String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** + String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),algorithm,stagingCluster,rProtoVREName,category,algorithm_type); this.logger = LoggerFactory.getLogger(StagingJob.class); -***REMOVED*** + } -***REMOVED*** - protected void execute() ***REMOVED*** + @Override + protected void execute() { this.logger.debug("Executing staging job..."); super.execute(new NotificationHelperStaging(), new CheckMethodStaging()); -***REMOVED*** + } @@ -40,4 +40,4 @@ public class StagingJob extends DMPMJob ***REMOVED*** -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java index 2de1a4d..57e43e2 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java @@ -2,24 +2,24 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException; -public class AnsibleException extends DMPMException ***REMOVED*** +public class AnsibleException extends DMPMException { -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 6772009633547404120L; private int returnCode; - public AnsibleException(int returnCode) ***REMOVED*** + public AnsibleException(int returnCode) { super ("Ansible work failed"); this.returnCode =returnCode; -***REMOVED*** + } -***REMOVED*** - public String getErrorMessage() ***REMOVED*** + @Override + public String getErrorMessage() { return "Installation failed. Return code=" + this.returnCode; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java index e6875dd..e09371b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java @@ -4,26 +4,26 @@ import java.util.Collection; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException; -public class UndefinedDependenciesException extends DMPMException ***REMOVED*** +public class UndefinedDependenciesException extends DMPMException { private String message; -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 4504593796352609191L; - public UndefinedDependenciesException(Collection undefinedDependencies) ***REMOVED*** + public UndefinedDependenciesException(Collection undefinedDependencies) { super ("Some dependencies are not defined"); this.message = "Following dependencies are not defined:\n"; - for (String n : undefinedDependencies) ***REMOVED*** + for (String n : undefinedDependencies) { message += "\n" + n +"\n"; - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED*** - public String getErrorMessage() ***REMOVED*** + @Override + public String getErrorMessage() { return this.message; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index 7207b50..61c9c03 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -1,47 +1,47 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -***REMOVED*** +import java.io.IOException; /** * Created by ggiammat on 5/9/17. */ -public class AlgorithmBuilder ***REMOVED*** +public class AlgorithmBuilder { - public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException ***REMOVED*** + public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException { return create(algorithmPackageURL, null, null, null, null, null, null, null); - ***REMOVED*** + } public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, - String category, String algorithmType, String skipJava) throws IOException, InterruptedException ***REMOVED*** + String category, String algorithmType, String skipJava) throws IOException, InterruptedException { Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); - if(category != null)***REMOVED*** + if(category != null){ algo.setCategory(category); - ***REMOVED*** - if(algorithmType != null)***REMOVED*** + } + if(algorithmType != null){ algo.setAlgorithmType(algorithmType); - ***REMOVED*** - if(skipJava != null)***REMOVED*** + } + if(skipJava != null){ algo.setSkipJava(skipJava); - ***REMOVED*** - if(skipJava != null)***REMOVED*** + } + if(skipJava != null){ algo.setSkipJava(skipJava); - ***REMOVED*** - if(name != null)***REMOVED*** + } + if(name != null){ algo.setName(name); - ***REMOVED*** - if(description != null)***REMOVED*** + } + if(description != null){ algo.setDescription(description); - ***REMOVED*** + } return algo; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 52ffbdb..069f448 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -4,21 +4,21 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileWriter; import java.io.InputStreamReader; -***REMOVED*** +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; -***REMOVED*** +import java.util.List; import java.util.Properties; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; import com.jcraft.jsch.Channel; import com.jcraft.jsch.ChannelSftp; @@ -27,7 +27,7 @@ import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import com.jcraft.jsch.SftpException; -public abstract class CheckMethod ***REMOVED*** +public abstract class CheckMethod { private Logger logger; private Configuration configuration; @@ -40,27 +40,27 @@ public abstract class CheckMethod ***REMOVED*** private final Properties sshConfig; public CheckMethod(Configuration configuration) - ***REMOVED*** + { this.logger = LoggerFactory.getLogger(CheckMethod.class); this.configuration = configuration; sshConfig = new java.util.Properties(); sshConfig.put("StrictHostKeyChecking", "no"); -***REMOVED*** + } - public void checkMethod(String machine, String token) throws AlgorithmException ***REMOVED*** - try ***REMOVED*** + public void checkMethod(String machine, String token) throws AlgorithmException { + try { this.logger.debug("Checking method for machine "+machine); this.logger.debug("By using tocken "+token); this.logger.debug("Machine: " + machine); -***REMOVED*** String getCapabilitesRequest = new String(); -***REMOVED*** String getCapabilitesResponse = new String(); +// String getCapabilitesRequest = new String(); +// String getCapabilitesResponse = new String(); this.logger.debug(" Token: " + token); - String request = "http:***REMOVED***" + machine + String request = "http://" + machine + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; String response = machine + "___" + token + ".xml"; -***REMOVED*** getCapabilitesRequest = request; -***REMOVED*** getCapabilitesResponse = response; - String baseDescriptionRequest = "http:***REMOVED***" + machine +// getCapabilitesRequest = request; +// getCapabilitesResponse = response; + String baseDescriptionRequest = "http://" + machine + "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token=" + token + "&Identifier="; URL requestURL = new URL(request); @@ -70,13 +70,13 @@ public abstract class CheckMethod ***REMOVED*** String line; boolean flag = true; this.logger.debug("Writing file"); - while (flag && (line = bufferedReader.readLine()) != null) ***REMOVED*** + while (flag && (line = bufferedReader.readLine()) != null) { this.logger.debug(line); fileWriter.write(line); fileWriter.write(System.lineSeparator()); if (line.contains("ows:Identifier")) - ***REMOVED*** + { this.logger.debug("Identifier found"); String operatorName = line.substring(line.indexOf(">") + 1); operatorName = operatorName.substring(0, operatorName.indexOf("<")); @@ -88,45 +88,45 @@ public abstract class CheckMethod ***REMOVED*** this.logger.debug("Inner line "+innerLine); boolean innerFlag = true; while (innerFlag && (innerLine = innerBufferedReader.readLine()) != null) - ***REMOVED*** + { if (innerLine.contains("ows:Abstract")) - ***REMOVED*** + { this.logger.debug("Abstract found"); String operatorDescription = innerLine.substring(innerLine.indexOf(">") + 1); operatorDescription = operatorDescription.substring(0, operatorDescription.indexOf("<")); this.logger.debug("Operator descriptor "+operatorDescription); this.logger.debug(" " + operatorDescription); innerFlag = false; - ***REMOVED*** else if (innerLine.contains("ows:ExceptionText")) - ***REMOVED*** + } else if (innerLine.contains("ows:ExceptionText")) + { this.logger.debug("Exception found"); this.logger.debug(" " + "error retrieving operator description"); innerFlag = false; flag = false; - ***REMOVED*** else - ***REMOVED*** + } else + { innerLine = innerBufferedReader.readLine(); this.logger.debug("Inner line completed "+innerLine); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } this.logger.debug("Operation successful"); fileWriter.close(); - ***REMOVED*** catch (Exception e) ***REMOVED*** + } catch (Exception e) { throw new AlgorithmException("Error "+e.getMessage(),e); - ***REMOVED*** -***REMOVED*** + } + } public void copyAlgorithms(Algorithm algo/*, String env*/) throws SVNCommitException, GenericException, AlgorithmException - ***REMOVED*** + { this.logger.debug("Looking if algo "+algo.getName()+ " exists"); File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); @@ -135,45 +135,45 @@ public abstract class CheckMethod ***REMOVED*** boolean fileExists = false; try - ***REMOVED*** + { fileExists = (this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { throw new GenericException(e); - ***REMOVED*** + } if (fileExists) - ***REMOVED*** + { try - ***REMOVED*** + { this.logger.debug("Files found"); this.copyFromDmToSVN(file/*,env*/); this.copyFromDmToSVN(file2/*,env*/); this.logger.debug("Files have been copied to SVN"); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { throw new GenericException(e); - ***REMOVED*** + } - ***REMOVED*** + } else - ***REMOVED*** + { this.logger.debug("Files not found"); this.logger.debug("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); throw new AlgorithmException("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); - ***REMOVED*** + } -***REMOVED*** + } public void deleteFiles(Algorithm a/*,String env*/) throws GenericException - ***REMOVED*** + { try - ***REMOVED*** + { Session session = generateSession(); this.logger.debug("checking existing in env: " + this.configuration.getHost()); @@ -195,29 +195,29 @@ public abstract class CheckMethod ***REMOVED*** ChannelSftp c = (ChannelSftp) channel; - if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED*** + if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){ c.rm(file.getPath()); c.rm(file2.getPath()); this.logger.debug("Both the files have been deleted"); - ***REMOVED*** + } else this.logger.debug("Files not found"); channel.disconnect(); c.disconnect(); session.disconnect(); - ***REMOVED*** catch (Exception e) - ***REMOVED*** + } catch (Exception e) + { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } - public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED*** + public boolean doesExist(String path/*, String env*/) throws Exception { Session session = generateSession(); boolean success = false; @@ -228,29 +228,29 @@ public abstract class CheckMethod ***REMOVED*** ChannelSftp c = (ChannelSftp) channel; this.logger.debug(path); - try ***REMOVED*** + try { c.lstat(path); success = true; - ***REMOVED*** catch (SftpException e) ***REMOVED*** - if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) ***REMOVED*** - ***REMOVED*** file doesn't exist + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + // file doesn't exist success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** + } + //success = true; // something else went wrong + } channel.disconnect(); c.disconnect(); session.disconnect(); this.logger.debug("Operation result "+success); return success; -***REMOVED*** + } protected abstract void copyFromDmToSVN(File a) throws SVNCommitException, Exception; - protected void copyFromDmToSVN(File algorithmsFile/*,String env*/,SVNUpdater svnUpdater) throws SVNException, SVNCommitException, JSchException, SftpException ***REMOVED*** + protected void copyFromDmToSVN(File algorithmsFile/*,String env*/,SVNUpdater svnUpdater) throws SVNException, SVNCommitException, JSchException, SftpException { this.logger.debug("Copying algorithm file from Data Miner to SVN"); String fileName = algorithmsFile.getName(); @@ -271,10 +271,10 @@ public abstract class CheckMethod ***REMOVED*** File f = new File(localFile); svnUpdater.updateAlgorithmFiles(f); f.delete(); -***REMOVED*** + } private Session generateSession () throws JSchException - ***REMOVED*** + { JSch jsch = new JSch(); jsch.setKnownHosts(KNOWN_HOSTS); jsch.addIdentity(PRIVATE_KEY); @@ -283,23 +283,23 @@ public abstract class CheckMethod ***REMOVED*** this.logger.debug("session created."); session.setConfig(this.sshConfig); return session; -***REMOVED*** + } - public static List getFiles(String a)***REMOVED*** + public static List getFiles(String a){ String[] array = a.split(","); ArrayList list = new ArrayList<>(Arrays.asList(array)); List ls = new LinkedList(); - for (String s: list)***REMOVED*** + for (String s: list){ ls.add(s.trim()); - ***REMOVED*** + } return ls; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java index e201384..3fb4bf8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java @@ -2,27 +2,27 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import static org.gcube.common.authorization.client.Constants.authorizationService; -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.library.AuthorizationEntry; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class CheckPermission ***REMOVED*** +public class CheckPermission { private static final Logger logger = LoggerFactory.getLogger(CheckPermission.class); public static boolean apply(String VREToken, String vre) throws ObjectNotFound, Exception - ***REMOVED*** + { AuthorizationEntry entry = authorizationService().get(VREToken); - if (entry.getContext().equals(vre)) ***REMOVED*** + if (entry.getContext().equals(vre)) { logger.info("Authorization OK!"); return true; - ***REMOVED*** + } logger.info("Not a valid token recognized for the VRE: "+vre); return false; -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java index 47ba533..bc1f77f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -1,26 +1,26 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -public abstract class ClusterBuilder ***REMOVED*** +public abstract class ClusterBuilder { private Configuration configuration; public ClusterBuilder (Configuration configuration) - ***REMOVED*** + { this.configuration = configuration; -***REMOVED*** + } - ***REMOVED***1. to complete - public Cluster getDataminerCluster() throws FileNotFoundException***REMOVED*** + //1. to complete + public Cluster getDataminerCluster() throws FileNotFoundException{ Cluster cluster = new Cluster(); Host h = new Host(); @@ -30,29 +30,29 @@ public abstract class ClusterBuilder ***REMOVED*** return cluster; -***REMOVED*** + } - public Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED*** + public Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{ Cluster cluster = new Cluster(); - for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) ***REMOVED*** + for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) { cluster.addHost(h); - ***REMOVED*** + } return cluster; -***REMOVED*** + } - public Cluster getRProtoCluster() throws IOException***REMOVED*** - ***REMOVED***Assumes the service is running in RPrototypingLab + public Cluster getRProtoCluster() throws IOException{ + //Assumes the service is running in RPrototypingLab String token = SecurityTokenProvider.instance.get(); String targetVRE = ScopeProvider.instance.get(); return this.getVRECluster(token, targetVRE); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index 379a22a..91c1dcb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -1,79 +1,79 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; -public abstract class NotificationHelper ***REMOVED*** +public abstract class NotificationHelper { private String subjectHeader; protected NotificationHelper (String subjectHeader) - ***REMOVED*** + { this.subjectHeader = subjectHeader; -***REMOVED*** + } -***REMOVED*** private Exception executionException; +// private Exception executionException; -***REMOVED*** private boolean isError() ***REMOVED*** -***REMOVED*** return this.executionException!=null; -***REMOVED*** ***REMOVED*** +// private boolean isError() { +// return this.executionException!=null; +// } -***REMOVED*** public void setExecutionException(Exception executionException) ***REMOVED*** -***REMOVED*** this.executionException = executionException; -***REMOVED*** ***REMOVED*** +// public void setExecutionException(Exception executionException) { +// this.executionException = executionException; +// } - public String getSuccessSubject() ***REMOVED*** + public String getSuccessSubject() { return this.subjectHeader+" is SUCCESS"; - ***REMOVED*** + } - public String getFailedSubject() ***REMOVED*** + public String getFailedSubject() { return String.format(this.subjectHeader+" is FAILED"); - ***REMOVED*** + } - public static String getSuccessBody(String info) ***REMOVED*** + public static String getSuccessBody(String info) { String message = String.format("The installation of the algorithm is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info; return message; - ***REMOVED*** + } - public static String getFailedBody(String message) ***REMOVED*** + public static String getFailedBody(String message) { String body = String.format("An error occurred while deploying your algorithm"); body+= "\n\nHere are the error details:\n\n" + message; return body; - ***REMOVED*** + } -***REMOVED*** public String getSuccessBodyRelease(String info) ***REMOVED*** -***REMOVED*** String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); -***REMOVED*** message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; -***REMOVED*** return message; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public String getFailedBodyRelease(String info) ***REMOVED*** -***REMOVED*** String message = String.format("SVN REPOSITORY UPDATE FAILED."); -***REMOVED*** message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; -***REMOVED*** return message; -***REMOVED*** ***REMOVED*** +// public String getSuccessBodyRelease(String info) { +// String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); +// message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; +// return message; +// } +// +// public String getFailedBodyRelease(String info) { +// String message = String.format("SVN REPOSITORY UPDATE FAILED."); +// message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; +// return message; +// } -***REMOVED*** public String getSubject() ***REMOVED*** -***REMOVED*** if(this.isError()) ***REMOVED*** -***REMOVED*** return this.getFailedSubject(); -***REMOVED*** ***REMOVED*** else ***REMOVED*** -***REMOVED*** return this.getSuccessSubject(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public String getBody() ***REMOVED*** -***REMOVED*** if(this.isError()) ***REMOVED*** -***REMOVED*** return this.getFailedBody(); -***REMOVED*** ***REMOVED*** else ***REMOVED*** -***REMOVED*** return this.getSuccessBody(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** +// public String getSubject() { +// if(this.isError()) { +// return this.getFailedSubject(); +// } else { +// return this.getSuccessSubject(); +// } +// } +// +// public String getBody() { +// if(this.isError()) { +// return this.getFailedBody(); +// } else { +// return this.getSuccessBody(); +// } +// } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java index 8c8e71b..924a995 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED*** +import java.io.IOException; import java.net.Authenticator; import java.net.InetSocketAddress; import java.net.PasswordAuthentication; @@ -10,13 +10,13 @@ import java.net.SocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; -***REMOVED*** +import java.util.List; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; -interface NetworkConfiguration ***REMOVED*** +interface NetworkConfiguration { public String getProxyHost(); @@ -28,118 +28,118 @@ interface NetworkConfiguration ***REMOVED*** public String getNonProxyHosts(); -***REMOVED*** +} -class FileBasedProxyConfiguration implements NetworkConfiguration ***REMOVED*** +class FileBasedProxyConfiguration implements NetworkConfiguration { private static PropertiesConfiguration configuration; - public FileBasedProxyConfiguration(String path) ***REMOVED*** - try ***REMOVED*** - ***REMOVED*** load the configuration + public FileBasedProxyConfiguration(String path) { + try { + // load the configuration configuration = new PropertiesConfiguration(path); - ***REMOVED*** set the reloading strategy to enable hot-configuration + // set the reloading strategy to enable hot-configuration FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); configuration.setReloadingStrategy(fcrs); - ***REMOVED*** catch (ConfigurationException e) ***REMOVED*** + } catch (ConfigurationException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** + } + } @Override - public String getProxyHost() ***REMOVED*** + public String getProxyHost() { return configuration.getString("proxyHost"); - ***REMOVED*** + } @Override - public String getProxyPort() ***REMOVED*** + public String getProxyPort() { return configuration.getString("proxyPort"); - ***REMOVED*** + } @Override - public String getProxyUser() ***REMOVED*** + public String getProxyUser() { return configuration.getString("proxyUser"); - ***REMOVED*** + } @Override - public String getProxyPassword() ***REMOVED*** + public String getProxyPassword() { return configuration.getString("proxyPassword"); - ***REMOVED*** + } @Override - public String getNonProxyHosts() ***REMOVED*** + public String getNonProxyHosts() { return configuration.getString("nonProxyHosts"); - ***REMOVED*** + } -***REMOVED*** +} -public class PropertiesBasedProxySelector extends ProxySelector ***REMOVED*** +public class PropertiesBasedProxySelector extends ProxySelector { List proxies = null; List nonProxyHosts = null; - public PropertiesBasedProxySelector(String proxySettingsPath) ***REMOVED*** + public PropertiesBasedProxySelector(String proxySettingsPath) { this(new FileBasedProxyConfiguration(proxySettingsPath)); - ***REMOVED*** + } - public PropertiesBasedProxySelector(NetworkConfiguration config) ***REMOVED*** - if (config == null || config.getProxyHost() == null) ***REMOVED*** + public PropertiesBasedProxySelector(NetworkConfiguration config) { + if (config == null || config.getProxyHost() == null) { this.proxies = null; return; - ***REMOVED*** + } String host = config.getProxyHost(); int port = 80; - if (config.getProxyPort() != null) ***REMOVED*** + if (config.getProxyPort() != null) { port = Integer.valueOf(config.getProxyPort()); - ***REMOVED*** + } - if (config.getNonProxyHosts() != null) ***REMOVED*** + if (config.getNonProxyHosts() != null) { this.nonProxyHosts = Arrays .asList(config.getNonProxyHosts().split("\\|")); - ***REMOVED*** + } this.proxies = new ArrayList(); this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, port))); - if (config.getProxyUser() != null) ***REMOVED*** + if (config.getProxyUser() != null) { final String username = config.getProxyUser(); final String password = config.getProxyPassword(); - Authenticator.setDefault(new Authenticator() ***REMOVED*** + Authenticator.setDefault(new Authenticator() { @Override - protected PasswordAuthentication getPasswordAuthentication() ***REMOVED*** + protected PasswordAuthentication getPasswordAuthentication() { return new PasswordAuthentication(username, password.toCharArray()); - ***REMOVED*** - ***REMOVED***); + } + }); - ***REMOVED*** - ***REMOVED*** + } + } @Override - public List select(URI uri) ***REMOVED*** - if (this.nonProxyHosts == null) ***REMOVED*** + public List select(URI uri) { + if (this.nonProxyHosts == null) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** else ***REMOVED*** - for (String entry : this.nonProxyHosts) ***REMOVED*** + } else { + for (String entry : this.nonProxyHosts) { entry = entry.trim(); - if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) ***REMOVED*** + if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - if (uri.getHost().equals(entry)) ***REMOVED*** + } + if (uri.getHost().equals(entry)) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - ***REMOVED*** + } + } return this.proxies; - ***REMOVED*** - ***REMOVED*** + } + } @Override - public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) ***REMOVED*** + public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) { - ***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java index a9a7d08..1bfa46c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java @@ -1,56 +1,56 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.io.SVNRepository; import org.tmatesoft.svn.core.io.SVNRepositoryFactory; import org.tmatesoft.svn.core.wc.SVNWCUtil; -public class SVNRepositoryManager ***REMOVED*** +public class SVNRepositoryManager { private SVNRepository svnRepository; private static SVNRepositoryManager instance; private Logger logger; private SVNRepositoryManager (Configuration configuration) throws SVNException - ***REMOVED*** + { this.logger = LoggerFactory.getLogger(SVNRepositoryManager.class); org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.SVNRepository repository = configuration.getSVNRepository(); this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(repository.getBaseUrl())); ISVNAuthenticationManager authManager = null; if (repository.getUsername() == null) - ***REMOVED*** + { this.logger.debug("Using SVN default credentials"); authManager = SVNWCUtil.createDefaultAuthenticationManager(); - ***REMOVED*** + } else - ***REMOVED*** + { this.logger.debug("Using IS credentials"); authManager = SVNWCUtil.createDefaultAuthenticationManager(repository.getUsername(),repository.getPassword()); - ***REMOVED*** + } this.svnRepository.setAuthenticationManager(authManager); -***REMOVED*** + } public static SVNRepositoryManager getInstance (Configuration configuration) throws SVNException - ***REMOVED*** + { if (instance == null) instance = new SVNRepositoryManager(configuration); return instance; -***REMOVED*** + } - public SVNRepository getSvnRepository() ***REMOVED*** + public SVNRepository getSvnRepository() { return svnRepository; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 480a518..1d55682 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -5,7 +5,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -17,20 +17,20 @@ import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.LinkedList; -***REMOVED*** -***REMOVED*** +import java.util.List; +import java.util.Set; import java.util.TimeZone; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNCommitInfo; import org.tmatesoft.svn.core.SVNErrorMessage; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNNodeKind; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; @@ -41,41 +41,41 @@ import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; /** * Created by ggiammat on 5/9/17. */ -public abstract class SVNUpdater ***REMOVED*** +public abstract class SVNUpdater { private SVNRepository svnRepository; private Configuration configuration; private Logger logger; - public SVNUpdater(Configuration configuration) throws SVNException ***REMOVED*** + public SVNUpdater(Configuration configuration) throws SVNException { this.configuration = configuration; this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); this.logger = LoggerFactory.getLogger(SVNUpdater.class); -***REMOVED*** + } -***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED*** -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); -***REMOVED******REMOVED*** +// public void updateRPRotoDeps(Algorithm algorithm) { +// this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); +// this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); +// this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); +// } public String getDependencyFile(String language/*, String env*/) - ***REMOVED*** + { return getDependencyFile(this.configuration,language); -***REMOVED*** + } private String getDependencyFile (Configuration configuration, String language) - ***REMOVED*** + { this.logger.debug("Getting dependency file for language "+language); switch (language) - ***REMOVED*** + { case "R": return configuration.getSVNCRANDepsList(); case "R-blackbox": @@ -99,22 +99,22 @@ public abstract class SVNUpdater ***REMOVED*** case "Windows-compiled": return configuration.getSVNWCDepsList(); default: - ***REMOVED*** - ***REMOVED*** -***REMOVED*** + return null; + } + } public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) - ***REMOVED*** + { return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); -***REMOVED*** + } - public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException***REMOVED*** - ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); + public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException{ + //this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); this.updateAlgorithmList(this.configuration.getRepository(), a); -***REMOVED*** + } @@ -122,33 +122,33 @@ public abstract class SVNUpdater ***REMOVED*** private void updateAlgorithmList(String svnMainAlgoRepo, File algorithmsFile) throws SVNException, SVNCommitException - ***REMOVED*** + { this.logger.debug("Adding .jar file: " + algorithmsFile + " to repository " + svnMainAlgoRepo); try - ***REMOVED*** + { if (fileExists(svnMainAlgoRepo+File.separator+algorithmsFile.getName(), -1)) - ***REMOVED*** + { this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName()); - ***REMOVED*** + } else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName()); - ***REMOVED*** + } catch (FileNotFoundException e) - ***REMOVED*** + { this.logger.error("Temporary algorithm file not found: this exception should not happen",e); - ***REMOVED*** + } finally - ***REMOVED*** + { this.svnRepository.closeSession(); - ***REMOVED*** -***REMOVED*** + } + } public void putFile(FileInputStream fileInputSreeam, String destinationFolder, String fileName) throws SVNException, SVNCommitException - ***REMOVED*** + { this.logger.debug("Putting new file on the SVN repository"); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); commitEditor.openRoot(-1); @@ -165,27 +165,27 @@ public abstract class SVNUpdater ***REMOVED*** SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) - ***REMOVED*** + { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName); - ***REMOVED*** + } this.logger.debug("Operation completed"); -***REMOVED*** + } - public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException ***REMOVED*** + public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException { this.logger.debug("Updating existing file on the SVN repository"); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Updating algorithm", null); commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); String filePath = destinationFolder + "/" + fileName; - ***REMOVED*** if (fileExists(filePath, -1)) ***REMOVED*** ***REMOVED*** updating existing file + // if (fileExists(filePath, -1)) { // updating existing file commitEditor.openFile(filePath, -1); - ***REMOVED******REMOVED*** else ***REMOVED*** ***REMOVED*** creating new file - ***REMOVED***commitEditor.addFile(filePath, null, -1); - ***REMOVED******REMOVED*** + //} else { // creating new file + //commitEditor.addFile(filePath, null, -1); + //} commitEditor.applyTextDelta(filePath, null); SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true); @@ -196,41 +196,41 @@ public abstract class SVNUpdater ***REMOVED*** SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) - ***REMOVED*** + { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName+" to be updated"); - ***REMOVED*** + } this.logger.debug("Operation completed"); -***REMOVED*** + } - public boolean fileExists(String path, long revision) throws SVNException ***REMOVED*** + public boolean fileExists(String path, long revision) throws SVNException { SVNNodeKind kind = svnRepository.checkPath(path, revision); - if (kind == SVNNodeKind.FILE) ***REMOVED*** + if (kind == SVNNodeKind.FILE) { return true; - ***REMOVED*** + } return false; - ***REMOVED*** + } -***REMOVED*** public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); -***REMOVED******REMOVED*** +// public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException +// { +// +// return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); +// } private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) - ***REMOVED*** + { boolean response = false; - try ***REMOVED*** + try { this.logger.debug("Updating algorithm list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); @@ -238,18 +238,18 @@ public abstract class SVNUpdater ***REMOVED*** List newContent = new LinkedList<>(Arrays.asList(lines)); - ***REMOVED*** check if the algorithm is already in the list (match the class name) and delete the content - for (String l : lines) ***REMOVED*** - if (l.contains(algorithm.getClazz())) ***REMOVED*** + // check if the algorithm is already in the list (match the class name) and delete the content + for (String l : lines) { + if (l.contains(algorithm.getClazz())) { newContent.remove(l); - ***REMOVED***System.out.println("Not updating algorithm list beacuse already present"); - ***REMOVED***return; - ***REMOVED*** - ***REMOVED*** + //System.out.println("Not updating algorithm list beacuse already present"); + //return; + } + } - ***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it + // the algorithm is not in the list or must be overwritten cause some modification. Add it newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type/*, env*/)); - ***REMOVED*** Collections.sort(newContent); + // Collections.sort(newContent); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); @@ -261,10 +261,10 @@ public abstract class SVNUpdater ***REMOVED*** ByteArrayOutputStream baos = new ByteArrayOutputStream(); - for (String line : newContent) ***REMOVED*** + for (String line : newContent) { baos.write(line.getBytes()); baos.write("\n".getBytes()); - ***REMOVED*** + } byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); @@ -276,32 +276,32 @@ public abstract class SVNUpdater ***REMOVED*** SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) - ***REMOVED*** + { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); response = false; - ***REMOVED*** + } else response = true; - ***REMOVED*** + } catch (Exception ex) - ***REMOVED*** + { this.logger.error("Unable to commit algorithm list",ex); response = false; - ***REMOVED*** + } finally - ***REMOVED*** + { svnRepository.closeSession(); - ***REMOVED*** + } return response; -***REMOVED*** + } - public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException ***REMOVED*** - ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); - ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException { + //Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + //long unixTime = System.currentTimeMillis() / 1000L; StringBuffer sb = new StringBuffer("| "); sb.append(algorithm.getName() + " | "); @@ -317,24 +317,24 @@ public abstract class SVNUpdater ***REMOVED*** this.logger.info("Algo details: "+sb.toString()); return sb.toString(); -***REMOVED*** + } public Collection getUndefinedDependencies(String file, Collection deps) throws GenericException - ***REMOVED*** + { try - ***REMOVED*** -***REMOVED*** SendMail sm = new SendMail(); -***REMOVED*** NotificationHelper nh = new NotificationHelper(); + { +// SendMail sm = new SendMail(); +// NotificationHelper nh = new NotificationHelper(); List undefined = new LinkedList(); - ***REMOVED***to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string - ***REMOVED***just to uncomment the following lines + //to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string + //just to uncomment the following lines - if(file.isEmpty())***REMOVED*** + if(file.isEmpty()){ return undefined; - ***REMOVED*** + } this.logger.debug("Checking dependencies list: " + file); @@ -342,128 +342,128 @@ public abstract class SVNUpdater ***REMOVED*** List validDependencies = new LinkedList(); - for (String singlefile: CheckMethod.getFiles(file))***REMOVED*** + for (String singlefile: CheckMethod.getFiles(file)){ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - for(String l: byteArrayOutputStream.toString().split("\\r?\\n"))***REMOVED*** + for(String l: byteArrayOutputStream.toString().split("\\r?\\n")){ validDependencies.add(l.trim()); - ***REMOVED******REMOVED*** + }} this.logger.debug("Valid dependencies are: "+validDependencies); - for(Dependency d: deps)***REMOVED*** + for(Dependency d: deps){ String depName = d.getName(); - if(!validDependencies.contains(depName))***REMOVED*** + if(!validDependencies.contains(depName)){ undefined.add(depName); - ***REMOVED*** - ***REMOVED*** + } + } return undefined; -***REMOVED*** -***REMOVED*** -***REMOVED*** for (String a : lines) ***REMOVED*** -***REMOVED*** for (String b : ldep) ***REMOVED*** -***REMOVED*** if (b.equals(a)) ***REMOVED*** -***REMOVED*** System.out.println("The following dependency is correctly written: " + b); -***REMOVED*** ***REMOVED*** else - ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** boolean check = false; -***REMOVED*** try ***REMOVED*** -***REMOVED*** System.out.println("Checking dependencies list: " + file); -***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); -***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); -***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED*** -***REMOVED*** ***REMOVED*** if(deps.isEmpty())***REMOVED*** -***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); -***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this -***REMOVED*** ***REMOVED*** algorithm"); -***REMOVED*** ***REMOVED*** throw e; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** ***REMOVED*** - ***REMOVED*** -***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED*** -***REMOVED*** List ldep = new LinkedList<>(); -***REMOVED*** for (Dependency d : deps) ***REMOVED*** -***REMOVED*** ldep.add(d.getName()); -***REMOVED*** ***REMOVED*** -***REMOVED*** for (String a : lines) ***REMOVED*** -***REMOVED*** for (String b : ldep) ***REMOVED*** -***REMOVED*** if (b.equals(a)) ***REMOVED*** -***REMOVED*** System.out.println("The following dependency is correctly written: " + b); -***REMOVED*** check = true; -***REMOVED*** ***REMOVED*** else -***REMOVED*** check = false; - ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** catch (Exception a) ***REMOVED*** -***REMOVED*** a.getMessage(); -***REMOVED*** ***REMOVED*** - ***REMOVED*** -***REMOVED*** return check; - ***REMOVED*** catch (SVNException e) - ***REMOVED*** +// +// +// for (String a : lines) { +// for (String b : ldep) { +// if (b.equals(a)) { +// System.out.println("The following dependency is correctly written: " + b); +// } else + // +// } +// } +// +// boolean check = false; +// try { +// System.out.println("Checking dependencies list: " + file); +// final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +// svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); +// String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + // +// // if(deps.isEmpty()){ +// // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); +// // Exception e = new Exception("No dependency specified for this +// // algorithm"); +// // throw e; +// // +// // } + // +// // else if (!deps.isEmpty()) { +// List ldep = new LinkedList<>(); +// for (Dependency d : deps) { +// ldep.add(d.getName()); +// } +// for (String a : lines) { +// for (String b : ldep) { +// if (b.equals(a)) { +// System.out.println("The following dependency is correctly written: " + b); +// check = true; +// } else +// check = false; + // +// } +// } +// // } +// } catch (Exception a) { +// a.getMessage(); +// } + // +// return check; + } catch (SVNException e) + { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } - public boolean checkIfAvaialable(String file, Collection deps) throws SVNException ***REMOVED*** - ***REMOVED***SendMail sm = new SendMail(); - ***REMOVED***NotificationHelper nh = new NotificationHelper(); + public boolean checkIfAvaialable(String file, Collection deps) throws SVNException { + //SendMail sm = new SendMail(); + //NotificationHelper nh = new NotificationHelper(); boolean check = false; - try ***REMOVED*** + try { this.logger.info("Checking dependencies list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED*** if(deps.isEmpty())***REMOVED*** - ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); - ***REMOVED*** Exception e = new Exception("No dependency specified for this - ***REMOVED*** algorithm"); - ***REMOVED*** throw e; - ***REMOVED*** - ***REMOVED*** ***REMOVED*** + // if(deps.isEmpty()){ + // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); + // Exception e = new Exception("No dependency specified for this + // algorithm"); + // throw e; + // + // } - ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED*** + // else if (!deps.isEmpty()) { List ldep = new LinkedList<>(); - for (Dependency d : deps) ***REMOVED*** + for (Dependency d : deps) { ldep.add(d.getName()); - ***REMOVED*** - for (String a : lines) ***REMOVED*** - for (String b : ldep) ***REMOVED*** - if (b.equals(a)) ***REMOVED*** + } + for (String a : lines) { + for (String b : ldep) { + if (b.equals(a)) { System.out.println("The following dependency is correctly written: " + b); check = true; - ***REMOVED*** else + } else check = false; - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** catch (Exception a) - ***REMOVED*** + } + } + // } + } catch (Exception a) + { this.logger.error(a.getMessage(),a); - ***REMOVED*** + } return check; -***REMOVED*** + } - public void updateSVN(String file, Collection deps) ***REMOVED*** - try ***REMOVED*** + public void updateSVN(String file, Collection deps) { + try { this.logger.info("Updating dependencies list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -471,9 +471,9 @@ public abstract class SVNUpdater ***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); List ldep = new LinkedList<>(); - for (Dependency d : deps) ***REMOVED*** + for (Dependency d : deps) { ldep.add(d.getName()); - ***REMOVED*** + } List aa = this.checkMatch(lines, ldep); Collections.sort(aa); @@ -487,11 +487,11 @@ public abstract class SVNUpdater ***REMOVED*** ByteArrayOutputStream baos = new ByteArrayOutputStream(); - for (String line : aa) ***REMOVED*** + for (String line : aa) { baos.write(line.getBytes()); baos.write("\n".getBytes()); - ***REMOVED*** + } byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); @@ -503,40 +503,40 @@ public abstract class SVNUpdater ***REMOVED*** commitEditor.closeFile(file, checksum); commitEditor.closeEdit(); - ***REMOVED*** catch (Exception ex) ***REMOVED*** + } catch (Exception ex) { ex.printStackTrace(); - ***REMOVED*** + } - finally ***REMOVED*** + finally { svnRepository.closeSession(); - ***REMOVED*** -***REMOVED*** + } + } - public static String md5(byte[] contents) ***REMOVED*** + public static String md5(byte[] contents) { final byte[] tmp = new byte[1024]; final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); - try ***REMOVED*** - while (checksumStream.read(tmp) > 0) ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + try { + while (checksumStream.read(tmp) > 0) { + // + } return checksumStream.getDigest(); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED*** never happens + } catch (IOException e) { + // never happens e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** finally ***REMOVED*** + return null; + } finally { SVNFileUtil.closeFile(checksumStream); - ***REMOVED*** -***REMOVED*** - public List checkMatch(String[] lines, List ls) ***REMOVED*** + } + } + public List checkMatch(String[] lines, List ls) { Set ss = new HashSet(ls); ss.addAll(Arrays.asList(lines)); return new ArrayList<>(ss); -***REMOVED*** + } - public String getTimeZone() throws ParseException***REMOVED*** + public String getTimeZone() throws ParseException{ Calendar cal = Calendar.getInstance(); cal.getTime(); DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy"); @@ -545,21 +545,21 @@ public abstract class SVNUpdater ***REMOVED*** formatter.setTimeZone(central); this.logger.info(formatter.format(fromDate)); return formatter.format(fromDate); -***REMOVED*** + } - public static void main(String[] args) throws SVNException, ParseException ***REMOVED*** + public static void main(String[] args) throws SVNException, ParseException { - ***REMOVED*** SVNUpdater c = new SVNUpdater(); + // SVNUpdater c = new SVNUpdater(); - ***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro"); - ***REMOVED***File b = new File ("/home/ngalante/Desktop/testB"); - ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; - ***REMOVED***System.out.println(unixTime); - ***REMOVED***c.updateAlgorithmFiles(a); - ***REMOVED***c.updateAlgorithmFiles(b); - ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + //File a = new File("/home/ngalante/Desktop/testCiro"); + //File b = new File ("/home/ngalante/Desktop/testB"); + //long unixTime = System.currentTimeMillis() / 1000L; + //System.out.println(unixTime); + //c.updateAlgorithmFiles(a); + //c.updateAlgorithmFiles(b); + //Timestamp timestamp = new Timestamp(System.currentTimeMillis()); Calendar cal = Calendar.getInstance(); cal.getTime(); @@ -571,9 +571,9 @@ public abstract class SVNUpdater ***REMOVED*** -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 999fe9c..8aeba45 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -5,23 +5,23 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.BufferedReader; -***REMOVED*** +import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -***REMOVED*** -***REMOVED*** +import java.net.MalformedURLException; +import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.Iterator; -***REMOVED*** +import java.util.List; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.library.AuthorizationEntry; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.encryption.encrypter.StringEncrypter; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint; @@ -35,10 +35,10 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class SendMail ***REMOVED*** +public class SendMail { private static final Logger logger = LoggerFactory.getLogger(SendMail.class); @@ -52,11 +52,11 @@ public class SendMail ***REMOVED*** private String socialServiceAddress; private String applicationToken; - public SendMail() ***REMOVED*** + public SendMail() { -***REMOVED*** + } - public void sendNotification(String subject, String body) throws EMailException ***REMOVED*** + public void sendNotification(String subject, String body) throws EMailException { logger.debug("SendNotification"); logger.debug("Notification Subject: " + subject); logger.debug("Notification Body: " + body); @@ -69,63 +69,63 @@ public class SendMail ***REMOVED*** sendPostRequest(postBody); -***REMOVED*** + } - private String createPostBody(String subject, String body) throws EMailException ***REMOVED*** - try ***REMOVED*** + private String createPostBody(String subject, String body) throws EMailException { + try { List recipientsList = getRecipients(); - if (recipientsList == null || recipientsList.isEmpty()) ***REMOVED*** + if (recipientsList == null || recipientsList.isEmpty()) { logger.error("Invalid recipient list: " + recipientsList); throw new EMailException("Unable to send email notification. Invalid recipient list:" + recipientsList); - ***REMOVED*** + } - ***REMOVED*** ***REMOVED***"subject": "subject-content", "body": "body-content", - ***REMOVED*** "recipients":[***REMOVED***"id":"userid"***REMOVED***]***REMOVED*** + // {"subject": "subject-content", "body": "body-content", + // "recipients":[{"id":"userid"}]} JSONObject data = new JSONObject(); data.put("subject", subject); data.put("body", body); JSONArray recipients = new JSONArray(); - for (String recipient : recipientsList) ***REMOVED*** + for (String recipient : recipientsList) { JSONObject d = new JSONObject(); d.put("id", recipient); recipients.put(d); - ***REMOVED*** + } data.put("recipients", recipients); logger.debug("Post Body: " + data); return data.toString(); - ***REMOVED*** catch (EMailException e) ***REMOVED*** + } catch (EMailException e) { throw e; - ***REMOVED*** catch (Throwable e) ***REMOVED*** + } catch (Throwable e) { logger.error("Error creating the notification body: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } - private String retrieveApplicationToken() throws EMailException ***REMOVED*** - try ***REMOVED*** + private String retrieveApplicationToken() throws EMailException { + try { logger.info("Retrieve Application Token"); SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition(DMPOOLMANAGER_SERVICE_QUERY_CONDITION); DiscoveryClient client = clientFor(ServiceEndpoint.class); List resources = client.submit(query); - if (resources.isEmpty()) ***REMOVED*** + if (resources.isEmpty()) { logger.error("No services resource found on IS!"); - ***REMOVED*** else ***REMOVED*** + } else { logger.debug("Retrieved: " + resources.get(0)); - ***REMOVED*** + } Group accessPoints = resources.get(0).profile().accessPoints(); - if (!accessPoints.isEmpty()) ***REMOVED*** + if (!accessPoints.isEmpty()) { Iterator iterator = accessPoints.iterator(); AccessPoint ap = iterator.next(); Group props = ap.properties(); - if (!props.isEmpty()) ***REMOVED*** + if (!props.isEmpty()) { Iterator iteratorProps = props.iterator(); Property p = iteratorProps.next(); String applicationToken = StringEncrypter.getEncrypter().decrypt(p.value()); @@ -133,55 +133,55 @@ public class SendMail ***REMOVED*** logger.info("Application Token retrieved"); return applicationToken; - ***REMOVED*** else ***REMOVED*** + } else { String error = "DMPoolManager application token not found in service resource on IS!"; logger.error(error); throw new EMailException(error); - ***REMOVED*** - ***REMOVED*** else ***REMOVED*** + } + } else { String error = "DMPoolManager invalid service resource on IS!"; logger.error(error); throw new EMailException(error); - ***REMOVED*** + } - ***REMOVED*** catch (Throwable e) ***REMOVED*** + } catch (Throwable e) { logger.error("DMPoolManager application token not found: " + e.getLocalizedMessage(), e); throw new EMailException("DMPoolManager application token not found: " + e.getLocalizedMessage(), e); - ***REMOVED*** + } -***REMOVED*** + } - private void retrieveSocialService() throws EMailException ***REMOVED*** - try ***REMOVED*** + private void retrieveSocialService() throws EMailException { + try { SimpleQuery query = queryFor(GCoreEndpoint.class); query.addCondition(SOCIAL_SERVICE_QUERY_CONDITION); DiscoveryClient client = clientFor(GCoreEndpoint.class); List resources = client.submit(query); socialServiceAddress = resources.get(0).profile().endpointMap().get(SOCIAL_SERVICE_URI).uri().toString(); logger.info("Retrieved Social Service Address: " + socialServiceAddress); - if (socialServiceAddress == null || socialServiceAddress.isEmpty()) ***REMOVED*** + if (socialServiceAddress == null || socialServiceAddress.isEmpty()) { throw new EMailException( "Unable to send email notification. Invalid address in GCoreEndpoint resource on IS."); - ***REMOVED*** - ***REMOVED*** catch (EMailException e) ***REMOVED*** + } + } catch (EMailException e) { logger.error(e.getLocalizedMessage(), e); throw e; - ***REMOVED*** catch (Throwable e) ***REMOVED*** + } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } - private String username(String token) throws ObjectNotFound, Exception ***REMOVED*** + private String username(String token) throws ObjectNotFound, Exception { AuthorizationEntry entry = authorizationService().get(token); logger.debug(entry.getClientInfo().getId()); return entry.getClientInfo().getId(); -***REMOVED*** + } - private void sendPostRequest(String postBody) throws EMailException ***REMOVED*** + private void sendPostRequest(String postBody) throws EMailException { - try ***REMOVED*** + try { logger.info("Execute Post Body:" + postBody); StringBuilder requestMessageBuilder = new StringBuilder(socialServiceAddress); @@ -194,7 +194,7 @@ public class SendMail ***REMOVED*** requestMessageBuilder.append(applicationToken); String endpoint = requestMessageBuilder.toString(); - ***REMOVED*** Send the request + // Send the request URL url = new URL(endpoint); URLConnection conn = url.openConnection(); conn.setRequestProperty("Accept", JSON_MIME_TYPE); @@ -205,13 +205,13 @@ public class SendMail ***REMOVED*** writer.write(postBody); writer.flush(); - ***REMOVED*** Get the response + // Get the response StringBuffer answer = new StringBuffer(); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); String line; - while ((line = reader.readLine()) != null) ***REMOVED*** + while ((line = reader.readLine()) != null) { answer.append(line); - ***REMOVED*** + } writer.close(); reader.close(); @@ -220,76 +220,76 @@ public class SendMail ***REMOVED*** logger.info("Notification Response: " + response); checkResponse(response); - ***REMOVED*** catch (EMailException e) ***REMOVED*** + } catch (EMailException e) { throw e; - ***REMOVED*** catch (MalformedURLException e) ***REMOVED*** + } catch (MalformedURLException e) { logger.error("Invalid URL: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** catch (IOException e) ***REMOVED*** + } catch (IOException e) { logger.error("Error in the IO process: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** catch (Throwable e) ***REMOVED*** + } catch (Throwable e) { logger.error("Error executing post:" + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } - private void checkResponse(String response) throws EMailException ***REMOVED*** - if (response == null) ***REMOVED*** + private void checkResponse(String response) throws EMailException { + if (response == null) { logger.error("Invalid notification response: " + response); throw new EMailException(); - ***REMOVED*** else ***REMOVED*** - try ***REMOVED*** + } else { + try { JSONObject res = new JSONObject(response); boolean success = res.getBoolean("success"); - if (!success) ***REMOVED*** + if (!success) { String message = res.getString("message"); logger.error("Error in send email notification: " + message); throw new EMailException("Error in send email notification: " + message); - ***REMOVED*** + } - ***REMOVED*** catch (JSONException e) ***REMOVED*** + } catch (JSONException e) { logger.error("Invalid notification response: " + response); throw new EMailException(e); - ***REMOVED*** + } - ***REMOVED*** -***REMOVED*** + } + } - private List getRecipients() ***REMOVED*** - try ***REMOVED*** + private List getRecipients() { + try { List recipients = new ArrayList(); String dataMinerManagers = retrieveDataMinerManagers(); logger.debug("Retrieved DataMiner Managers: " + dataMinerManagers); - if (dataMinerManagers != null && !dataMinerManagers.isEmpty()) ***REMOVED*** + if (dataMinerManagers != null && !dataMinerManagers.isEmpty()) { JSONObject obj = new JSONObject(dataMinerManagers); JSONArray data = obj.getJSONArray("result"); - if (data != null) ***REMOVED*** - for (int i = 0; i < data.length(); i++) ***REMOVED*** + if (data != null) { + for (int i = 0; i < data.length(); i++) { recipients.add(data.getString(i)); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** else ***REMOVED*** + } + } + } else { logger.info("Use the default admins how workaround "); List defaultManagers = DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); recipients.addAll(defaultManagers); - ***REMOVED*** + } recipients.add(this.username(SecurityTokenProvider.instance.get())); logger.info("Retrieved Recipients: " + recipients); return recipients; - ***REMOVED*** catch (Exception e) ***REMOVED*** + } catch (Exception e) { logger.error("Error retrieving recipients: " + e.getLocalizedMessage(), e); logger.info("Use the default admins how workaround "); return DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); - ***REMOVED*** + } -***REMOVED*** + } - private String retrieveDataMinerManagers() throws Exception ***REMOVED*** - ***REMOVED*** Try to retrieve a url like this: - ***REMOVED*** https:***REMOVED***api.d4science.org/social-networking-library-ws/rest/2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=xxx-xxxx-xxxx-xxx + private String retrieveDataMinerManagers() throws Exception { + // Try to retrieve a url like this: + // https://api.d4science.org/social-networking-library-ws/rest/2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=xxx-xxxx-xxxx-xxx StringBuilder requestMessageBuilder = new StringBuilder(socialServiceAddress); @@ -298,7 +298,7 @@ public class SendMail ***REMOVED*** requestMessageBuilder.append(USER_ROLES_ADDRESS_PATH); logger.info("Request Admins Url: " + requestMessageBuilder.toString()); - ***REMOVED*** SecurityTokenProvider.instance.get() + // SecurityTokenProvider.instance.get() requestMessageBuilder.append(applicationToken); String requestAdminsUrl = requestMessageBuilder.toString(); @@ -311,6 +311,6 @@ public class SendMail ***REMOVED*** return EntityUtils.toString(client.execute(getReq).getEntity()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java index 05fa3a5..d8d6b2d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java @@ -1,35 +1,35 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; -public class AlgorithmException extends DMPMException***REMOVED*** +public class AlgorithmException extends DMPMException{ -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = -5678597187512954288L; private String algorithmName; public AlgorithmException (String algorithmName) - ***REMOVED*** + { super ("Algorithm exception"); this.algorithmName = algorithmName; -***REMOVED*** + } public AlgorithmException (String algorithmName, Throwable cause) - ***REMOVED*** + { super ("Algorithm exception", cause); this.algorithmName = algorithmName; -***REMOVED*** + } -***REMOVED*** - public String getErrorMessage() ***REMOVED*** + @Override + public String getErrorMessage() { return "Installation completed but DataMiner Interface not working correctly or files " + this.algorithmName + ".jar and " + this.algorithmName + "_interface.jar not availables at the expected path"; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java index 25419ac..f614e7d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java @@ -1,20 +1,20 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; -public abstract class DMPMException extends Exception***REMOVED*** +public abstract class DMPMException extends Exception{ -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 1L; public DMPMException (String errorMessage) - ***REMOVED*** + { super (errorMessage); -***REMOVED*** + } - public DMPMException(String errorMessage,Throwable cause) ***REMOVED*** + public DMPMException(String errorMessage,Throwable cause) { super (errorMessage,cause); -***REMOVED*** + } public abstract String getErrorMessage (); -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java index 854fa29..b0d02d9 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java @@ -1,28 +1,28 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; -public class EMailException extends Exception ***REMOVED*** +public class EMailException extends Exception { private static final String MESSAGE = "Unable to send email notification"; -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 1L; - public EMailException() ***REMOVED*** + public EMailException() { super(MESSAGE); -***REMOVED*** + } - public EMailException(String message) ***REMOVED*** + public EMailException(String message) { super(message); -***REMOVED*** + } - public EMailException(String message, Throwable e) ***REMOVED*** + public EMailException(String message, Throwable e) { super(message, e); -***REMOVED*** + } - public EMailException(Throwable e) ***REMOVED*** + public EMailException(Throwable e) { super(MESSAGE, e); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java index 20e3e93..d7b99dd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java @@ -1,26 +1,26 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; -public class GenericException extends DMPMException ***REMOVED*** +public class GenericException extends DMPMException { -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = 6772009633547404120L; - public GenericException(Throwable cause) ***REMOVED*** + public GenericException(Throwable cause) { super ("Generic exception",cause); -***REMOVED*** + } -***REMOVED*** - public String getErrorMessage() ***REMOVED*** + @Override + public String getErrorMessage() { return this.getCause().getMessage(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java index a882451..c3383a7 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java @@ -2,43 +2,43 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; import org.tmatesoft.svn.core.SVNErrorMessage; -public class SVNCommitException extends DMPMException ***REMOVED*** +public class SVNCommitException extends DMPMException { -***REMOVED**** + /** * -***REMOVED*** + */ private static final long serialVersionUID = -5225403308313619585L; private SVNErrorMessage svnErrorMessage; private String fileName; - public SVNCommitException(SVNErrorMessage errorMessage, String fileName) ***REMOVED*** + public SVNCommitException(SVNErrorMessage errorMessage, String fileName) { super ("Unable to commit"); this.svnErrorMessage = errorMessage; this.fileName = fileName; -***REMOVED*** + } - public SVNCommitException(String message,SVNErrorMessage errorMessage,String fileName) ***REMOVED*** + public SVNCommitException(String message,SVNErrorMessage errorMessage,String fileName) { super (message); this.svnErrorMessage = errorMessage; this.fileName = fileName; -***REMOVED*** + } - public SVNErrorMessage getSvnErrorMessage() ***REMOVED*** + public SVNErrorMessage getSvnErrorMessage() { return svnErrorMessage; -***REMOVED*** + } -***REMOVED*** - public String getErrorMessage() ***REMOVED*** + @Override + public String getErrorMessage() { return "Commit operation failed for "+this.fileName + "the message of the SVN Server is the following:\n"+this.svnErrorMessage.getMessage(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java index 7852766..b3c762e 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java @@ -5,101 +5,101 @@ import java.io.File; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.SftpException; -public class CheckMethodProduction extends CheckMethod***REMOVED*** +public class CheckMethodProduction extends CheckMethod{ public CheckMethodProduction() - ***REMOVED*** + { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } -***REMOVED*** - protected void copyFromDmToSVN(File a) throws SVNException, SVNCommitException, JSchException, SftpException ***REMOVED*** + @Override + protected void copyFromDmToSVN(File a) throws SVNException, SVNCommitException, JSchException, SftpException { super.copyFromDmToSVN(a, new SVNUpdaterProduction()); -***REMOVED*** + } - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); -***REMOVED*** System.out.println(a.getStagingHost()); + public static void main(String[] args) throws Exception { +// ServiceConfiguration a = new ServiceConfiguration(); +// System.out.println(a.getStagingHost()); CheckMethodProduction a = new CheckMethodProduction(); - ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + //a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); -***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); -***REMOVED*** System.out.println(aa.getName()); -***REMOVED*** System.out.println(aa.getPath()); +// File aa = new File("OCTAVEBLACKBOX.jar"); +// System.out.println(aa.getName()); +// System.out.println(aa.getPath()); - ***REMOVED***a.copyFromDmToSVN(aa); -***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED*** -***REMOVED*** System.out.println("AAA");***REMOVED*** -***REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED*** -***REMOVED*** System.out.println("BBBB"); -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED*** -***REMOVED*** System.out.println("CCCC");***REMOVED*** -***REMOVED*** -***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); -***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + //a.copyFromDmToSVN(aa); +// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxx-xxx-xxx-xxx")){ +// System.out.println("AAA"); } +// +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ +// System.out.println("BBBB"); +// +// } +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ +// System.out.println("CCCC");} +// +// File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +// a.copyFromDmToSVN(aa, "Dev"); -***REMOVED*** +// -***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", -***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); +//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +// "xxxx-xxx-xxx-xxx")); -***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", - ***REMOVED*** "***REMOVED***")); +//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", + // "xxx-xxxx-xxx-xxx")); try - ***REMOVED*** + { a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "***REMOVED***"); -***REMOVED*** catch (Exception e) - ***REMOVED*** + "xxx-xxx-xxx-xxx"); + } catch (Exception e) + { e.printStackTrace(); -***REMOVED*** + } -***REMOVED***Algorithm aa = new Algorithm(); -***REMOVED***aa.setName("UDPIPE_WRAPPER"); -***REMOVED***System.out.println(a.algoExists(aa)); -***REMOVED******REMOVED*** -***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); -***REMOVED******REMOVED*** -***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); -***REMOVED******REMOVED*** -***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED*** -***REMOVED***System.out.println("ciao"); -***REMOVED*** -***REMOVED******REMOVED*** +//Algorithm aa = new Algorithm(); +//aa.setName("UDPIPE_WRAPPER"); +//System.out.println(a.algoExists(aa)); +//// +//ServiceConfiguration bp = new ServiceConfiguration(); +//// +//SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx"); +//// +//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { +//System.out.println("ciao"); +// +//} -***REMOVED*** -***REMOVED***Algorithm al = new Algorithm(); -***REMOVED*** al.setName("UDPIPE_WRAPPER"); -***REMOVED*** a.deleteFiles(al); +// +//Algorithm al = new Algorithm(); +// al.setName("UDPIPE_WRAPPER"); +// a.deleteFiles(al); -***REMOVED*** -***REMOVED*** +} +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java index cf56ce0..5463078 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java @@ -5,93 +5,93 @@ import java.io.File; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; -public class CheckMethodStaging extends CheckMethod***REMOVED*** +public class CheckMethodStaging extends CheckMethod{ public CheckMethodStaging() - ***REMOVED*** + { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } -***REMOVED*** - protected void copyFromDmToSVN(File a) throws Exception***REMOVED*** + @Override + protected void copyFromDmToSVN(File a) throws Exception{ super.copyFromDmToSVN(a, new SVNUpdaterStaging()); -***REMOVED*** + } - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); -***REMOVED*** System.out.println(a.getStagingHost()); + public static void main(String[] args) throws Exception { +// ServiceConfiguration a = new ServiceConfiguration(); +// System.out.println(a.getStagingHost()); CheckMethodStaging a = new CheckMethodStaging(); - ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + //a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); -***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); -***REMOVED*** System.out.println(aa.getName()); -***REMOVED*** System.out.println(aa.getPath()); +// File aa = new File("OCTAVEBLACKBOX.jar"); +// System.out.println(aa.getName()); +// System.out.println(aa.getPath()); - ***REMOVED***a.copyFromDmToSVN(aa); -***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED*** -***REMOVED*** System.out.println("AAA");***REMOVED*** -***REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED*** -***REMOVED*** System.out.println("BBBB"); -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED*** -***REMOVED*** System.out.println("CCCC");***REMOVED*** -***REMOVED*** -***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); -***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + //a.copyFromDmToSVN(aa); +// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxx-xxx-xxx")){ +// System.out.println("AAA"); } +// +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ +// System.out.println("BBBB"); +// +// } +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ +// System.out.println("CCCC");} +// +// File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +// a.copyFromDmToSVN(aa, "Dev"); -***REMOVED*** +// -***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", -***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); +//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +// "xxxx-xxx-xxx-xxx")); -***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", - ***REMOVED*** "***REMOVED***")); +//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", + // "xxx-xxx-xxx-xxx")); try - ***REMOVED*** + { a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "***REMOVED***"); -***REMOVED*** catch (Exception e) - ***REMOVED*** + "xxx-xxx-xxxx-xxx"); + } catch (Exception e) + { e.printStackTrace(); -***REMOVED*** + } -***REMOVED***Algorithm aa = new Algorithm(); -***REMOVED***aa.setName("UDPIPE_WRAPPER"); -***REMOVED***System.out.println(a.algoExists(aa)); -***REMOVED******REMOVED*** -***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); -***REMOVED******REMOVED*** -***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); -***REMOVED******REMOVED*** -***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED*** -***REMOVED***System.out.println("ciao"); -***REMOVED*** -***REMOVED******REMOVED*** +//Algorithm aa = new Algorithm(); +//aa.setName("UDPIPE_WRAPPER"); +//System.out.println(a.algoExists(aa)); +//// +//ServiceConfiguration bp = new ServiceConfiguration(); +//// +//SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx"); +//// +//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { +//System.out.println("ciao"); +// +//} -***REMOVED*** -***REMOVED***Algorithm al = new Algorithm(); -***REMOVED*** al.setName("UDPIPE_WRAPPER"); -***REMOVED*** a.deleteFiles(al); +// +//Algorithm al = new Algorithm(); +// al.setName("UDPIPE_WRAPPER"); +// a.deleteFiles(al); -***REMOVED*** -***REMOVED*** +} +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java index 694e97a..8fa38c3 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java @@ -3,12 +3,12 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; -public class ClusterBuilderProduction extends ClusterBuilder***REMOVED*** +public class ClusterBuilderProduction extends ClusterBuilder{ - public ClusterBuilderProduction() ***REMOVED*** + public ClusterBuilderProduction() { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java index 662250d..0d1aa7b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java @@ -3,11 +3,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; -public class ClusterBuilderStaging extends ClusterBuilder***REMOVED*** +public class ClusterBuilderStaging extends ClusterBuilder{ - public ClusterBuilderStaging() ***REMOVED*** + public ClusterBuilderStaging() { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java index 2fc9978..2b6b89a 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java @@ -2,18 +2,18 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; -public class NotificationHelperProduction extends NotificationHelper***REMOVED*** +public class NotificationHelperProduction extends NotificationHelper{ -***REMOVED*** private Exception executionException; +// private Exception executionException; - public NotificationHelperProduction() ***REMOVED*** + public NotificationHelperProduction() { super ("[DataMinerGhostProductionInstallationRequestReport]"); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java index 32cda99..372b55a 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java @@ -2,17 +2,17 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; -public class NotificationHelperStaging extends NotificationHelper ***REMOVED*** +public class NotificationHelperStaging extends NotificationHelper { -***REMOVED*** private Exception executionException; +// private Exception executionException; - public NotificationHelperStaging() ***REMOVED*** + public NotificationHelperStaging() { super ("[DataMinerGhostStagingInstallationRequestReport]"); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java index 3b7918f..2fdbaeb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java @@ -2,20 +2,20 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; /** * Created by ggiammat on 5/9/17. */ -public class SVNUpdaterProduction extends SVNUpdater***REMOVED*** +public class SVNUpdaterProduction extends SVNUpdater{ - public SVNUpdaterProduction() throws SVNException ***REMOVED*** + public SVNUpdaterProduction() throws SVNException { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java index 79d405b..d6d93a5 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java @@ -2,20 +2,20 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; /** * Created by ggiammat on 5/9/17. */ -public class SVNUpdaterStaging extends SVNUpdater***REMOVED*** +public class SVNUpdaterStaging extends SVNUpdater{ - public SVNUpdaterStaging() throws SVNException ***REMOVED*** + public SVNUpdaterStaging() throws SVNException { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/resources/custom/custom-flr-Flash/meta/main.yaml b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml index 680d85a..efa0ff5 100755 --- a/src/main/resources/custom/custom-flr-Flash/meta/main.yaml +++ b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml @@ -1,4 +1,4 @@ #--- #dependencies: # -# - ***REMOVED*** role: digest ***REMOVED*** +# - { role: digest } diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index 238de49..3819f20 100755 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -1,7 +1,7 @@ #YML node file -SVN_REPO = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ +SVN_REPO = https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ -svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube +svn.repository = https://svn.d4science.research-infrastructures.eu/gcube svn.algo.main.repo = /trunk/data-analysis/DataMinerConfiguration/algorithms diff --git a/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml index b811206..e3ea93e 100755 --- a/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml +++ b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml @@ -1,4 +1,4 @@ dependencies: -# - ***REMOVED*** role: gcube-dataminer ***REMOVED*** -# - ***REMOVED*** role: os-unzip ***REMOVED*** -# - ***REMOVED*** role: os-java-1.7.0 ***REMOVED*** +# - { role: gcube-dataminer } +# - { role: os-unzip } +# - { role: os-java-1.7.0 } diff --git a/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml index 7bb84f9..13f8109 100755 --- a/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml +++ b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml @@ -12,14 +12,14 @@ #- name: download the installer zip # get_url: -# url: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip +# url: https://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip # dest: /home/dpm/algorithmInstaller.zip # validate_certs: no #- name: download and unzip the package # unarchive: -# src: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller1_1.zip -## src: http:***REMOVED***maven.research-infrastructures.eu:8081/nexus/service/local/artifact/maven/redirect?r=gcube-snapshots&g=org.gcube.dataanalysis&a=dataminer-algorithms-importer&v=1.1.1-SNAPSHOT&e=tar.gz +# src: https://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller1_1.zip +## src: http://maven.research-infrastructures.eu:8081/nexus/service/local/artifact/maven/redirect?r=gcube-snapshots&g=org.gcube.dataanalysis&a=dataminer-algorithms-importer&v=1.1.1-SNAPSHOT&e=tar.gz # dest: /home/gcube # remote_src: yes # validate_certs: no diff --git a/src/main/resources/static/gcube-dataminer/meta/main.yaml b/src/main/resources/static/gcube-dataminer/meta/main.yaml index 2d96d9f..bab366c 100755 --- a/src/main/resources/static/gcube-dataminer/meta/main.yaml +++ b/src/main/resources/static/gcube-dataminer/meta/main.yaml @@ -1,2 +1,2 @@ dependencies: - - ***REMOVED*** role: gcube-ghn ***REMOVED*** + - { role: gcube-ghn } diff --git a/src/main/resources/static/os-java-1.7.0/tasks/main.yaml b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml index 2b6334b..2906525 100755 --- a/src/main/resources/static/os-java-1.7.0/tasks/main.yaml +++ b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml @@ -3,5 +3,5 @@ - name: adding OS Package java-1.7.0 package: name: java-1.7.0-openjdk - state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/resources/static/os-unzip/tasks/main.yaml b/src/main/resources/static/os-unzip/tasks/main.yaml index 4f3e56c..6887424 100755 --- a/src/main/resources/static/os-unzip/tasks/main.yaml +++ b/src/main/resources/static/os-unzip/tasks/main.yaml @@ -3,5 +3,5 @@ - name: adding OS Package unzip package: name: unzip - state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/resources/templates/cran-package-add/meta/main.yaml b/src/main/resources/templates/cran-package-add/meta/main.yaml index 3e2a070..7179b15 100755 --- a/src/main/resources/templates/cran-package-add/meta/main.yaml +++ b/src/main/resources/templates/cran-package-add/meta/main.yaml @@ -1,4 +1,4 @@ --- #dependencies: # -# - ***REMOVED*** role: os-r ***REMOVED*** +# - { role: os-r } diff --git a/src/main/resources/templates/cran-package-add/tasks/main.yaml b/src/main/resources/templates/cran-package-add/tasks/main.yaml index 42dd680..5b9f563 100755 --- a/src/main/resources/templates/cran-package-add/tasks/main.yaml +++ b/src/main/resources/templates/cran-package-add/tasks/main.yaml @@ -1,4 +1,4 @@ # tasks file for r --- - name: Install algorithm - command: Rscript -e "install.packages('', repos=c('http:***REMOVED***ftp.heanet.ie/mirrors/cran.r-project.org/'))" + command: Rscript -e "install.packages('', repos=c('http://ftp.heanet.ie/mirrors/cran.r-project.org/'))" diff --git a/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml index 8c27de1..9396c53 100755 --- a/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml +++ b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml @@ -1,4 +1,4 @@ --- dependencies: - - ***REMOVED*** role: gcube-algorithm-installer ***REMOVED*** + - { role: gcube-algorithm-installer } diff --git a/src/main/resources/templates/github-package-add/meta/main.yaml b/src/main/resources/templates/github-package-add/meta/main.yaml index 3e2a070..7179b15 100755 --- a/src/main/resources/templates/github-package-add/meta/main.yaml +++ b/src/main/resources/templates/github-package-add/meta/main.yaml @@ -1,4 +1,4 @@ --- #dependencies: # -# - ***REMOVED*** role: os-r ***REMOVED*** +# - { role: os-r } diff --git a/src/main/resources/templates/os-package-add/tasks/main.yaml b/src/main/resources/templates/os-package-add/tasks/main.yaml index 4fbc8a0..7df04bc 100755 --- a/src/main/resources/templates/os-package-add/tasks/main.yaml +++ b/src/main/resources/templates/os-package-add/tasks/main.yaml @@ -3,6 +3,6 @@ - name: adding OS Package package: name: - state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index c599c4d..7faa928 100755 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -1,6 +1,6 @@ + "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" + "http://java.sun.com/dtd/web-app_2_3.dtd" > Archetype Created Web Application @@ -41,7 +41,7 @@ diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java index e366bd6..fb14c71 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java @@ -1,31 +1,31 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; -***REMOVED*** -***REMOVED*** +import java.net.URL; +import java.util.List; import java.util.Vector; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -public class AlgorithmPackageParserTest ***REMOVED*** +public class AlgorithmPackageParserTest { private static int BUFFER_SIZE = 2048; - public void extractAllAlgorithms() throws IOException ***REMOVED*** - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; + public void extractAllAlgorithms() throws IOException { + String url = "http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; List commands = this.extractAddAlgorithmCommands(url); AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) ***REMOVED*** + for (String cmd : commands) { System.out.println("-----------------------------------------"); System.out.println(cmd); AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); System.out.println(aac); - ***REMOVED*** start creating the algo from the command + // start creating the algo from the command Algorithm algo = new Algorithm(); algo.setAlgorithmType(aac.getAlgorithmType()); algo.setCategory(aac.getCategory()); @@ -35,20 +35,20 @@ public class AlgorithmPackageParserTest ***REMOVED*** algo.setPackageURL(aac.getUrl()); algo.setSkipJava(aac.getSkipjava()); - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) ***REMOVED*** + // then override with info from the package + if (aac.getUrl().length() > 4) { Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) ***REMOVED*** + if (packagedAlgo != null) { algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** + } + } algorithms.addAlgorithm(algo); break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** + } + //to uncomment + // new DataminerPoolManager().addAlgorithmsToVRE(algorithms, + // "/gcube/devNext/NextNext"); + } /** * Extract 'addAlgorithm' commands from a file containing wiki-table-style @@ -58,43 +58,43 @@ public class AlgorithmPackageParserTest ***REMOVED*** * @throws IOException */ private List extractAddAlgorithmCommands(String listUrl) - throws IOException ***REMOVED*** + throws IOException { URL url = new URL(listUrl); InputStream is = url.openStream(); StringBuilder s = new StringBuilder(); byte[] buffer = new byte[BUFFER_SIZE]; int read = 0; - while ((read = is.read(buffer)) != -1) ***REMOVED*** + while ((read = is.read(buffer)) != -1) { s.append(new String(buffer, 0, read)); - ***REMOVED*** + } List out = new Vector<>(); String[] lines = s.toString().split("\n"); - for (String line : lines) ***REMOVED*** + for (String line : lines) { System.out.println("--------------------"); - if (!line.isEmpty()) ***REMOVED*** + if (!line.isEmpty()) { String[] parts = line.split("\\|"); int c = 1; - for (String part : parts) ***REMOVED*** - if (part == null || part.trim().isEmpty()) ***REMOVED*** + for (String part : parts) { + if (part == null || part.trim().isEmpty()) { continue; - ***REMOVED*** + } System.out.println(c + ". " + part); c++; - if (part.contains("addAlgorithm.sh")) ***REMOVED*** + if (part.contains("addAlgorithm.sh")) { String cmd = part.trim(); cmd = cmd.replaceAll("", ""); cmd = cmd.replaceAll("", ""); System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); + // AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); + // System.out.println(aac); out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } return out; - ***REMOVED*** + } /** * Create an Algorithm starting from the algorithm jar. @@ -103,18 +103,18 @@ public class AlgorithmPackageParserTest ***REMOVED*** * @return * @throws IOException */ - private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** + private Algorithm extractAlgorithm(String url) throws IOException { return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** + } - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); + public static void main(String[] args) throws Exception { +// ProxySelector.setDefault(new PropertiesBasedProxySelector( +// "/home/ngalante/.proxy-settings")); new AlgorithmPackageParserTest().extractAllAlgorithms(); -***REMOVED*** AlgorithmPackageParserTest at = new AlgorithmPackageParserTest(); -***REMOVED*** Algorithm a = at.extractAlgorithm("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); -***REMOVED*** System.out.println(a.getFullname()); - ***REMOVED*** +// AlgorithmPackageParserTest at = new AlgorithmPackageParserTest(); +// Algorithm a = at.extractAlgorithm("http://data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); +// System.out.println(a.getFullname()); + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java index 0d2a716..0fd378b 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.UUID; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; @@ -10,9 +10,9 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; -public class AnsibleWorkerTest ***REMOVED*** +public class AnsibleWorkerTest { - public static void main(String[] args) throws IOException ***REMOVED*** + public static void main(String[] args) throws IOException { AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); System.out.println("created worker named " + worker.getWorkerId()); @@ -27,7 +27,7 @@ public class AnsibleWorkerTest ***REMOVED*** r.addTaskFile(tf); worker.addRole(r); - ***REMOVED***worker.apply(); - ***REMOVED*** + //worker.apply(); + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java index 31c9853..507683f 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java @@ -4,66 +4,66 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.util.Iterator; -***REMOVED*** +import java.util.List; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.encryption.encrypter.StringEncrypter; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.utils.Group; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -public class ApplicationTokenTest ***REMOVED*** +public class ApplicationTokenTest { private static final String DMPOOLMANAGER_SERVICE_QUERY_CONDITION = "$resource/Profile/Name/text() eq 'DMPoolManager'"; - public static void main(String[] args) ***REMOVED*** + public static void main(String[] args) { ScopeProvider.instance.set("/gcube/devsec/devVRE"); SecurityTokenProvider.instance.set("xxxx-xxxx-xxx"); decryptToken(); -***REMOVED*** + } - private static void decryptToken() ***REMOVED*** - try ***REMOVED*** + private static void decryptToken() { + try { SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition(DMPOOLMANAGER_SERVICE_QUERY_CONDITION); DiscoveryClient client = clientFor(ServiceEndpoint.class); List resources = client.submit(query); - if (resources.isEmpty()) ***REMOVED*** + if (resources.isEmpty()) { System.out.println("No services resource found on IS!"); - ***REMOVED*** else ***REMOVED*** + } else { System.out.println("Retrieved: " + resources.get(0)); - ***REMOVED*** + } Group accessPoints = resources.get(0).profile().accessPoints(); - if (!accessPoints.isEmpty()) ***REMOVED*** + if (!accessPoints.isEmpty()) { Iterator iterator = accessPoints.iterator(); AccessPoint ap = iterator.next(); Group props = ap.properties(); - if (!props.isEmpty()) ***REMOVED*** + if (!props.isEmpty()) { Iterator iteratorProps = props.iterator(); Property p = iteratorProps.next(); String applicationToken = StringEncrypter.getEncrypter().decrypt(p.value()); System.out.println("Application token found: " + applicationToken); - ***REMOVED*** else ***REMOVED*** + } else { System.out.println("No application token found in service resource on IS!"); - ***REMOVED*** - ***REMOVED*** else ***REMOVED*** + } + } else { System.out.println("Invalid service resource on IS!"); - ***REMOVED*** + } - ***REMOVED*** catch (Exception e) ***REMOVED*** + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java index bc2ec8d..8c735e0 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java @@ -2,106 +2,106 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.net.ProxySelector; -***REMOVED*** -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; -public class DataminerPoolManagerTest ***REMOVED*** +public class DataminerPoolManagerTest { /* private static Map domains = new HashMap<>(); private static Map clusters = new HashMap<>(); - private static Dependency createDependency(String depName) ***REMOVED*** + private static Dependency createDependency(String depName) { String[] parts = depName.split(":"); Dependency out = new Dependency(); - if(parts.length>1) ***REMOVED*** + if(parts.length>1) { out.setType(parts[0]); out.setName(parts[1]); - ***REMOVED*** else ***REMOVED*** + } else { out.setType("os"); out.setName(depName); - ***REMOVED*** + } return out; - ***REMOVED*** + } - private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED*** + private static Algorithm createAlgorithm(String name, String ... deps) { Algorithm a = new Algorithm(); a.setName(name); - for(String dep:deps) ***REMOVED*** + for(String dep:deps) { a.addDependency(createDependency(dep)); - ***REMOVED*** + } return a; - ***REMOVED*** + } - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED*** + private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { AlgorithmSet out = new AlgorithmSet(); out.setName(name); - for(Algorithm a:algs) ***REMOVED*** + for(Algorithm a:algs) { out.addAlgorithm(a); - ***REMOVED*** + } return out; - ***REMOVED*** + } - private static Domain getDomain(String name) ***REMOVED*** - if(domains.get(name)==null) ***REMOVED*** + private static Domain getDomain(String name) { + if(domains.get(name)==null) { Domain d = new Domain(); d.setName(name); domains.put(name, d); return d; - ***REMOVED*** else ***REMOVED*** + } else { return domains.get(name); - ***REMOVED*** - ***REMOVED*** + } + } - private static Host createHost(String hostname, String domainName) ***REMOVED*** + private static Host createHost(String hostname, String domainName) { Host out = new Host(); out.setName(hostname); Domain d = getDomain(domainName); out.setDomain(d); return out; - ***REMOVED*** + } - private static Cluster getCluster(String name) ***REMOVED*** - if(clusters.get(name)==null) ***REMOVED*** + private static Cluster getCluster(String name) { + if(clusters.get(name)==null) { Cluster d = new Cluster(); d.setName(name); clusters.put(name, d); return d; - ***REMOVED*** else ***REMOVED*** + } else { return clusters.get(name); - ***REMOVED*** - ***REMOVED*** + } + } - private static Collection extractDependencies() ***REMOVED*** + private static Collection extractDependencies() { Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - for(Dependency d:a.getDependencies()) ***REMOVED*** + for(Cluster c:clusters.values()) { + for(AlgorithmSet as:c.getAlgorithmSets()) { + for(Algorithm a:as.getAlgorithms()) { + for(Dependency d:a.getDependencies()) { out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } return out; - ***REMOVED*** + } - private static Collection extractAlgorithms() ***REMOVED*** + private static Collection extractAlgorithms() { Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** + for(Cluster c:clusters.values()) { + for(AlgorithmSet as:c.getAlgorithmSets()) { + for(Algorithm a:as.getAlgorithms()) { out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } return out; - ***REMOVED*** + } - static ***REMOVED*** + static { Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); Algorithm ensemble = createAlgorithm("ensemble", "python"); @@ -132,10 +132,10 @@ public class DataminerPoolManagerTest ***REMOVED*** cluster2.addAlgorithmSet(as1); cluster2.addAlgorithmSet(as3); - ***REMOVED*** + } */ - public static void main(String[] args) throws Exception ***REMOVED*** + public static void main(String[] args) throws Exception { /* AnsibleBridge ab = new AnsibleBridge(); ab.printInventoryByDomainAndSets(clusters.values()); @@ -144,70 +144,70 @@ public class DataminerPoolManagerTest ***REMOVED*** AnsibleWorker worker = ab.createWorker(); - for(Algorithm a:extractAlgorithms()) ***REMOVED*** - for(Role r:ab.generateRoles(a)) ***REMOVED*** + for(Algorithm a:extractAlgorithms()) { + for(Role r:ab.generateRoles(a)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** + } + } - for(Dependency d:extractDependencies()) ***REMOVED*** - for(Role r:ab.generateRoles(d)) ***REMOVED*** + for(Dependency d:extractDependencies()) { + for(Role r:ab.generateRoles(d)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** + } + } */ ScopeProvider.instance.set("/gcube/devNext/NextNext"); ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); +// +// // create the algorithm (download it, etc etc) +// Algorithm algorithm = new Algorithm(); +// algorithm.setName("ichtyop"); +// algorithm.setClazz("org.gcube..."); +// algorithm.setDescription("some description"); +// +// Dependency d = new Dependency(); +// d.setName("libpng"); +// d.setType("os"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("some-r-package"); +// d.setType("cran"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("some-other-r-package"); +// d.setType("cran"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("voodoo"); +// d.setType("custom"); +// algorithm.addDependency(d); +// +// // create the algorithm (download it, etc etc) +// Algorithm ewe = new Algorithm(); +// ewe.setName("ewe"); +// +// d = new Dependency(); +// d.setName("voodoo"); +// d.setType("custom"); +// ewe.addDependency(d); AlgorithmSet algorithms = new AlgorithmSet(); algorithms.setName("dummy-set"); -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); +// algorithms.addAlgorithm(algorithm); +// algorithms.addAlgorithm(ewe); Algorithm ensemble = new Algorithm(); ensemble.setName("ensemble"); ensemble.setCategory("ICHTHYOP_MODEL"); ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); + ensemble.setPackageURL("http://data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); ensemble.setSkipJava("N"); ensemble.setDescription("test"); @@ -218,8 +218,8 @@ public class DataminerPoolManagerTest ***REMOVED*** ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); + //new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java index d4c5dbc..211fd04 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java @@ -2,16 +2,16 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.net.ProxySelector; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; -public class ISClientTest ***REMOVED*** +public class ISClientTest { - public static void main(String[] args) ***REMOVED*** + public static void main(String[] args) { ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ScopeProvider.instance.set("/gcube/devNext/NextNext"); System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index 2d40976..50b1c20 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -1,34 +1,34 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; -public class JobTest ***REMOVED*** +public class JobTest { - public static void main(String[] args) throws SVNException, IOException, InterruptedException***REMOVED*** + public static void main(String[] args) throws SVNException, IOException, InterruptedException{ ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); SVNUpdater svnUpdater = new SVNUpdaterProduction(); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); + Algorithm algo = AlgorithmBuilder.create("http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); - ***REMOVED***test phase - ***REMOVED***Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); - ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); - ***REMOVED***DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get()); - ***REMOVED***job.start(); + //test phase + //Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + //Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + //DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get()); + //job.start(); - ***REMOVED***release phase - ***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); - ***REMOVED***DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); - ***REMOVED***job2.start(); + //release phase + //Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); + //DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); + //job2.start(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index eccc491..89e7d8c 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -1,36 +1,36 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; -***REMOVED*** -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging; -***REMOVED*** +import org.tmatesoft.svn.core.SVNException; /** * Created by ggiammat on 5/17/17. */ -public class SVNTests ***REMOVED*** +public class SVNTests { - public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED*** + public static void main(String[] args) throws SVNException, IOException, InterruptedException { SVNUpdater svnUpdater = new SVNUpdaterStaging(); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); - ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); + Algorithm algo = AlgorithmBuilder.create("http://data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); + //algo.setClazz(algo.getClazz() + "TEST"); algo.setAlgorithmType("transducerers"); algo.setCategory("NLP"); algo.setFullname("Ondřej Košarko"); - ***REMOVED***System.out.println(algo.getCategory()); - ***REMOVED***System.out.println(algo.getSkipJava()); + //System.out.println(algo.getCategory()); + //System.out.println(algo.getSkipJava()); - ***REMOVED***svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); + //svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); - ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); - ***REMOVED***svnUpdater.readRPRotoDeps(algo); - ***REMOVED*** -***REMOVED*** + //svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + //svnUpdater.readRPRotoDeps(algo); + } +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java index 15d94b3..ebf23e4 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java @@ -1,38 +1,38 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; -public class SendMailTest ***REMOVED*** +public class SendMailTest { - public static void main(String[] args) ***REMOVED*** - try ***REMOVED*** + public static void main(String[] args) { + try { - ***REMOVED*** NotificationHelper nh = new NotificationHelper(); + // NotificationHelper nh = new NotificationHelper(); SendMail sm = new SendMail(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); SecurityTokenProvider.instance.set("xxxx-xxxx-xxx"); - ***REMOVED*** System.out.println(sm.getRootToken()); - ***REMOVED*** sm.getGenericResourceByName(""); + // System.out.println(sm.getRootToken()); + // sm.getGenericResourceByName(""); - ***REMOVED***ScopeProvider.instance.set("/gcube/devsec/devVRE"); - ***REMOVED***SecurityTokenProvider.instance.set("xxxx-xxxx-xxxx"); + //ScopeProvider.instance.set("/gcube/devsec/devVRE"); + //SecurityTokenProvider.instance.set("xxxx-xxxx-xxxx"); - ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test - ***REMOVED*** failed")); - ***REMOVED*** sm.username(SecurityTokenProvider.instance.get()); - ***REMOVED*** sm.retrieveAdminRole(); - ***REMOVED*** sm.getAdminRoles(); + // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test + // failed")); + // sm.username(SecurityTokenProvider.instance.get()); + // sm.retrieveAdminRole(); + // sm.getAdminRoles(); sm.sendNotification("Test DMPoolManager Notification", "This is only a test please cancel this message."); - ***REMOVED*** System.out.println(sm.getAdmins()); - ***REMOVED*** sm.sendNotification("test", "test"); - ***REMOVED*** System.out.println(sm.getSocialService()); - ***REMOVED*** catch (Exception e) ***REMOVED*** + // System.out.println(sm.getAdmins()); + // sm.sendNotification("test", "test"); + // System.out.println(sm.getSocialService()); + } catch (Exception e) { System.out.println("Error in sent mail: "+e.getLocalizedMessage()); e.printStackTrace(); - ***REMOVED*** -***REMOVED*** -***REMOVED*** + } + } +}