From 1d630bc331e7ec3814a48646049ee83d249b9e1a Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Wed, 18 Jan 2023 17:33:11 +0100 Subject: [PATCH] Removed sensitive data --- .settings/org.eclipse.core.resources.prefs | 8 +- CHANGELOG.md | 2 +- FUNDING.md | 38 +-- README.md | 14 +- descriptor.xml | 10 +- gcube-app.xml | 8 +- pom.xml | 18 +- profile.xml | 18 +- .../poolmanager/ansible/AnsibleWorker.java | 60 ++-- .../ansible/model/AnsibleHost.java | 8 +- .../poolmanager/ansible/model/HostGroup.java | 10 +- .../poolmanager/ansible/model/Inventory.java | 16 +- .../poolmanager/ansible/model/Playbook.java | 22 +- .../poolmanager/ansible/model/Role.java | 18 +- .../poolmanager/ansible/model/RoleFile.java | 18 +- .../ansiblebridge/AnsibleBridge.java | 316 +++++++++--------- .../ansiblebridge/AnsibleSerializeHelper.java | 54 +-- .../template/AlgorithmPackage.java | 40 +-- .../template/CranDependencyPackage.java | 4 +- .../template/CustomDependencyPackage.java | 54 +-- .../template/CustomRoleManager.java | 14 +- .../template/DependencyPackage.java | 34 +- .../template/OSDependencyPackage.java | 4 +- .../template/StaticRoleManager.java | 16 +- .../template/TemplateManager.java | 60 ++-- .../poolmanager/clients/HAProxy.java | 170 ++++++++++ .../poolmanager/clients/ISClient.java | 216 ++++++------ .../poolmanager/clients/ScopedCacheMap.java | 4 +- .../ClientConfigurationCache.java | 34 +- .../clients/configuration/Configuration.java | 2 +- .../configuration/ConfigurationImpl.java | 74 ++-- .../DMPMClientConfiguratorManager.java | 46 +-- .../clients/configuration/SVNRepository.java | 18 +- .../configurations/AbstractConfiguration.java | 94 +++--- .../clients/configurations/Prod.java | 6 +- .../clients/configurations/Stage.java | 6 +- .../poolmanager/datamodel/Action.java | 14 +- .../poolmanager/datamodel/AlgoResource.java | 10 +- .../poolmanager/datamodel/Algorithm.java | 84 ++--- .../poolmanager/datamodel/AlgorithmSet.java | 22 +- .../poolmanager/datamodel/Cluster.java | 24 +- .../poolmanager/datamodel/Dependency.java | 12 +- .../poolmanager/datamodel/Domain.java | 6 +- .../dataminer/poolmanager/datamodel/Host.java | 24 +- .../comparator/AlgorithmComparator.java | 4 +- .../comparator/DependencyComparator.java | 4 +- .../datamodel/comparator/HostComparator.java | 6 +- .../process/AddAlgorithmCommand.java | 30 +- .../process/AlgorithmPackageParser.java | 270 +++++++-------- .../poolmanager/rest/PoolManager.java | 56 ++-- .../poolmanager/rest/RestPoolManager.java | 136 ++++---- .../poolmanager/service/DMPMJob.java | 86 ++--- .../service/DataminerPoolManager.java | 34 +- .../service/ProductionPublishingJob.java | 20 +- .../poolmanager/service/StagingJob.java | 14 +- .../service/exceptions/AnsibleException.java | 6 +- .../UndefinedDependenciesException.java | 8 +- .../poolmanager/util/AlgorithmBuilder.java | 20 +- .../poolmanager/util/CheckMethod.java | 92 ++--- .../poolmanager/util/CheckPermission.java | 12 +- .../poolmanager/util/ClusterBuilder.java | 26 +- .../poolmanager/util/NotificationHelper.java | 80 ++--- .../util/PropertiesBasedProxySelector.java | 60 ++-- .../util/SVNRepositoryManager.java | 18 +- .../poolmanager/util/SVNUpdater.java | 310 ++++++++--------- .../dataminer/poolmanager/util/SendMail.java | 122 +++---- .../util/exception/AlgorithmException.java | 8 +- .../util/exception/DMPMException.java | 6 +- .../util/exception/EMailException.java | 10 +- .../util/exception/GenericException.java | 6 +- .../util/exception/SVNCommitException.java | 10 +- .../util/impl/CheckMethodProduction.java | 92 +++-- .../util/impl/CheckMethodStaging.java | 94 +++--- .../util/impl/ClusterBuilderProduction.java | 4 +- .../util/impl/ClusterBuilderStaging.java | 4 +- .../impl/NotificationHelperProduction.java | 8 +- .../util/impl/NotificationHelperStaging.java | 8 +- .../util/impl/SVNUpdaterProduction.java | 4 +- .../util/impl/SVNUpdaterStaging.java | 4 +- .../custom/custom-flr-Flash/meta/main.yaml | 2 +- src/main/resources/service.properties | 4 +- .../gcube-algorithm-installer/meta/main.yaml | 6 +- .../gcube-algorithm-installer/tasks/main.yaml | 6 +- .../static/gcube-dataminer/meta/main.yaml | 2 +- .../static/os-java-1.7.0/tasks/main.yaml | 2 +- .../resources/static/os-unzip/tasks/main.yaml | 2 +- .../templates/cran-package-add/meta/main.yaml | 2 +- .../cran-package-add/tasks/main.yaml | 2 +- .../gcube-algorithm-add/meta/main.yaml | 2 +- .../github-package-add/meta/main.yaml | 2 +- .../templates/os-package-add/tasks/main.yaml | 2 +- src/main/webapp/WEB-INF/web.xml | 6 +- .../AlgorithmPackageParserTest.java | 60 ++-- .../AnsibleWorkerTest.java | 8 +- .../ApplicationTokenTest.java | 28 +- .../DataminerPoolManagerTest.java | 140 ++++---- .../dataminerpoolmanager/ISClientTest.java | 6 +- .../dataminerpoolmanager/JobTest.java | 28 +- .../dataminerpoolmanager/SVNTests.java | 20 +- .../dataminerpoolmanager/SendMailTest.java | 38 +-- 100 files changed, 1968 insertions(+), 1802 deletions(-) create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs index a931bf8..29abf99 100755 --- a/.settings/org.eclipse.core.resources.prefs +++ b/.settings/org.eclipse.core.resources.prefs @@ -1,6 +1,6 @@ eclipse.preferences.version=1 -encoding***REMOVED***src/main/java=UTF-8 -encoding***REMOVED***src/main/resources=UTF-8 -encoding***REMOVED***src/test/java=UTF-8 -encoding***REMOVED***src/test/resources=UTF-8 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 encoding/=UTF-8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cfe8d7..ae9662a 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,4 +60,4 @@ - First Release -This project adheres to [Semantic Versioning](https:***REMOVED***semver.org/spec/v2.0.0.html). +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/FUNDING.md b/FUNDING.md index 3805322..6fa9eac 100755 --- a/FUNDING.md +++ b/FUNDING.md @@ -3,24 +3,24 @@ The projects leading to this software have received funding from a series of European Union programmes including: - the Sixth Framework Programme for Research and Technological Development - - [DILIGENT](https:***REMOVED***cordis.europa.eu/project/id/004260) (grant no. 004260). + - [DILIGENT](https://cordis.europa.eu/project/id/004260) (grant no. 004260). - the Seventh Framework Programme for research, technological development and demonstration - - [D4Science](https:***REMOVED***cordis.europa.eu/project/id/212488) (grant no. 212488); - - [D4Science-II](https:***REMOVED***cordis.europa.eu/project/id/239019) (grant no.239019); - - [ENVRI](https:***REMOVED***cordis.europa.eu/project/id/283465) (grant no. 283465); - - [iMarine](https:***REMOVED***cordis.europa.eu/project/id/283644) (grant no. 283644); - - [EUBrazilOpenBio](https:***REMOVED***cordis.europa.eu/project/id/288754) (grant no. 288754). + - [D4Science](https://cordis.europa.eu/project/id/212488) (grant no. 212488); + - [D4Science-II](https://cordis.europa.eu/project/id/239019) (grant no.239019); + - [ENVRI](https://cordis.europa.eu/project/id/283465) (grant no. 283465); + - [iMarine](https://cordis.europa.eu/project/id/283644) (grant no. 283644); + - [EUBrazilOpenBio](https://cordis.europa.eu/project/id/288754) (grant no. 288754). - the H2020 research and innovation programme - - [SoBigData](https:***REMOVED***cordis.europa.eu/project/id/654024) (grant no. 654024); - - [PARTHENOS](https:***REMOVED***cordis.europa.eu/project/id/654119) (grant no. 654119); - - [EGI-Engage](https:***REMOVED***cordis.europa.eu/project/id/654142) (grant no. 654142); - - [ENVRI PLUS](https:***REMOVED***cordis.europa.eu/project/id/654182) (grant no. 654182); - - [BlueBRIDGE](https:***REMOVED***cordis.europa.eu/project/id/675680) (grant no. 675680); - - [PerformFISH](https:***REMOVED***cordis.europa.eu/project/id/727610) (grant no. 727610); - - [AGINFRA PLUS](https:***REMOVED***cordis.europa.eu/project/id/731001) (grant no. 731001); - - [DESIRA](https:***REMOVED***cordis.europa.eu/project/id/818194) (grant no. 818194); - - [ARIADNEplus](https:***REMOVED***cordis.europa.eu/project/id/823914) (grant no. 823914); - - [RISIS 2](https:***REMOVED***cordis.europa.eu/project/id/824091) (grant no. 824091); - - [EOSC-Pillar](https:***REMOVED***cordis.europa.eu/project/id/857650) (grant no. 857650); - - [Blue Cloud](https:***REMOVED***cordis.europa.eu/project/id/862409) (grant no. 862409); - - [SoBigData-PlusPlus](https:***REMOVED***cordis.europa.eu/project/id/871042) (grant no. 871042); \ No newline at end of file + - [SoBigData](https://cordis.europa.eu/project/id/654024) (grant no. 654024); + - [PARTHENOS](https://cordis.europa.eu/project/id/654119) (grant no. 654119); + - [EGI-Engage](https://cordis.europa.eu/project/id/654142) (grant no. 654142); + - [ENVRI PLUS](https://cordis.europa.eu/project/id/654182) (grant no. 654182); + - [BlueBRIDGE](https://cordis.europa.eu/project/id/675680) (grant no. 675680); + - [PerformFISH](https://cordis.europa.eu/project/id/727610) (grant no. 727610); + - [AGINFRA PLUS](https://cordis.europa.eu/project/id/731001) (grant no. 731001); + - [DESIRA](https://cordis.europa.eu/project/id/818194) (grant no. 818194); + - [ARIADNEplus](https://cordis.europa.eu/project/id/823914) (grant no. 823914); + - [RISIS 2](https://cordis.europa.eu/project/id/824091) (grant no. 824091); + - [EOSC-Pillar](https://cordis.europa.eu/project/id/857650) (grant no. 857650); + - [Blue Cloud](https://cordis.europa.eu/project/id/862409) (grant no. 862409); + - [SoBigData-PlusPlus](https://cordis.europa.eu/project/id/871042) (grant no. 871042); \ No newline at end of file diff --git a/README.md b/README.md index f9e9ce2..3be2c89 100755 --- a/README.md +++ b/README.md @@ -8,12 +8,12 @@ DataMiner Pool Manager is a service to support the integration of algorithms in ## Built With -* [OpenJDK](https:***REMOVED***openjdk.java.net/) - The JDK used -* [Maven](https:***REMOVED***maven.apache.org/) - Dependency Management +* [OpenJDK](https://openjdk.java.net/) - The JDK used +* [Maven](https://maven.apache.org/) - Dependency Management ## Documentation -* Use of this widget is is described on [Wiki](https:***REMOVED***wiki.gcube-system.org/gcube/How_to_use_the_DataMiner_Pool_Manager). +* Use of this widget is is described on [Wiki](https://wiki.gcube-system.org/gcube/How_to_use_the_DataMiner_Pool_Manager). ## Change log @@ -21,9 +21,9 @@ See [CHANGELOG.md](CHANGELOG.md). ## Authors -* **Paolo Fabriani** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) -* **Nunzio Andrea Galante** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) -* **Ciro Formisano** - [Engineering Ingegneria Informatica S.p.A., Italy](https:***REMOVED***www.eng.it/) +* **Paolo Fabriani** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) +* **Nunzio Andrea Galante** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) +* **Ciro Formisano** - [Engineering Ingegneria Informatica S.p.A., Italy](https://www.eng.it/) ## License @@ -32,7 +32,7 @@ This project is licensed under the EUPL V.1.1 License - see the [LICENSE.md](LIC ## About the gCube Framework -This software is part of the [gCubeFramework](https:***REMOVED***www.gcube-system.org/ "gCubeFramework"): an +This software is part of the [gCubeFramework](https://www.gcube-system.org/ "gCubeFramework"): an open-source software toolkit used for building and operating Hybrid Data Infrastructures enabling the dynamic deployment of Virtual Research Environments by favouring the realisation of reuse oriented policies. diff --git a/descriptor.xml b/descriptor.xml index cb0690e..8af47b2 100755 --- a/descriptor.xml +++ b/descriptor.xml @@ -1,7 +1,7 @@ + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> servicearchive tar.gz @@ -24,8 +24,8 @@ - target/${build.finalName***REMOVED***.${project.packaging***REMOVED*** - /${artifactId***REMOVED*** + target/${build.finalName}.${project.packaging} + /${artifactId} \ No newline at end of file diff --git a/gcube-app.xml b/gcube-app.xml index 41b7ee5..4979186 100755 --- a/gcube-app.xml +++ b/gcube-app.xml @@ -1,9 +1,9 @@ - ${serviceName***REMOVED*** - ${serviceClass***REMOVED*** - ${version***REMOVED*** - ${description***REMOVED*** + ${serviceName} + ${serviceClass} + ${version} + ${description} /api/swagger.* \ No newline at end of file diff --git a/pom.xml b/pom.xml index 787a2f1..60427d3 100755 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ - + 4.0.0 @@ -18,9 +18,9 @@ DataMiner Pool Manager is a service to support the integration of algorithms in D4Science Infrastructure - scm:git:https:***REMOVED***code-repo.d4science.org/gCubeSystem/${project.artifactId***REMOVED***.git - scm:git:https:***REMOVED***code-repo.d4science.org/gCubeSystem/${project.artifactId***REMOVED***.git - https:***REMOVED***code-repo.d4science.org/gCubeSystem/${project.artifactId***REMOVED*** + scm:git:https://code-repo.d4science.org/gCubeSystem/${project.artifactId}.git + scm:git:https://code-repo.d4science.org/gCubeSystem/${project.artifactId}.git + https://code-repo.d4science.org/gCubeSystem/${project.artifactId} @@ -28,7 +28,7 @@ dataminer-pool-manager DataAnalysis - ${project.basedir***REMOVED***/src/main/webapp/WEB-INF + ${project.basedir}/src/main/webapp/WEB-INF distro UTF-8 1.5.13 @@ -103,7 +103,7 @@ 1.10 - + com.jcraft jsch @@ -172,10 +172,10 @@ process-resources - ${webappDirectory***REMOVED*** + ${webappDirectory} - ${project.basedir***REMOVED*** + ${project.basedir} true src diff --git a/profile.xml b/profile.xml index be1a45f..270b2eb 100755 --- a/profile.xml +++ b/profile.xml @@ -1,23 +1,23 @@ - + Service - ${description***REMOVED*** + ${description} DataminerPoolManager - ${artifactId***REMOVED*** + ${artifactId} 1.0.0 - ${artifactId***REMOVED*** - ${version***REMOVED*** + ${artifactId} + ${version} - ${groupId***REMOVED*** - ${artifactId***REMOVED*** - ${version***REMOVED*** + ${groupId} + ${artifactId} + ${version} - ${build.finalName***REMOVED***.jar + ${build.finalName}.jar diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 885736b..b840ec6 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.util.Scanner; @@ -45,53 +45,53 @@ public class AnsibleWorker { public AnsibleWorker(File root) { this.workerRoot = root; this.ensureWorkStructure(); - ***REMOVED*** + } -***REMOVED*** public File getWorkdir() { -***REMOVED*** return this.workerRoot; -***REMOVED*** ***REMOVED*** +// public File getWorkdir() { +// return this.workerRoot; +// } public File getRolesDir() { return new File(this.workerRoot, ROLES_DIR); - ***REMOVED*** + } public String getWorkerId() { return this.workerRoot.getName(); - ***REMOVED*** + } public void ensureWorkStructure() { - ***REMOVED*** generate root + // generate root this.workerRoot.mkdirs(); - ***REMOVED*** + } public void removeWorkStructure() { - ***REMOVED*** remove the working dir + // remove the working dir this.workerRoot.delete(); - ***REMOVED*** + } public File getPlaybookFile() { return new File(this.workerRoot, PLAYBOOK_NAME); - ***REMOVED*** + } public File getInventoryFile() { return new File(this.workerRoot, INVENTORY_NAME); - ***REMOVED*** + } public void setInventory(Inventory inventory) throws IOException { - ***REMOVED*** serialize the string to the 'inventory' file + // serialize the string to the 'inventory' file AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); - ***REMOVED*** + } public void setPlaybook(Playbook playbook) throws IOException { - ***REMOVED*** serialize the string to the 'playbook' file + // serialize the string to the 'playbook' file AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); - ***REMOVED*** + } public void addRole(Role r) throws IOException { - ***REMOVED*** Serialize role in the workdir + // Serialize role in the workdir AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); - ***REMOVED*** + } public int execute(PrintStream ps) @@ -105,17 +105,17 @@ public class AnsibleWorker { inheritIO(p.getInputStream(), ps); inheritIO(p.getErrorStream(), ps); -***REMOVED*** writer.println(this.getStatus(p.waitFor())); -***REMOVED*** writer.close(); +// writer.println(this.getStatus(p.waitFor())); +// writer.close(); return p.waitFor(); - ***REMOVED*** catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - ***REMOVED*** + } return -1; -***REMOVED*** + } private static void inheritIO(final InputStream src, final PrintStream dest) { new Thread(new Runnable() { @@ -123,11 +123,11 @@ public class AnsibleWorker { Scanner sc = new Scanner(src); while (sc.hasNextLine()) { dest.println(sc.nextLine()); - ***REMOVED*** + } sc.close(); - ***REMOVED*** - ***REMOVED***).start(); -***REMOVED*** + } + }).start(); + } /** * Destroy the worker: @@ -135,6 +135,6 @@ public class AnsibleWorker { */ public void destroy() { this.removeWorkStructure(); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java index 5b318f3..9ed34b3 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java @@ -6,14 +6,14 @@ public class AnsibleHost { public AnsibleHost(String name) { this.name = name; - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java index c1b5852..077e1a1 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java @@ -12,18 +12,18 @@ public class HostGroup { public HostGroup(String name) { this.name = name; this.hosts = new Vector<>(); - ***REMOVED*** + } public void addHost(AnsibleHost h) { this.hosts.add(h); - ***REMOVED*** + } public String getName() { return this.name; - ***REMOVED*** + } public Collection getHosts() { return new Vector<>(this.hosts); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java index 351bc22..4d6400d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java @@ -9,29 +9,29 @@ public class Inventory { public Inventory() { this.groups = new Vector<>(); - ***REMOVED*** + } public void addGroup(HostGroup group) { this.groups.add(group); - ***REMOVED*** + } public void addHost(AnsibleHost h, String groupName) { this.getGroup(groupName).addHost(h); - ***REMOVED*** + } private HostGroup getGroup(String groupName) { for (HostGroup hg : this.groups) { if (groupName.equals(hg.getName())) { return hg; - ***REMOVED*** - ***REMOVED*** + } + } HostGroup hg = new HostGroup(groupName); this.groups.add(hg); return hg; - ***REMOVED*** + } public Collection getHostGroups() { return new Vector<>(this.groups); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java index d25ce80..c08c7e2 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; -***REMOVED*** +import java.util.List; import java.util.Vector; public class Playbook { @@ -13,38 +13,38 @@ public class Playbook { public Playbook() { this.roles = new Vector<>(); - ***REMOVED*** + } public void addRole(String role) { roles.add(role); - ***REMOVED*** + } public void applyTo(String hostGroupName) { this.hostGroupName = hostGroupName; - ***REMOVED*** + } public String getHostGroupName() { return hostGroupName; - ***REMOVED*** + } public List getRoles() { return new Vector<>(roles); - ***REMOVED*** + } public String getRemote_user() { return remote_user; -***REMOVED*** +} public void setRemote_user(String remote_user) { this.remote_user = remote_user; -***REMOVED*** +} public void setHostGroupName(String hostGroupName) { this.hostGroupName = hostGroupName; -***REMOVED*** +} public void setRoles(List roles) { this.roles = roles; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java index 1a4c899..3f3aa33 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java @@ -17,35 +17,35 @@ public class Role { public Role() { this.tasks = new Vector<>(); this.meta = new Vector<>(); - ***REMOVED*** + } public Role(String name) { this(); this.name = name; - ***REMOVED*** + } public void addTaskFile(RoleFile tf) { this.tasks.add(tf); - ***REMOVED*** + } public void addMeta(RoleFile tf) { this.meta.add(tf); - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public Collection getTaskFiles() { return new Vector<>(this.tasks); - ***REMOVED*** + } public Collection getMeta() { return new Vector<>(this.meta); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java index 963a46d..3cff918 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java @@ -19,36 +19,36 @@ public class RoleFile { private String content; public RoleFile() { - ***REMOVED*** + } public RoleFile(String name, String content) { this(); this.setName(name); this.setContent(content); - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public String getContent() { return content; - ***REMOVED*** + } public void setContent(String content) { this.content = content; - ***REMOVED*** + } public String getPath() { return path; - ***REMOVED*** + } public void setPath(String path) { this.path = path; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 44bf2df..e2b5d23 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -1,9 +1,9 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Collection; -***REMOVED*** +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -25,57 +25,57 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.Stati import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; public class AnsibleBridge { - ***REMOVED*** private static final org.slf4j.Logger LOGGER = - ***REMOVED*** LoggerFactory.getLogger(AnsibleBridge.class); + // private static final org.slf4j.Logger LOGGER = + // LoggerFactory.getLogger(AnsibleBridge.class); private String dpmRoot; - ***REMOVED*** public AnsibleBridge() { - ***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); - ***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** + // public AnsibleBridge() { + // this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); + // //this(System.getProperty("/home/gcube/dataminer-pool-manager")); + // + // } public AnsibleBridge(String root) { this.dpmRoot = root; this.ensureServiceRoot(); -***REMOVED*** + } private void ensureServiceRoot() { - ***REMOVED*** generate root + // generate root new File(dpmRoot).mkdirs(); - ***REMOVED*** 'template' is for template roles - ***REMOVED*** this.getTemplatesDir().mkdirs(); - ***REMOVED*** 'static' is for custom roles - ***REMOVED*** this.getCustomDir().mkdirs(); - ***REMOVED*** 'work' is for temporary working directories + // 'template' is for template roles + // this.getTemplatesDir().mkdirs(); + // 'static' is for custom roles + // this.getCustomDir().mkdirs(); + // 'work' is for temporary working directories this.getWorkDir().mkdirs(); -***REMOVED*** + } private File getWorkDir() { return new File(this.dpmRoot, "work"); -***REMOVED*** + } - ***REMOVED*** private String getTemplatesDir() { - ***REMOVED*** String input = null; - ***REMOVED*** input = - ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); - ***REMOVED*** return input; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** private String getCustomDir() { - ***REMOVED*** String input = null; - ***REMOVED*** input = - ***REMOVED*** AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); - ***REMOVED*** return input; - ***REMOVED*** ***REMOVED*** + // private String getTemplatesDir() { + // String input = null; + // input = + // AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); + // return input; + // } + // + // + // private String getCustomDir() { + // String input = null; + // input = + // AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); + // return input; + // } public AnsibleWorker createWorker(Algorithm algorithm, Cluster dataminerCluster, boolean includeAlgorithmDependencies, String user) throws IOException { @@ -84,50 +84,50 @@ public class AnsibleBridge { List algoRoles = new Vector<>(); - ***REMOVED*** add algorithms and dependencies to the worker + // add algorithms and dependencies to the worker for (Role r : this.generateRoles(algorithm, includeAlgorithmDependencies)) { algoRoles.add(r); worker.addRole(r); - ***REMOVED*** + } - ***REMOVED*** to comment the for in case of just install algo + // to comment the for in case of just install algo if (includeAlgorithmDependencies) { for (Dependency d : algorithm.getDependencies()) { for (Role r : this.generateRoles(d)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } - ***REMOVED*** add static roles + // add static roles for (Role r : this.getStaticRoleManager().getStaticRoles()) { worker.addRole(r); - ***REMOVED*** + } - ***REMOVED*** generate the inventory + // generate the inventory Inventory inventory = new Inventory(); for (Host h : dataminerCluster.getHosts()) { AnsibleHost ah = new AnsibleHost(h.getName()); inventory.addHost(ah, "universe"); inventory.addHost(ah, "d4science"); - ***REMOVED*** + } worker.setInventory(inventory); - ***REMOVED*** generate the playbook + // generate the playbook Playbook playbook = new Playbook(); playbook.setRemote_user(user); playbook.applyTo("universe"); for (Role r : algoRoles) { - ***REMOVED*** add only 'add' roles + // add only 'add' roles if (!r.getName().endsWith("remove")) { playbook.addRole(r.getName()); - ***REMOVED*** - ***REMOVED*** + } + } worker.setPlaybook(playbook); return worker; -***REMOVED*** + } public void printInventoryByDomainAndSets(Collection clusters) { Map> inventory = new TreeMap<>(); @@ -141,21 +141,21 @@ public class AnsibleBridge { if (hosts == null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); - ***REMOVED*** + } hosts.add(h); - ***REMOVED*** + } - ***REMOVED*** - ***REMOVED*** + } + } for (String key : inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); for (Host h : hosts) { System.out.println(h.getName() + "." + h.getDomain().getName()); - ***REMOVED*** + } System.out.println(); - ***REMOVED*** -***REMOVED*** + } + } public void printInventoryBySets(Collection clusters) { Map> inventory = new TreeMap<>(); @@ -168,116 +168,116 @@ public class AnsibleBridge { if (hosts == null) { hosts = new TreeSet<>(new HostComparator()); inventory.put(key, hosts); - ***REMOVED*** + } hosts.add(h); - ***REMOVED*** + } - ***REMOVED*** - ***REMOVED*** + } + } for (String key : inventory.keySet()) { System.out.println(key); Collection hosts = inventory.get(key); for (Host h : hosts) { System.out.println(h.getName() + "." + h.getDomain().getName()); - ***REMOVED*** + } System.out.println(); - ***REMOVED*** -***REMOVED*** + } + } - ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster - ***REMOVED*** cluster, /*boolean updateSVN,*/ boolean test) throws IOException, - ***REMOVED*** InterruptedException, SVNException { - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** return applyAlgorithmSetToCluster - ***REMOVED*** (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); - ***REMOVED*** ***REMOVED*** + // public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster + // cluster, /*boolean updateSVN,*/ boolean test) throws IOException, + // InterruptedException, SVNException { + // + // + // return applyAlgorithmSetToCluster + // (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); + // } - ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster - ***REMOVED*** cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws - ***REMOVED*** IOException, InterruptedException, SVNException { - ***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), - ***REMOVED*** uuid)); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** List algoRoles = new Vector<>(); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker - ***REMOVED*** for (Algorithm a : as.getAlgorithms()) { - ***REMOVED*** for (Role r : this.generateRoles(a)) { - ***REMOVED*** algoRoles.add(r); - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED***to comment the for in case of just install algo - ***REMOVED*** if(test){ - ***REMOVED*** for (Dependency d : a.getDependencies()) { - ***REMOVED*** for (Role r : this.generateRoles(d)) { - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** add static roles - ***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) { - ***REMOVED*** worker.addRole(r); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** generate the inventory - ***REMOVED*** Inventory inventory = new Inventory(); - ***REMOVED*** for (Host h : cluster.getHosts()) { - ***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName()); - ***REMOVED*** inventory.addHost(ah, "universe"); - ***REMOVED*** inventory.addHost(ah, "d4science"); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** worker.setInventory(inventory); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** generate the playbook - ***REMOVED*** Playbook playbook = new Playbook(); - ***REMOVED*** if(test){ - ***REMOVED*** playbook.setRemote_user("root");***REMOVED*** - ***REMOVED*** playbook.setRemote_user("gcube"); - ***REMOVED*** playbook.applyTo("universe"); - ***REMOVED*** for(Role r:algoRoles) { - ***REMOVED*** ***REMOVED*** add only 'add' roles - ***REMOVED*** if(!r.getName().endsWith("remove")) { - ***REMOVED*** playbook.addRole(r.getName()); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** worker.setPlaybook(playbook); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** execute and save log locally - ***REMOVED*** ***REMOVED***PrintStream console = System.out; - ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); - ***REMOVED*** path.mkdirs(); - ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); - ***REMOVED*** FileOutputStream fos = new FileOutputStream(n); - ***REMOVED*** PrintStream ps = new PrintStream(fos); - ***REMOVED*** - ***REMOVED*** ***REMOVED***System.setErr(console); - ***REMOVED*** - ***REMOVED*** worker.apply(as,ps,test); - ***REMOVED*** ***REMOVED***System.setOut(console); - ***REMOVED*** ***REMOVED***worker.apply(); - ***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); - ***REMOVED*** - ***REMOVED*** ***REMOVED*** destroy the worker - ***REMOVED*** worker.destroy(); - ***REMOVED*** return worker; - ***REMOVED*** ***REMOVED*** + // public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster + // cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws + // IOException, InterruptedException, SVNException { + // AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), + // uuid)); + // + // + // List algoRoles = new Vector<>(); + // + // // add algorithms and dependencies to the worker + // for (Algorithm a : as.getAlgorithms()) { + // for (Role r : this.generateRoles(a)) { + // algoRoles.add(r); + // worker.addRole(r); + // } + // //to comment the for in case of just install algo + // if(test){ + // for (Dependency d : a.getDependencies()) { + // for (Role r : this.generateRoles(d)) { + // worker.addRole(r); + // } + // } + // } + // } + // + // // add static roles + // for(Role r:this.getStaticRoleManager().getStaticRoles()) { + // worker.addRole(r); + // } + // + // // generate the inventory + // Inventory inventory = new Inventory(); + // for (Host h : cluster.getHosts()) { + // AnsibleHost ah = new AnsibleHost(h.getName()); + // inventory.addHost(ah, "universe"); + // inventory.addHost(ah, "d4science"); + // } + // worker.setInventory(inventory); + // + // // generate the playbook + // Playbook playbook = new Playbook(); + // if(test){ + // playbook.setRemote_user("root");} + // playbook.setRemote_user("gcube"); + // playbook.applyTo("universe"); + // for(Role r:algoRoles) { + // // add only 'add' roles + // if(!r.getName().endsWith("remove")) { + // playbook.addRole(r.getName()); + // } + // } + // + // worker.setPlaybook(playbook); + // + // // execute and save log locally + // //PrintStream console = System.out; + // File path = new File(worker.getWorkdir() + File.separator + "logs"); + // path.mkdirs(); + // File n = new File(path + File.separator + worker.getWorkerId()); + // FileOutputStream fos = new FileOutputStream(n); + // PrintStream ps = new PrintStream(fos); + // + // //System.setErr(console); + // + // worker.apply(as,ps,test); + // //System.setOut(console); + // //worker.apply(); + // System.out.println("Log stored to to " + n.getAbsolutePath()); + // + // // destroy the worker + // worker.destroy(); + // return worker; + // } private TemplateManager getTemplateManager() { return new TemplateManager(); -***REMOVED*** + } private CustomRoleManager getCustomRoleManager() { return new CustomRoleManager(); -***REMOVED*** + } private StaticRoleManager getStaticRoleManager() { return new StaticRoleManager(); -***REMOVED*** + } /** * Generate all roles for this dependency @@ -293,32 +293,32 @@ public class AnsibleBridge { OSDependencyPackage pkg = new OSDependencyPackage(dep); if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** + } - ***REMOVED*** else if ("custom".equalsIgnoreCase(dep.getType())) { + } else if ("custom".equalsIgnoreCase(dep.getType())) { CustomDependencyPackage pkg = new CustomDependencyPackage(dep); if (pkg != null) { roles.addAll(pkg.getRoles(this.getCustomRoleManager())); - ***REMOVED*** - ***REMOVED*** else if ("github".equalsIgnoreCase(dep.getType())) { + } + } else if ("github".equalsIgnoreCase(dep.getType())) { CranDependencyPackage pkg = new CranDependencyPackage(dep); if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - ***REMOVED*** else if ("cran".equalsIgnoreCase(dep.getType())) { + } + } else if ("cran".equalsIgnoreCase(dep.getType())) { CranDependencyPackage pkg = new CranDependencyPackage(dep); if (pkg != null) { roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** + } - ***REMOVED*** + } return roles; -***REMOVED*** + } public Collection generateRoles(Algorithm a, boolean includeAlgorithmDependencies) { AlgorithmPackage pkg = new AlgorithmPackage(a, includeAlgorithmDependencies); return pkg.getRoles(this.getTemplateManager()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java index 48d9197..fa0496c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.io.PrintWriter; import org.apache.commons.io.IOUtils; @@ -22,12 +22,12 @@ public class AnsibleSerializeHelper { out+=String.format("[%s]\n", hg.getName()); for(AnsibleHost h:hg.getHosts()) { out+=h.getName()+"\n"; - ***REMOVED*** + } out+="\n"; - ***REMOVED*** + } out = out.trim(); serialize(out, inventoryFile); - ***REMOVED*** + } public static void serialize(Playbook playbook, File playbookFile) throws IOException { String out = "- hosts: " + playbook.getHostGroupName() + "\n"; @@ -35,47 +35,47 @@ public class AnsibleSerializeHelper { out+=" roles:\n"; for(String r:playbook.getRoles()) { out+=" - " + r+"\n"; - ***REMOVED*** + } out+=" vars:\n"; out+=" os_package_state: present\n"; out = out.trim(); serialize(out, playbookFile); - ***REMOVED*** + } public static void serializeRole(Role r, File dir) throws IOException { - ***REMOVED*** create root + // create root File root = new File(dir, r.getName()); root.mkdirs(); - ***REMOVED*** create tasks + // create tasks if(r.getTaskFiles().size()>0) { File tasks = new File(root, "tasks"); tasks.mkdirs(); for(RoleFile tf: r.getTaskFiles()) { serializeTask(tf, tasks); - ***REMOVED*** - ***REMOVED*** + } + } - ***REMOVED*** create meta + // create meta if(r.getMeta().size()>0) { File meta = new File(root, "meta"); meta.mkdirs(); for(RoleFile tf: r.getMeta()) { serializeTask(tf, meta); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } public static void serializeTask(RoleFile tf, File dir) throws IOException { File f = new File(dir, tf.getName()); serialize(tf.getContent().trim(), f); - ***REMOVED*** + } public static void serialize(String s, File f) throws IOException { PrintWriter out = new PrintWriter(f); out.println(s); out.close(); - ***REMOVED*** + } public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException { Role out = new Role(); @@ -83,7 +83,7 @@ public class AnsibleSerializeHelper { if(!roleDir.exists()) { throw new FileNotFoundException(); - ***REMOVED*** + } try { File tasksDir = new File(roleDir, "tasks"); @@ -93,11 +93,11 @@ public class AnsibleSerializeHelper { RoleFile tf = new RoleFile(main.getName(), content); tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); out.addTaskFile(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) { + } + } + } catch(FileNotFoundException e) { e.printStackTrace(); - ***REMOVED*** + } try { File metaDir = new File(roleDir, "meta"); @@ -107,13 +107,13 @@ public class AnsibleSerializeHelper { RoleFile tf = new RoleFile(main.getName(), content); tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); out.addMeta(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) { + } + } + } catch(FileNotFoundException e) { e.printStackTrace(); - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java index d84f27c..35c5575 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java @@ -6,12 +6,12 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Vector; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AlgorithmPackage { @@ -25,7 +25,7 @@ public class AlgorithmPackage { this.algorithm = a; this.includeAlgorithmDependencies = includeAlgorithmDependencies; - ***REMOVED*** + } protected Map getDictionary(Algorithm a) { @@ -36,47 +36,47 @@ public class AlgorithmPackage { out.put("atype", a.getAlgorithmType()); out.put("skipjava", a.getSkipJava()); out.put("vre", ScopeProvider.instance.get()); - ***REMOVED***out.put("vre", "FAKE_VRE"); + //out.put("vre", "FAKE_VRE"); out.put("packageurl", a.getPackageURL()); out.put("description", a.getDescription()); String deps = ""; if(includeAlgorithmDependencies){ for(Dependency d:a.getDependencies()) { - deps+=String.format("- { role: %s ***REMOVED***\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); - ***REMOVED******REMOVED*** + deps+=String.format("- { role: %s }\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); + }} deps = deps.trim(); out.put("dependencies", deps); return out; - ***REMOVED*** + } protected Algorithm getAlgorithm() { return this.algorithm; - ***REMOVED*** + } public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" + for(String mode:new String[]{"add"}) { // "remove", "update" String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); try { - ***REMOVED*** find template + // find template Role template = tm.getRoleTemplate("gcube-algorithm-" + mode); - ***REMOVED*** + // if(template!=null) { Map dictionary = this.getDictionary(this.getAlgorithm()); Role r = tm.fillRoleTemplate(template, dictionary); r.setName(roleName); out.add(r); - ***REMOVED*** else + } else { this.logger.warn("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java index 48e4c64..baa4910 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java @@ -6,6 +6,6 @@ public class CranDependencyPackage extends DependencyPackage { public CranDependencyPackage(Dependency d) { super(d); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java index 3fe1c36..ae51255 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java @@ -6,64 +6,64 @@ import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** +import org.slf4j.Logger; public class CustomDependencyPackage extends DependencyPackage { -***REMOVED*** + private Logger logger; public CustomDependencyPackage(Dependency dependency) { super(dependency); - ***REMOVED*** + } -***REMOVED*** private String getCustomRepositoryLocation(String ansibleRoot) { -***REMOVED*** return ansibleRoot+"/custom"; -***REMOVED*** ***REMOVED*** +// private String getCustomRepositoryLocation(String ansibleRoot) { +// return ansibleRoot+"/custom"; +// } /* public void serializeTo(String ansibleRoot) { - for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { - ***REMOVED*** look for roles in the 'custom' repository + for(String mode:new String[]{"add", "remove", "update"}) { + // look for roles in the 'custom' repository try { - ***REMOVED*** role name + // role name String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - ***REMOVED*** look for the custom role + // look for the custom role File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); System.out.println("** CUSTOM ** " + src); if(src.exists()) { - ***REMOVED*** do copy + // do copy System.out.println("copying CUSTOM role"); File dest = new File(ansibleRoot+"/work/"+roleName); FileUtils.copyDirectory(src, dest); - ***REMOVED*** - ***REMOVED*** catch(IOException e) { + } + } catch(IOException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } */ public Collection getRoles(CustomRoleManager crm) { Collection out = new Vector<>(); -***REMOVED*** for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" - ***REMOVED*** role name +// for(String mode:new String[]{"add", "remove", "update"}) { + for(String mode:new String[]{"add"}) { // "remove", "update" + // role name String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); try { - ***REMOVED*** look for custom role + // look for custom role Role role = crm.getRole(roleName); if(role!=null) { out.add(role); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no custom role found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java index 80490ad..b49851c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.NoSuchElementException; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; @@ -16,16 +16,16 @@ public class CustomRoleManager { String input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); return input; - ***REMOVED*** + } public Role getRole(String roleName) throws NoSuchElementException { File f = new File(this.getRoot(), roleName); try { return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) { -***REMOVED*** e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java index 6bf00bf..b5b44fd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java @@ -8,53 +8,53 @@ import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DependencyPackage { -***REMOVED*** + private Logger logger; private Dependency dependency; public DependencyPackage(Dependency dependency) { this.dependency = dependency; this.logger = LoggerFactory.getLogger(DependencyPackage.class); - ***REMOVED*** + } protected Map getDictionary(Dependency d) { Map out = new HashMap(); out.put("name", d.getName()); out.put("type", d.getType()); return out; - ***REMOVED*** + } protected Dependency getDependency() { return this.dependency; - ***REMOVED*** + } public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" + for(String mode:new String[]{"add"}) { // "remove", "update" String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode); try { - ***REMOVED*** find template + // find template Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); - ***REMOVED*** + // if(template!=null) { Map dictionary = this.getDictionary(this.getDependency()); Role r = tm.fillRoleTemplate(template, dictionary); r.setName(roleName); out.add(r); - ***REMOVED*** else { + } else { this.logger.warn("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); + } + } catch (NoSuchElementException e) { +// e.printStackTrace(); this.logger.warn("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java index e816641..61648f0 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java @@ -6,6 +6,6 @@ public class OSDependencyPackage extends DependencyPackage { public OSDependencyPackage(Dependency dependency) { super(dependency); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java index 6f88500..98f377e 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Collection; import java.util.Vector; @@ -14,25 +14,25 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSeriali public StaticRoleManager() { - ***REMOVED*** + } public String getRoot() { String input = AnsibleBridge.class.getClassLoader().getResource("static").getPath(); return input; - ***REMOVED*** + } public Collection getStaticRoles() { Collection out = new Vector<>(); for(File f: new File(this.getRoot()).listFiles()) { try { out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); - ***REMOVED*** catch(IOException e) { + } catch(IOException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } - ***REMOVED*** + } diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java index 257c01b..855c83c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.Map; import java.util.NoSuchElementException; @@ -15,71 +15,71 @@ public class TemplateManager { public TemplateManager() { - ***REMOVED*** + } public String getTemplateRoot() { String input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); return input; - ***REMOVED*** + } -***REMOVED*** private String readTemplate(String templateName) throws IOException { -***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); -***REMOVED*** System.out.println("looking for file " + templateFile.getName()); -***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** +// private String readTemplate(String templateName) throws IOException { +// File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); +// System.out.println("looking for file " + templateFile.getName()); +// String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); +// return out; +// } -***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException { -***REMOVED*** String template = null; -***REMOVED*** try { -***REMOVED*** template = this.readTemplate(templateName); -***REMOVED*** ***REMOVED*** catch (IOException e) { -***REMOVED*** throw new NoSuchElementException(); -***REMOVED*** ***REMOVED*** -***REMOVED*** return template; -***REMOVED*** ***REMOVED*** +// public String getTemplate(String templateName) throws NoSuchElementException { +// String template = null; +// try { +// template = this.readTemplate(templateName); +// } catch (IOException e) { +// throw new NoSuchElementException(); +// } +// return template; +// } public Role fillRoleTemplate(Role template, Map dictionary) { Role out = new Role(); out.setName(template.getName()); for(RoleFile tf:template.getTaskFiles()) { out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** + } for(RoleFile tf:template.getMeta()) { out.addMeta(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** + } return out; - ***REMOVED*** + } private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) { RoleFile out = new RoleFile(); out.setName(template.getName()); out.setContent(this.fillTemplate(template.getContent(), dictionary)); return out; - ***REMOVED*** + } private String fillTemplate(String template, Map dictionary) { if (template != null) { ST t = new ST(template); for (String key : dictionary.keySet()) { t.add(key, dictionary.get(key)); - ***REMOVED*** + } String output = t.render(); return output; - ***REMOVED*** + } return template; - ***REMOVED*** + } public Role getRoleTemplate(String roleName) throws NoSuchElementException { File f = new File(this.getTemplateRoot(), roleName); try { return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) { -***REMOVED*** e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java new file mode 100755 index 0000000..940746e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java @@ -0,0 +1,170 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.clients; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.LinkedList; +import java.util.List; + +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; +import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckPermission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import au.com.bytecode.opencsv.CSVReader; + +public class HAProxy { + + private static final String token1 = ""; + private static final String token2 = ""; + + private Logger logger; + + private CSVReader reader; + + public HAProxy() { + this.logger = LoggerFactory.getLogger(HAProxy.class); + } + +// public Cluster getClusterByHProxy() throws IOException { +// Cluster cl = new Cluster(); +// String HProxy = ISClient.getHProxy(); +// URL stockURL = new URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); +// BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); +// reader = new CSVReader(in); +// String[] nextLine; +// while ((nextLine = reader.readNext()) != null) { +// // rules to add +// if (HProxy.contains(nextLine[0])) { +// cl.setName(nextLine[0]); +// } +// } +// return cl; +// +// } + + public Cluster MapCluster() throws IOException { + Cluster cl = new Cluster(); + String HProxy = ISClient.getHProxy(); + //Prod HAproxy + if (HProxy.equals("dataminer-cluster1.d4science.org")) { + cl.setName("dataminer_cluster1"); + } + if (HProxy.equals("dataminer-bigdata.d4science.org")) { + cl.setName("bigdata"); + } + if (HProxy.equals("dataminer-cluster1.d4science.org")) { + cl.setName("dataminer_cluster1"); + } + if (HProxy.equals("dataminer-cloud1.d4science.org")) { + cl.setName("dataminer_cloud1"); + } + if (HProxy.equals("dataminer-prototypes.d4science.org")) { + cl.setName("prototypes"); + } + if (HProxy.equals("dataminer.d4science.org")) { + cl.setName("gcubeapps"); + } + if (HProxy.equals("dataminer-genericworkers.d4science.org")) { + cl.setName("genericworkers"); + } + if (HProxy.equals("dataminer-genericworkers-proto.d4science.org")) { + cl.setName("genericworkers_proto"); + } + //dev HAProxy + if (HProxy.equals("dataminer-d-workers.d4science.org")||(HProxy.equals("dataminer-d-d4s.d4science.org"))) { + cl.setName("devnext_backend"); + } + //preProd HAProxy +// if (HProxy.equals("dataminer1-pre.d4science.org")) { +// cl.setName("dataminer1-pre.d4science.org"); +// } + return cl; + + } + + + public List listDataMinersByCluster(String targetVREToken,String targetVRE) throws IOException { + + SecurityTokenProvider.instance.set(targetVREToken); + ScopeProvider.instance.set(targetVRE); + + // next op to use when Cluster info available in the IS + // Cluster cluster = this.getClusterByHProxy(); + Cluster cluster = this.MapCluster(); + List out = new LinkedList(); + Host a = new Host(); + + //no proxy dataminer (preprod) + if (cluster.getName() == null){ + a.setName(ISClient.getHProxy()); + out.add(a); + } + + // if preprod, just one dm available +// if (cluster.getName().equals("dataminer1-pre.d4science.org")) { +// a.setName("dataminer1-pre.d4science.org"); +// out.add(a); + //} + else { + // prod + //URL stockURL = new + //URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); + URL stockURL = new URL("http://"+ ISClient.getHProxy() +":8880/;csv"); + //URL stockURL = new URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); + //System.out.println(stockURL); + // dev + //URL stockURL = new + //URL("http://data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); + BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); + reader = new CSVReader(in, ','); + String[] nextLine; + while ((nextLine = reader.readNext()) != null) { + if (nextLine[1].equals("BACKEND") || (nextLine[1].equals("FRONTEND"))) { + continue; + } + if (nextLine[0].equals(cluster.getName())) { + Host b = new Host(); + b.setName(nextLine[1]); + out.add(b); + this.logger.info(b.getFullyQualifiedName()); + } + } + } + this.logger.info(out.toString()); + return out; + + } + + public static void main(String[] args) throws ObjectNotFound, Exception { + HAProxy a = new HAProxy(); + //ScopeProvider.instance.set("/gcube/devNext/NextNext"); + //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set(token1); + + CheckPermission test = new CheckPermission(); + CheckPermission.apply(token2, "/gcube/devNext/NextNext"); + + + //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + // System.out.println(a.getHProxy()); + // System.out.println(a.MapCluster()); + //System.out.println(a.listDataMinersByCluster("xxx-xxx-xxx-xxx","/gcube/devNext/NextNext")); + // System.out.println(a.listDataMinersByCluster()); + + // List list = new LinkedList(); + // Dependency aa = new Dependency(); + // aa.setName("testnunzio"); + // aa.setType("cran:"); + // list.add(aa); + + // a.checkSVNdep(); + // System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); + // System.out.println(a.listDataminersInVRE()); + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index e413782..a299693 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -1,38 +1,38 @@ -***REMOVED*** +package org.gcube.dataanalysis.dataminer.poolmanager.clients; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -***REMOVED*** +import java.io.IOException; import java.io.StringWriter; import java.util.Arrays; import java.util.Collection; -***REMOVED*** +import java.util.List; import java.util.Vector; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.Resources; import org.gcube.common.resources.gcore.ServiceEndpoint; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; import org.gcube.informationsystem.publisher.ScopedPublisher; import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; public class ISClient { -***REMOVED*** + private Logger logger; public ISClient() { this.logger = LoggerFactory.getLogger(ISClient.class); -***REMOVED*** + } public Host getDataminer(String hostname) { @@ -42,20 +42,20 @@ public class ISClient { if (!remote2) { h.setName("dataminer1-devnext.d4science.org"); return h; - ***REMOVED*** else { - ***REMOVED***SimpleQuery query = queryFor(ServiceEndpoint.class); - ***REMOVED***query.addCondition("$resource/Profile/RunTime/HostedOn/text() eq '" + hostname + "'"); - ***REMOVED***DiscoveryClient client = clientFor(ServiceEndpoint.class); - ***REMOVED***List resources = client.submit(query); - ***REMOVED***ServiceEndpoint a = resources.get(0); - ***REMOVED***h.setName(a.profile().runtime().hostedOn()); + } else { + //SimpleQuery query = queryFor(ServiceEndpoint.class); + //query.addCondition("$resource/Profile/RunTime/HostedOn/text() eq '" + hostname + "'"); + //DiscoveryClient client = clientFor(ServiceEndpoint.class); + //List resources = client.submit(query); + //ServiceEndpoint a = resources.get(0); + //h.setName(a.profile().runtime().hostedOn()); h.setName(hostname); - ***REMOVED*** + } return h; -***REMOVED*** + } - ***REMOVED*** return the HProxy hostname in the VRE + // return the HProxy hostname in the VRE public static String getHProxy() { Host h = new Host(); SimpleQuery query = queryFor(ServiceEndpoint.class); @@ -64,7 +64,7 @@ public class ISClient { List resources = client.submit(query); h.setName(resources.get(0).profile().runtime().hostedOn()); return h.getName(); -***REMOVED*** + } public Collection listDataminersInVRE() { @@ -74,19 +74,19 @@ public class ISClient { if (!remote) { Collection out = new Vector<>(); Host h = new Host(); - ***REMOVED*** h.setName("bb-dataminer.res.eng.it"); - ***REMOVED*** h.setName("vm101.ui.savba.sk"); + // h.setName("bb-dataminer.res.eng.it"); + // h.setName("vm101.ui.savba.sk"); h.setName("dataminer1-devnext.d4science.org"); out.add(h); - ***REMOVED*** - ***REMOVED*** else { + return out; + } else { SimpleQuery query = queryFor(ServiceEndpoint.class); - ***REMOVED*** old version - ***REMOVED*** query.addCondition("$resource/Profile/Category/text() eq - ***REMOVED*** 'DataAnalysis'") - ***REMOVED*** .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); + // old version + // query.addCondition("$resource/Profile/Category/text() eq + // 'DataAnalysis'") + // .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); @@ -99,10 +99,10 @@ public class ISClient { Host h = new Host(); h.setName(r.profile().runtime().hostedOn()); out.add(h); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** -***REMOVED*** + } + return out; + } + } public void updateAlg(Algorithm algo) { @@ -115,15 +115,15 @@ public class ISClient { List ds = client.submit(query); if (ds.isEmpty()) { return; - ***REMOVED*** + } GenericResource a = ds.get(0); a.profile().newBody(this.getAlgoBody(algo)); try { scopedPublisher.update(a); - ***REMOVED*** catch (RegistryNotFoundException e) { + } catch (RegistryNotFoundException e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } private String getAlgoBody(Algorithm algo) { @@ -131,20 +131,20 @@ public class ISClient { + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" + "\n" + "" + algo.getDependencies() + ""; -***REMOVED*** + } -***REMOVED*** public void addAlgToIs(Algorithm algo) { -***REMOVED*** GenericResource a = new GenericResource(); -***REMOVED*** a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); -***REMOVED*** a.profile().newBody(this.getAlgoBody(algo)); -***REMOVED*** try { -***REMOVED*** publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***)); -***REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** +// public void addAlgToIs(Algorithm algo) { +// GenericResource a = new GenericResource(); +// a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); +// a.profile().newBody(this.getAlgoBody(algo)); +// try { +// publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() })); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } @@ -154,22 +154,22 @@ public class ISClient { a.profile().newBody(this.getAlgoBody(algo)); try { SecurityTokenProvider.instance.set(token); - publishScopedResource(a, Arrays.asList(new String[] { SecurityTokenProvider.instance.get() ***REMOVED***)); - ***REMOVED*** catch (Exception e) { + publishScopedResource(a, Arrays.asList(new String[] { SecurityTokenProvider.instance.get() })); + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); String id = resource.id(); - this.logger.info("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id, + this.logger.info("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id, ScopeProvider.instance.get()); - ***REMOVED*** scopedPublisher.remove(resource, scopes); + // scopedPublisher.remove(resource, scopes); advancedScopedPublisher.forceRemove(resource); - this.logger.info("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id); -***REMOVED*** + this.logger.info("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id); + } public void publishScopedResource(GenericResource a, List scopes) throws RegistryNotFoundException, Exception { @@ -181,69 +181,69 @@ public class ISClient { this.logger.debug(scopes.toString()); this.logger.debug(stringWriter.toString()); scopedPublisher.create(a, scopes); - ***REMOVED*** catch (RegistryNotFoundException e) { + } catch (RegistryNotFoundException e) { this.logger.error("Registry not found",e); throw e; - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED*** public Set getAlgoFromIs() { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** -***REMOVED*** Set out = new HashSet(); -***REMOVED*** SimpleQuery query = queryFor(GenericResource.class); -***REMOVED*** query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") -***REMOVED*** .setResult("$resource"); -***REMOVED*** DiscoveryClient client = clientFor(GenericResource.class); -***REMOVED*** List ds = client.submit(query); -***REMOVED*** for (GenericResource a : ds) { -***REMOVED*** out.add(this.convertAlgo(a)); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** +// public Set getAlgoFromIs() { +// // TODO Auto-generated method stub +// +// Set out = new HashSet(); +// SimpleQuery query = queryFor(GenericResource.class); +// query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") +// .setResult("$resource"); +// DiscoveryClient client = clientFor(GenericResource.class); +// List ds = client.submit(query); +// for (GenericResource a : ds) { +// out.add(this.convertAlgo(a)); +// } +// return out; +// } -***REMOVED*** private Algorithm convertAlgo(GenericResource a) { -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** -***REMOVED*** ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); -***REMOVED*** out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); -***REMOVED*** out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); -***REMOVED*** out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); -***REMOVED*** out.setName(a.profile().name()); -***REMOVED*** out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); -***REMOVED*** out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); -***REMOVED*** out.setDescription(a.profile().description()); -***REMOVED*** -***REMOVED*** Set deps = new HashSet(); -***REMOVED*** for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { -***REMOVED*** org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); -***REMOVED*** d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); -***REMOVED*** deps.add(d1); -***REMOVED*** ***REMOVED*** -***REMOVED*** out.setDependencies(deps); -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** +// private Algorithm convertAlgo(GenericResource a) { +// Algorithm out = new Algorithm(); +// +// // out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); +// out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); +// out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); +// out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); +// out.setName(a.profile().name()); +// out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); +// out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); +// out.setDescription(a.profile().description()); +// +// Set deps = new HashSet(); +// for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { +// org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); +// d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); +// deps.add(d1); +// } +// out.setDependencies(deps); +// return out; +// } public static void main(String[] args) throws IOException, SVNException { ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - ***REMOVED*** System.out.println(a.getHProxy()); - ***REMOVED*** System.out.println(a.MapCluster()); - ***REMOVED*** System.out.println(a.listDataMinersByCluster()); - ***REMOVED*** System.out.println(a.listDataMinersByCluster()); + // System.out.println(a.getHProxy()); + // System.out.println(a.MapCluster()); + // System.out.println(a.listDataMinersByCluster()); + // System.out.println(a.listDataMinersByCluster()); - ***REMOVED*** List list = new LinkedList(); - ***REMOVED*** Dependency aa = new Dependency(); - ***REMOVED*** aa.setName("testnunzio"); - ***REMOVED*** aa.setType("cran:"); - ***REMOVED*** list.add(aa); + // List list = new LinkedList(); + // Dependency aa = new Dependency(); + // aa.setName("testnunzio"); + // aa.setType("cran:"); + // list.add(aa); - ***REMOVED*** a.checkSVNdep(); - ***REMOVED***System.out.println(a.getDataminer("dataminer1-d-d4s.d4science.org").getDomain()); - ***REMOVED*** System.out.println(a.listDataminersInVRE()); -***REMOVED*** -***REMOVED*** + // a.checkSVNdep(); + //System.out.println(a.getDataminer("dataminer1-d-d4s.d4science.org").getDomain()); + // System.out.println(a.listDataminersInVRE()); + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java index 64adecc..941c63f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ScopedCacheMap.java @@ -1,4 +1,4 @@ -***REMOVED*** +package org.gcube.dataanalysis.dataminer.poolmanager.clients; import java.util.HashMap; @@ -15,4 +15,4 @@ public class ScopedCacheMap extends HashMap { -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java index c65ef9a..774a33d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java @@ -6,7 +6,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.util.Date; import java.util.HashMap; import java.util.Iterator; -***REMOVED*** +import java.util.List; import java.util.Map; import org.gcube.common.resources.gcore.GenericResource; @@ -14,19 +14,19 @@ import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Config import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.AbstractConfiguration; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientConfigurationCache { -***REMOVED*** + private Logger logger; private SVNRepository svnRepository; private Map configurations; private long svnRepositoryTimeout; private Map configurationsTimeouts; - private final long duration = 120000; ***REMOVED***2 minutes + private final long duration = 120000; //2 minutes ClientConfigurationCache () { @@ -35,7 +35,7 @@ public class ClientConfigurationCache this.configurations = new HashMap<>(); this.svnRepositoryTimeout = 0; this.configurationsTimeouts = new HashMap<>(); -***REMOVED*** + } public AbstractConfiguration getConfiguration (CONFIGURATIONS configuration) { @@ -46,10 +46,10 @@ public class ClientConfigurationCache { this.configurations.put(configuration.toString(), getConfiguration (configuration.getType())); this.configurationsTimeouts.put(configuration.toString(), currentTime); - ***REMOVED*** + } return this.configurations.get(configuration.toString()); -***REMOVED*** + } public SVNRepository getSVNRepository () @@ -60,10 +60,10 @@ public class ClientConfigurationCache { this.svnRepository = queryForRepository(); this.svnRepositoryTimeout = currentTime; - ***REMOVED*** + } return this.svnRepository; -***REMOVED*** + } private SVNRepository queryForRepository() @@ -100,22 +100,22 @@ public class ClientConfigurationCache this.logger.debug("Repository username "+repositoryUsername); this.logger.debug("Repository password "+repositoryPassword); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { this.logger.debug("SVN Username and password not present"); - ***REMOVED*** + } this.logger.debug("SVN Repository URL: "+repositoryURL); this.logger.debug("SVN Repository path: "+repositoryPath); response = new SVNRepository(repositoryURL, repositoryPath,repositoryUsername, repositoryPassword); - ***REMOVED*** - ***REMOVED*** + } + } return response; -***REMOVED*** + } private AbstractConfiguration getConfiguration (AbstractConfiguration type) { @@ -125,6 +125,6 @@ public class ClientConfigurationCache List configurations = client.submit(query); if (configurations != null && !configurations.isEmpty()) return configurations.get(0); else return null; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java index c99b563..94357af 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java @@ -35,4 +35,4 @@ public interface Configuration { public String getGhostAlgoDirectory(); -***REMOVED*** \ No newline at end of file +} \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java index 23d314a..363ad1d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java @@ -14,37 +14,37 @@ public class ConfigurationImpl implements Configuration { private CONFIGURATIONS(AbstractConfiguration type) { this.type = type; - ***REMOVED*** + } @Override public String toString() { return this.type.getType(); - ***REMOVED*** + } public AbstractConfiguration getType () { return this.type; - ***REMOVED*** + } -***REMOVED*** + } -***REMOVED*** enum REPOSITORIES { -***REMOVED*** REPO ("svn.repository"), -***REMOVED*** MAIN_ALGO ("svn.algo.main.repo"); -***REMOVED*** -***REMOVED*** private String type; -***REMOVED*** -***REMOVED*** private REPOSITORIES(String type) { -***REMOVED*** this.type = type; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public String toString() { -***REMOVED*** return this.type; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** +// enum REPOSITORIES { +// REPO ("svn.repository"), +// MAIN_ALGO ("svn.algo.main.repo"); +// +// private String type; +// +// private REPOSITORIES(String type) { +// this.type = type; +// } +// +// @Override +// public String toString() { +// return this.type; +// } +// } private CONFIGURATIONS type; @@ -53,100 +53,100 @@ public class ConfigurationImpl implements Configuration { public ConfigurationImpl(CONFIGURATIONS type,ClientConfigurationCache cache) { this.type = type; this.cache = cache; -***REMOVED*** + } @Override public String getHost() { return this.cache.getConfiguration(this.type).getHost (); -***REMOVED*** + } @Override public String getSVNAlgorithmsList() { return this.cache.getConfiguration(this.type).getAlgorithmsList(); -***REMOVED*** + } @Override public String getRepository() { return this.cache.getConfiguration(this.type).getSoftwareRepo(); -***REMOVED*** + } @Override public String getSVNLinuxCompiledDepsList() { return this.cache.getConfiguration(this.type).getDepsLinuxCompiled(); -***REMOVED*** + } @Override public String getSVNPreInstalledDepsList() { return this.cache.getConfiguration(this.type).getDepsPreInstalled(); -***REMOVED*** + } @Override public String getSVNRBDepsList() { return this.cache.getConfiguration(this.type).getDepsRBlackbox(); -***REMOVED*** + } @Override public String getSVNCRANDepsList() { return this.cache.getConfiguration(this.type).getDepsR(); -***REMOVED*** + } @Override public String getSVNJavaDepsList() { return this.cache.getConfiguration(this.type).getDepsJava(); -***REMOVED*** + } @Override public String getSVNKWDepsList() { return this.cache.getConfiguration(this.type).getDepsKnimeWorkflow(); -***REMOVED*** + } @Override public String getSVNKW4_1DepsList() { return this.cache.getConfiguration(this.type).getDepsKnimeWorkflow4_1(); -***REMOVED*** + } @Override public String getSVNOctaveDepsList() { return this.cache.getConfiguration(this.type).getDepsOctave(); -***REMOVED*** + } @Override public String getSVNPythonDepsList() { return this.cache.getConfiguration(this.type).getDepsPython(); -***REMOVED*** + } @Override public String getSVNPython3_6DepsList() { return this.cache.getConfiguration(this.type).getDepsPython3_6(); -***REMOVED*** + } @Override public String getSVNWCDepsList() { return this.cache.getConfiguration(this.type).getDepsWindowsCompiled(); -***REMOVED*** + } @Override public SVNRepository getSVNRepository() { return this.cache.getSVNRepository(); -***REMOVED*** + } @@ -154,5 +154,5 @@ public class ConfigurationImpl implements Configuration { public String getGhostAlgoDirectory() { return this.cache.getConfiguration(this.type).getGhostRepo(); -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java index d67ebac..25f8f3f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java @@ -1,16 +1,16 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; -***REMOVED*** +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; -***REMOVED*** +import java.util.List; import java.util.Properties; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ScopedCacheMap; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.CONFIGURATIONS; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; public class DMPMClientConfiguratorManager { @@ -29,10 +29,10 @@ public class DMPMClientConfiguratorManager { try { this.defaultAdmins.load(this.getClass().getResourceAsStream("/default.admins")); this.logger.debug("Default users successfully loaded"); - ***REMOVED*** catch (Exception e) { + } catch (Exception e) { this.logger.error("Unable to get default users", e); - ***REMOVED*** -***REMOVED*** + } + } private ClientConfigurationCache getCurrentCache() { String currentScope = ScopeProvider.instance.get(); @@ -45,58 +45,58 @@ public class DMPMClientConfiguratorManager { cache = new ClientConfigurationCache(); this.cacheMap.put(currentScope, cache); - ***REMOVED*** + } return cache; -***REMOVED*** + } public static DMPMClientConfiguratorManager getInstance() { if (instance == null) instance = new DMPMClientConfiguratorManager(); return instance; -***REMOVED*** + } public Configuration getProductionConfiguration() { return new ConfigurationImpl(CONFIGURATIONS.PROD, getCurrentCache()); -***REMOVED*** + } public Configuration getStagingConfiguration() { return new ConfigurationImpl(CONFIGURATIONS.STAGE, getCurrentCache()); -***REMOVED*** + } public List getDefaultAdmins() { List admins = new ArrayList(); if (defaultAdmins == null || defaultAdmins.isEmpty()) { admins.add("statistical.manager"); - ***REMOVED*** else { + } else { Iterator keys = this.defaultAdmins.keySet().iterator(); while (keys.hasNext()) { String key = (String) keys.next(); admins.add(defaultAdmins.getProperty(key)); - ***REMOVED*** + } - ***REMOVED*** + } this.logger.debug("Default admins list: " + admins); return admins; -***REMOVED*** + } public static void main(String[] args) throws IOException, SVNException { DMPMClientConfiguratorManager a = new DMPMClientConfiguratorManager(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - ***REMOVED*** SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + // SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); System.out.println("RESULT 1" + a.getStagingConfiguration().getSVNCRANDepsList()); System.out.println("RESULT 2" + a.getProductionConfiguration().getRepository()); System.out.println("RESULT 3" + a.getStagingConfiguration().getSVNRepository().getPath()); - ***REMOVED*** System.out.println(a.getRepo()); - ***REMOVED*** System.out.println(a.getAlgoRepo()); - ***REMOVED*** System.out.println(a.getSVNRepo()); + // System.out.println(a.getRepo()); + // System.out.println(a.getAlgoRepo()); + // System.out.println(a.getSVNRepo()); -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java index f7ef1a6..8323706 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java @@ -18,36 +18,36 @@ public class SVNRepository this.path = path; this.username = username; this.password = password; -***REMOVED*** + } SVNRepository(String baseUrl, String path) { this (baseUrl, path, null, null); -***REMOVED*** + } public String getUsername() { return username; -***REMOVED*** + } public void setUsername(String username) { this.username = username; -***REMOVED*** + } public String getPassword() { return password; -***REMOVED*** + } public void setPassword(String password) { this.password = password; -***REMOVED*** + } public String getBaseUrl() { return baseUrl; -***REMOVED*** + } public String getPath() { return path; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java index 92a973f..dac6a94 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java @@ -26,152 +26,152 @@ public abstract class AbstractConfiguration @XmlElement (name="host") public String getHost() { return host; -***REMOVED*** + } public void setHost(String host) { this.host = host; -***REMOVED*** + } @XmlElement (name="algorithms-list") public String getAlgorithmsList() { return algorithmsList; -***REMOVED*** + } public void setAlgorithmsList(String algorithmsList) { this.algorithmsList = algorithmsList; -***REMOVED*** + } @XmlElement (name="software-repo") public String getSoftwareRepo() { return softwareRepo; -***REMOVED*** + } public void setSoftwareRepo(String softwareRepo) { this.softwareRepo = softwareRepo; -***REMOVED*** + } @XmlElement (name="ghost-repo") public String getGhostRepo() { return ghostRepo; -***REMOVED*** + } public void setGhostRepo(String ghostRepo) { this.ghostRepo = ghostRepo; -***REMOVED*** + } @XmlElement (name="deps-linux-compiled") public String getDepsLinuxCompiled() { return depsLinuxCompiled; -***REMOVED*** + } public void setDepsLinuxCompiled(String depsLinuxCompiled) { this.depsLinuxCompiled = depsLinuxCompiled; -***REMOVED*** + } @XmlElement (name="deps-pre-installed") public String getDepsPreInstalled() { return depsPreInstalled; -***REMOVED*** + } public void setDepsPreInstalled(String depsPreInstalled) { this.depsPreInstalled = depsPreInstalled; -***REMOVED*** + } @XmlElement (name="deps-r-blackbox") public String getDepsRBlackbox() { return depsRBlackbox; -***REMOVED*** + } public void setDepsRBlackbox(String depsRBlackbox) { this.depsRBlackbox = depsRBlackbox; -***REMOVED*** + } @XmlElement (name="deps-r") public String getDepsR() { return depsR; -***REMOVED*** + } public void setDepsR(String depsR) { this.depsR = depsR; -***REMOVED*** + } @XmlElement (name="deps-java") public String getDepsJava() { return depsJava; -***REMOVED*** + } public void setDepsJava(String depsJava) { this.depsJava = depsJava; -***REMOVED*** + } @XmlElement (name="deps-knime-workflow") public String getDepsKnimeWorkflow() { return depsKnimeWorkflow; -***REMOVED*** + } public void setDepsKnimeWorkflow(String depsKnimeWorkflow) { this.depsKnimeWorkflow = depsKnimeWorkflow; -***REMOVED*** + } @XmlElement (name="deps-knime-workflow4_1") public String getDepsKnimeWorkflow4_1() { return depsKnimeWorkflow4_1; -***REMOVED*** + } public void setDepsKnimeWorkflow4_1(String depsKnimeWorkflow4_1) { this.depsKnimeWorkflow4_1 = depsKnimeWorkflow4_1; -***REMOVED*** + } @XmlElement (name="deps-octave") public String getDepsOctave() { return depsOctave; -***REMOVED*** + } public void setDepsOctave(String depsOctave) { this.depsOctave = depsOctave; -***REMOVED*** + } @XmlElement (name="deps-python") public String getDepsPython() { return depsPython; -***REMOVED*** + } public void setDepsPython(String depsPython) { this.depsPython = depsPython; -***REMOVED*** + } @XmlElement (name="deps-python3_6") public String getDepsPython3_6() { return depsPython3_6; -***REMOVED*** + } public void setDepsPython3_6(String depsPython3_6) { this.depsPython3_6 = depsPython3_6; -***REMOVED*** + } @XmlElement (name="deps-windows-compiled") public String getDepsWindowsCompiled() { return depsWindowsCompiled; -***REMOVED*** + } public void setDepsWindowsCompiled(String depsWindowsCompiled) { this.depsWindowsCompiled = depsWindowsCompiled; -***REMOVED*** + } protected String getXML (String type) { - return "<"+type+">{$resource/Profile/Body/"+type+"/ghost/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/algorithms-list/text()***REMOVED***"+ - " {$resource/Profile/Body/"+type+"/software.repo/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/algo.ghost.repo/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-linux-compiled/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-pre-installed/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-r-blackbox/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-r/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-java/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-knime-workflow/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-knime-workflow4_1/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-octave/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-python/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-python3_6/text()***REMOVED***"+ - "{$resource/Profile/Body/"+type+"/deps-windows-compiled/text()***REMOVED***"; -***REMOVED*** + return "<"+type+">{$resource/Profile/Body/"+type+"/ghost/text()}"+ + "{$resource/Profile/Body/"+type+"/algorithms-list/text()}"+ + " {$resource/Profile/Body/"+type+"/software.repo/text()}"+ + "{$resource/Profile/Body/"+type+"/algo.ghost.repo/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-linux-compiled/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-pre-installed/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-r-blackbox/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-r/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-java/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-knime-workflow/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-knime-workflow4_1/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-octave/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-python/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-python3_6/text()}"+ + "{$resource/Profile/Body/"+type+"/deps-windows-compiled/text()}"; + } abstract public String getXMLModel (); abstract public String getType (); -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java index 9da7aef..b01fd05 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Prod.java @@ -12,12 +12,12 @@ public class Prod extends AbstractConfiguration { public String getXMLModel () { return super.getXML(TYPE); -***REMOVED*** + } @Override public String getType() { return TYPE; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java index e70a2fe..2b02b74 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/Stage.java @@ -12,12 +12,12 @@ public class Stage extends AbstractConfiguration { public String getXMLModel () { return super.getXML(TYPE); -***REMOVED*** + } @Override public String getType() { return TYPE; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java index c4ef421..989cec4 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java @@ -8,26 +8,26 @@ public class Action { public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public String getDescription() { return description; - ***REMOVED*** + } public void setDescription(String description) { this.description = description; - ***REMOVED*** + } public String getScript() { return script; - ***REMOVED*** + } public void setScript(String script) { this.script = script; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java index c899055..5144963 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java @@ -5,18 +5,18 @@ public class AlgoResource { protected String id; public AlgoResource() { -***REMOVED*** + } public AlgoResource(String id) { this.id = id; -***REMOVED*** + } public String getId() { return this.id; -***REMOVED*** + } public void setId(String id) { this.id = id; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index a85f359..48e564b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -26,86 +26,86 @@ public class Algorithm { public Algorithm() { this.actions = new Vector<>(); this.dependencies = new Vector<>(); - ***REMOVED*** Dependency p = new Dependency(); + // Dependency p = new Dependency(); - ***REMOVED***init with default values + //init with default values this.skipJava = "N"; this.algorithmType = "transducerers"; - ***REMOVED*** + } public void addDependency(Dependency dep) { this.dependencies.add(dep); - ***REMOVED*** + } public void addAction(Action action) { this.actions.add(action); - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public String getDescription() { return description; - ***REMOVED*** + } public void setDescription(String description) { this.description = description.replace(": ", " "); - ***REMOVED*** + } public String getCategory() { return category; - ***REMOVED*** + } public void setCategory(String category) { this.category = category; - ***REMOVED*** + } public Collection getActions() { return actions; - ***REMOVED*** + } public Collection getDependencies() { return dependencies; - ***REMOVED*** + } public Collection getOSDependencies(){ Set deps = new HashSet<>(); for(Dependency d: this.getDependencies()){ if(d.getType().equals("os")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } public Collection getCranDependencies(){ Set deps = new HashSet<>(); for(Dependency d: this.getDependencies()){ if(d.getType().equals("cran")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } public Collection getGitHubDependencies(){ Set deps = new HashSet<>(); for(Dependency d: this.getDependencies()){ if(d.getType().equals("github")){ deps.add(d); - ***REMOVED*** - ***REMOVED*** + } + } return deps; - ***REMOVED*** + } public void setDependencies(Collection deps) { this.dependencies = deps; - ***REMOVED*** + } public String toString() { String out = "Algorithm: " + this.getName()+"\n"; @@ -113,76 +113,76 @@ public class Algorithm { out+=" Description: " + this.getDescription()+"\n"; out+=" Dependencies: " + this.getDependencies()+"\n"; return out; - ***REMOVED*** + } public String getClazz() { return clazz; - ***REMOVED*** + } public void setClazz(String clazz) { this.clazz = clazz; - ***REMOVED*** + } public String getPackageURL() { return packageURL; - ***REMOVED*** + } public void setPackageURL(String packageURL) { this.packageURL = packageURL; - ***REMOVED*** + } public String getAlgorithmType() { return algorithmType; - ***REMOVED*** + } public void setAlgorithmType(String algorithmType) { this.algorithmType = algorithmType; - ***REMOVED*** + } public String getSkipJava() { return skipJava; - ***REMOVED*** + } public void setSkipJava(String skipJava) { this.skipJava = skipJava; - ***REMOVED*** + } public String getUsername() { return username; -***REMOVED*** +} public void setUsername(String username) { this.username = username; -***REMOVED*** +} public String getFullname() { return fullname; -***REMOVED*** +} public void setFullname(String fullname) { this.fullname = fullname; -***REMOVED*** +} public String getEmail() { return email; -***REMOVED*** +} public void setEmail(String email) { this.email = email; -***REMOVED*** +} public String getLanguage() { return language; -***REMOVED*** +} public void setLanguage(String language) { this.language = language; -***REMOVED*** +} public void setActions(Collection actions) { this.actions = actions; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java index f42c11c..7144103 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java @@ -11,27 +11,27 @@ public class AlgorithmSet { public AlgorithmSet() { this.algorithms = new Vector<>(); - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public Collection getAlgorithms() { return new Vector<>(algorithms); - ***REMOVED*** + } public void addAlgorithm(Algorithm algoritm) { this.algorithms.add(algoritm); - ***REMOVED*** + } public Boolean hasAlgorithm(Algorithm algorithm) { @@ -39,18 +39,18 @@ public class AlgorithmSet { if (a.getName().equals(algorithm.getName())) { return true; - ***REMOVED*** - ***REMOVED*** + } + } return false; - ***REMOVED*** + } public String toString() { String out = "ALGOSET: " + this.name + "\n"; for(Algorithm a:this.algorithms) { out+=a+"\n"; - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java index 50e4039..11f5771 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java @@ -29,54 +29,54 @@ public class Cluster { { this.hosts = new Vector<>(); this.algoSets = new Vector<>(); - ***REMOVED*** + } public void addAlgorithmSet(AlgorithmSet set) { this.algoSets.add(set); - ***REMOVED*** + } public void addHost(Host host) { this.hosts.add(host); - ***REMOVED*** + } public Collection getHosts() { return hosts; - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public String getDescription() { return description; - ***REMOVED*** + } public void setDescription(String description) { this.description = description; - ***REMOVED*** + } public Collection getAlgorithmSets() { return algoSets; - ***REMOVED*** + } public String toString() { String out = "Cluster: "+this.name+"\n"; for(Host h:this.getHosts()) { out+=" "+h+"\n"; - ***REMOVED*** + } return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java index d6f22ee..bd4a65b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java @@ -8,22 +8,22 @@ public class Dependency { public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public String getType() { return type; - ***REMOVED*** + } public void setType(String type) { this.type = type; - ***REMOVED*** + } public String toString() { return this.type+":"+this.name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java index b8e55ae..49f463b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java @@ -6,10 +6,10 @@ public class Domain { public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java index 20ac71c..f37dc80 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java @@ -8,41 +8,41 @@ public class Host { public Host(String hostname) { this.setName(hostname); - ***REMOVED*** + } public Host() { -***REMOVED*** +} public String getFullyQualifiedName() { if(this.domain!=null && this.domain.getName()!=null) return this.getName()+"."+this.getDomain().getName(); else return this.getName(); - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public void setName(String name) { this.name = name; - ***REMOVED*** + } public Domain getDomain() { return domain; - ***REMOVED*** + } public void setDomain(Domain domain) { this.domain = domain; - ***REMOVED*** + } -***REMOVED*** public String toString() { -***REMOVED*** return this.name + "@" + this.domain; -***REMOVED*** ***REMOVED*** +// public String toString() { +// return this.name + "@" + this.domain; +// } public String toString() { return this.name; -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java index 41eb913..024c2d5 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java @@ -9,7 +9,7 @@ public class AlgorithmComparator implements Comparator { @Override public int compare(Algorithm a1, Algorithm a2) { return a1.getName().compareTo(a2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java index a5ff1e1..abbfbbd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java @@ -12,7 +12,7 @@ public class DependencyComparator implements Comparator { if(out!=0) return out; return a1.getName().compareTo(a2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java index 0d27a62..1e159b8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java @@ -2,7 +2,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; import java.util.Comparator; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; public class HostComparator implements Comparator { @@ -12,6 +12,6 @@ public class HostComparator implements Comparator { if(out!=0) return out; return h1.getName().compareTo(h2.getName()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java index 39a4f88..b6b3c27 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java @@ -38,53 +38,53 @@ public class AddAlgorithmCommand { d = d + st.nextToken() + " "; this.setDescription(d); - ***REMOVED*** + } public void setDescription(String d) { if(d!=null) { d = d.trim(); if(d.startsWith("\"") && d.endsWith("\"")) { d = d.substring(1, d.length()-1).trim(); - ***REMOVED*** - ***REMOVED*** + } + } this.description = d; - ***REMOVED*** + } public String getCommand() { return command; - ***REMOVED*** + } public String getName() { return name; - ***REMOVED*** + } public String getCategory() { return category; - ***REMOVED*** + } public String getClazz() { return clazz; - ***REMOVED*** + } public String getVRE() { return scope; - ***REMOVED*** + } public String getAlgorithmType() { return algorithmType; - ***REMOVED*** + } public String getSkipjava() { return skipJava; - ***REMOVED*** + } public String getUrl() { return url; - ***REMOVED*** + } public String getDescription() { return description; - ***REMOVED*** + } public String toString() { String out = ""; @@ -98,6 +98,6 @@ public class AddAlgorithmCommand { out += String.format("%-12s: %s\n", "url", url); out += String.format("%-12s: %s\n", "description", this.description); return out; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index fd29714..e5e186b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -1,10 +1,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.process; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; -***REMOVED*** +import java.net.URL; import java.util.HashMap; -***REMOVED*** +import java.util.List; import java.util.Map; import java.util.Vector; import java.util.zip.ZipEntry; @@ -12,8 +12,8 @@ import java.util.zip.ZipInputStream; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AlgorithmPackageParser { @@ -31,7 +31,7 @@ public class AlgorithmPackageParser { METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description", METADATA_CLASS_NAME = "Class Name", - ***REMOVED*** private static final String METADATA_PACKAGES = "Packages"; + // private static final String METADATA_PACKAGES = "Packages"; METADATA_KEY_VALUE_SEPARATOR = ":"; @@ -41,20 +41,20 @@ public class AlgorithmPackageParser { public AlgorithmPackageParser() { this.logger = LoggerFactory.getLogger(AlgorithmPackageParser.class); - ***REMOVED*** + } public Algorithm parsePackage(String url) throws IOException { String packageMetadata = this.getPackageMetadata(url); if (packageMetadata == null) { this.logger.warn("WARNING: No metadata found for " + url); return null; - ***REMOVED*** else { + } else { Map> parsedMetadata = this.parseMetadata(packageMetadata); Algorithm a = this.createAlgorithm(parsedMetadata); a.setPackageURL(url); return a; - ***REMOVED*** - ***REMOVED*** + } + } private String getPackageMetadata(String url) throws IOException { @@ -66,13 +66,13 @@ public class AlgorithmPackageParser { if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) { out = this.getEntryContent(zipIs); break; - ***REMOVED*** + } entry = zipIs.getNextEntry(); - ***REMOVED*** + } is.close(); zipIs.close(); return out; - ***REMOVED*** + } private String getEntryContent(ZipInputStream zipIn) throws IOException { @@ -81,9 +81,9 @@ public class AlgorithmPackageParser { int read = 0; while ((read = zipIn.read(buffer)) != -1) { s.append(new String(buffer, 0, read)); - ***REMOVED*** + } return s.toString(); - ***REMOVED*** + } private Map> parseMetadata(String metadata) { @@ -94,58 +94,58 @@ public class AlgorithmPackageParser { String value = null; for (String line : lines) { - ***REMOVED*** skip empty lines + // skip empty lines if (line.trim().isEmpty()) { continue; - ***REMOVED*** - ***REMOVED*** scan lines one by one, looking for key and values + } + // scan lines one by one, looking for key and values String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); if (parts.length > 1) { - ***REMOVED*** key and value on the same line + // key and value on the same line key = parts[0].trim(); value = line.substring(parts[0].length() + 1).trim(); - ***REMOVED*** else if (parts.length == 1) { - ***REMOVED*** either a key or a value + } else if (parts.length == 1) { + // either a key or a value if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) { - ***REMOVED*** key + // key key = parts[0].trim(); value = null; - ***REMOVED*** else { - ***REMOVED*** value + } else { + // value value = line.trim(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** add key+value to the map + } + } + // add key+value to the map if (key != null && value != null) { List values = out.get(key); if (values == null) { values = new Vector<>(); out.put(key, values); - ***REMOVED*** + } values.add(value); this.logger.debug(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); - ***REMOVED*** - ***REMOVED*** + } + } return out; - ***REMOVED*** + } -***REMOVED*** private Algorithm createAlgorithm(Map> metadata) { -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); -***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); -***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); -***REMOVED*** List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); -***REMOVED*** if (dependencies != null) { -***REMOVED*** for (String pkg : dependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** +// private Algorithm createAlgorithm(Map> metadata) { +// Algorithm out = new Algorithm(); +// out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); +// out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); +// out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); +// List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); +// if (dependencies != null) { +// for (String pkg : dependencies) { +// Dependency dep = new Dependency(); +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// return out; +// } private Algorithm createAlgorithm(Map> metadata) { @@ -166,8 +166,8 @@ public class AlgorithmPackageParser { Dependency dep = new Dependency(); dep.setName(pkg); out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** + } + } @@ -176,90 +176,90 @@ public class AlgorithmPackageParser { -***REMOVED*** List rdependencies = extractMultipleValues(metadata, "cran"); -***REMOVED*** if (rdependencies != null) { -***REMOVED*** for (String pkg : rdependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List defdependencies = extractMultipleValues(metadata, "Packages"); -***REMOVED*** if (defdependencies != null) { -***REMOVED*** for (String pkg : defdependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** List osdependencies = extractMultipleValues(metadata, "os"); -***REMOVED*** if (osdependencies != null) { -***REMOVED*** for (String pkg : osdependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List gitdependencies = extractMultipleValues(metadata, "github"); -***REMOVED*** if (gitdependencies != null) { -***REMOVED*** for (String pkg : gitdependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("github"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** List cdependencies = extractMultipleValues(metadata, "custom"); -***REMOVED*** if (cdependencies != null) { -***REMOVED*** for (String pkg : cdependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** -***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** +// List rdependencies = extractMultipleValues(metadata, "cran"); +// if (rdependencies != null) { +// for (String pkg : rdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("cran"); +// out.addDependency(dep); +// } +// } +// +// +// List defdependencies = extractMultipleValues(metadata, "Packages"); +// if (defdependencies != null) { +// for (String pkg : defdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// +// List osdependencies = extractMultipleValues(metadata, "os"); +// if (osdependencies != null) { +// for (String pkg : osdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("os"); +// out.addDependency(dep); +// } +// } +// +// +// +// List gitdependencies = extractMultipleValues(metadata, "github"); +// if (gitdependencies != null) { +// for (String pkg : gitdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("github"); +// out.addDependency(dep); +// } +// } +// +// +// +// List cdependencies = extractMultipleValues(metadata, "custom"); +// if (cdependencies != null) { +// for (String pkg : cdependencies) { +// Dependency dep = new Dependency(); +// +// //if (pkg.startsWith("os:")){ +// dep.setName(pkg); +// dep.setType("custom"); +// out.addDependency(dep); +// } +// } -***REMOVED*** if (pkg.startsWith("r:")){ -***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (pkg.startsWith("custom:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** ***REMOVED*** +// if (pkg.startsWith("r:")){ +// //String results = StringEscapeUtils.escapeJava(pkg); +// dep.setName(pkg); +// dep.setType("cran"); +// } +// if (pkg.startsWith("custom:")){ +// dep.setName(pkg); +// dep.setType("custom"); +// } +// if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ +// dep.setName(pkg); +// dep.setType("os"); +// } return out; - ***REMOVED*** + } @@ -269,10 +269,10 @@ public class AlgorithmPackageParser { List l = metadata.get(key); if (l != null && l.size() == 1) { return l.get(0); - ***REMOVED*** else { + } else { return null; - ***REMOVED*** - ***REMOVED*** + } + } @@ -282,10 +282,10 @@ public class AlgorithmPackageParser { List l = metadata.get(key); if (l != null) { return new Vector<>(l); - ***REMOVED*** else { + } else { return null; - ***REMOVED*** - ***REMOVED*** + } + } public static void main(String[] args) { @@ -312,6 +312,6 @@ public class AlgorithmPackageParser { ap.parseMetadata(txt); -***REMOVED*** +} -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 353f52a..653597f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -1,37 +1,37 @@ -***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import java.net.MalformedURLException; -***REMOVED******REMOVED*** -***REMOVED***import java.net.UnknownHostException; -***REMOVED*** -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** -***REMOVED***public interface PoolManager { -***REMOVED*** -***REMOVED*** String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; -***REMOVED*** -***REMOVED*** Algorithm extractAlgorithm(String url) throws IOException; -***REMOVED*** -***REMOVED*** String getLogById(String logId) throws IOException; -***REMOVED*** -***REMOVED*** void getLogId(Algorithm algo, String vre); -***REMOVED*** -***REMOVED*** String getScriptFromURL(URL logId) throws IOException; -***REMOVED*** -***REMOVED*** URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; -***REMOVED*** -***REMOVED******REMOVED*** +//package org.gcube.dataanalysis.dataminer.poolmanager.rest; +// +//import java.io.IOException; +//import java.net.MalformedURLException; +//import java.net.URL; +//import java.net.UnknownHostException; +// +//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +// +//public interface PoolManager { +// +// String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; +// +// Algorithm extractAlgorithm(String url) throws IOException; +// +// String getLogById(String logId) throws IOException; +// +// void getLogId(Algorithm algo, String vre); +// +// String getScriptFromURL(URL logId) throws IOException; +// +// URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; +// +//} package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** +import java.io.IOException; import java.net.MalformedURLException; -***REMOVED*** +import java.net.URL; import java.net.UnknownHostException; -***REMOVED*** +import java.util.List; import java.util.Set; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; @@ -64,4 +64,4 @@ public interface PoolManager List updateSVN(String file, List ldep) throws SVNException, IOException; -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 64d57c1..014d22a 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -1,11 +1,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** +import java.io.IOException; import java.net.MalformedURLException; -***REMOVED*** +import java.net.URL; import java.net.UnknownHostException; -***REMOVED*** +import java.util.List; import java.util.Set; import javax.ws.rs.GET; @@ -13,33 +13,33 @@ import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; @Path("/") public class RestPoolManager implements PoolManager { - ***REMOVED***@Context ServletContext context; + //@Context ServletContext context; private final Logger logger; private DataminerPoolManager service; - ***REMOVED***@Context - ***REMOVED***private ApplicationContext context = ContextProvider.get(); + //@Context + //private ApplicationContext context = ContextProvider.get(); public RestPoolManager() { this.logger = LoggerFactory.getLogger(RestPoolManager.class); this.service = new DataminerPoolManager(); -***REMOVED*** + } @GET @@ -52,9 +52,9 @@ public class RestPoolManager implements PoolManager @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { this.logger.debug("Stage algorithm method called"); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); - ***REMOVED***String env = context.application().getInitParameter("Environment"); + //String env = context.application().getInitParameter("Environment"); return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type/*,env*/); -***REMOVED*** + } @GET @@ -62,41 +62,41 @@ public class RestPoolManager implements PoolManager @Produces("text/plain") public String publishAlgorithm( @QueryParam("algorithmPackageURL") String algorithmPackageURL, - ***REMOVED***@QueryParam("targetVREToken") String targetVREToken, + //@QueryParam("targetVREToken") String targetVREToken, @QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category, @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { this.logger.debug("Publish algorithm method called"); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); - ***REMOVED***String env = context.application().getInitParameter("Environment"); + //String env = context.application().getInitParameter("Environment"); return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type/*,env*/); -***REMOVED*** + } /* - * /scopes/ POST ***REMOVED*** add an algorithm to all dataminers in the scope - * /hosts/ POST ***REMOVED*** add an algorithm to the given host + * /scopes/ POST // add an algorithm to all dataminers in the scope + * /hosts/ POST // add an algorithm to the given host */ @GET @Path("/log") @Produces("text/plain") public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub this.logger.debug("Get log by id method called"); this.logger.debug("Returning Log =" + logUrl); return service.getLogById(logUrl); -***REMOVED*** + } @GET @Path("/monitor") @Produces("text/plain") public String getMonitorById(@QueryParam("logUrl") String logUrl) throws IOException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub this.logger.debug("Get monitor by id method called"); this.logger.debug("Returning Log =" + logUrl); return service.getMonitorById(logUrl); -***REMOVED*** + } @@ -105,134 +105,132 @@ public class RestPoolManager implements PoolManager @Override public Algorithm extractAlgorithm(String url) throws IOException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } -***REMOVED*** + public static void main(String[] args) throws ObjectNotFound, Exception { - ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); -***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new -***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); -***REMOVED*** SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); + // System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); +// // ProxySelector.setDefault(new +// // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); +// ScopeProvider.instance.set("/gcube/devNext"); - SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + SecurityTokenProvider.instance.set(""); -***REMOVED*** AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); -***REMOVED*** System.out.println(entry.getContext()); +// AuthorizationEntry entry = authorizationService().get("xxxx-xxx-xxxxx-xxxx"); +// System.out.println(entry.getContext()); RestPoolManager a = new RestPoolManager(); - a.stageAlgorithm("http:***REMOVED***data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0"); -***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"); -***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager(); -***REMOVED*** + a.stageAlgorithm("http://data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0"); +// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxxx-xxxx","/gcube/devNext/NextNext"); +// // PoolManager aa = new DataminerPoolManager(); +// -***REMOVED*** + } - ***REMOVED***Production Testing + //Production Testing /* stageAlgorithm(Rproto caller token,pacchetto, category) - http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre) - node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab + node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=xxxx-xxxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxxx-xxxx&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab getLogById(Rproto caller token, logid) - http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&logUrl= + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxxx-xxx-xxx-xxx&logUrl= */ - ***REMOVED***dev Testing + //dev Testing /* stageAlgorithm(dev_caller_vre_token,pacchetto, category) - http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxxx-xxxx-xxxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token, target prod vre) - http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&logUrl=450bb7f9-9e38-4bde-8f4d-f3296f95deba + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxxx-xxxx-xxx-xxxx&logUrl=450bb7f9-9e38-4bde-8f4d-f3296f95deba getLogById(dev_caller_vre_token, logid) - http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&logUrl=426c8e35-a624-4710-b612-c90929c32c27 */ + http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-c90929c32c27 */ @Override public void getLogId(Algorithm algo, String vre) { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub -***REMOVED*** + } @Override public String getScriptFromURL(URL logId) throws IOException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public void addAlgToIs(Algorithm algo) { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub -***REMOVED*** + } @Override public Set getAlgoFromIs() { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public List updateSVN(String file, List ldep) throws SVNException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public String addAlgorithmToHost(Algorithm algo, String host, boolean test) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @Override public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub + // TODO Auto-generated method stub return null; -***REMOVED*** + } @@ -242,4 +240,4 @@ public class RestPoolManager implements PoolManager -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index 705d75b..8179c72 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -2,19 +2,19 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import java.io.File; import java.io.FileOutputStream; -***REMOVED*** +import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.util.Collection; import java.util.UUID; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.AnsibleException; import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.UndefinedDependenciesException; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; @@ -24,8 +24,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class DMPMJob { @@ -40,7 +40,7 @@ public abstract class DMPMJob { private String vREName; private String category; private String algorithm_type; -***REMOVED*** + private Logger logger; private enum STATUS { @@ -53,8 +53,8 @@ public abstract class DMPMJob { STATUS (String status) { this.status = status; - ***REMOVED*** -***REMOVED*** + } + } public DMPMJob(SVNUpdater svnUpdater,Configuration configuration,Algorithm algorithm, Cluster cluster,String vREName, String category, String algorithm_type){ @@ -71,10 +71,10 @@ public abstract class DMPMJob { this.dmpmHomeDirectory = new String (System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); this.id = UUID.randomUUID().toString(); - ***REMOVED***TODO: dmpm work directory should be loaded from configuration file + //TODO: dmpm work directory should be loaded from configuration file this.jobLogs = new File(this.dmpmHomeDirectory+File.separator+"jobs"); this.jobLogs.mkdirs(); -***REMOVED*** + } public String start() @@ -86,14 +86,14 @@ public abstract class DMPMJob { public void run() { try { execute(); - ***REMOVED*** catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); + } + } + }).start(); return this.id; -***REMOVED*** + } protected AnsibleWorker createWorker(Algorithm algo, Cluster dataminerCluster, @@ -102,12 +102,12 @@ public abstract class DMPMJob { AnsibleBridge ansibleBridge = new AnsibleBridge(this.dmpmHomeDirectory); try { return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); - ***REMOVED*** catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - ***REMOVED*** + } return null; -***REMOVED*** + } public void setStatusInformation(STATUS exitStatus) { @@ -115,18 +115,18 @@ public abstract class DMPMJob { try { File statusFile = new File (this.jobLogs,this.id + "_exitStatus"); - ***REMOVED***File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus"); + //File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus"); PrintWriter writer = new PrintWriter(statusFile, "UTF-8"); writer.println(exitStatus.status); writer.close(); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { this.logger.error ("Unable to update exit status file with status "+exitStatus.status,e); - ***REMOVED*** + } -***REMOVED*** + } private void updateLogFile (File logFile, String message) { @@ -135,13 +135,13 @@ public abstract class DMPMJob { PrintWriter writer = new PrintWriter(logFile,"UTF-8"); writer.print(message); writer.close(); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { this.logger.error("Unable to log the error message: "+message,e); - ***REMOVED*** + } -***REMOVED*** + } protected abstract void execute (); @@ -158,9 +158,9 @@ public abstract class DMPMJob { this.logger.debug("Some dependencies are not defined"); throw new UndefinedDependenciesException(undefinedDependencies); - ***REMOVED*** + } -***REMOVED*** + } private String installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws DMPMException { @@ -174,7 +174,7 @@ public abstract class DMPMJob { else { this.logger.debug("Operation completed"); - ***REMOVED***this.setStatusInformation(STATUS.PROGRESS); + //this.setStatusInformation(STATUS.PROGRESS); this.logger.debug("Checking the method..."); methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get()); methodChecker.copyAlgorithms(this.algorithm); @@ -185,8 +185,8 @@ public abstract class DMPMJob { this.algorithm.getFullname()); this.setStatusInformation(STATUS.COMPLETED); return algorithmListResult ?"":"\nWARNING: algorithm list could not be updated on SVN"; - ***REMOVED*** -***REMOVED*** + } + } protected void execute(NotificationHelper nh, CheckMethod methodChecker) { @@ -213,7 +213,7 @@ public abstract class DMPMJob { - ***REMOVED*** catch (DMPMException dmpme) + } catch (DMPMException dmpme) { this.logger.error("Operation failed: "+dmpme.getMessage()); this.logger.error("Exception: ",dmpme); @@ -222,18 +222,18 @@ public abstract class DMPMJob { this.updateLogFile(logFile, errorMessage); sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage); - ***REMOVED*** + } - ***REMOVED*** catch (EMailException eme) + } catch (EMailException eme) { this.logger.error("Unable to send notification email",eme); - ***REMOVED*** + } -***REMOVED*** + } @@ -244,17 +244,17 @@ public abstract class DMPMJob { FileOutputStream fos = new FileOutputStream(logFile, true); PrintStream ps = new PrintStream(fos); -***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); -***REMOVED*** PrintWriter fos2 = new PrintWriter(m, "UTF-8"); +// File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); +// PrintWriter fos2 = new PrintWriter(m, "UTF-8"); return worker.execute(ps); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } public String buildInfo() { @@ -266,8 +266,8 @@ public abstract class DMPMJob { "Staging DataMiner Host: "+ this.configuration.getHost()+"\n"+ "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ "Target VRE: "+this.vREName+"\n"; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 3442b21..5ee25a8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -2,37 +2,37 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import java.io.File; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.util.Scanner; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderStaging; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; public class DataminerPoolManager { -***REMOVED*** + private Logger logger; private SVNUpdaterStaging svnUpdaterStaging; private SVNUpdaterProduction svnUpdaterProduction; public DataminerPoolManager() { this.logger = LoggerFactory.getLogger(this.getClass()); try { - ***REMOVED***TODO: read this from configuration + //TODO: read this from configuration this.svnUpdaterStaging = new SVNUpdaterStaging(); this.svnUpdaterProduction = new SVNUpdaterProduction(); - ***REMOVED*** catch (SVNException e) { + } catch (SVNException e) { this.logger.error("SVN Exception",e); - ***REMOVED*** -***REMOVED*** + } + } public String stageAlgorithm(Algorithm algo,String targetVRE, String category, String algorithm_type/*,String env*/) throws IOException, InterruptedException @@ -43,11 +43,11 @@ public class DataminerPoolManager { this.logger.debug("Algo type "+algorithm_type); ClusterBuilder stagingClusterBuilder = new ClusterBuilderStaging(); Cluster stagingCluster = stagingClusterBuilder.getDataminerCluster(); - ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + //Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); DMPMJob job = new StagingJob(this.svnUpdaterStaging, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; -***REMOVED*** + } public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException { @@ -60,12 +60,12 @@ public class DataminerPoolManager { DMPMJob job = new ProductionPublishingJob(this.svnUpdaterProduction, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; -***REMOVED*** + } public String getLogById(String id) throws FileNotFoundException{ - ***REMOVED***TODO: load dir from configuration file + //TODO: load dir from configuration file this.logger.debug("Getting log by id "+id); File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + id); @@ -75,13 +75,13 @@ public class DataminerPoolManager { this.logger.debug("Response "+response); scanner.close(); return response; -***REMOVED*** + } public String getMonitorById(String id) throws FileNotFoundException{ this.logger.debug("Getting monitor by id "+id); - ***REMOVED***TODO: load dir from configuration file + //TODO: load dir from configuration file File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + id + "_exitStatus"); Scanner scanner = new Scanner(path); @@ -89,9 +89,9 @@ public class DataminerPoolManager { this.logger.debug("Response "+response); scanner.close(); return response; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index 0ec7029..071dbdb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -5,33 +5,33 @@ import java.io.UnsupportedEncodingException; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodProduction; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperProduction; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ProductionPublishingJob extends DMPMJob { - ***REMOVED***private String targetVREToken; - ***REMOVED***private String env; -***REMOVED*** + //private String targetVREToken; + //private String env; + private Logger logger; public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),algorithm,prodCluster,targetVREName,category,algorithm_type); - this.logger = LoggerFactory.getLogger(StagingJob.class);***REMOVED*** this.jobLogs = new File( + this.logger = LoggerFactory.getLogger(StagingJob.class);// this.jobLogs = new File( -***REMOVED*** + } @Override protected void execute() { this.logger.debug("Executing staging job..."); super.execute(new NotificationHelperProduction(), new CheckMethodProduction()); -***REMOVED*** + } @@ -42,4 +42,4 @@ public class ProductionPublishingJob extends DMPMJob { -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index 5809928..b7e761f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -5,25 +5,25 @@ import java.io.UnsupportedEncodingException; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodStaging; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperStaging; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class StagingJob extends DMPMJob { -***REMOVED*** + private Logger logger; public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, /* Cluster rProtoCluster, */ String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),algorithm,stagingCluster,rProtoVREName,category,algorithm_type); this.logger = LoggerFactory.getLogger(StagingJob.class); -***REMOVED*** + } @Override protected void execute() { @@ -31,7 +31,7 @@ public class StagingJob extends DMPMJob { this.logger.debug("Executing staging job..."); super.execute(new NotificationHelperStaging(), new CheckMethodStaging()); -***REMOVED*** + } @@ -40,4 +40,4 @@ public class StagingJob extends DMPMJob { -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java index 7b25ee5..57e43e2 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/AnsibleException.java @@ -15,11 +15,11 @@ public class AnsibleException extends DMPMException { public AnsibleException(int returnCode) { super ("Ansible work failed"); this.returnCode =returnCode; -***REMOVED*** + } @Override public String getErrorMessage() { return "Installation failed. Return code=" + this.returnCode; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java index 8cdd8ff..e09371b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/exceptions/UndefinedDependenciesException.java @@ -18,12 +18,12 @@ public class UndefinedDependenciesException extends DMPMException { this.message = "Following dependencies are not defined:\n"; for (String n : undefinedDependencies) { message += "\n" + n +"\n"; - ***REMOVED*** -***REMOVED*** + } + } @Override public String getErrorMessage() { return this.message; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index b55a520..61c9c03 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -***REMOVED*** +import java.io.IOException; /** * Created by ggiammat on 5/9/17. @@ -13,7 +13,7 @@ public class AlgorithmBuilder { public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException { return create(algorithmPackageURL, null, null, null, null, null, null, null); - ***REMOVED*** + } public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, String category, String algorithmType, String skipJava) throws IOException, InterruptedException { @@ -24,24 +24,24 @@ public class AlgorithmBuilder { if(category != null){ algo.setCategory(category); - ***REMOVED*** + } if(algorithmType != null){ algo.setAlgorithmType(algorithmType); - ***REMOVED*** + } if(skipJava != null){ algo.setSkipJava(skipJava); - ***REMOVED*** + } if(skipJava != null){ algo.setSkipJava(skipJava); - ***REMOVED*** + } if(name != null){ algo.setName(name); - ***REMOVED*** + } if(description != null){ algo.setDescription(description); - ***REMOVED*** + } return algo; - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 864efde..069f448 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -1,14 +1,14 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED*** +import java.io.BufferedReader; import java.io.File; import java.io.FileWriter; -***REMOVED*** -***REMOVED*** +import java.io.InputStreamReader; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -***REMOVED*** -***REMOVED*** +import java.util.LinkedList; +import java.util.List; import java.util.Properties; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; @@ -16,8 +16,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; import com.jcraft.jsch.Channel; @@ -29,7 +29,7 @@ import com.jcraft.jsch.SftpException; public abstract class CheckMethod { -***REMOVED*** + private Logger logger; private Configuration configuration; private final String KNOWN_HOSTS= "~/.ssh/known_hosts", @@ -45,22 +45,22 @@ public abstract class CheckMethod { this.configuration = configuration; sshConfig = new java.util.Properties(); sshConfig.put("StrictHostKeyChecking", "no"); -***REMOVED*** + } public void checkMethod(String machine, String token) throws AlgorithmException { try { this.logger.debug("Checking method for machine "+machine); this.logger.debug("By using tocken "+token); this.logger.debug("Machine: " + machine); -***REMOVED*** String getCapabilitesRequest = new String(); -***REMOVED*** String getCapabilitesResponse = new String(); +// String getCapabilitesRequest = new String(); +// String getCapabilitesResponse = new String(); this.logger.debug(" Token: " + token); - String request = "http:***REMOVED***" + machine + String request = "http://" + machine + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; String response = machine + "___" + token + ".xml"; -***REMOVED*** getCapabilitesRequest = request; -***REMOVED*** getCapabilitesResponse = response; - String baseDescriptionRequest = "http:***REMOVED***" + machine +// getCapabilitesRequest = request; +// getCapabilitesResponse = response; + String baseDescriptionRequest = "http://" + machine + "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token=" + token + "&Identifier="; URL requestURL = new URL(request); @@ -97,31 +97,31 @@ public abstract class CheckMethod { this.logger.debug("Operator descriptor "+operatorDescription); this.logger.debug(" " + operatorDescription); innerFlag = false; - ***REMOVED*** else if (innerLine.contains("ows:ExceptionText")) + } else if (innerLine.contains("ows:ExceptionText")) { this.logger.debug("Exception found"); this.logger.debug(" " + "error retrieving operator description"); innerFlag = false; flag = false; - ***REMOVED*** else + } else { innerLine = innerBufferedReader.readLine(); this.logger.debug("Inner line completed "+innerLine); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } this.logger.debug("Operation successful"); fileWriter.close(); - ***REMOVED*** catch (Exception e) { + } catch (Exception e) { throw new AlgorithmException("Error "+e.getMessage(),e); - ***REMOVED*** -***REMOVED*** + } + } @@ -138,10 +138,10 @@ public abstract class CheckMethod { { fileExists = (this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { throw new GenericException(e); - ***REMOVED*** + } if (fileExists) @@ -153,20 +153,20 @@ public abstract class CheckMethod { this.copyFromDmToSVN(file/*,env*/); this.copyFromDmToSVN(file2/*,env*/); this.logger.debug("Files have been copied to SVN"); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { throw new GenericException(e); - ***REMOVED*** + } - ***REMOVED*** + } else { this.logger.debug("Files not found"); this.logger.debug("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); throw new AlgorithmException("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); - ***REMOVED*** + } -***REMOVED*** + } @@ -200,19 +200,19 @@ public abstract class CheckMethod { c.rm(file.getPath()); c.rm(file2.getPath()); this.logger.debug("Both the files have been deleted"); - ***REMOVED*** + } else this.logger.debug("Files not found"); channel.disconnect(); c.disconnect(); session.disconnect(); - ***REMOVED*** catch (Exception e) + } catch (Exception e) { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } @@ -231,20 +231,20 @@ public abstract class CheckMethod { try { c.lstat(path); success = true; - ***REMOVED*** catch (SftpException e) { + } catch (SftpException e) { if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { - ***REMOVED*** file doesn't exist + // file doesn't exist success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** + } + //success = true; // something else went wrong + } channel.disconnect(); c.disconnect(); session.disconnect(); this.logger.debug("Operation result "+success); return success; -***REMOVED*** + } protected abstract void copyFromDmToSVN(File a) throws SVNCommitException, Exception; @@ -271,7 +271,7 @@ public abstract class CheckMethod { File f = new File(localFile); svnUpdater.updateAlgorithmFiles(f); f.delete(); -***REMOVED*** + } private Session generateSession () throws JSchException { @@ -283,7 +283,7 @@ public abstract class CheckMethod { this.logger.debug("session created."); session.setConfig(this.sshConfig); return session; -***REMOVED*** + } @@ -295,11 +295,11 @@ public abstract class CheckMethod { for (String s: list){ ls.add(s.trim()); - ***REMOVED*** + } return ls; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java index b5a3bb0..3fb4bf8 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java @@ -2,10 +2,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import static org.gcube.common.authorization.client.Constants.authorizationService; -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.library.AuthorizationEntry; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CheckPermission { @@ -20,9 +20,9 @@ public class CheckPermission { if (entry.getContext().equals(vre)) { logger.info("Authorization OK!"); return true; - ***REMOVED*** + } logger.info("Not a valid token recognized for the VRE: "+vre); return false; -***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java index b6d02a0..bc1f77f 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -1,14 +1,14 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; public abstract class ClusterBuilder { @@ -17,9 +17,9 @@ public abstract class ClusterBuilder { public ClusterBuilder (Configuration configuration) { this.configuration = configuration; -***REMOVED*** + } - ***REMOVED***1. to complete + //1. to complete public Cluster getDataminerCluster() throws FileNotFoundException{ Cluster cluster = new Cluster(); @@ -30,7 +30,7 @@ public abstract class ClusterBuilder { return cluster; -***REMOVED*** + } @@ -40,19 +40,19 @@ public abstract class ClusterBuilder { Cluster cluster = new Cluster(); for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) { cluster.addHost(h); - ***REMOVED*** + } return cluster; -***REMOVED*** + } public Cluster getRProtoCluster() throws IOException{ - ***REMOVED***Assumes the service is running in RPrototypingLab + //Assumes the service is running in RPrototypingLab String token = SecurityTokenProvider.instance.get(); String targetVRE = ScopeProvider.instance.get(); return this.getVRECluster(token, targetVRE); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index 4cfcbd3..91c1dcb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; public abstract class NotificationHelper { @@ -9,71 +9,71 @@ public abstract class NotificationHelper { protected NotificationHelper (String subjectHeader) { this.subjectHeader = subjectHeader; -***REMOVED*** + } -***REMOVED*** private Exception executionException; +// private Exception executionException; -***REMOVED*** private boolean isError() { -***REMOVED*** return this.executionException!=null; -***REMOVED*** ***REMOVED*** +// private boolean isError() { +// return this.executionException!=null; +// } -***REMOVED*** public void setExecutionException(Exception executionException) { -***REMOVED*** this.executionException = executionException; -***REMOVED*** ***REMOVED*** +// public void setExecutionException(Exception executionException) { +// this.executionException = executionException; +// } public String getSuccessSubject() { return this.subjectHeader+" is SUCCESS"; - ***REMOVED*** + } public String getFailedSubject() { return String.format(this.subjectHeader+" is FAILED"); - ***REMOVED*** + } public static String getSuccessBody(String info) { String message = String.format("The installation of the algorithm is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info; return message; - ***REMOVED*** + } public static String getFailedBody(String message) { String body = String.format("An error occurred while deploying your algorithm"); body+= "\n\nHere are the error details:\n\n" + message; return body; - ***REMOVED*** + } -***REMOVED*** public String getSuccessBodyRelease(String info) { -***REMOVED*** String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); -***REMOVED*** message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; -***REMOVED*** return message; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public String getFailedBodyRelease(String info) { -***REMOVED*** String message = String.format("SVN REPOSITORY UPDATE FAILED."); -***REMOVED*** message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; -***REMOVED*** return message; -***REMOVED*** ***REMOVED*** +// public String getSuccessBodyRelease(String info) { +// String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); +// message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; +// return message; +// } +// +// public String getFailedBodyRelease(String info) { +// String message = String.format("SVN REPOSITORY UPDATE FAILED."); +// message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; +// return message; +// } -***REMOVED*** public String getSubject() { -***REMOVED*** if(this.isError()) { -***REMOVED*** return this.getFailedSubject(); -***REMOVED*** ***REMOVED*** else { -***REMOVED*** return this.getSuccessSubject(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public String getBody() { -***REMOVED*** if(this.isError()) { -***REMOVED*** return this.getFailedBody(); -***REMOVED*** ***REMOVED*** else { -***REMOVED*** return this.getSuccessBody(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** +// public String getSubject() { +// if(this.isError()) { +// return this.getFailedSubject(); +// } else { +// return this.getSuccessSubject(); +// } +// } +// +// public String getBody() { +// if(this.isError()) { +// return this.getFailedBody(); +// } else { +// return this.getSuccessBody(); +// } +// } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java index 98118a3..924a995 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -***REMOVED*** +import java.io.IOException; import java.net.Authenticator; import java.net.InetSocketAddress; import java.net.PasswordAuthentication; @@ -10,7 +10,7 @@ import java.net.SocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; -***REMOVED*** +import java.util.List; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; @@ -28,7 +28,7 @@ interface NetworkConfiguration { public String getNonProxyHosts(); -***REMOVED*** +} class FileBasedProxyConfiguration implements NetworkConfiguration { @@ -36,42 +36,42 @@ class FileBasedProxyConfiguration implements NetworkConfiguration { public FileBasedProxyConfiguration(String path) { try { - ***REMOVED*** load the configuration + // load the configuration configuration = new PropertiesConfiguration(path); - ***REMOVED*** set the reloading strategy to enable hot-configuration + // set the reloading strategy to enable hot-configuration FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); configuration.setReloadingStrategy(fcrs); - ***REMOVED*** catch (ConfigurationException e) { + } catch (ConfigurationException e) { e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** + } + } @Override public String getProxyHost() { return configuration.getString("proxyHost"); - ***REMOVED*** + } @Override public String getProxyPort() { return configuration.getString("proxyPort"); - ***REMOVED*** + } @Override public String getProxyUser() { return configuration.getString("proxyUser"); - ***REMOVED*** + } @Override public String getProxyPassword() { return configuration.getString("proxyPassword"); - ***REMOVED*** + } @Override public String getNonProxyHosts() { return configuration.getString("nonProxyHosts"); - ***REMOVED*** + } -***REMOVED*** +} public class PropertiesBasedProxySelector extends ProxySelector { @@ -81,13 +81,13 @@ public class PropertiesBasedProxySelector extends ProxySelector { public PropertiesBasedProxySelector(String proxySettingsPath) { this(new FileBasedProxyConfiguration(proxySettingsPath)); - ***REMOVED*** + } public PropertiesBasedProxySelector(NetworkConfiguration config) { if (config == null || config.getProxyHost() == null) { this.proxies = null; return; - ***REMOVED*** + } String host = config.getProxyHost(); @@ -95,12 +95,12 @@ public class PropertiesBasedProxySelector extends ProxySelector { if (config.getProxyPort() != null) { port = Integer.valueOf(config.getProxyPort()); - ***REMOVED*** + } if (config.getNonProxyHosts() != null) { this.nonProxyHosts = Arrays .asList(config.getNonProxyHosts().split("\\|")); - ***REMOVED*** + } this.proxies = new ArrayList(); this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, @@ -114,32 +114,32 @@ public class PropertiesBasedProxySelector extends ProxySelector { @Override protected PasswordAuthentication getPasswordAuthentication() { return new PasswordAuthentication(username, password.toCharArray()); - ***REMOVED*** - ***REMOVED***); + } + }); - ***REMOVED*** - ***REMOVED*** + } + } @Override public List select(URI uri) { if (this.nonProxyHosts == null) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** else { + } else { for (String entry : this.nonProxyHosts) { entry = entry.trim(); if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** + } if (uri.getHost().equals(entry)) { return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - ***REMOVED*** + } + } return this.proxies; - ***REMOVED*** - ***REMOVED*** + } + } @Override public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) { - ***REMOVED*** -***REMOVED*** + } +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java index 9ff0239..1bfa46c 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java @@ -1,8 +1,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; @@ -14,7 +14,7 @@ public class SVNRepositoryManager { private SVNRepository svnRepository; private static SVNRepositoryManager instance; -***REMOVED*** + private Logger logger; private SVNRepositoryManager (Configuration configuration) throws SVNException { @@ -27,16 +27,16 @@ public class SVNRepositoryManager { { this.logger.debug("Using SVN default credentials"); authManager = SVNWCUtil.createDefaultAuthenticationManager(); - ***REMOVED*** + } else { this.logger.debug("Using IS credentials"); authManager = SVNWCUtil.createDefaultAuthenticationManager(repository.getUsername(),repository.getPassword()); - ***REMOVED*** + } this.svnRepository.setAuthenticationManager(authManager); -***REMOVED*** + } public static SVNRepositoryManager getInstance (Configuration configuration) throws SVNException @@ -44,13 +44,13 @@ public class SVNRepositoryManager { if (instance == null) instance = new SVNRepositoryManager(configuration); return instance; -***REMOVED*** + } public SVNRepository getSvnRepository() { return svnRepository; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 0a32f81..1d55682 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -5,7 +5,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -***REMOVED*** +import java.io.IOException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -16,8 +16,8 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; -***REMOVED*** -***REMOVED*** +import java.util.LinkedList; +import java.util.List; import java.util.Set; import java.util.TimeZone; @@ -26,8 +26,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNCommitInfo; import org.tmatesoft.svn.core.SVNErrorMessage; import org.tmatesoft.svn.core.SVNException; @@ -45,26 +45,26 @@ public abstract class SVNUpdater { private SVNRepository svnRepository; private Configuration configuration; -***REMOVED*** + private Logger logger; public SVNUpdater(Configuration configuration) throws SVNException { this.configuration = configuration; this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); this.logger = LoggerFactory.getLogger(SVNUpdater.class); -***REMOVED*** + } -***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) { -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); -***REMOVED******REMOVED*** +// public void updateRPRotoDeps(Algorithm algorithm) { +// this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); +// this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); +// this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); +// } public String getDependencyFile(String language/*, String env*/) { return getDependencyFile(this.configuration,language); -***REMOVED*** + } @@ -100,21 +100,21 @@ public abstract class SVNUpdater { return configuration.getSVNWCDepsList(); default: return null; - ***REMOVED*** -***REMOVED*** + } + } public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); -***REMOVED*** + } public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException{ - ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); + //this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); this.updateAlgorithmList(this.configuration.getRepository(), a); -***REMOVED*** + } @@ -132,18 +132,18 @@ public abstract class SVNUpdater { { this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName()); - ***REMOVED*** + } else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName()); - ***REMOVED*** + } catch (FileNotFoundException e) { this.logger.error("Temporary algorithm file not found: this exception should not happen",e); - ***REMOVED*** + } finally { this.svnRepository.closeSession(); - ***REMOVED*** -***REMOVED*** + } + } @@ -168,10 +168,10 @@ public abstract class SVNUpdater { { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName); - ***REMOVED*** + } this.logger.debug("Operation completed"); -***REMOVED*** + } public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException { @@ -181,11 +181,11 @@ public abstract class SVNUpdater { commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); String filePath = destinationFolder + "/" + fileName; - ***REMOVED*** if (fileExists(filePath, -1)) { ***REMOVED*** updating existing file + // if (fileExists(filePath, -1)) { // updating existing file commitEditor.openFile(filePath, -1); - ***REMOVED******REMOVED*** else { ***REMOVED*** creating new file - ***REMOVED***commitEditor.addFile(filePath, null, -1); - ***REMOVED******REMOVED*** + //} else { // creating new file + //commitEditor.addFile(filePath, null, -1); + //} commitEditor.applyTextDelta(filePath, null); SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true); @@ -200,10 +200,10 @@ public abstract class SVNUpdater { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName+" to be updated"); - ***REMOVED*** + } this.logger.debug("Operation completed"); -***REMOVED*** + } @@ -212,18 +212,18 @@ public abstract class SVNUpdater { SVNNodeKind kind = svnRepository.checkPath(path, revision); if (kind == SVNNodeKind.FILE) { return true; - ***REMOVED*** + } return false; - ***REMOVED*** + } -***REMOVED*** public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException -***REMOVED*** { -***REMOVED*** -***REMOVED*** return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); -***REMOVED******REMOVED*** +// public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException +// { +// +// return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); +// } private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) @@ -238,18 +238,18 @@ public abstract class SVNUpdater { List newContent = new LinkedList<>(Arrays.asList(lines)); - ***REMOVED*** check if the algorithm is already in the list (match the class name) and delete the content + // check if the algorithm is already in the list (match the class name) and delete the content for (String l : lines) { if (l.contains(algorithm.getClazz())) { newContent.remove(l); - ***REMOVED***System.out.println("Not updating algorithm list beacuse already present"); - ***REMOVED***return; - ***REMOVED*** - ***REMOVED*** + //System.out.println("Not updating algorithm list beacuse already present"); + //return; + } + } - ***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it + // the algorithm is not in the list or must be overwritten cause some modification. Add it newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type/*, env*/)); - ***REMOVED*** Collections.sort(newContent); + // Collections.sort(newContent); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); @@ -264,7 +264,7 @@ public abstract class SVNUpdater { for (String line : newContent) { baos.write(line.getBytes()); baos.write("\n".getBytes()); - ***REMOVED*** + } byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); @@ -279,29 +279,29 @@ public abstract class SVNUpdater { { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); response = false; - ***REMOVED*** + } else response = true; - ***REMOVED*** + } catch (Exception ex) { this.logger.error("Unable to commit algorithm list",ex); response = false; - ***REMOVED*** + } finally { svnRepository.closeSession(); - ***REMOVED*** + } return response; -***REMOVED*** + } public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException { - ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); - ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; + //Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + //long unixTime = System.currentTimeMillis() / 1000L; StringBuffer sb = new StringBuffer("| "); sb.append(algorithm.getName() + " | "); @@ -317,24 +317,24 @@ public abstract class SVNUpdater { this.logger.info("Algo details: "+sb.toString()); return sb.toString(); -***REMOVED*** + } public Collection getUndefinedDependencies(String file, Collection deps) throws GenericException { try { -***REMOVED*** SendMail sm = new SendMail(); -***REMOVED*** NotificationHelper nh = new NotificationHelper(); +// SendMail sm = new SendMail(); +// NotificationHelper nh = new NotificationHelper(); List undefined = new LinkedList(); - ***REMOVED***to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string - ***REMOVED***just to uncomment the following lines + //to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string + //just to uncomment the following lines if(file.isEmpty()){ return undefined; - ***REMOVED*** + } this.logger.debug("Checking dependencies list: " + file); @@ -349,77 +349,77 @@ public abstract class SVNUpdater { for(String l: byteArrayOutputStream.toString().split("\\r?\\n")){ validDependencies.add(l.trim()); - ***REMOVED******REMOVED*** + }} this.logger.debug("Valid dependencies are: "+validDependencies); for(Dependency d: deps){ String depName = d.getName(); if(!validDependencies.contains(depName)){ undefined.add(depName); - ***REMOVED*** - ***REMOVED*** + } + } return undefined; -***REMOVED*** -***REMOVED*** -***REMOVED*** for (String a : lines) { -***REMOVED*** for (String b : ldep) { -***REMOVED*** if (b.equals(a)) { -***REMOVED*** System.out.println("The following dependency is correctly written: " + b); -***REMOVED*** ***REMOVED*** else - ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** boolean check = false; -***REMOVED*** try { -***REMOVED*** System.out.println("Checking dependencies list: " + file); -***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); -***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); -***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED*** -***REMOVED*** ***REMOVED*** if(deps.isEmpty()){ -***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); -***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this -***REMOVED*** ***REMOVED*** algorithm"); -***REMOVED*** ***REMOVED*** throw e; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** ***REMOVED*** - ***REMOVED*** -***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) { -***REMOVED*** List ldep = new LinkedList<>(); -***REMOVED*** for (Dependency d : deps) { -***REMOVED*** ldep.add(d.getName()); -***REMOVED*** ***REMOVED*** -***REMOVED*** for (String a : lines) { -***REMOVED*** for (String b : ldep) { -***REMOVED*** if (b.equals(a)) { -***REMOVED*** System.out.println("The following dependency is correctly written: " + b); -***REMOVED*** check = true; -***REMOVED*** ***REMOVED*** else -***REMOVED*** check = false; - ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** catch (Exception a) { -***REMOVED*** a.getMessage(); -***REMOVED*** ***REMOVED*** - ***REMOVED*** -***REMOVED*** return check; - ***REMOVED*** catch (SVNException e) +// +// +// for (String a : lines) { +// for (String b : ldep) { +// if (b.equals(a)) { +// System.out.println("The following dependency is correctly written: " + b); +// } else + // +// } +// } +// +// boolean check = false; +// try { +// System.out.println("Checking dependencies list: " + file); +// final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +// svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); +// String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + // +// // if(deps.isEmpty()){ +// // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); +// // Exception e = new Exception("No dependency specified for this +// // algorithm"); +// // throw e; +// // +// // } + // +// // else if (!deps.isEmpty()) { +// List ldep = new LinkedList<>(); +// for (Dependency d : deps) { +// ldep.add(d.getName()); +// } +// for (String a : lines) { +// for (String b : ldep) { +// if (b.equals(a)) { +// System.out.println("The following dependency is correctly written: " + b); +// check = true; +// } else +// check = false; + // +// } +// } +// // } +// } catch (Exception a) { +// a.getMessage(); +// } + // +// return check; + } catch (SVNException e) { throw new GenericException(e); - ***REMOVED*** + } -***REMOVED*** + } public boolean checkIfAvaialable(String file, Collection deps) throws SVNException { - ***REMOVED***SendMail sm = new SendMail(); - ***REMOVED***NotificationHelper nh = new NotificationHelper(); + //SendMail sm = new SendMail(); + //NotificationHelper nh = new NotificationHelper(); boolean check = false; try { @@ -428,39 +428,39 @@ public abstract class SVNUpdater { svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED*** if(deps.isEmpty()){ - ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); - ***REMOVED*** Exception e = new Exception("No dependency specified for this - ***REMOVED*** algorithm"); - ***REMOVED*** throw e; - ***REMOVED*** - ***REMOVED*** ***REMOVED*** + // if(deps.isEmpty()){ + // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); + // Exception e = new Exception("No dependency specified for this + // algorithm"); + // throw e; + // + // } - ***REMOVED*** else if (!deps.isEmpty()) { + // else if (!deps.isEmpty()) { List ldep = new LinkedList<>(); for (Dependency d : deps) { ldep.add(d.getName()); - ***REMOVED*** + } for (String a : lines) { for (String b : ldep) { if (b.equals(a)) { System.out.println("The following dependency is correctly written: " + b); check = true; - ***REMOVED*** else + } else check = false; - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** catch (Exception a) + } + } + // } + } catch (Exception a) { this.logger.error(a.getMessage(),a); - ***REMOVED*** + } return check; -***REMOVED*** + } public void updateSVN(String file, Collection deps) { try { @@ -473,7 +473,7 @@ public abstract class SVNUpdater { List ldep = new LinkedList<>(); for (Dependency d : deps) { ldep.add(d.getName()); - ***REMOVED*** + } List aa = this.checkMatch(lines, ldep); Collections.sort(aa); @@ -491,7 +491,7 @@ public abstract class SVNUpdater { baos.write(line.getBytes()); baos.write("\n".getBytes()); - ***REMOVED*** + } byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); @@ -503,14 +503,14 @@ public abstract class SVNUpdater { commitEditor.closeFile(file, checksum); commitEditor.closeEdit(); - ***REMOVED*** catch (Exception ex) { + } catch (Exception ex) { ex.printStackTrace(); - ***REMOVED*** + } finally { svnRepository.closeSession(); - ***REMOVED*** -***REMOVED*** + } + } public static String md5(byte[] contents) { final byte[] tmp = new byte[1024]; @@ -518,22 +518,22 @@ public abstract class SVNUpdater { "md5"); try { while (checksumStream.read(tmp) > 0) { - ***REMOVED*** - ***REMOVED*** + // + } return checksumStream.getDigest(); - ***REMOVED*** catch (IOException e) { - ***REMOVED*** never happens + } catch (IOException e) { + // never happens e.printStackTrace(); return null; - ***REMOVED*** finally { + } finally { SVNFileUtil.closeFile(checksumStream); - ***REMOVED*** -***REMOVED*** + } + } public List checkMatch(String[] lines, List ls) { Set ss = new HashSet(ls); ss.addAll(Arrays.asList(lines)); return new ArrayList<>(ss); -***REMOVED*** + } public String getTimeZone() throws ParseException{ @@ -545,21 +545,21 @@ public abstract class SVNUpdater { formatter.setTimeZone(central); this.logger.info(formatter.format(fromDate)); return formatter.format(fromDate); -***REMOVED*** + } public static void main(String[] args) throws SVNException, ParseException { - ***REMOVED*** SVNUpdater c = new SVNUpdater(); + // SVNUpdater c = new SVNUpdater(); - ***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro"); - ***REMOVED***File b = new File ("/home/ngalante/Desktop/testB"); - ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; - ***REMOVED***System.out.println(unixTime); - ***REMOVED***c.updateAlgorithmFiles(a); - ***REMOVED***c.updateAlgorithmFiles(b); - ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + //File a = new File("/home/ngalante/Desktop/testCiro"); + //File b = new File ("/home/ngalante/Desktop/testB"); + //long unixTime = System.currentTimeMillis() / 1000L; + //System.out.println(unixTime); + //c.updateAlgorithmFiles(a); + //c.updateAlgorithmFiles(b); + //Timestamp timestamp = new Timestamp(System.currentTimeMillis()); Calendar cal = Calendar.getInstance(); cal.getTime(); @@ -571,9 +571,9 @@ public abstract class SVNUpdater { -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 9e4f447..8aeba45 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -4,24 +4,24 @@ import static org.gcube.common.authorization.client.Constants.authorizationServi import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -***REMOVED*** -***REMOVED*** -***REMOVED*** +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.MalformedURLException; -***REMOVED*** +import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.Iterator; -***REMOVED*** +import java.util.List; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -***REMOVED*** +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.library.AuthorizationEntry; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.encryption.encrypter.StringEncrypter; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint; @@ -35,8 +35,8 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -***REMOVED*** -***REMOVED*** +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SendMail { @@ -54,7 +54,7 @@ public class SendMail { public SendMail() { -***REMOVED*** + } public void sendNotification(String subject, String body) throws EMailException { logger.debug("SendNotification"); @@ -69,7 +69,7 @@ public class SendMail { sendPostRequest(postBody); -***REMOVED*** + } private String createPostBody(String subject, String body) throws EMailException { try { @@ -77,10 +77,10 @@ public class SendMail { if (recipientsList == null || recipientsList.isEmpty()) { logger.error("Invalid recipient list: " + recipientsList); throw new EMailException("Unable to send email notification. Invalid recipient list:" + recipientsList); - ***REMOVED*** + } - ***REMOVED*** {"subject": "subject-content", "body": "body-content", - ***REMOVED*** "recipients":[{"id":"userid"***REMOVED***]***REMOVED*** + // {"subject": "subject-content", "body": "body-content", + // "recipients":[{"id":"userid"}]} JSONObject data = new JSONObject(); data.put("subject", subject); @@ -91,21 +91,21 @@ public class SendMail { JSONObject d = new JSONObject(); d.put("id", recipient); recipients.put(d); - ***REMOVED*** + } data.put("recipients", recipients); logger.debug("Post Body: " + data); return data.toString(); - ***REMOVED*** catch (EMailException e) { + } catch (EMailException e) { throw e; - ***REMOVED*** catch (Throwable e) { + } catch (Throwable e) { logger.error("Error creating the notification body: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } private String retrieveApplicationToken() throws EMailException { try { @@ -116,9 +116,9 @@ public class SendMail { List resources = client.submit(query); if (resources.isEmpty()) { logger.error("No services resource found on IS!"); - ***REMOVED*** else { + } else { logger.debug("Retrieved: " + resources.get(0)); - ***REMOVED*** + } Group accessPoints = resources.get(0).profile().accessPoints(); if (!accessPoints.isEmpty()) { @@ -133,23 +133,23 @@ public class SendMail { logger.info("Application Token retrieved"); return applicationToken; - ***REMOVED*** else { + } else { String error = "DMPoolManager application token not found in service resource on IS!"; logger.error(error); throw new EMailException(error); - ***REMOVED*** - ***REMOVED*** else { + } + } else { String error = "DMPoolManager invalid service resource on IS!"; logger.error(error); throw new EMailException(error); - ***REMOVED*** + } - ***REMOVED*** catch (Throwable e) { + } catch (Throwable e) { logger.error("DMPoolManager application token not found: " + e.getLocalizedMessage(), e); throw new EMailException("DMPoolManager application token not found: " + e.getLocalizedMessage(), e); - ***REMOVED*** + } -***REMOVED*** + } private void retrieveSocialService() throws EMailException { try { @@ -162,22 +162,22 @@ public class SendMail { if (socialServiceAddress == null || socialServiceAddress.isEmpty()) { throw new EMailException( "Unable to send email notification. Invalid address in GCoreEndpoint resource on IS."); - ***REMOVED*** - ***REMOVED*** catch (EMailException e) { + } + } catch (EMailException e) { logger.error(e.getLocalizedMessage(), e); throw e; - ***REMOVED*** catch (Throwable e) { + } catch (Throwable e) { logger.error(e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } private String username(String token) throws ObjectNotFound, Exception { AuthorizationEntry entry = authorizationService().get(token); logger.debug(entry.getClientInfo().getId()); return entry.getClientInfo().getId(); -***REMOVED*** + } private void sendPostRequest(String postBody) throws EMailException { @@ -194,7 +194,7 @@ public class SendMail { requestMessageBuilder.append(applicationToken); String endpoint = requestMessageBuilder.toString(); - ***REMOVED*** Send the request + // Send the request URL url = new URL(endpoint); URLConnection conn = url.openConnection(); conn.setRequestProperty("Accept", JSON_MIME_TYPE); @@ -205,13 +205,13 @@ public class SendMail { writer.write(postBody); writer.flush(); - ***REMOVED*** Get the response + // Get the response StringBuffer answer = new StringBuffer(); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); String line; while ((line = reader.readLine()) != null) { answer.append(line); - ***REMOVED*** + } writer.close(); reader.close(); @@ -220,26 +220,26 @@ public class SendMail { logger.info("Notification Response: " + response); checkResponse(response); - ***REMOVED*** catch (EMailException e) { + } catch (EMailException e) { throw e; - ***REMOVED*** catch (MalformedURLException e) { + } catch (MalformedURLException e) { logger.error("Invalid URL: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** catch (IOException e) { + } catch (IOException e) { logger.error("Error in the IO process: " + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** catch (Throwable e) { + } catch (Throwable e) { logger.error("Error executing post:" + e.getLocalizedMessage(), e); throw new EMailException(e); - ***REMOVED*** + } -***REMOVED*** + } private void checkResponse(String response) throws EMailException { if (response == null) { logger.error("Invalid notification response: " + response); throw new EMailException(); - ***REMOVED*** else { + } else { try { JSONObject res = new JSONObject(response); boolean success = res.getBoolean("success"); @@ -247,15 +247,15 @@ public class SendMail { String message = res.getString("message"); logger.error("Error in send email notification: " + message); throw new EMailException("Error in send email notification: " + message); - ***REMOVED*** + } - ***REMOVED*** catch (JSONException e) { + } catch (JSONException e) { logger.error("Invalid notification response: " + response); throw new EMailException(e); - ***REMOVED*** + } - ***REMOVED*** -***REMOVED*** + } + } private List getRecipients() { try { @@ -268,28 +268,28 @@ public class SendMail { if (data != null) { for (int i = 0; i < data.length(); i++) { recipients.add(data.getString(i)); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** else { + } + } + } else { logger.info("Use the default admins how workaround "); List defaultManagers = DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); recipients.addAll(defaultManagers); - ***REMOVED*** + } recipients.add(this.username(SecurityTokenProvider.instance.get())); logger.info("Retrieved Recipients: " + recipients); return recipients; - ***REMOVED*** catch (Exception e) { + } catch (Exception e) { logger.error("Error retrieving recipients: " + e.getLocalizedMessage(), e); logger.info("Use the default admins how workaround "); return DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); - ***REMOVED*** + } -***REMOVED*** + } private String retrieveDataMinerManagers() throws Exception { - ***REMOVED*** Try to retrieve a url like this: - ***REMOVED*** https:***REMOVED***api.d4science.org/social-networking-library-ws/rest/2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=xxx-xxxx-xxxx-xxx + // Try to retrieve a url like this: + // https://api.d4science.org/social-networking-library-ws/rest/2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=xxx-xxxx-xxxx-xxx StringBuilder requestMessageBuilder = new StringBuilder(socialServiceAddress); @@ -298,7 +298,7 @@ public class SendMail { requestMessageBuilder.append(USER_ROLES_ADDRESS_PATH); logger.info("Request Admins Url: " + requestMessageBuilder.toString()); - ***REMOVED*** SecurityTokenProvider.instance.get() + // SecurityTokenProvider.instance.get() requestMessageBuilder.append(applicationToken); String requestAdminsUrl = requestMessageBuilder.toString(); @@ -311,6 +311,6 @@ public class SendMail { return EntityUtils.toString(client.execute(getReq).getEntity()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java index 5c29084..d8d6b2d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java @@ -13,7 +13,7 @@ public class AlgorithmException extends DMPMException{ super ("Algorithm exception"); this.algorithmName = algorithmName; -***REMOVED*** + } @@ -21,7 +21,7 @@ public class AlgorithmException extends DMPMException{ { super ("Algorithm exception", cause); this.algorithmName = algorithmName; -***REMOVED*** + } @Override public String getErrorMessage() { @@ -29,7 +29,7 @@ public class AlgorithmException extends DMPMException{ return "Installation completed but DataMiner Interface not working correctly or files " + this.algorithmName + ".jar and " + this.algorithmName + "_interface.jar not availables at the expected path"; -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java index 6d8916d..f614e7d 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/DMPMException.java @@ -10,11 +10,11 @@ public abstract class DMPMException extends Exception{ public DMPMException (String errorMessage) { super (errorMessage); -***REMOVED*** + } public DMPMException(String errorMessage,Throwable cause) { super (errorMessage,cause); -***REMOVED*** + } public abstract String getErrorMessage (); -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java index d84ddf3..b0d02d9 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java @@ -11,18 +11,18 @@ public class EMailException extends Exception { public EMailException() { super(MESSAGE); -***REMOVED*** + } public EMailException(String message) { super(message); -***REMOVED*** + } public EMailException(String message, Throwable e) { super(message, e); -***REMOVED*** + } public EMailException(Throwable e) { super(MESSAGE, e); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java index aa1c2e4..d7b99dd 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/GenericException.java @@ -12,15 +12,15 @@ public class GenericException extends DMPMException { public GenericException(Throwable cause) { super ("Generic exception",cause); -***REMOVED*** + } @Override public String getErrorMessage() { return this.getCause().getMessage(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java index da4ec02..c3383a7 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java @@ -17,18 +17,18 @@ public class SVNCommitException extends DMPMException { super ("Unable to commit"); this.svnErrorMessage = errorMessage; this.fileName = fileName; -***REMOVED*** + } public SVNCommitException(String message,SVNErrorMessage errorMessage,String fileName) { super (message); this.svnErrorMessage = errorMessage; this.fileName = fileName; -***REMOVED*** + } public SVNErrorMessage getSvnErrorMessage() { return svnErrorMessage; -***REMOVED*** + } @@ -38,7 +38,7 @@ public class SVNCommitException extends DMPMException { return "Commit operation failed for "+this.fileName + "the message of the SVN Server is the following:\n"+this.svnErrorMessage.getMessage(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java index fa22a76..10a1014 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java @@ -17,89 +17,87 @@ public class CheckMethodProduction extends CheckMethod{ { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } @Override protected void copyFromDmToSVN(File a) throws SVNException, SVNCommitException, JSchException, SftpException { super.copyFromDmToSVN(a, new SVNUpdaterProduction()); -***REMOVED*** + } public static void main(String[] args) throws Exception { -***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); -***REMOVED*** System.out.println(a.getStagingHost()); +// ServiceConfiguration a = new ServiceConfiguration(); +// System.out.println(a.getStagingHost()); CheckMethodProduction a = new CheckMethodProduction(); - ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + //a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); -***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); -***REMOVED*** System.out.println(aa.getName()); -***REMOVED*** System.out.println(aa.getPath()); +// File aa = new File("OCTAVEBLACKBOX.jar"); +// System.out.println(aa.getName()); +// System.out.println(aa.getPath()); - ***REMOVED***a.copyFromDmToSVN(aa); -***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){ -***REMOVED*** System.out.println("AAA");***REMOVED*** -***REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ -***REMOVED*** System.out.println("BBBB"); -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ -***REMOVED*** System.out.println("CCCC");***REMOVED*** -***REMOVED*** -***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); -***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + //a.copyFromDmToSVN(aa); +// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxx-xxx-xxx-xxx")){ +// System.out.println("AAA"); } +// +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ +// System.out.println("BBBB"); +// +// } +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ +// System.out.println("CCCC");} +// +// File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +// a.copyFromDmToSVN(aa, "Dev"); -***REMOVED*** +// -***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", -***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); +//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +// "xxx-xxxx-xxx-xx")); -***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", - ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); try { a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); -***REMOVED*** catch (Exception e) + "xxx-xxx-xxx-xxx"); + } catch (Exception e) { e.printStackTrace(); -***REMOVED*** + } -***REMOVED***Algorithm aa = new Algorithm(); -***REMOVED***aa.setName("UDPIPE_WRAPPER"); -***REMOVED***System.out.println(a.algoExists(aa)); -***REMOVED******REMOVED*** -***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); -***REMOVED******REMOVED*** -***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); -***REMOVED******REMOVED*** -***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { -***REMOVED***System.out.println("ciao"); -***REMOVED*** -***REMOVED******REMOVED*** +//Algorithm aa = new Algorithm(); +//aa.setName("UDPIPE_WRAPPER"); +//System.out.println(a.algoExists(aa)); +//// +//ServiceConfiguration bp = new ServiceConfiguration(); +//// +//SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx"); +//// +//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { +//System.out.println("ciao"); +// +//} -***REMOVED*** -***REMOVED***Algorithm al = new Algorithm(); -***REMOVED*** al.setName("UDPIPE_WRAPPER"); -***REMOVED*** a.deleteFiles(al); +// +//Algorithm al = new Algorithm(); +// al.setName("UDPIPE_WRAPPER"); +// a.deleteFiles(al); -***REMOVED*** -***REMOVED*** +} +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java index 680cda6..e36001e 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java @@ -11,87 +11,87 @@ public class CheckMethodStaging extends CheckMethod{ public CheckMethodStaging() { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } @Override protected void copyFromDmToSVN(File a) throws Exception{ super.copyFromDmToSVN(a, new SVNUpdaterStaging()); -***REMOVED*** + } public static void main(String[] args) throws Exception { -***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); -***REMOVED*** System.out.println(a.getStagingHost()); +// ServiceConfiguration a = new ServiceConfiguration(); +// System.out.println(a.getStagingHost()); CheckMethodStaging a = new CheckMethodStaging(); - ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + //a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); -***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); -***REMOVED*** System.out.println(aa.getName()); -***REMOVED*** System.out.println(aa.getPath()); +// File aa = new File("OCTAVEBLACKBOX.jar"); +// System.out.println(aa.getName()); +// System.out.println(aa.getPath()); - ***REMOVED***a.copyFromDmToSVN(aa); -***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){ -***REMOVED*** System.out.println("AAA");***REMOVED*** -***REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ -***REMOVED*** System.out.println("BBBB"); -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ -***REMOVED*** System.out.println("CCCC");***REMOVED*** -***REMOVED*** -***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); -***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + //a.copyFromDmToSVN(aa); +// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxxx-xxx-xxx")){ +// System.out.println("AAA"); } +// +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ +// System.out.println("BBBB"); +// +// } +// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){ +// System.out.println("CCCC");} +// +// File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +// a.copyFromDmToSVN(aa, "Dev"); -***REMOVED*** +// -***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", -***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); +//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +// "xxxx-xxx-xxx-xxx")); -***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", - ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); +//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", + // "xxx-xxx-xxx-xxx")); try { a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); -***REMOVED*** catch (Exception e) + "xxxx-xxxx-xxxx-xxxx"); + } catch (Exception e) { e.printStackTrace(); -***REMOVED*** + } -***REMOVED***Algorithm aa = new Algorithm(); -***REMOVED***aa.setName("UDPIPE_WRAPPER"); -***REMOVED***System.out.println(a.algoExists(aa)); -***REMOVED******REMOVED*** -***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); -***REMOVED******REMOVED*** -***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); -***REMOVED******REMOVED*** -***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { -***REMOVED***System.out.println("ciao"); -***REMOVED*** -***REMOVED******REMOVED*** +//Algorithm aa = new Algorithm(); +//aa.setName("UDPIPE_WRAPPER"); +//System.out.println(a.algoExists(aa)); +//// +//ServiceConfiguration bp = new ServiceConfiguration(); +//// +//SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx"); +//// +//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { +//System.out.println("ciao"); +// +//} -***REMOVED*** -***REMOVED***Algorithm al = new Algorithm(); -***REMOVED*** al.setName("UDPIPE_WRAPPER"); -***REMOVED*** a.deleteFiles(al); +// +//Algorithm al = new Algorithm(); +// al.setName("UDPIPE_WRAPPER"); +// a.deleteFiles(al); -***REMOVED*** -***REMOVED*** +} +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java index e24d470..8fa38c3 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java @@ -8,7 +8,7 @@ public class ClusterBuilderProduction extends ClusterBuilder{ public ClusterBuilderProduction() { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java index 2fafead..0d1aa7b 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java @@ -7,7 +7,7 @@ public class ClusterBuilderStaging extends ClusterBuilder{ public ClusterBuilderStaging() { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java index 5f103c0..2b6b89a 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java @@ -2,18 +2,18 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; public class NotificationHelperProduction extends NotificationHelper{ -***REMOVED*** private Exception executionException; +// private Exception executionException; public NotificationHelperProduction() { super ("[DataMinerGhostProductionInstallationRequestReport]"); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java index cacf7d4..372b55a 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java @@ -2,17 +2,17 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; -***REMOVED***import scala.actors.threadpool.Arrays; +//import scala.actors.threadpool.Arrays; public class NotificationHelperStaging extends NotificationHelper { -***REMOVED*** private Exception executionException; +// private Exception executionException; public NotificationHelperStaging() { super ("[DataMinerGhostStagingInstallationRequestReport]"); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java index 36b4d3a..2fdbaeb 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java @@ -14,8 +14,8 @@ public class SVNUpdaterProduction extends SVNUpdater{ public SVNUpdaterProduction() throws SVNException { super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java index eaa6cbf..d6d93a5 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java @@ -12,10 +12,10 @@ public class SVNUpdaterStaging extends SVNUpdater{ public SVNUpdaterStaging() throws SVNException { super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/main/resources/custom/custom-flr-Flash/meta/main.yaml b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml index b24c771..efa0ff5 100755 --- a/src/main/resources/custom/custom-flr-Flash/meta/main.yaml +++ b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml @@ -1,4 +1,4 @@ #--- #dependencies: # -# - { role: digest ***REMOVED*** +# - { role: digest } diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index 238de49..3819f20 100755 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -1,7 +1,7 @@ #YML node file -SVN_REPO = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ +SVN_REPO = https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ -svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube +svn.repository = https://svn.d4science.research-infrastructures.eu/gcube svn.algo.main.repo = /trunk/data-analysis/DataMinerConfiguration/algorithms diff --git a/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml index 27a57fb..e3ea93e 100755 --- a/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml +++ b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml @@ -1,4 +1,4 @@ dependencies: -# - { role: gcube-dataminer ***REMOVED*** -# - { role: os-unzip ***REMOVED*** -# - { role: os-java-1.7.0 ***REMOVED*** +# - { role: gcube-dataminer } +# - { role: os-unzip } +# - { role: os-java-1.7.0 } diff --git a/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml index 7bb84f9..13f8109 100755 --- a/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml +++ b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml @@ -12,14 +12,14 @@ #- name: download the installer zip # get_url: -# url: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip +# url: https://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip # dest: /home/dpm/algorithmInstaller.zip # validate_certs: no #- name: download and unzip the package # unarchive: -# src: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller1_1.zip -## src: http:***REMOVED***maven.research-infrastructures.eu:8081/nexus/service/local/artifact/maven/redirect?r=gcube-snapshots&g=org.gcube.dataanalysis&a=dataminer-algorithms-importer&v=1.1.1-SNAPSHOT&e=tar.gz +# src: https://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller1_1.zip +## src: http://maven.research-infrastructures.eu:8081/nexus/service/local/artifact/maven/redirect?r=gcube-snapshots&g=org.gcube.dataanalysis&a=dataminer-algorithms-importer&v=1.1.1-SNAPSHOT&e=tar.gz # dest: /home/gcube # remote_src: yes # validate_certs: no diff --git a/src/main/resources/static/gcube-dataminer/meta/main.yaml b/src/main/resources/static/gcube-dataminer/meta/main.yaml index 2f68d3c..bab366c 100755 --- a/src/main/resources/static/gcube-dataminer/meta/main.yaml +++ b/src/main/resources/static/gcube-dataminer/meta/main.yaml @@ -1,2 +1,2 @@ dependencies: - - { role: gcube-ghn ***REMOVED*** + - { role: gcube-ghn } diff --git a/src/main/resources/static/os-java-1.7.0/tasks/main.yaml b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml index 3acc5a3..2906525 100755 --- a/src/main/resources/static/os-java-1.7.0/tasks/main.yaml +++ b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml @@ -3,5 +3,5 @@ - name: adding OS Package java-1.7.0 package: name: java-1.7.0-openjdk - state: "{{os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/resources/static/os-unzip/tasks/main.yaml b/src/main/resources/static/os-unzip/tasks/main.yaml index 25d085d..6887424 100755 --- a/src/main/resources/static/os-unzip/tasks/main.yaml +++ b/src/main/resources/static/os-unzip/tasks/main.yaml @@ -3,5 +3,5 @@ - name: adding OS Package unzip package: name: unzip - state: "{{os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/resources/templates/cran-package-add/meta/main.yaml b/src/main/resources/templates/cran-package-add/meta/main.yaml index dbe153f..7179b15 100755 --- a/src/main/resources/templates/cran-package-add/meta/main.yaml +++ b/src/main/resources/templates/cran-package-add/meta/main.yaml @@ -1,4 +1,4 @@ --- #dependencies: # -# - { role: os-r ***REMOVED*** +# - { role: os-r } diff --git a/src/main/resources/templates/cran-package-add/tasks/main.yaml b/src/main/resources/templates/cran-package-add/tasks/main.yaml index 42dd680..5b9f563 100755 --- a/src/main/resources/templates/cran-package-add/tasks/main.yaml +++ b/src/main/resources/templates/cran-package-add/tasks/main.yaml @@ -1,4 +1,4 @@ # tasks file for r --- - name: Install algorithm - command: Rscript -e "install.packages('', repos=c('http:***REMOVED***ftp.heanet.ie/mirrors/cran.r-project.org/'))" + command: Rscript -e "install.packages('', repos=c('http://ftp.heanet.ie/mirrors/cran.r-project.org/'))" diff --git a/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml index b8f3340..9396c53 100755 --- a/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml +++ b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml @@ -1,4 +1,4 @@ --- dependencies: - - { role: gcube-algorithm-installer ***REMOVED*** + - { role: gcube-algorithm-installer } diff --git a/src/main/resources/templates/github-package-add/meta/main.yaml b/src/main/resources/templates/github-package-add/meta/main.yaml index dbe153f..7179b15 100755 --- a/src/main/resources/templates/github-package-add/meta/main.yaml +++ b/src/main/resources/templates/github-package-add/meta/main.yaml @@ -1,4 +1,4 @@ --- #dependencies: # -# - { role: os-r ***REMOVED*** +# - { role: os-r } diff --git a/src/main/resources/templates/os-package-add/tasks/main.yaml b/src/main/resources/templates/os-package-add/tasks/main.yaml index 71a2609..7df04bc 100755 --- a/src/main/resources/templates/os-package-add/tasks/main.yaml +++ b/src/main/resources/templates/os-package-add/tasks/main.yaml @@ -3,6 +3,6 @@ - name: adding OS Package package: name: - state: "{{os_package_state***REMOVED******REMOVED***" + state: "{{os_package_state}}" become: true diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index c599c4d..7faa928 100755 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -1,6 +1,6 @@ + "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" + "http://java.sun.com/dtd/web-app_2_3.dtd" > Archetype Created Web Application @@ -41,7 +41,7 @@ diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java index 3373bf9..fb14c71 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java @@ -1,9 +1,9 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; import java.io.InputStream; -***REMOVED*** -***REMOVED*** +import java.net.URL; +import java.util.List; import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; @@ -16,7 +16,7 @@ public class AlgorithmPackageParserTest { private static int BUFFER_SIZE = 2048; public void extractAllAlgorithms() throws IOException { - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; + String url = "http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; List commands = this.extractAddAlgorithmCommands(url); AlgorithmSet algorithms = new AlgorithmSet(); for (String cmd : commands) { @@ -25,7 +25,7 @@ public class AlgorithmPackageParserTest { AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); System.out.println(aac); - ***REMOVED*** start creating the algo from the command + // start creating the algo from the command Algorithm algo = new Algorithm(); algo.setAlgorithmType(aac.getAlgorithmType()); algo.setCategory(aac.getCategory()); @@ -35,20 +35,20 @@ public class AlgorithmPackageParserTest { algo.setPackageURL(aac.getUrl()); algo.setSkipJava(aac.getSkipjava()); - ***REMOVED*** then override with info from the package + // then override with info from the package if (aac.getUrl().length() > 4) { Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); if (packagedAlgo != null) { algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** + } + } algorithms.addAlgorithm(algo); break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** + } + //to uncomment + // new DataminerPoolManager().addAlgorithmsToVRE(algorithms, + // "/gcube/devNext/NextNext"); + } /** * Extract 'addAlgorithm' commands from a file containing wiki-table-style @@ -67,7 +67,7 @@ public class AlgorithmPackageParserTest { int read = 0; while ((read = is.read(buffer)) != -1) { s.append(new String(buffer, 0, read)); - ***REMOVED*** + } List out = new Vector<>(); String[] lines = s.toString().split("\n"); for (String line : lines) { @@ -78,7 +78,7 @@ public class AlgorithmPackageParserTest { for (String part : parts) { if (part == null || part.trim().isEmpty()) { continue; - ***REMOVED*** + } System.out.println(c + ". " + part); c++; if (part.contains("addAlgorithm.sh")) { @@ -86,15 +86,15 @@ public class AlgorithmPackageParserTest { cmd = cmd.replaceAll("", ""); cmd = cmd.replaceAll("", ""); System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); + // AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); + // System.out.println(aac); out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } return out; - ***REMOVED*** + } /** * Create an Algorithm starting from the algorithm jar. @@ -105,16 +105,16 @@ public class AlgorithmPackageParserTest { */ private Algorithm extractAlgorithm(String url) throws IOException { return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** + } public static void main(String[] args) throws Exception { -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); +// ProxySelector.setDefault(new PropertiesBasedProxySelector( +// "/home/ngalante/.proxy-settings")); new AlgorithmPackageParserTest().extractAllAlgorithms(); -***REMOVED*** AlgorithmPackageParserTest at = new AlgorithmPackageParserTest(); -***REMOVED*** Algorithm a = at.extractAlgorithm("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); -***REMOVED*** System.out.println(a.getFullname()); - ***REMOVED*** +// AlgorithmPackageParserTest at = new AlgorithmPackageParserTest(); +// Algorithm a = at.extractAlgorithm("http://data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); +// System.out.println(a.getFullname()); + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java index 0f1a0e7..0fd378b 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.io.File; -***REMOVED*** +import java.io.IOException; import java.util.UUID; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; @@ -27,7 +27,7 @@ public class AnsibleWorkerTest { r.addTaskFile(tf); worker.addRole(r); - ***REMOVED***worker.apply(); - ***REMOVED*** + //worker.apply(); + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java index 83f41a3..507683f 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ApplicationTokenTest.java @@ -4,15 +4,15 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.util.Iterator; -***REMOVED*** +import java.util.List; -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.encryption.encrypter.StringEncrypter; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.utils.Group; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; @@ -25,7 +25,7 @@ public class ApplicationTokenTest { SecurityTokenProvider.instance.set("xxxx-xxxx-xxx"); decryptToken(); -***REMOVED*** + } private static void decryptToken() { try { @@ -37,9 +37,9 @@ public class ApplicationTokenTest { if (resources.isEmpty()) { System.out.println("No services resource found on IS!"); - ***REMOVED*** else { + } else { System.out.println("Retrieved: " + resources.get(0)); - ***REMOVED*** + } Group accessPoints = resources.get(0).profile().accessPoints(); @@ -53,17 +53,17 @@ public class ApplicationTokenTest { String applicationToken = StringEncrypter.getEncrypter().decrypt(p.value()); System.out.println("Application token found: " + applicationToken); - ***REMOVED*** else { + } else { System.out.println("No application token found in service resource on IS!"); - ***REMOVED*** - ***REMOVED*** else { + } + } else { System.out.println("Invalid service resource on IS!"); - ***REMOVED*** + } - ***REMOVED*** catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); - ***REMOVED*** -***REMOVED*** + } + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java index 8e812e9..8c735e0 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java @@ -2,7 +2,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.net.ProxySelector; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; @@ -20,30 +20,30 @@ public class DataminerPoolManagerTest { if(parts.length>1) { out.setType(parts[0]); out.setName(parts[1]); - ***REMOVED*** else { + } else { out.setType("os"); out.setName(depName); - ***REMOVED*** + } return out; - ***REMOVED*** + } private static Algorithm createAlgorithm(String name, String ... deps) { Algorithm a = new Algorithm(); a.setName(name); for(String dep:deps) { a.addDependency(createDependency(dep)); - ***REMOVED*** + } return a; - ***REMOVED*** + } private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { AlgorithmSet out = new AlgorithmSet(); out.setName(name); for(Algorithm a:algs) { out.addAlgorithm(a); - ***REMOVED*** + } return out; - ***REMOVED*** + } private static Domain getDomain(String name) { if(domains.get(name)==null) { @@ -51,10 +51,10 @@ public class DataminerPoolManagerTest { d.setName(name); domains.put(name, d); return d; - ***REMOVED*** else { + } else { return domains.get(name); - ***REMOVED*** - ***REMOVED*** + } + } private static Host createHost(String hostname, String domainName) { Host out = new Host(); @@ -62,7 +62,7 @@ public class DataminerPoolManagerTest { Domain d = getDomain(domainName); out.setDomain(d); return out; - ***REMOVED*** + } private static Cluster getCluster(String name) { if(clusters.get(name)==null) { @@ -70,10 +70,10 @@ public class DataminerPoolManagerTest { d.setName(name); clusters.put(name, d); return d; - ***REMOVED*** else { + } else { return clusters.get(name); - ***REMOVED*** - ***REMOVED*** + } + } private static Collection extractDependencies() { Collection out = new TreeSet<>(new DependencyComparator()); @@ -82,12 +82,12 @@ public class DataminerPoolManagerTest { for(Algorithm a:as.getAlgorithms()) { for(Dependency d:a.getDependencies()) { out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } + } return out; - ***REMOVED*** + } private static Collection extractAlgorithms() { Collection out = new TreeSet<>(new AlgorithmComparator()); @@ -95,11 +95,11 @@ public class DataminerPoolManagerTest { for(AlgorithmSet as:c.getAlgorithmSets()) { for(Algorithm a:as.getAlgorithms()) { out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** + } + } + } return out; - ***REMOVED*** + } static { @@ -132,7 +132,7 @@ public class DataminerPoolManagerTest { cluster2.addAlgorithmSet(as1); cluster2.addAlgorithmSet(as3); - ***REMOVED*** + } */ public static void main(String[] args) throws Exception { @@ -147,67 +147,67 @@ public class DataminerPoolManagerTest { for(Algorithm a:extractAlgorithms()) { for(Role r:ab.generateRoles(a)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** + } + } for(Dependency d:extractDependencies()) { for(Role r:ab.generateRoles(d)) { worker.addRole(r); - ***REMOVED*** - ***REMOVED*** + } + } */ ScopeProvider.instance.set("/gcube/devNext/NextNext"); ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); +// +// // create the algorithm (download it, etc etc) +// Algorithm algorithm = new Algorithm(); +// algorithm.setName("ichtyop"); +// algorithm.setClazz("org.gcube..."); +// algorithm.setDescription("some description"); +// +// Dependency d = new Dependency(); +// d.setName("libpng"); +// d.setType("os"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("some-r-package"); +// d.setType("cran"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("some-other-r-package"); +// d.setType("cran"); +// algorithm.addDependency(d); +// +// d = new Dependency(); +// d.setName("voodoo"); +// d.setType("custom"); +// algorithm.addDependency(d); +// +// // create the algorithm (download it, etc etc) +// Algorithm ewe = new Algorithm(); +// ewe.setName("ewe"); +// +// d = new Dependency(); +// d.setName("voodoo"); +// d.setType("custom"); +// ewe.addDependency(d); AlgorithmSet algorithms = new AlgorithmSet(); algorithms.setName("dummy-set"); -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); +// algorithms.addAlgorithm(algorithm); +// algorithms.addAlgorithm(ewe); Algorithm ensemble = new Algorithm(); ensemble.setName("ensemble"); ensemble.setCategory("ICHTHYOP_MODEL"); ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); + ensemble.setPackageURL("http://data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); ensemble.setSkipJava("N"); ensemble.setDescription("test"); @@ -218,8 +218,8 @@ public class DataminerPoolManagerTest { ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); + //new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java index 426cc6c..211fd04 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java @@ -2,7 +2,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; import java.net.ProxySelector; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; @@ -12,6 +12,6 @@ public class ISClientTest { ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ScopeProvider.instance.set("/gcube/devNext/NextNext"); System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index 8ce58f2..50b1c20 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -1,8 +1,8 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; -***REMOVED*** +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; @@ -16,19 +16,19 @@ public class JobTest { ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); SVNUpdater svnUpdater = new SVNUpdaterProduction(); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); + Algorithm algo = AlgorithmBuilder.create("http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); - ***REMOVED***test phase - ***REMOVED***Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); - ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); - ***REMOVED***DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get()); - ***REMOVED***job.start(); + //test phase + //Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + //Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + //DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get()); + //job.start(); - ***REMOVED***release phase - ***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); - ***REMOVED***DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); - ***REMOVED***job2.start(); + //release phase + //Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); + //DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); + //job2.start(); -***REMOVED*** + } -***REMOVED*** +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 779c489..89e7d8c 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** +import java.io.IOException; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; @@ -20,17 +20,17 @@ public class SVNTests { SVNUpdater svnUpdater = new SVNUpdaterStaging(); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); - ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); + Algorithm algo = AlgorithmBuilder.create("http://data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); + //algo.setClazz(algo.getClazz() + "TEST"); algo.setAlgorithmType("transducerers"); algo.setCategory("NLP"); algo.setFullname("Ondřej Košarko"); - ***REMOVED***System.out.println(algo.getCategory()); - ***REMOVED***System.out.println(algo.getSkipJava()); + //System.out.println(algo.getCategory()); + //System.out.println(algo.getSkipJava()); - ***REMOVED***svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); + //svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); - ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); - ***REMOVED***svnUpdater.readRPRotoDeps(algo); - ***REMOVED*** -***REMOVED*** + //svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + //svnUpdater.readRPRotoDeps(algo); + } +} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java index efb2a1f..ebf23e4 100755 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SendMailTest.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; -***REMOVED*** -***REMOVED*** +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; public class SendMailTest { @@ -9,30 +9,30 @@ public class SendMailTest { public static void main(String[] args) { try { - ***REMOVED*** NotificationHelper nh = new NotificationHelper(); + // NotificationHelper nh = new NotificationHelper(); SendMail sm = new SendMail(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); SecurityTokenProvider.instance.set("xxxx-xxxx-xxx"); - ***REMOVED*** System.out.println(sm.getRootToken()); - ***REMOVED*** sm.getGenericResourceByName(""); + // System.out.println(sm.getRootToken()); + // sm.getGenericResourceByName(""); - ***REMOVED***ScopeProvider.instance.set("/gcube/devsec/devVRE"); - ***REMOVED***SecurityTokenProvider.instance.set("xxxx-xxxx-xxxx"); + //ScopeProvider.instance.set("/gcube/devsec/devVRE"); + //SecurityTokenProvider.instance.set("xxxx-xxxx-xxxx"); - ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test - ***REMOVED*** failed")); - ***REMOVED*** sm.username(SecurityTokenProvider.instance.get()); - ***REMOVED*** sm.retrieveAdminRole(); - ***REMOVED*** sm.getAdminRoles(); + // sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test + // failed")); + // sm.username(SecurityTokenProvider.instance.get()); + // sm.retrieveAdminRole(); + // sm.getAdminRoles(); sm.sendNotification("Test DMPoolManager Notification", "This is only a test please cancel this message."); - ***REMOVED*** System.out.println(sm.getAdmins()); - ***REMOVED*** sm.sendNotification("test", "test"); - ***REMOVED*** System.out.println(sm.getSocialService()); - ***REMOVED*** catch (Exception e) { + // System.out.println(sm.getAdmins()); + // sm.sendNotification("test", "test"); + // System.out.println(sm.getSocialService()); + } catch (Exception e) { System.out.println("Error in sent mail: "+e.getLocalizedMessage()); e.printStackTrace(); - ***REMOVED*** -***REMOVED*** -***REMOVED*** + } + } +}