Jenkinsfile updated

connection-properties/iis-cdh5-test-gw.properties

@@ -0,0 +1,26 @@
+#dnet-hadoop connection properties for iis-cdh5-test-gw
+
+dhp.hadoop.frontend.temp.dir=/home/jenkins
+dhp.hadoop.frontend.user.name=jenkins
+dhp.hadoop.frontend.host.name=iis-cdh5-test-gw.ocean.icm.edu.pl
+dhp.hadoop.frontend.port.ssh=22
+oozieServiceLoc=http://iis-cdh5-test-m3:11000/oozie
+jobTracker=yarnRM
+nameNode=hdfs://nameservice1
+oozie.execution.log.file.location = target/extract-and-run-on-remote-host.log
+maven.executable=mvn
+
+#common spark related
+sparkDriverMemory=7G
+sparkExecutorMemory=7G
+sparkExecutorCores=4
+
+#spark 1 related
+oozieActionShareLibForSpark1=spark
+spark1YarnHistoryServerAddress=http://iis-cdh5-test-gw.ocean.icm.edu.pl:18088
+spark1EventLogDir=/user/spark/applicationHistory
+
+#spark 2 related
+oozieActionShareLibForSpark2=spark2
+spark2YarnHistoryServerAddress=http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089
+spark2EventLogDir=/user/spark/spark2ApplicationHistory

jenkins/deploy/Jenkinsfile

@@ -21,24 +21,24 @@ pipeline {
             }
         }
 
-//         stage("Upload") {
+        stage("00-Upload-aggregatorGraph") {
-//             steps {
+            steps {
-//                 configFileProvider([configFile(fileId: "iis-build-properties", variable: 'BUILD_PROPERTIES')]) {
+                configFileProvider([configFile(fileId: "dnet-hadoop-build.properties", variable: 'BUILD_PROPERTIES')]) {
-//                     load "${BUILD_PROPERTIES}"
+                    load "${BUILD_PROPERTIES}"
-//                     withEnv(["JAVA_HOME=${ tool type: 'jdk', name: "${JDK_VERSION}" }",
+                    withEnv(["JAVA_HOME=${ tool type: 'jdk', name: "${JDK_VERSION}" }",
-//                              "PATH+MAVEN=${tool type: 'maven', name: "${MAVEN_VERSION}"}/bin:${env.JAVA_HOME}/bin"]) {
+                             "PATH+MAVEN=${tool type: 'maven', name: "${MAVEN_VERSION}"}/bin:${env.JAVA_HOME}/bin"]) {
-//                         sshagent(['iis-deployment-credentials']) {
+                        sshagent(['dnet-hadoop-deployment-credentials']) {
-//                             sh """
+                            sh """
-//                                 cd iis-wf/iis-wf-primary && \
+                                cd dhp-workflows/dhp-graph-mapper && \
-//                                 mvn clean package -Poozie-package,deploy \
+                                mvn clean package -Poozie-package,deploy \
-//                                     -Dworkflow.source.dir=eu/dnetlib/iis/wf/primary/main \
+                                    -Dworkflow.source.dir=eu/dnetlib/dhp/oa/graph/raw_all \
-//                                     -DiisConnectionProperties=${WORKSPACE}/connection-properties/iis-cdh5-test-gw.properties
+                                    -DdhpConnectionProperties=${WORKSPACE}/connection-properties/iis-cdh5-test-gw.properties
-//                             """
+                            """
-//                         }
+                        }
-//                     }
+                    }
-//                 }
+                }
-//             }
+            }
-//         }
+        }
 
 //         stage("Deploy") {
 //             steps {