From 0ed3f62ebc77055679fe651efc9b0fb1533b0968 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Tue, 26 Jan 2021 20:49:36 -0500 Subject: [PATCH 01/16] Change default values for paremeters to refer gCubeCI. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d847418..d9eda37 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,10 +45,10 @@ pipeline { defaultValue: 'https://code-repo.d4science.org/TestActions/', description: 'The root URL of the repositories') string(name: 'repo_list', - defaultValue: 'https://code-repo.d4science.org/manuele.simi/gCubeActionsFiles/raw/branch/master/repos/default_list.txt', + defaultValue: 'https://code-repo.d4science.org/gCubeCI/gCubeActions/raw/branch/master/repos/default_list.txt', description: 'The file with the list of repositories to update') string(name: 'action_root', - defaultValue: 'https://code-repo.d4science.org/manuele.simi/gCubeActionsFiles/raw/branch/master/', + defaultValue: 'https://code-repo.d4science.org/gCubeCI/gCubeActions/raw/branch/master/', description: 'The root URL of the Bash fragment to execute.') string(name: 'action_file', defaultValue: '', -- 2.17.1 From 7bd5d0808add3886710504bd98304740536dd3bb Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 17:13:31 -0500 Subject: [PATCH 02/16] Catch the stdout of the Action. --- Jenkinsfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9eda37..540852e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -158,17 +158,17 @@ def exec(actions, repo_url, repo_name) { withCredentials([usernamePassword(credentialsId: '88b54962-1c0e-49cb-8155-22276860f346', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) { def complete_url = "${repo_url}.git" def repository = complete_url.replaceFirst(".+://", "https://${GIT_USERNAME}:${GIT_PASSWORD}@") - sh(""" - git remote set-url origin $repository - git remote -v - git config user.email "git.gcube@localhost" - git config user.name "git.gcube" - curl "${ACTION_URL}" -o actions.sh - chmod a+x actions.sh - source actions.sh - rm actions.sh - git push --force origin HEAD:master || true - """) + output = sh(script: """ + git remote set-url origin $repository + git remote -v + git config user.email "git.gcube@localhost" + git config user.name "git.gcube" + curl "${ACTION_URL}" -o actions.sh + chmod a+x actions.sh + source actions.sh + rm actions.sh + git push --force origin HEAD:master || true + """), returnStdout: true)?.trim() } } } -- 2.17.1 From 451474974ae8d09e16072b36fb2a70a60002ce6b Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 17:46:15 -0500 Subject: [PATCH 03/16] Use a bash wrapper. Compile the report with the standard output from all the executed actions. --- Jenkinsfile | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 540852e..978387c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,6 +34,7 @@ pipeline { AGENT_ROOT_FOLDER = "${agent_root_folder}" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" ACTION_REPORT = "${agent_root_folder}/actions.${env.BUILD_NUMBER}.csv" + ACTION_OUTPUT = "${agent_root_folder}/action-output.${env.BUILD_NUMBER}.txt" ACTION_URL="${actionURL}" REPO_ROOT="${git_root}" @@ -74,7 +75,7 @@ pipeline { for (int i = 0; i < projects.size(); i++) { stage(projects[i]) { echo "About to execute over ${projects[i]}" - checkout_and_exec(projects[i], action_code) + checkout_and_exec(projects[i]) sh "echo -e ${projects[i]},${git_root}/${projects[i]},Completed >> $ACTION_REPORT" } } @@ -90,6 +91,8 @@ pipeline { sh ''' cp $ACTION_REPORT ./actions.${PIPELINE_BUILD_NUMBER}.csv cat ./actions.${PIPELINE_BUILD_NUMBER}.csv + cp $ACTION_OUTPUT ./action-output.${PIPELINE_BUILD_NUMBER}.txt + cat ./action-output.${PIPELINE_BUILD_NUMBER}.txt ''' } } @@ -117,7 +120,7 @@ pipeline { * Clones the repository and executes the fragment * NOTE: 'credentialsId' be manually configured in Jenkins to access all the repos */ -def checkout_and_exec(repo_name, actions) { +def checkout_and_exec(repo_name) { def repo_url = "${git_root}/${repo_name}" sh(script: "rm -r ${repo_name} || true", returnStdout: true)?.trim() checkout([ @@ -137,7 +140,7 @@ def checkout_and_exec(repo_name, actions) { get_last_commit(repo_name) //exec the action - exec(actions, repo_url, repo_name) + exec(repo_url, repo_name) } @@ -152,13 +155,13 @@ String get_last_commit(repo_name) { /** Execs the bash fragment */ -def exec(actions, repo_url, repo_name) { - +def exec(repo_url, repo_name) { + def output = ''; dir(repo_name) { withCredentials([usernamePassword(credentialsId: '88b54962-1c0e-49cb-8155-22276860f346', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) { def complete_url = "${repo_url}.git" def repository = complete_url.replaceFirst(".+://", "https://${GIT_USERNAME}:${GIT_PASSWORD}@") - output = sh(script: """ + def bashWrapper = """ git remote set-url origin $repository git remote -v git config user.email "git.gcube@localhost" @@ -168,9 +171,14 @@ def exec(actions, repo_url, repo_name) { source actions.sh rm actions.sh git push --force origin HEAD:master || true - """), returnStdout: true)?.trim() + """ + output = sh(script: bashWrapper, returnStdout: true) } } + sh "echo -e --- STDOUT FROM REPO ${repo_url} --- >> $ACTION_OUTPUT" + sh "echo -e ${output} >> $ACTION_OUTPUT" + sh "echo -e --- END REPO ${repo_url} --- >> $ACTION_OUTPUT" + } //a non CPS method is necessary for the usage of splitEachLine() -- 2.17.1 From 69ef5c1180395043f2feafee586ad80de0d1021c Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 21:21:46 -0500 Subject: [PATCH 04/16] Escape stdout. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 978387c..83db420 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,7 +176,7 @@ def exec(repo_url, repo_name) { } } sh "echo -e --- STDOUT FROM REPO ${repo_url} --- >> $ACTION_OUTPUT" - sh "echo -e ${output} >> $ACTION_OUTPUT" + sh "echo -e '${output}' >> $ACTION_OUTPUT" sh "echo -e --- END REPO ${repo_url} --- >> $ACTION_OUTPUT" } -- 2.17.1 From e9369099fb3fe364c4346992f94cedfec92cd69a Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 21:41:38 -0500 Subject: [PATCH 05/16] Update default repo list and README. --- Jenkinsfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 83db420..313d2ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { defaultValue: 'https://code-repo.d4science.org/TestActions/', description: 'The root URL of the repositories') string(name: 'repo_list', - defaultValue: 'https://code-repo.d4science.org/gCubeCI/gCubeActions/raw/branch/master/repos/default_list.txt', + defaultValue: 'https://code-repo.d4science.org/gCubeCI/gCubeActions/raw/branch/master/repos/TestActions_all_sorted.txt', description: 'The file with the list of repositories to update') string(name: 'action_root', defaultValue: 'https://code-repo.d4science.org/gCubeCI/gCubeActions/raw/branch/master/', diff --git a/README.md b/README.md index c53523f..db4fd5d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ fi * [Pipeline Syntax](https://jenkins.io/doc/book/pipeline/syntax/) * [Shared Libraries](https://jenkins.io/doc/book/pipeline/shared-libraries/) * [SCM Step](https://jenkins.io/doc/pipeline/steps/workflow-scm-step/) - +* [Workflow Durable Task Step](https://www.jenkins.io/doc/pipeline/steps/workflow-durable-task-step/) ## Wiki doc * [CI: gCube Actions Pipeline](https://wiki.gcube-system.org/gcube/Continuous_Integration:_Actions_Jenkins_Pipeline) -- 2.17.1 From 446721255ba8f616a7a1ee9f50427468f830b246 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:04:11 -0500 Subject: [PATCH 06/16] Format stdout report as XML. Attach it to the email. --- Jenkinsfile | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 313d2ff..f610b36 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,7 +34,7 @@ pipeline { AGENT_ROOT_FOLDER = "${agent_root_folder}" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" ACTION_REPORT = "${agent_root_folder}/actions.${env.BUILD_NUMBER}.csv" - ACTION_OUTPUT = "${agent_root_folder}/action-output.${env.BUILD_NUMBER}.txt" + ACTION_OUTPUT = "${agent_root_folder}/action-output.${env.BUILD_NUMBER}.xml" ACTION_URL="${actionURL}" REPO_ROOT="${git_root}" @@ -66,6 +66,8 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT + sh "echo -e '' > $ACTION_OUTPUT" + ''' } } @@ -79,6 +81,7 @@ pipeline { sh "echo -e ${projects[i]},${git_root}/${projects[i]},Completed >> $ACTION_REPORT" } } + sh "echo -e ''>> $ACTION_OUTPUT" } } } @@ -89,17 +92,17 @@ pipeline { always { script { sh ''' - cp $ACTION_REPORT ./actions.${PIPELINE_BUILD_NUMBER}.csv - cat ./actions.${PIPELINE_BUILD_NUMBER}.csv - cp $ACTION_OUTPUT ./action-output.${PIPELINE_BUILD_NUMBER}.txt - cat ./action-output.${PIPELINE_BUILD_NUMBER}.txt + cp $ACTION_REPORT ./action.${PIPELINE_BUILD_NUMBER}.csv + cat ./action.${PIPELINE_BUILD_NUMBER}.csv + cp $ACTION_OUTPUT ./action-output.${PIPELINE_BUILD_NUMBER}.xml + cat ./action-output.${PIPELINE_BUILD_NUMBER}.xml ''' } } success { echo 'The actions pipeline worked!' - emailext attachmentsPattern: "**/actions.${env.BUILD_NUMBER}.csv", + emailext attachmentsPattern: "**/action*${env.BUILD_NUMBER}.*", to: 'manuele.simi@isti.cnr.it', subject: "Actions report(build #${PIPELINE_BUILD_NUMBER})", body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}" @@ -172,12 +175,14 @@ def exec(repo_url, repo_name) { rm actions.sh git push --force origin HEAD:master || true """ - output = sh(script: bashWrapper, returnStdout: true) + output = sh(script: bashWrapper, returnStdout: true)?.trim() } } - sh "echo -e --- STDOUT FROM REPO ${repo_url} --- >> $ACTION_OUTPUT" + sh "echo -e '"' --- >> $ACTION_OUTPUT" + sh "echo -e '' >> $ACTION_OUTPUT" sh "echo -e '${output}' >> $ACTION_OUTPUT" - sh "echo -e --- END REPO ${repo_url} --- >> $ACTION_OUTPUT" + sh "echo -e '' >> $ACTION_OUTPUT" + sh "echo -e '' --- >> $ACTION_OUTPUT" } -- 2.17.1 From a2b8c5c6fcec4df54e7191f834085b28b6fa16bd Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:07:32 -0500 Subject: [PATCH 07/16] Fix syntax. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f610b36..33aee94 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -178,7 +178,7 @@ def exec(repo_url, repo_name) { output = sh(script: bashWrapper, returnStdout: true)?.trim() } } - sh "echo -e '"' --- >> $ACTION_OUTPUT" + sh "echo -e '' --- >> $ACTION_OUTPUT" sh "echo -e '' >> $ACTION_OUTPUT" sh "echo -e '${output}' >> $ACTION_OUTPUT" sh "echo -e '' >> $ACTION_OUTPUT" -- 2.17.1 From 4f3305496de53cfbd31bd71b7dfd811a899ed939 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:21:41 -0500 Subject: [PATCH 08/16] Write as XML action. --- Jenkinsfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33aee94..ab59a6e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,8 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - sh "echo -e '' > $ACTION_OUTPUT" - + //sh "echo -e '' > $ACTION_OUTPUT" ''' } } @@ -81,7 +80,7 @@ pipeline { sh "echo -e ${projects[i]},${git_root}/${projects[i]},Completed >> $ACTION_REPORT" } } - sh "echo -e ''>> $ACTION_OUTPUT" + // sh "echo -e '' >> $ACTION_OUTPUT" } } } @@ -178,11 +177,14 @@ def exec(repo_url, repo_name) { output = sh(script: bashWrapper, returnStdout: true)?.trim() } } - sh "echo -e '' --- >> $ACTION_OUTPUT" - sh "echo -e '' >> $ACTION_OUTPUT" - sh "echo -e '${output}' >> $ACTION_OUTPUT" - sh "echo -e '' >> $ACTION_OUTPUT" - sh "echo -e '' --- >> $ACTION_OUTPUT" + def xml_action = """ + + + ${output} + + + """ + sh "echo -e '${xml_action}' >> $ACTION_OUTPUT" } -- 2.17.1 From b228f723bb5c24f9df187880b81dd81879c4986b Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:24:07 -0500 Subject: [PATCH 09/16] Temporary remove root element. --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ab59a6e..ac7a8bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,6 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - //sh "echo -e '' > $ACTION_OUTPUT" ''' } } @@ -80,7 +79,7 @@ pipeline { sh "echo -e ${projects[i]},${git_root}/${projects[i]},Completed >> $ACTION_REPORT" } } - // sh "echo -e '' >> $ACTION_OUTPUT" + } } } -- 2.17.1 From cf27635c5903e96fe547f4760a1ae3879e337f5a Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:26:27 -0500 Subject: [PATCH 10/16] Add back root element. --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ac7a8bc..571f0ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,6 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT + sh "echo -e '' > $ACTION_OUTPUT" ''' } } @@ -79,7 +80,7 @@ pipeline { sh "echo -e ${projects[i]},${git_root}/${projects[i]},Completed >> $ACTION_REPORT" } } - + sh "echo -e '' >> $ACTION_OUTPUT" } } } -- 2.17.1 From 6581d726153437cb29b48d9e5a4a5990d42c3773 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:28:22 -0500 Subject: [PATCH 11/16] Fix syntax. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 571f0ec..d229de9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - sh "echo -e '' > $ACTION_OUTPUT" + echo '' > $ACTION_OUTPUT" ''' } } -- 2.17.1 From 64871fa96afd066c4d6e0503e6b8a6209fc7dd8f Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:29:41 -0500 Subject: [PATCH 12/16] Fix syntax. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d229de9..1bf5ae8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - echo '' > $ACTION_OUTPUT" + echo '' > $ACTION_OUTPUT ''' } } -- 2.17.1 From cdd185bbd63d141f3e46c490b0a012532cbc8c38 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:37:43 -0500 Subject: [PATCH 13/16] Escape root attribute. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1bf5ae8..d2e3fa4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - echo '' > $ACTION_OUTPUT + echo "" > $ACTION_OUTPUT ''' } } -- 2.17.1 From a94fd80ce71214d023634aef3bef78adab6044b6 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:47:55 -0500 Subject: [PATCH 14/16] Add -e to echo. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d2e3fa4..fb2e3ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - echo "" > $ACTION_OUTPUT + echo -e "" > $ACTION_OUTPUT ''' } } -- 2.17.1 From 0942dfc7fb5b574968d7d3ba3389ae0a39ee7d27 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 22:57:53 -0500 Subject: [PATCH 15/16] Define root element as var. --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb2e3ad..9d78568 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,13 +66,14 @@ pipeline { echo "#Build ${PIPELINE_BUILD_NUMBER},," > $ACTION_REPORT echo "#StartTime ${date},," >> $ACTION_REPORT echo "Project,Repo,Result" >> $ACTION_REPORT - echo -e "" > $ACTION_OUTPUT ''' } } stage('clone and exec') { steps { script { + def start_el = "" + sh "echo -e '${start_el}' >> $ACTION_OUTPUT" for (int i = 0; i < projects.size(); i++) { stage(projects[i]) { echo "About to execute over ${projects[i]}" -- 2.17.1 From 04ae09d5917ce2e58889044f7b398efa711b4b1b Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Sun, 31 Jan 2021 23:36:01 -0500 Subject: [PATCH 16/16] Add recipient. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9d78568..5658f6e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,14 +103,14 @@ pipeline { echo 'The actions pipeline worked!' emailext attachmentsPattern: "**/action*${env.BUILD_NUMBER}.*", - to: 'manuele.simi@isti.cnr.it', + to: 'manuele.simi@isti.cnr.it,roberto.cirillo@isti.cnr.it', subject: "Actions report(build #${PIPELINE_BUILD_NUMBER})", body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}" } failure { echo 'The actions pipeline has failed' emailext attachLog: true, - to: 'manuele.simi@isti.cnr.it', + to: 'manuele.simi@isti.cnr.it,roberto.cirillo@isti.cnr.it', subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed", body: "Something is wrong with ${env.BUILD_URL}" } -- 2.17.1