From 2485ec13c8b383fc8c1ec966f511da30f0309d88 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 3 Aug 2022 17:32:20 +0200 Subject: [PATCH 001/118] add method for extracting components from a cdv folder --- Jenkinsfile | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3ad9b2b..30699d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!groovy - +import groovy.io.FileType /** * Deploy components to the D4Science Infrastructure (dev-only) * @@ -119,10 +119,13 @@ pipeline { } stage('Nothing to do ') { when{ - allOf{ - triggeredBy 'TimerTrigger' - environment name: 'IS_CRON', value: 'False' + anyOf{ + allOf{ + triggeredBy 'TimerTrigger' + environment name: 'IS_CRON', value: 'False' + } } + } steps { echo 'Do Nothing: cron build disabled' @@ -282,4 +285,29 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ echo "deploy file empty" fi ''' +} + +#experimental test + +def getComponentsFromCSVDir(def dirPath){ + File folder = new File(dirPath) + folder.eachFileRecurse FileType.FILES, { file -> + // do nothing if the file ends with a .txt extension + if (file.name.endsWith(".csv")) { + println "Processing file ${file.absolutePath}" + readFile(file).split('\n').each { line, count-> + if (line.startsWith('#')) + return + def fields = line.split(',') + components.add([ + name : fields[0], + version : fields[1], + host : fields[2] + ] + ) + } + + } + } + return components } \ No newline at end of file From 3e41dac45c7ca1beeb1c7ee397ecc2bd606d840a Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 10:13:03 +0200 Subject: [PATCH 002/118] add build number to every deploy file. Delete the file after processing it --- Jenkinsfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30699d5..930f0c5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,8 +9,8 @@ import groovy.io.FileType // related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/ def agent_root_folder = '/var/lib/jenkins' -def agent_deploy_filename = 'deploy.csv' -def agent_deploy_backup_filename = 'deploy.bck' +def agent_deploy_filename = 'deploy.${env.BUILD_NUMBER}.csv' +def agent_deploy_backup_filename = 'deploy.${env.BUILD_NUMBER}.bck' def deployList def backupList @@ -52,6 +52,7 @@ pipeline { environment { AGENT_ROOT_FOLDER = "${agent_root_folder}" DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" + PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" DEPLOY_FILE = "${agent_root_folder}/${agent_deploy_filename}" BACKUP_FILE = "${agent_root_folder}/${agent_deploy_backup_filename}" TRIGGER_JOB= "${params.TRIGGER_JOB}" @@ -99,7 +100,9 @@ pipeline { echo 'Cron build enabled. Deploy from system ongoing' script { // parse the report and extract the data - def components = getComponentsFromCSV(deployList) + // def components = getComponentsFromCSV(deployList) + println "Going to check the deploy file in ${DEPLOY_ROOT_FOLDER}" + def components =getComponentsFromCSVDir("${DEPLOY_ROOT_FOLDER}") if (components.size() > 0) { for (component in components) { stage(component){ @@ -306,8 +309,9 @@ def getComponentsFromCSVDir(def dirPath){ ] ) } - } + // remove the file here if possible + file.delete(); } return components } \ No newline at end of file From 0759dccca502ce5c434e5c0c0e0f758fb00d8f13 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 10:25:49 +0200 Subject: [PATCH 003/118] fix sintax --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 930f0c5..956b3c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -290,7 +290,7 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ ''' } -#experimental test +//experimental test def getComponentsFromCSVDir(def dirPath){ File folder = new File(dirPath) From fbd860296ded894fdaf4f178fb7c474cdaaef535 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 13:00:41 +0200 Subject: [PATCH 004/118] change agent label from CD to ansible --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 956b3c0..49022c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ backupList = agent_root_folder+'/'+agent_deploy_backup_filename; pipeline { agent { - label 'CD' + label 'ansible' } options { ansiColor('xterm') From d7f99a58fc2fc5ac21ee736cf38c707893972b1e Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 15:19:39 +0200 Subject: [PATCH 005/118] create separated folder for deploy files --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 49022c2..45c4071 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,6 +85,7 @@ pipeline { steps { sh ''' date=`date`; + mkdir -p /${DEPLOY_ROOT_FOLDER} ''' } } From 73adf8b6561c82cc8bc43b5aab5a04c2432391af Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 15:23:39 +0200 Subject: [PATCH 006/118] add userid cause to pending job --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 45c4071..0667d37 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -141,6 +141,7 @@ pipeline { anyOf{ triggeredBy 'BuildUpstreamCause' triggeredBy 'UpstreamCause' + triggeredBy 'UserIdCause' } } steps { From 146dd63cb6daad930e684630b1abe64f15dc0589 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 15:44:33 +0200 Subject: [PATCH 007/118] fix deploy filename --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0667d37..483bd7c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,8 +9,8 @@ import groovy.io.FileType // related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/ def agent_root_folder = '/var/lib/jenkins' -def agent_deploy_filename = 'deploy.${env.BUILD_NUMBER}.csv' -def agent_deploy_backup_filename = 'deploy.${env.BUILD_NUMBER}.bck' +//def agent_deploy_filename = 'deploy.${env.BUILD_NUMBER}.csv' +//def agent_deploy_backup_filename = 'deploy.${env.BUILD_NUMBER}.bck' def deployList def backupList @@ -53,8 +53,8 @@ pipeline { AGENT_ROOT_FOLDER = "${agent_root_folder}" DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" - DEPLOY_FILE = "${agent_root_folder}/${agent_deploy_filename}" - BACKUP_FILE = "${agent_root_folder}/${agent_deploy_backup_filename}" + DEPLOY_FILE = "${agent_root_folder}/deploy.${PIPELINE_BUILD_NUMBER}.csv" + BACKUP_FILE = "${agent_root_folder}/deploy.${PIPELINE_BUILD_NUMBER}.bck" TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" From 10c932cc83e1118f81d0194f5baa9a663b6faa15 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 15:47:16 +0200 Subject: [PATCH 008/118] removed param deploy list management --- Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 483bd7c..751a32e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,13 +17,7 @@ def backupList if (params.deployFile) { println "Using custom deploy file" deployList = params.deployFile -} else { - println "Using local deploy file" - //load the report from local - deployList = agent_root_folder+'/'+agent_deploy_filename; - println "Load from local file ${deployList}" } -backupList = agent_root_folder+'/'+agent_deploy_backup_filename; //locate the targetHost file //String targetHostURL = "https://code-repo.d4science.org/gCubeCI/gCubeDeployer/raw/branch/master/open/gcube-${gCube_release_version}.yaml" From 879193b653ffcdfbbba1c0ad0b70b4524cce9a50 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 15:54:27 +0200 Subject: [PATCH 009/118] fix DEPLOY_FILE_ROOT_FOLDER variable --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 751a32e..978f49b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,10 +45,11 @@ pipeline { environment { AGENT_ROOT_FOLDER = "${agent_root_folder}" - DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" + DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster" + DEPLOY_FILE_ROOT_FOLDER="${agent_root_folder}/CD" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" - DEPLOY_FILE = "${agent_root_folder}/deploy.${PIPELINE_BUILD_NUMBER}.csv" - BACKUP_FILE = "${agent_root_folder}/deploy.${PIPELINE_BUILD_NUMBER}.bck" + DEPLOY_FILE = "${DEPLOY_FILE_ROOT_FOLDER}/deploy.${PIPELINE_BUILD_NUMBER}.csv" + BACKUP_FILE = "${DEPLOY_FILE_ROOT_FOLDER}/deploy.${PIPELINE_BUILD_NUMBER}.bck" TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" @@ -79,7 +80,7 @@ pipeline { steps { sh ''' date=`date`; - mkdir -p /${DEPLOY_ROOT_FOLDER} + mkdir -p /${DEPLOY_FILE_ROOT_FOLDER} ''' } } @@ -97,7 +98,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_ROOT_FOLDER}" - def components =getComponentsFromCSVDir("${DEPLOY_ROOT_FOLDER}") + def components =getComponentsFromCSVDir("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { for (component in components) { stage(component){ From f0383b7fa6ef1296663daad95293a00038a4d514 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 16:18:28 +0200 Subject: [PATCH 010/118] processing deploy files: convert list to set --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 978f49b..7c89f68 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -100,6 +100,7 @@ pipeline { println "Going to check the deploy file in ${DEPLOY_ROOT_FOLDER}" def components =getComponentsFromCSVDir("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { + components.components.toSet(); for (component in components) { stage(component){ println "Deploy on going of component: $component" From f3efd14a02bd58df73bb82e371a72c886385b818 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 16:30:10 +0200 Subject: [PATCH 011/118] set cron every 15, just for test --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7c89f68..a824143 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) -// cron('H/15 * * * *') + cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) - cron('H H(9-16)/6 * * 1-5') +// cron('H H(9-16)/6 * * 1-5') } environment { From 63cf98a6f038e6706d91fecc90f14f68b854fb40 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 16:31:43 +0200 Subject: [PATCH 012/118] change var name components --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a824143..ab8ba76 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -100,8 +100,8 @@ pipeline { println "Going to check the deploy file in ${DEPLOY_ROOT_FOLDER}" def components =getComponentsFromCSVDir("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { - components.components.toSet(); - for (component in components) { + def componentSet=components.toSet(); + for (component in componentSet) { stage(component){ println "Deploy on going of component: $component" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { From ccae2577cb988992ceac460de10239cb51895951 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 17:02:22 +0200 Subject: [PATCH 013/118] add some logs --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ab8ba76..f465599 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,7 +80,7 @@ pipeline { steps { sh ''' date=`date`; - mkdir -p /${DEPLOY_FILE_ROOT_FOLDER} + mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} ''' } } @@ -97,7 +97,7 @@ pipeline { script { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) - println "Going to check the deploy file in ${DEPLOY_ROOT_FOLDER}" + println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" def components =getComponentsFromCSVDir("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { def componentSet=components.toSet(); @@ -308,6 +308,7 @@ def getComponentsFromCSVDir(def dirPath){ ) } } + println "removing current deploy file ${file.absolutePath}" // remove the file here if possible file.delete(); } From dfc6899bd6bd878ed5b491c590e35e414baedda1 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 17:11:18 +0200 Subject: [PATCH 014/118] add some logs --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f465599..0b4879e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -292,8 +292,9 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ def getComponentsFromCSVDir(def dirPath){ File folder = new File(dirPath) + println "folder ready ${folder.absolutePath}" folder.eachFileRecurse FileType.FILES, { file -> - // do nothing if the file ends with a .txt extension + // check it if the file ends with a .csv extension if (file.name.endsWith(".csv")) { println "Processing file ${file.absolutePath}" readFile(file).split('\n').each { line, count-> From 6f27e2727401bba3d9e1e3375c2de2171d74a597 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 17:22:09 +0200 Subject: [PATCH 015/118] remove file.getAbsolutePath method --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0b4879e..fb51f83 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -292,11 +292,11 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ def getComponentsFromCSVDir(def dirPath){ File folder = new File(dirPath) - println "folder ready ${folder.absolutePath}" + println "folder ready " folder.eachFileRecurse FileType.FILES, { file -> // check it if the file ends with a .csv extension if (file.name.endsWith(".csv")) { - println "Processing file ${file.absolutePath}" + println "Processing file " readFile(file).split('\n').each { line, count-> if (line.startsWith('#')) return @@ -309,7 +309,7 @@ def getComponentsFromCSVDir(def dirPath){ ) } } - println "removing current deploy file ${file.absolutePath}" + println "removing current deploy file" // remove the file here if possible file.delete(); } From 15b19c4cd323123ac3255c4dd39e1663798562fe Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 4 Aug 2022 17:50:01 +0200 Subject: [PATCH 016/118] remove file deletion --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb51f83..87c0fa2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -311,7 +311,7 @@ def getComponentsFromCSVDir(def dirPath){ } println "removing current deploy file" // remove the file here if possible - file.delete(); + // file.delete(); } return components } \ No newline at end of file From 58660da4623c799d09aa34ce73c887a39f12914b Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Mon, 29 Aug 2022 09:52:04 +0200 Subject: [PATCH 017/118] change cronjob --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 87c0fa2..cc1e513 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/15 * * * *') +// cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) -// cron('H H(9-16)/6 * * 1-5') + cron('H H(9-16)/6 * * 1-5') } environment { From 9ac38b137ecaf136b4329e5bee35dc5e0356ee87 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Tue, 30 Aug 2022 15:03:35 +0200 Subject: [PATCH 018/118] change var IS_CRON to IS_CRON_TMP temporarily --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cc1e513..9f6dd59 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -89,7 +89,7 @@ pipeline { allOf{ triggeredBy 'TimerTrigger' // maybe we can add a new condition in order to consider the manual execution of this pipeline - environment name: 'IS_CRON', value: 'True' + environment name: 'IS_CRON_TMP', value: 'True' } } steps { @@ -122,7 +122,7 @@ pipeline { anyOf{ allOf{ triggeredBy 'TimerTrigger' - environment name: 'IS_CRON', value: 'False' + environment name: 'IS_CRON_TMP', value: 'False' } } @@ -133,7 +133,7 @@ pipeline { } stage('New pending deploy ') { when{ - environment name: 'IS_CRON', value: 'True' + environment name: 'IS_CRON_TMP', value: 'True' anyOf{ triggeredBy 'BuildUpstreamCause' triggeredBy 'UpstreamCause' @@ -161,7 +161,7 @@ pipeline { } stage('Deploy from job ') { when{ - environment name: 'IS_CRON', value: 'False'; + environment name: 'IS_CRON_TMP', value: 'False'; expression { env.TRIGGER_CD.toBoolean() } @@ -184,7 +184,7 @@ pipeline { stage('Deploy manually ') { when{ allOf{ - environment name: 'IS_CRON', value: 'False'; + environment name: 'IS_CRON_TMP', value: 'False'; triggeredBy 'UserIdCause' } } From c87343479ec5f3335d2e9ca905a8d1847734cfae Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Thu, 8 Sep 2022 14:40:49 +0200 Subject: [PATCH 019/118] add empty folder check --- Jenkinsfile | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9f6dd59..3ae44a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -291,27 +291,29 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ //experimental test def getComponentsFromCSVDir(def dirPath){ - File folder = new File(dirPath) - println "folder ready " - folder.eachFileRecurse FileType.FILES, { file -> - // check it if the file ends with a .csv extension - if (file.name.endsWith(".csv")) { - println "Processing file " - readFile(file).split('\n').each { line, count-> - if (line.startsWith('#')) - return - def fields = line.split(',') - components.add([ - name : fields[0], - version : fields[1], - host : fields[2] - ] - ) - } - } - println "removing current deploy file" - // remove the file here if possible - // file.delete(); - } - return components + File folder = new File(dirPath) + println "folder ready " + if (folder){ + folder.eachFileRecurse FileType.FILES, { file -> + // check it if the file ends with a .csv extension + if (file.name.endsWith(".csv")) { + println "Processing file " + readFile(file).split('\n').each { line, count-> + if (line.startsWith('#')) + return + def fields = line.split(',') + components.add([ + name : fields[0], + version : fields[1], + host : fields[2] + ] + ) + } + } + println "removing current deploy file: "+file.name + // remove the file here if possible + // file.delete(); + } + } + return components } \ No newline at end of file From 478a55e7414d1950c095b8f51a5d08f85c1eb0ff Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Thu, 8 Sep 2022 14:56:14 +0200 Subject: [PATCH 020/118] set cron every 15' --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3ae44a8..7841f6c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) -// cron('H/15 * * * *') + cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) - cron('H H(9-16)/6 * * 1-5') +// cron('H H(9-16)/6 * * 1-5') } environment { From 3ab0b4bf241cb940f16988d515710861ab935d25 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Thu, 8 Sep 2022 17:32:18 +0200 Subject: [PATCH 021/118] restored old cron --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7841f6c..3ae44a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/15 * * * *') +// cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) -// cron('H H(9-16)/6 * * 1-5') + cron('H H(9-16)/6 * * 1-5') } environment { From 51746ad51a0111809249e382341acb9394b1e571 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 11:24:19 +0200 Subject: [PATCH 022/118] fix println sintax; restored cron every 15' --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3ae44a8..c2f4283 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) -// cron('H/15 * * * *') + cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) - cron('H H(9-16)/6 * * 1-5') +// cron('H H(9-16)/6 * * 1-5') } environment { @@ -292,8 +292,9 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ def getComponentsFromCSVDir(def dirPath){ File folder = new File(dirPath) - println "folder ready " + println ("folder ready "); if (folder){ + println (" processing folder "+folder); folder.eachFileRecurse FileType.FILES, { file -> // check it if the file ends with a .csv extension if (file.name.endsWith(".csv")) { From 7d2c0529e6e2ef9ac2116cf0cfadb8519c72506b Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 11:48:07 +0200 Subject: [PATCH 023/118] fix println sintax --- .project | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .project diff --git a/.project b/.project new file mode 100644 index 0000000..7818f37 --- /dev/null +++ b/.project @@ -0,0 +1,11 @@ + + + gCubeDeployer + + + + + + + + From 0ba5b543fac595e9e3abdc5f5af586e6d877c64b Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 11:49:36 +0200 Subject: [PATCH 024/118] small change: add new log --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c2f4283..66b6e31 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -297,8 +297,9 @@ def getComponentsFromCSVDir(def dirPath){ println (" processing folder "+folder); folder.eachFileRecurse FileType.FILES, { file -> // check it if the file ends with a .csv extension + println("checking file: "+file); if (file.name.endsWith(".csv")) { - println "Processing file " + println ("Processing file "); readFile(file).split('\n').each { line, count-> if (line.startsWith('#')) return @@ -311,7 +312,7 @@ def getComponentsFromCSVDir(def dirPath){ ) } } - println "removing current deploy file: "+file.name + println ("removing current deploy file: "+file.name); // remove the file here if possible // file.delete(); } From 6ea001f0f61ac86e42cc44c5fb5c6c3032f7a884 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 12:35:22 +0200 Subject: [PATCH 025/118] add import FileType.FILES --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 66b6e31..96bb4fc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,6 @@ #!groovy import groovy.io.FileType +import static groovy.io.FileType.FILES /** * Deploy components to the D4Science Infrastructure (dev-only) * From eec8436621f7f064ffb0f488e7e1340e8d100ca8 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 14:35:25 +0200 Subject: [PATCH 026/118] new test method for parsing the file --- Jenkinsfile | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 96bb4fc..64f1295 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,7 +99,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir("${DEPLOY_FILE_ROOT_FOLDER}") + def components =getComponentsFromCSVDir2("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -319,4 +319,35 @@ def getComponentsFromCSVDir(def dirPath){ } } return components -} \ No newline at end of file +} + +def getComponentsFromCSVDir2(def dirPath){ + File folder = new File(dirPath) + println ("folder ready "); + if (folder){ + println (" processing folder "+folder); + folder.eachFileRecurse( FileType.FILES) { + if (it.name.endsWith(".csv")) { + println ("Processing file "+it); + readFile(it).split('\n').each { line, count-> + if (line.startsWith('#')) + return + def fields = line.split(',') + components.add([ + name : fields[0], + version : fields[1], + host : fields[2] + ] + ) + } + } + println ("removing current deploy file: "+it.name); + // remove the file here if possible + // file.delete(); + } + } + return components +} + + + From 815f302d89dd5c9ea7e38841ce2fa7d5120d8825 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 14:46:50 +0200 Subject: [PATCH 027/118] add / to the folder path --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 64f1295..e92b712 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,10 +47,10 @@ pipeline { environment { AGENT_ROOT_FOLDER = "${agent_root_folder}" DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster" - DEPLOY_FILE_ROOT_FOLDER="${agent_root_folder}/CD" + DEPLOY_FILE_ROOT_FOLDER="${agent_root_folder}/CD/" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" - DEPLOY_FILE = "${DEPLOY_FILE_ROOT_FOLDER}/deploy.${PIPELINE_BUILD_NUMBER}.csv" - BACKUP_FILE = "${DEPLOY_FILE_ROOT_FOLDER}/deploy.${PIPELINE_BUILD_NUMBER}.bck" + DEPLOY_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" + BACKUP_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" From 78a62bc9e3f2e7ead3c1ae2155d9d505102c17f4 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 9 Sep 2022 17:02:27 +0200 Subject: [PATCH 028/118] restored old cron --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e92b712..f4778af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,9 +39,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/15 * * * *') +// cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) -// cron('H H(9-16)/6 * * 1-5') + cron('H H(9-16)/6 * * 1-5') } environment { From c2c8d566885da732e2277fd3f0aaa24fe70cdb80 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 10:06:53 +0200 Subject: [PATCH 029/118] working on getComponentsFromCSVDir method --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f4778af..b7919a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -322,13 +322,15 @@ def getComponentsFromCSVDir(def dirPath){ } def getComponentsFromCSVDir2(def dirPath){ - File folder = new File(dirPath) + def list = [] + def folder = new File(dirPath) println ("folder ready "); if (folder){ println (" processing folder "+folder); folder.eachFileRecurse( FileType.FILES) { if (it.name.endsWith(".csv")) { println ("Processing file "+it); + list << it readFile(it).split('\n').each { line, count-> if (line.startsWith('#')) return @@ -346,6 +348,10 @@ def getComponentsFromCSVDir2(def dirPath){ // file.delete(); } } + println "files to remove:"; + list.each { + println it.path + } return components } From e4a683cbd6a270931e7467d38b327bf85798040c Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 10:07:20 +0200 Subject: [PATCH 030/118] set cron every 15' --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7919a4..342a3ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,9 +39,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) -// cron('H/15 * * * *') + cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) - cron('H H(9-16)/6 * * 1-5') +// cron('H H(9-16)/6 * * 1-5') } environment { From 8094698bd17a90ca476753cad7a369ed0fb9feb2 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 10:37:54 +0200 Subject: [PATCH 031/118] working on folder method --- Jenkinsfile | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 342a3ce..1732f20 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!groovy -import groovy.io.FileType +import groovy.io.FileType.* import static groovy.io.FileType.FILES /** * Deploy components to the D4Science Infrastructure (dev-only) @@ -99,7 +99,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir2("${DEPLOY_FILE_ROOT_FOLDER}") + def components =getComponentsFromCSVDir3("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -355,5 +355,21 @@ def getComponentsFromCSVDir2(def dirPath){ return components } +def getComponentsFromCSVDir3(def dirPath){ + println "checking current folder" + dh = new File('.') + dh.eachFileRecurse { + println it + } + println "checking CD folder"; + def startDir = new File("/var/lib/jenkins/CD") + startDir.eachFileRecurse(FILES) { + if (it.name.endsWith('.csv')) { + println it + } + } +} + + From 9f7b104220c502ae267df6b56e7458ef63fbd622 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 11:36:11 +0200 Subject: [PATCH 032/118] add @NonCPS to method --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 1732f20..ae3eca8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -291,6 +291,7 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ //experimental test + @NonCPS def getComponentsFromCSVDir(def dirPath){ File folder = new File(dirPath) println ("folder ready "); @@ -321,6 +322,7 @@ def getComponentsFromCSVDir(def dirPath){ return components } + @NonCPS def getComponentsFromCSVDir2(def dirPath){ def list = [] def folder = new File(dirPath) @@ -355,6 +357,7 @@ def getComponentsFromCSVDir2(def dirPath){ return components } + @NonCPS def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" dh = new File('.') From dc9ed3e958dde9db0d143f6f7e9e7c8701ebc5c5 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 14:01:10 +0200 Subject: [PATCH 033/118] update cron every 30' --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae3eca8..f5d5a08 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/15 * * * *') + cron('H/30 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) // cron('H H(9-16)/6 * * 1-5') } From 816f919a3a83ebf68a89a3bc25118c7e10f2e240 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 14:03:06 +0200 Subject: [PATCH 034/118] remove print for every file in folder --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f5d5a08..ce5945c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -360,10 +360,10 @@ def getComponentsFromCSVDir2(def dirPath){ @NonCPS def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" - dh = new File('.') - dh.eachFileRecurse { - println it - } + // dh = new File('.') +// dh.eachFileRecurse { +// println it +// } println "checking CD folder"; def startDir = new File("/var/lib/jenkins/CD") startDir.eachFileRecurse(FILES) { From 90ff91f07ecf2d61e150fea45ad2e9e9bf3f7af8 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 14:16:12 +0200 Subject: [PATCH 035/118] method fixed. switch to previous getComponentsFromCSVDir method version --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ce5945c..56448e8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,7 +99,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir3("${DEPLOY_FILE_ROOT_FOLDER}") + def components =getComponentsFromCSVDir2("${DEPLOY_FILE_ROOT_FOLDER}") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -352,7 +352,8 @@ def getComponentsFromCSVDir2(def dirPath){ } println "files to remove:"; list.each { - println it.path + println ("removing"+it.path); + it.delete(); } return components } From f3827aa956753c3061ff1c7194c7fde11bd71f48 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 14:48:07 +0200 Subject: [PATCH 036/118] fix method --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 56448e8..c035689 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -329,11 +329,11 @@ def getComponentsFromCSVDir2(def dirPath){ println ("folder ready "); if (folder){ println (" processing folder "+folder); - folder.eachFileRecurse( FileType.FILES) { - if (it.name.endsWith(".csv")) { + folder.eachFileRecurse FileType.FILES, { + if (file.name.endsWith(".csv")) { println ("Processing file "+it); - list << it - readFile(it).split('\n').each { line, count-> + list << file + readFile(file).split('\n').each { line, count-> if (line.startsWith('#')) return def fields = line.split(',') From 0c0cb5bda510d8f9907a1588f17f8b2c4109e308 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 15:10:09 +0200 Subject: [PATCH 037/118] small change --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c035689..7872652 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,8 @@ #!groovy import groovy.io.FileType.* import static groovy.io.FileType.FILES +import jenkins.model.* + /** * Deploy components to the D4Science Infrastructure (dev-only) * @@ -324,6 +326,7 @@ def getComponentsFromCSVDir(def dirPath){ @NonCPS def getComponentsFromCSVDir2(def dirPath){ + jenkins = Jenkins.instance def list = [] def folder = new File(dirPath) println ("folder ready "); From a39271efa3e95705f6dd0e1dba5da3b404480100 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 15:22:52 +0200 Subject: [PATCH 038/118] small change --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7872652..635dd12 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -326,7 +326,6 @@ def getComponentsFromCSVDir(def dirPath){ @NonCPS def getComponentsFromCSVDir2(def dirPath){ - jenkins = Jenkins.instance def list = [] def folder = new File(dirPath) println ("folder ready "); From b04945740124e2e5e7059f67723e6bbca6edb168 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 15:32:35 +0200 Subject: [PATCH 039/118] small change --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 635dd12..788b4e9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!groovy -import groovy.io.FileType.* +import groovy.io.FileType import static groovy.io.FileType.FILES import jenkins.model.* From 9dcb9773caa0f00dbc56568fb2df9d8e44a5d647 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 15:45:32 +0200 Subject: [PATCH 040/118] small change --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 788b4e9..6f88823 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir2("${DEPLOY_FILE_ROOT_FOLDER}") + def components =getComponentsFromCSVDir2("CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { From 8ed1b5b4eb7b06718376afcba92b5c0c51aea778 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 16:28:44 +0200 Subject: [PATCH 041/118] small change --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6f88823..59ec6e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,6 +84,8 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} + mkdir -p ./CD + mv -v ${DEPLOY_FILE_ROOT_FOLDER} ./CD ''' } } @@ -101,7 +103,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir2("CD") + def components =getComponentsFromCSVDir2("./CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { From 578d0d1e84b57cd2dec7d48bafa40a964af15301 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Fri, 16 Sep 2022 16:43:39 +0200 Subject: [PATCH 042/118] small test --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 59ec6e0..4944de0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,7 +103,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir2("./CD") + def components =getComponentsFromCSVDir3(${DEPLOY_FILE_ROOT_FOLDER}) if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -365,7 +365,8 @@ def getComponentsFromCSVDir2(def dirPath){ @NonCPS def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" - // dh = new File('.') + dh = new File('.') + println "current folder path: "+dh.path // dh.eachFileRecurse { // println it // } From fe0f9534aa5b42c227fdc052d71a3c6644f1e374 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Sat, 17 Sep 2022 12:25:35 +0200 Subject: [PATCH 043/118] restored old cron --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4944de0..958e857 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,9 +41,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/30 * * * *') +// cron('H/30 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) -// cron('H H(9-16)/6 * * 1-5') + cron('H H(9-16)/6 * * 1-5') } environment { From 87cc55ddf8cd40897fd154bb1f7e35990d3ad282 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 11:36:18 +0200 Subject: [PATCH 044/118] try to move the content of CD folder inside the ws --- Jenkinsfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 958e857..958b567 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,8 +84,8 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} - mkdir -p ./CD - mv -v ${DEPLOY_FILE_ROOT_FOLDER} ./CD + mkdir -p ${env.WORKSPACE}/CD + mv -v ${DEPLOY_FILE_ROOT_FOLDER} ${env.WORKSPACE}/CD ''' } } @@ -365,18 +365,17 @@ def getComponentsFromCSVDir2(def dirPath){ @NonCPS def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" - dh = new File('.') + dh = new File(${env.WORKSPACE}+"/CD") println "current folder path: "+dh.path // dh.eachFileRecurse { // println it // } println "checking CD folder"; - def startDir = new File("/var/lib/jenkins/CD") - startDir.eachFileRecurse(FILES) { - if (it.name.endsWith('.csv')) { - println it - } - } +// startDir.eachFileRecurse(FILES) { +// if (it.name.endsWith('.csv')) { +// println it +// } +// } } From 5f7f5d16e9cfa83426fdd5fa7dc8dd18b0924c50 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 11:39:53 +0200 Subject: [PATCH 045/118] set cron every 15' --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 958b567..c7cee78 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,9 +41,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) -// cron('H/30 * * * *') + cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) - cron('H H(9-16)/6 * * 1-5') +// cron('H H(9-16)/6 * * 1-5') } environment { From 6040738a1209ae306a370228e1bf37c7b546cd7e Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 12:40:36 +0200 Subject: [PATCH 046/118] change WORKSPACE var sintax --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c7cee78..2ab32be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} - mkdir -p ${env.WORKSPACE}/CD + mkdir -p "${WORKSPACE}/CD" mv -v ${DEPLOY_FILE_ROOT_FOLDER} ${env.WORKSPACE}/CD ''' } From 9fa9903bf48a22509d5a7903a3250e57129a6cdc Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 12:48:41 +0200 Subject: [PATCH 047/118] change WORKSPACE var sintax --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2ab32be..b4b0682 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v ${DEPLOY_FILE_ROOT_FOLDER} ${env.WORKSPACE}/CD + mv -v "${DEPLOY_FILE_ROOT_FOLDER}" "${WORKSPACE}/CD" ''' } } @@ -103,7 +103,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir3(${DEPLOY_FILE_ROOT_FOLDER}) + def components =getComponentsFromCSVDir3("${WORKSPACE}/CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -365,7 +365,7 @@ def getComponentsFromCSVDir2(def dirPath){ @NonCPS def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" - dh = new File(${env.WORKSPACE}+"/CD") + dh = new File(dirPath) println "current folder path: "+dh.path // dh.eachFileRecurse { // println it From 863fd1c4ecb58cd3e8daaaabc67636ba8b6dce53 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 14:35:08 +0200 Subject: [PATCH 048/118] fixed mv command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b4b0682..ca19120 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}" "${WORKSPACE}/CD" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}" ''' } } From c6d0b9d64672aa3a464a17ee7d7de07549b2d5ac Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 14:46:13 +0200 Subject: [PATCH 049/118] moving the whole folder under workspace --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca19120..596b8f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,8 +84,7 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} - mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}" "${WORKSPACE}" ''' } } From c70558a11f6caa0055d6398de536e714ffbc6b20 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 15:28:47 +0200 Subject: [PATCH 050/118] fix mv --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 596b8f8..7318191 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,8 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} - mv -v "${DEPLOY_FILE_ROOT_FOLDER}" "${WORKSPACE}" + mkdir -p "${WORKSPACE}/CD" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}" ... "${WORKSPACE}" ''' } } From 527822a1f2f6d9f1e4eea536ee037f66e95ac24e Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 15:59:19 +0200 Subject: [PATCH 051/118] fix mv --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7318191..9958357 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,8 +84,9 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} + mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}" ... "${WORKSPACE}" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" ... "${WORKSPACE}" ''' } } From 95b03417bcb23314e140acd089c24207ec210099 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 15:59:30 +0200 Subject: [PATCH 052/118] fix mv --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9958357..19c1910 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,6 @@ pipeline { sh ''' date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} - mkdir -p "${WORKSPACE}/CD" mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" ... "${WORKSPACE}" ''' From a21064c4e29d7c1a616ecaef27d24e8d8be9b7e5 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 16:00:22 +0200 Subject: [PATCH 053/118] fix mv --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 19c1910..ca19120 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" ... "${WORKSPACE}" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}" ''' } } From bc8c57ce91159262d1270822c0643c5c49b29157 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 16:20:38 +0200 Subject: [PATCH 054/118] fix mv --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca19120..c4efa30 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}" + 'mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}"' ''' } } From a55729d32cad1158f5f124d9e15739a3f2555929 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 16:48:33 +0200 Subject: [PATCH 055/118] fix mv --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c4efa30..6600263 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - 'mv -v "${DEPLOY_FILE_ROOT_FOLDER}*" "${WORKSPACE}"' + 'mv -v "${DEPLOY_FILE_ROOT_FOLDER}\*" "${WORKSPACE}"' ''' } } From 20f82cb65771c09850c892bc07c914b11c2d5c4f Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:05:10 +0200 Subject: [PATCH 056/118] restored old cron --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6600263..5f6fec0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,9 +41,9 @@ pipeline { triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) - cron('H/15 * * * *') +// cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) -// cron('H H(9-16)/6 * * 1-5') + cron('H H(9-16)/6 * * 1-5') } environment { @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - 'mv -v "${DEPLOY_FILE_ROOT_FOLDER}\*" "${WORKSPACE}"' + mv -v "${DEPLOY_FILE_ROOT_FOLDER}\*" "${WORKSPACE}" ''' } } From eb1efee8893a97442f86174d544adbb0bf9aebb3 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:07:21 +0200 Subject: [PATCH 057/118] fixed mv wildcard --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5f6fec0..6e3f622 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}\*" "${WORKSPACE}" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}'*''" "${WORKSPACE}" ''' } } From e284591aab1289dd83bc6eab30342297b367daa5 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:07:45 +0200 Subject: [PATCH 058/118] fixed mv wildcard --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6e3f622..6651356 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}'*''" "${WORKSPACE}" + mv -v "${DEPLOY_FILE_ROOT_FOLDER}'*'" "${WORKSPACE}" ''' } } From fcf685405999990815dd2d64c2395867e357edd4 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:35:10 +0200 Subject: [PATCH 059/118] use find command --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6651356..3dc6b48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,8 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - mv -v "${DEPLOY_FILE_ROOT_FOLDER}'*'" "${WORKSPACE}" + + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv {} "${WORKSPACE}" ''' } } From 16bc2ddcb776a8157c4d472530f231907a8103aa Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:36:42 +0200 Subject: [PATCH 060/118] fix find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3dc6b48..4d0a38d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv {} "${WORKSPACE}" + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv {} "${WORKSPACE}/CD" ''' } } From a4cd1755f98ff7b881c836185f8498119fccd16f Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Wed, 28 Sep 2022 17:40:31 +0200 Subject: [PATCH 061/118] fix find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4d0a38d..db625d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv {} "${WORKSPACE}/CD" + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv \{\} "${WORKSPACE}/CD" ''' } } From bac26e2f790b3da28c0a7a0ac90b4b5b1783a21a Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 11:34:45 +0200 Subject: [PATCH 062/118] fix find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index db625d1..777224b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv \{\} "${WORKSPACE}/CD" + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv \{\} "${WORKSPACE}/CD" \; ''' } } From 56e53ec4b07b4a79131de5b8a83f08ddb25654f5 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 11:37:32 +0200 Subject: [PATCH 063/118] small change --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 777224b..54b83a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,6 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv \{\} "${WORKSPACE}/CD" \; ''' } From 911b5e837d0acef03d2c1ffef84c7c08c75f2149 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 11:51:02 +0200 Subject: [PATCH 064/118] fix escape chars on find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 54b83a6..108cfbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv \{\} "${WORKSPACE}/CD" \; + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv '\'{'\'} "${WORKSPACE}/CD" '\'; ''' } } From 2ff971b0215204d6abb4a4f4f7b536db9fb7c31c Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 11:56:26 +0200 Subject: [PATCH 065/118] fix escape chars on find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 108cfbe..b8fabc3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv '\'{'\'} "${WORKSPACE}/CD" '\'; + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv '\'{'\'} "${WORKSPACE}/CD" '\'+ ''' } } From 748351d64ec1de079aa8c4abba49f76fca0bed0f Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 12:05:09 +0200 Subject: [PATCH 066/118] fix escape chars on find command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b8fabc3..1c41b45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { date=`date`; mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv '\'{'\'} "${WORKSPACE}/CD" '\'+ + find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv --target-directory="${WORKSPACE}/CD" '\'{'\'} '\'+ ''' } } From 670451a0280c0e268d6173babc642c985e2daaeb Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 12:51:14 +0200 Subject: [PATCH 067/118] find work properly. Going to pass to the original method --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1c41b45..2352cec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,7 +103,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir3("${WORKSPACE}/CD") + def components =getComponentsFromCSVDir("${WORKSPACE}/CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -367,9 +367,9 @@ def getComponentsFromCSVDir3(def dirPath){ println "checking current folder" dh = new File(dirPath) println "current folder path: "+dh.path -// dh.eachFileRecurse { -// println it -// } + dh.eachFileRecurse { + println it + } println "checking CD folder"; // startDir.eachFileRecurse(FILES) { // if (it.name.endsWith('.csv')) { From 3f607a0ac4ad14618f8a975d037574f4e966b41a Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 15:53:00 +0200 Subject: [PATCH 068/118] replace File with FilePath --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2352cec..1979956 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -295,9 +295,10 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ //experimental test - @NonCPS + //@NonCPS def getComponentsFromCSVDir(def dirPath){ - File folder = new File(dirPath) +// File folder = new File(dirPath) + folder= new FilePath(Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel(), dirPath); println ("folder ready "); if (folder){ println (" processing folder "+folder); From f48fbfc448c8661b4bf27990fdecd4a97d08485d Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 15:56:47 +0200 Subject: [PATCH 069/118] add FilePath import --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 1979956..8ffdb1d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,6 +2,8 @@ import groovy.io.FileType import static groovy.io.FileType.FILES import jenkins.model.* +import hudson.FilePath; +import jenkins.model.Jenkins; /** * Deploy components to the D4Science Infrastructure (dev-only) From 3c928cd070fcd9e27979c01fe2a1c13e9b25ba75 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Thu, 29 Sep 2022 17:08:00 +0200 Subject: [PATCH 070/118] add script function --- Jenkinsfile | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ffdb1d..6a371b5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,7 +105,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir("${WORKSPACE}/CD") + def components =getComponentsFromCSVDir4("${WORKSPACE}/CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -381,6 +381,24 @@ def getComponentsFromCSVDir3(def dirPath){ // } } +def getComponentsFromCSVDir4(def dirPath){ + sh ''' + echo "parsing $dirPath"; + for entry in `ls $dirPath`; do + INPUT=$entry + OLDIFS=$IFS + IFS=',' + [ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; } + while read componentname version + do + echo "Name : $componentname" + echo "DOB : $version" + done < $INPUT + IFS=$OLDIFS + done + ''' +} + From 8842ea81bb4f7bb17507d2894a5eef5f6cb4d168 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 09:53:28 +0200 Subject: [PATCH 071/118] changed method version --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6a371b5..bac763d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,7 +105,7 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir4("${WORKSPACE}/CD") + def components =getComponentsFromCSVDir("${WORKSPACE}/CD") if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { @@ -389,10 +389,11 @@ def getComponentsFromCSVDir4(def dirPath){ OLDIFS=$IFS IFS=',' [ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; } - while read componentname version + while read artifactid version host do - echo "Name : $componentname" - echo "DOB : $version" + echo "Component name : $artifactid" + echo "Component version : $version" + echo "Target host : $host" done < $INPUT IFS=$OLDIFS done From ccb34365cad3103398b93e237f2c9486077d0b96 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 14:57:58 +0200 Subject: [PATCH 072/118] add NonCPS tag to the method --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bac763d..779754b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -297,10 +297,10 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ //experimental test - //@NonCPS + @NonCPS def getComponentsFromCSVDir(def dirPath){ -// File folder = new File(dirPath) - folder= new FilePath(Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel(), dirPath); + File folder = new File(dirPath) +// folder= new FilePath(Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel(), dirPath); println ("folder ready "); if (folder){ println (" processing folder "+folder); From c32ef76a54e1fa642ffb6daa3531d69ad285320d Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 15:35:00 +0200 Subject: [PATCH 073/118] readCSV file --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 779754b..716a1ab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,7 +105,8 @@ pipeline { // parse the report and extract the data // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - def components =getComponentsFromCSVDir("${WORKSPACE}/CD") + //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") + def components = readCSV file: 'CD/deploy.csv' if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { From 7b0d9123c26aaece205f23735f00edf4f52b42d7 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 15:41:49 +0200 Subject: [PATCH 074/118] add print --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 716a1ab..530a6d0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,6 +107,7 @@ pipeline { println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") def components = readCSV file: 'CD/deploy.csv' + println(components) if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { From 8b5d8e97944057199e74ee7e772b558018485286 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 15:49:15 +0200 Subject: [PATCH 075/118] add print --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 530a6d0..46f142b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,6 +111,7 @@ pipeline { if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { + println(component) stage(component){ println "Deploy on going of component: $component" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { From 9f8f11018d733b65697c2e46f8b456a77b24e133 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 15:52:52 +0200 Subject: [PATCH 076/118] add print --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 46f142b..2fa45d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,7 +111,7 @@ pipeline { if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { - println(component) + println(component.values.0) stage(component){ println "Deploy on going of component: $component" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { From 9f1ebdbdd20d3ca61b7660848b77176e9c394413 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:27:21 +0200 Subject: [PATCH 077/118] fill components --- Jenkinsfile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2fa45d6..792e360 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,7 +106,18 @@ pipeline { // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") - def components = readCSV file: 'CD/deploy.csv' + //def components = readCSV file: 'CD/deploy.csv' + readCSV(file: 'CD/deploy.csv').each { line , count-> + if (line.startsWith('#')) + return + def fields = line.split(',') + components.add([ + name : fields[0], + version : fields[1], + host : fields[2] + ] + ) + } println(components) if (components.size() > 0) { def componentSet=components.toSet(); From f3133d34f6d6cb31bbe1112398d24f07ba1be7f5 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:28:54 +0200 Subject: [PATCH 078/118] wrong print --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 792e360..d7271d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,7 +122,6 @@ pipeline { if (components.size() > 0) { def componentSet=components.toSet(); for (component in componentSet) { - println(component.values.0) stage(component){ println "Deploy on going of component: $component" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { From bc1c7b319148e5623ce096127362122b6175bd38 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:31:48 +0200 Subject: [PATCH 079/118] wrong print --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d7271d4..068e302 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -108,9 +108,9 @@ pipeline { //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") //def components = readCSV file: 'CD/deploy.csv' readCSV(file: 'CD/deploy.csv').each { line , count-> - if (line.startsWith('#')) + if (line.toString().startsWith('#')) return - def fields = line.split(',') + def fields = line.toString().split(',') components.add([ name : fields[0], version : fields[1], From 115f6403baad5f3d0edb430a009b4ab064eaa526 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:34:20 +0200 Subject: [PATCH 080/118] small fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 068e302..0076552 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,7 +111,7 @@ pipeline { if (line.toString().startsWith('#')) return def fields = line.toString().split(',') - components.add([ + def components.add([ name : fields[0], version : fields[1], host : fields[2] From 48eacafc67f50156af18f5588d8b5eeaad2bcae5 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:37:41 +0200 Subject: [PATCH 081/118] small fix --- Jenkinsfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0076552..4ab1463 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,12 +111,15 @@ pipeline { if (line.toString().startsWith('#')) return def fields = line.toString().split(',') - def components.add([ - name : fields[0], - version : fields[1], - host : fields[2] - ] - ) + println(fileds[0]) + println(fileds[1]) + println(fileds[2]) + // def components.add([ + // name : fields[0], + // version : fields[1], + // host : fields[2] + // ] + // ) } println(components) if (components.size() > 0) { From ca74aeab831ef93bd2903a12115eaf461457c21e Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:43:08 +0200 Subject: [PATCH 082/118] small fix --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ab1463..f6a6682 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,9 +111,9 @@ pipeline { if (line.toString().startsWith('#')) return def fields = line.toString().split(',') - println(fileds[0]) - println(fileds[1]) - println(fileds[2]) + println(fields[0]) + println(fields[1]) + println(fields[2]) // def components.add([ // name : fields[0], // version : fields[1], From 58fd9640df4ac1eb6ce5230b41877593c13e798e Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:56:31 +0200 Subject: [PATCH 083/118] small fix --- Jenkinsfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6a6682..ae47224 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -110,17 +110,17 @@ pipeline { readCSV(file: 'CD/deploy.csv').each { line , count-> if (line.toString().startsWith('#')) return - def fields = line.toString().split(',') - println(fields[0]) - println(fields[1]) - println(fields[2]) - // def components.add([ - // name : fields[0], - // version : fields[1], - // host : fields[2] - // ] - // ) + // line.get(0); + // def fields = line.toString().split(',') + def components.add([ + name : line.get(0), + version : line.get(1), + host : line.get(2) + ] + ) } + //def records = readCSV file: 'CD/deploy.csv' + def components println(components) if (components.size() > 0) { def componentSet=components.toSet(); From 3cf615d21cfb554251cf130b6e0cbbb17ff1ef01 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 16:58:14 +0200 Subject: [PATCH 084/118] small fix --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae47224..a7e44aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -120,7 +120,6 @@ pipeline { ) } //def records = readCSV file: 'CD/deploy.csv' - def components println(components) if (components.size() > 0) { def componentSet=components.toSet(); From b30a726d2ae856095fa7dba58d2e7bf92bfd7eb7 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:15:45 +0200 Subject: [PATCH 085/118] small fix --- Jenkinsfile | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a7e44aa..1228837 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,19 +107,30 @@ pipeline { println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") //def components = readCSV file: 'CD/deploy.csv' - readCSV(file: 'CD/deploy.csv').each { line , count-> - if (line.toString().startsWith('#')) - return + //readCSV(file: 'CD/deploy.csv').each { line , count-> + // if (line.toString().startsWith('#')) + // return // line.get(0); // def fields = line.toString().split(',') - def components.add([ - name : line.get(0), - version : line.get(1), - host : line.get(2) - ] - ) - } - //def records = readCSV file: 'CD/deploy.csv' + // def components.add([ + // name : line.get(0), + // version : line.get(1), + // host : line.get(2) + // ] + // ) + // } + def records = readCSV file: 'CD/deploy.csv' + for (def record : records) { + println("Processing record: "+record) + if (record.toString().startsWith('#')) return; + def components.add([ + name : record.get(0), + version : record.get(1), + host : record.get(2) + ] + ) + + } println(components) if (components.size() > 0) { def componentSet=components.toSet(); From 6937a900b8f517cb4a5ccbc3995916b8ef75c364 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:22:17 +0200 Subject: [PATCH 086/118] small fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1228837..f48239d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,7 +122,7 @@ pipeline { def records = readCSV file: 'CD/deploy.csv' for (def record : records) { println("Processing record: "+record) - if (record.toString().startsWith('#')) return; + // if (record.toString().startsWith('#')) return; def components.add([ name : record.get(0), version : record.get(1), From b39f166d815f7883af5967ab2616f09d5223d742 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:23:35 +0200 Subject: [PATCH 087/118] small fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f48239d..45e10a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -123,7 +123,7 @@ pipeline { for (def record : records) { println("Processing record: "+record) // if (record.toString().startsWith('#')) return; - def components.add([ + components.add([ name : record.get(0), version : record.get(1), host : record.get(2) From 4868c68c96c716e7b3db85eed784dc537070be32 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:30:45 +0200 Subject: [PATCH 088/118] small fix --- Jenkinsfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 45e10a9..c5240be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -123,12 +123,9 @@ pipeline { for (def record : records) { println("Processing record: "+record) // if (record.toString().startsWith('#')) return; - components.add([ - name : record.get(0), - version : record.get(1), - host : record.get(2) - ] - ) + println ("field 0 "+record.get(0)) + println ("field 1 "+record.get(1)) + println ("field 2 "+record.get(2)) } println(components) From 081b52280e322833f24f2d2bede43f655b447e91 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:42:51 +0200 Subject: [PATCH 089/118] removed component array --- Jenkinsfile | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c5240be..ec4161d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,27 +122,16 @@ pipeline { def records = readCSV file: 'CD/deploy.csv' for (def record : records) { println("Processing record: "+record) - // if (record.toString().startsWith('#')) return; println ("field 0 "+record.get(0)) println ("field 1 "+record.get(1)) println ("field 2 "+record.get(2)) - - } - println(components) - if (components.size() > 0) { - def componentSet=components.toSet(); - for (component in componentSet) { - stage(component){ - println "Deploy on going of component: $component" - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - checkup("${component.name}", "${component.version}", "${component.host}"); - deploy("${component.name}", "${component.version}", "${component.host}"); - } - } + stage(record.get(0)){ + println "Deploy on going of component: record.get(0)" + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + checkup(record.get(0), record.get(1), record.get(2)); + deploy(record.get(0), record.get(1), record.get(2)s); + } } - }else{ - println "No components found" - } } cleanup(DEPLOY_FILE,BACKUP_FILE); } From a346ad116aa35b9d03a8f2fa9119a2427cf27556 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:45:53 +0200 Subject: [PATCH 090/118] fix sintax --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec4161d..afb0d40 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { println "Deploy on going of component: record.get(0)" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { checkup(record.get(0), record.get(1), record.get(2)); - deploy(record.get(0), record.get(1), record.get(2)s); + deploy(record.get(0), record.get(1), record.get(2)); } } } From 5c2faa5acd4f12c3a34824a7b47513f37e65b3e2 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:49:06 +0200 Subject: [PATCH 091/118] fix sintax --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index afb0d40..ca86ff8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -132,6 +132,7 @@ pipeline { deploy(record.get(0), record.get(1), record.get(2)); } } + } } cleanup(DEPLOY_FILE,BACKUP_FILE); } From ed52fdb21e9a4d992807433871d2f8cf18c3c755 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:52:42 +0200 Subject: [PATCH 092/118] fix sintax --- Jenkinsfile | 85 ----------------------------------------------------- 1 file changed, 85 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca86ff8..181f0ed 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -309,91 +309,6 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ } //experimental test - - @NonCPS -def getComponentsFromCSVDir(def dirPath){ - File folder = new File(dirPath) -// folder= new FilePath(Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel(), dirPath); - println ("folder ready "); - if (folder){ - println (" processing folder "+folder); - folder.eachFileRecurse FileType.FILES, { file -> - // check it if the file ends with a .csv extension - println("checking file: "+file); - if (file.name.endsWith(".csv")) { - println ("Processing file "); - readFile(file).split('\n').each { line, count-> - if (line.startsWith('#')) - return - def fields = line.split(',') - components.add([ - name : fields[0], - version : fields[1], - host : fields[2] - ] - ) - } - } - println ("removing current deploy file: "+file.name); - // remove the file here if possible - // file.delete(); - } - } - return components -} - - @NonCPS -def getComponentsFromCSVDir2(def dirPath){ - def list = [] - def folder = new File(dirPath) - println ("folder ready "); - if (folder){ - println (" processing folder "+folder); - folder.eachFileRecurse FileType.FILES, { - if (file.name.endsWith(".csv")) { - println ("Processing file "+it); - list << file - readFile(file).split('\n').each { line, count-> - if (line.startsWith('#')) - return - def fields = line.split(',') - components.add([ - name : fields[0], - version : fields[1], - host : fields[2] - ] - ) - } - } - println ("removing current deploy file: "+it.name); - // remove the file here if possible - // file.delete(); - } - } - println "files to remove:"; - list.each { - println ("removing"+it.path); - it.delete(); - } - return components -} - - @NonCPS -def getComponentsFromCSVDir3(def dirPath){ - println "checking current folder" - dh = new File(dirPath) - println "current folder path: "+dh.path - dh.eachFileRecurse { - println it - } - println "checking CD folder"; -// startDir.eachFileRecurse(FILES) { -// if (it.name.endsWith('.csv')) { -// println it -// } -// } -} - def getComponentsFromCSVDir4(def dirPath){ sh ''' echo "parsing $dirPath"; From 5debaa7edd8514eb3b52f0a5e70ac33001e7be77 Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 17:57:05 +0200 Subject: [PATCH 093/118] add new var CD_ROOT_FOLDER --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 181f0ed..18e61ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,6 +52,7 @@ pipeline { AGENT_ROOT_FOLDER = "${agent_root_folder}" DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster" DEPLOY_FILE_ROOT_FOLDER="${agent_root_folder}/CD/" + CD_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" DEPLOY_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" BACKUP_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" @@ -275,7 +276,7 @@ def getComponentsFromCSV(def deployList) { def deploy(String service, String version, String host){ def now = new Date(); println("Going to deploy the service "+service+" with version: "+version+" on target: "+host); - def statusCode = sh( script: "cd $DEPLOY_ROOT_FOLDER;./deployService.sh $service $version $host;", returnStdout: true); + def statusCode = sh( script: "cd $CD_ROOT_FOLDER;./deployService.sh $service $version $host;", returnStdout: true); sh(""" echo " last exit code \$?"; """) From c9db99700068a035a565c25b5ea6cd859273ad1f Mon Sep 17 00:00:00 2001 From: Roberto Cirillo Date: Fri, 30 Sep 2022 18:06:24 +0200 Subject: [PATCH 094/118] removed unused methods, clean code --- Jenkinsfile | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 18e61ff..988d3f2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,29 +103,10 @@ pipeline { steps { echo 'Cron build enabled. Deploy from system ongoing' script { - // parse the report and extract the data - // def components = getComponentsFromCSV(deployList) println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" - //def components =getComponentsFromCSVDir("${WORKSPACE}/CD") - //def components = readCSV file: 'CD/deploy.csv' - //readCSV(file: 'CD/deploy.csv').each { line , count-> - // if (line.toString().startsWith('#')) - // return - // line.get(0); - // def fields = line.toString().split(',') - // def components.add([ - // name : line.get(0), - // version : line.get(1), - // host : line.get(2) - // ] - // ) - // } def records = readCSV file: 'CD/deploy.csv' for (def record : records) { println("Processing record: "+record) - println ("field 0 "+record.get(0)) - println ("field 1 "+record.get(1)) - println ("field 2 "+record.get(2)) stage(record.get(0)){ println "Deploy on going of component: record.get(0)" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { From f72d3705fbd4cc52524f424604dc06ebe6d68731 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Mon, 3 Oct 2022 10:09:52 +0200 Subject: [PATCH 095/118] rename var DEPLOY_FILE_ROOT_FOLDER to PENDING_DEPLOY_FOLDER --- Jenkinsfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 988d3f2..c263a7c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,12 +50,11 @@ pipeline { environment { AGENT_ROOT_FOLDER = "${agent_root_folder}" - DEPLOY_ROOT_FOLDER ="${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster" - DEPLOY_FILE_ROOT_FOLDER="${agent_root_folder}/CD/" + PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/" CD_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" - DEPLOY_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" - BACKUP_FILE = "${DEPLOY_FILE_ROOT_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" + DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" + BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" @@ -86,9 +85,9 @@ pipeline { steps { sh ''' date=`date`; - mkdir -p ${DEPLOY_FILE_ROOT_FOLDER} + mkdir -p ${PENDING_DEPLOY_FOLDER} mkdir -p "${WORKSPACE}/CD" - find "${DEPLOY_FILE_ROOT_FOLDER}" -type f -exec mv --target-directory="${WORKSPACE}/CD" '\'{'\'} '\'+ + find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${WORKSPACE}/CD" '\'{'\'} '\'+ ''' } } @@ -103,7 +102,7 @@ pipeline { steps { echo 'Cron build enabled. Deploy from system ongoing' script { - println "Going to check the deploy file in ${DEPLOY_FILE_ROOT_FOLDER}" + println "Going to check the deploy file in ${PENDING_DEPLOY_FOLDER}" def records = readCSV file: 'CD/deploy.csv' for (def record : records) { println("Processing record: "+record) @@ -151,8 +150,7 @@ pipeline { else echo "${TRIGGER_JOB},${TRIGGER_VERSION},${TRIGGER_HOST}" >> ${DEPLOY_FILE} fi - ''' - + ''' // the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { sh(""" From 3b9e5e182634c9d5fbe435e37328896e408d82cb Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Mon, 3 Oct 2022 16:38:01 +0200 Subject: [PATCH 096/118] add loop on deploy folder --- Jenkinsfile | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c263a7c..5e65e6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,6 +55,7 @@ pipeline { PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" + DEPLOY_FOLDER= "${WORKSPACE}/CD-${PIPELINE_BUILD_NUMBER}" TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" @@ -103,17 +104,28 @@ pipeline { echo 'Cron build enabled. Deploy from system ongoing' script { println "Going to check the deploy file in ${PENDING_DEPLOY_FOLDER}" - def records = readCSV file: 'CD/deploy.csv' - for (def record : records) { - println("Processing record: "+record) - stage(record.get(0)){ - println "Deploy on going of component: record.get(0)" - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - checkup(record.get(0), record.get(1), record.get(2)); - deploy(record.get(0), record.get(1), record.get(2)); + // def records = readCSV file: 'CD/deploy.csv' +// def files = findFiles(glob: 'target/surefire-reports/*.csv') + def deployFolder="CD-${env.BUILD_NUMBER}"; + println ("searching files in folder ${deployFolder}"); + def files = findFiles(glob: "CD-${deployFolder}/*.csv") + for (def file : files){ + echo """ + Processing: ${files[0].name} ${files[0].path} ${files[0].directory} + ${files[0].length} ${files[0].lastModified} + """ + def records = readCSV file: 'file.path' + for (def record : records) { + println("Processing record: "+record) + stage(record.get(0)){ + println "Deploy on going of component: record.get(0)" + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + checkup(record.get(0), record.get(1), record.get(2)); + deploy(record.get(0), record.get(1), record.get(2)); + } } - } - } + } + } } cleanup(DEPLOY_FILE,BACKUP_FILE); } From 1245adac9a6157b60365879b9a794d6f1636c0d6 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Mon, 3 Oct 2022 17:12:19 +0200 Subject: [PATCH 097/118] set deploy-folder with build number --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e65e6a..2bdedc1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,8 +87,8 @@ pipeline { sh ''' date=`date`; mkdir -p ${PENDING_DEPLOY_FOLDER} - mkdir -p "${WORKSPACE}/CD" - find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${WORKSPACE}/CD" '\'{'\'} '\'+ + mkdir -p "${DEPLOY_FOLDER}" + find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${DEPLOY_FOLDER}" '\'{'\'} '\'+ ''' } } From 647b4e52ac48d21eab2fbcb244e467eb3261d70e Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Mon, 3 Oct 2022 17:17:38 +0200 Subject: [PATCH 098/118] moving echo print outside the loop --- Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 2bdedc1..d4ba4a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,6 +109,10 @@ pipeline { def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "CD-${deployFolder}/*.csv") + echo """ + Processing: ${files[0].name} ${files[0].path} ${files[0].directory} + ${files[0].length} ${files[0].lastModified} + """ for (def file : files){ echo """ Processing: ${files[0].name} ${files[0].path} ${files[0].directory} From 5e9c7a1e9f1ac703e7fda10b6fe3b4a32552efbe Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 09:46:54 +0200 Subject: [PATCH 099/118] fix findfiles method --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d4ba4a4..1bc4109 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -108,7 +108,7 @@ pipeline { // def files = findFiles(glob: 'target/surefire-reports/*.csv') def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); - def files = findFiles(glob: "CD-${deployFolder}/*.csv") + def files = findFiles(glob: "${deployFolder}/*.csv") echo """ Processing: ${files[0].name} ${files[0].path} ${files[0].directory} ${files[0].length} ${files[0].lastModified} From 51b48579ab3984f79b7007c7e3901559bb8c5219 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 09:56:24 +0200 Subject: [PATCH 100/118] fis readCSV method --- Jenkinsfile | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1bc4109..6e64102 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,29 +109,31 @@ pipeline { def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") - echo """ - Processing: ${files[0].name} ${files[0].path} ${files[0].directory} - ${files[0].length} ${files[0].lastModified} - """ - for (def file : files){ - echo """ - Processing: ${files[0].name} ${files[0].path} ${files[0].directory} - ${files[0].length} ${files[0].lastModified} - """ - def records = readCSV file: 'file.path' - for (def record : records) { - println("Processing record: "+record) - stage(record.get(0)){ - println "Deploy on going of component: record.get(0)" - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - checkup(record.get(0), record.get(1), record.get(2)); - deploy(record.get(0), record.get(1), record.get(2)); - } - } - } - } + if (!files.isEmpty()){ + for (def file : files){ + echo """ + Found: ${file.name} with path ${file.path} + length ${files[0].length} lastModified ${files[0].lastModified} + """ + def records = readCSV file: "${file.path}" + for (def record : records) { + println("Processing record: "+record) + stage(record.get(0)){ + println "Deploy on going of component: record.get(0)" + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + checkup(record.get(0), record.get(1), record.get(2)); + deploy(record.get(0), record.get(1), record.get(2)); + } + } + } + } + cleanup(DEPLOY_FILE,BACKUP_FILE); + }else{ + println ("Nothing to do"); + } + } - cleanup(DEPLOY_FILE,BACKUP_FILE); + } } stage('Nothing to do ') { From fa0928823d0e2186edaad75934937d138376b720 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:06:59 +0200 Subject: [PATCH 101/118] FIX ISeMPTY CHECK --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6e64102..0d6dbc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,7 +109,9 @@ pipeline { def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") - if (!files.isEmpty()){ + if (files.isEmpty()){ + println ("Nothing to do"); + }else{ for (def file : files){ echo """ Found: ${file.name} with path ${file.path} @@ -128,8 +130,7 @@ pipeline { } } cleanup(DEPLOY_FILE,BACKUP_FILE); - }else{ - println ("Nothing to do"); + } } From 99d8a9070636f753c92249d3f25bbdd02da105c0 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:14:38 +0200 Subject: [PATCH 102/118] small change --- Jenkinsfile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d6dbc0..723aee7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,7 +109,7 @@ pipeline { def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") - if (files.isEmpty()){ + if (files == null){ println ("Nothing to do"); }else{ for (def file : files){ @@ -127,9 +127,10 @@ pipeline { deploy(record.get(0), record.get(1), record.get(2)); } } - } + } + clean(${file.path}); } - cleanup(DEPLOY_FILE,BACKUP_FILE); + } @@ -307,6 +308,19 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ ''' } +/clean and update the local deploy file +def clean(def DEPLOY_FILE){ + sh ''' + echo "cleanup $DEPLOY_FILE"; + if [ -f ${DEPLOY_FILE} ]; then + rm ${DEPLOY_FILE}; + else + echo "deploy file empty" + fi + ''' +} + + //experimental test def getComponentsFromCSVDir4(def dirPath){ sh ''' From 75e36dcd24a926b452143ba3102c434ae645554f Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:15:56 +0200 Subject: [PATCH 103/118] fix sintax --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 723aee7..27b94bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -308,7 +308,7 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ ''' } -/clean and update the local deploy file +//clean and update the local deploy file def clean(def DEPLOY_FILE){ sh ''' echo "cleanup $DEPLOY_FILE"; From 3f3e10fa2fba5259c6579d181cec9d71b6fd95a1 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:23:18 +0200 Subject: [PATCH 104/118] fix clean method --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 27b94bd..a9c17ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -128,7 +128,7 @@ pipeline { } } } - clean(${file.path}); + clean("${file.path}"); } From 7a361393f1f7441671d94cdfada620d3872f8f53 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:40:02 +0200 Subject: [PATCH 105/118] fix clean method --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9c17ec..6b92ca2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -309,13 +309,13 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ } //clean and update the local deploy file -def clean(def DEPLOY_FILE){ +def clean(def file){ sh ''' - echo "cleanup $DEPLOY_FILE"; - if [ -f ${DEPLOY_FILE} ]; then - rm ${DEPLOY_FILE}; + echo "cleaning $file"; + if [ -f ${file} ]; then + rm ${file}; else - echo "deploy file empty" + echo "file not exist" fi ''' } From 360a7cf5be39262f73487f422e03f35b4fd853f7 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:48:23 +0200 Subject: [PATCH 106/118] fix clean method --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b92ca2..862020c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -312,8 +312,8 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ def clean(def file){ sh ''' echo "cleaning $file"; - if [ -f ${file} ]; then - rm ${file}; + if [ -f $file ]; then + rm $file; else echo "file not exist" fi From 350b9ccb8c86f563a0d98e899ca91853d7fb00ed Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 10:59:41 +0200 Subject: [PATCH 107/118] put remove actions outside the loop --- Jenkinsfile | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 862020c..972f10c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,6 +112,7 @@ pipeline { if (files == null){ println ("Nothing to do"); }else{ + def toRemove = [] as ArrayList for (def file : files){ echo """ Found: ${file.name} with path ${file.path} @@ -127,10 +128,10 @@ pipeline { deploy(record.get(0), record.get(1), record.get(2)); } } - } - clean("${file.path}"); + } + toRemove.add("${file.path}") } - + clean(toRemove) } @@ -309,14 +310,17 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ } //clean and update the local deploy file -def clean(def file){ +def clean(def files){ sh ''' - echo "cleaning $file"; - if [ -f $file ]; then - rm $file; - else - echo "file not exist" - fi + echo "parsing $files"; + for file in `ls $dirPath`; do + echo "cleaning $file"; + if [ -f $file ]; then + rm $file; + else + echo "file not exist" + fi + done ''' } From 7f1a2be84e540b13c7fb40b8bfcf82414e26131d Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 11:12:30 +0200 Subject: [PATCH 108/118] small fix --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 972f10c..bba9859 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -312,8 +312,8 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ //clean and update the local deploy file def clean(def files){ sh ''' - echo "parsing $files"; - for file in `ls $dirPath`; do + echo "parsing ${files}"; + for file in `ls $files`; do echo "cleaning $file"; if [ -f $file ]; then rm $file; From 79abb3f26ec8cfce14cbf38aa95d28d61fcc1680 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 11:20:07 +0200 Subject: [PATCH 109/118] small fix --- Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bba9859..7b9286d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,10 +129,8 @@ pipeline { } } } - toRemove.add("${file.path}") + sh "rm ${file.path}" } - clean(toRemove) - } } @@ -310,7 +308,7 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ } //clean and update the local deploy file -def clean(def files){ +def clean(files){ sh ''' echo "parsing ${files}"; for file in `ls $files`; do From 28fa798768ef4f86a17b0615771e90b0714e84ed Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 11:24:18 +0200 Subject: [PATCH 110/118] add folder delete --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7b9286d..7de5c0d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,7 +112,6 @@ pipeline { if (files == null){ println ("Nothing to do"); }else{ - def toRemove = [] as ArrayList for (def file : files){ echo """ Found: ${file.name} with path ${file.path} @@ -130,8 +129,9 @@ pipeline { } } sh "rm ${file.path}" - } - } + } + sh "rm -Rf ${deployFolder}" + } } From 1f93095e752842781ca029c867d5f763941012e6 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 11:42:36 +0200 Subject: [PATCH 111/118] merging test --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7de5c0d..a6914fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,6 +112,9 @@ pipeline { if (files == null){ println ("Nothing to do"); }else{ + for ( def file : files){ + sh " ${file.path} >> ${deployFolder}/testdeploy.csv" + } for (def file : files){ echo """ Found: ${file.name} with path ${file.path} @@ -130,7 +133,7 @@ pipeline { } sh "rm ${file.path}" } - sh "rm -Rf ${deployFolder}" + // sh "rm -Rf ${deployFolder}" } } From 092b4213d394e7ea94808b0be889586bfd42981c Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 11:49:50 +0200 Subject: [PATCH 112/118] merging deploy files test --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6914fa..911c1bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,10 @@ pipeline { println ("Nothing to do"); }else{ for ( def file : files){ - sh " ${file.path} >> ${deployFolder}/testdeploy.csv" + sh """ + touch ${deployFolder}/testdeploy.csv + ${file.path} >> ${deployFolder}/testdeploy.csv + """ } for (def file : files){ echo """ From d7bc7469e0ef2fc9e06a21991df1c59ab3f18824 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 12:15:46 +0200 Subject: [PATCH 113/118] small fix --- Jenkinsfile | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 911c1bc..8005fa4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -104,20 +104,13 @@ pipeline { echo 'Cron build enabled. Deploy from system ongoing' script { println "Going to check the deploy file in ${PENDING_DEPLOY_FOLDER}" - // def records = readCSV file: 'CD/deploy.csv' -// def files = findFiles(glob: 'target/surefire-reports/*.csv') def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") if (files == null){ println ("Nothing to do"); }else{ - for ( def file : files){ - sh """ - touch ${deployFolder}/testdeploy.csv - ${file.path} >> ${deployFolder}/testdeploy.csv - """ - } + def serviceList = [] for (def file : files){ echo """ Found: ${file.name} with path ${file.path} @@ -126,17 +119,20 @@ pipeline { def records = readCSV file: "${file.path}" for (def record : records) { println("Processing record: "+record) - stage(record.get(0)){ - println "Deploy on going of component: record.get(0)" - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - checkup(record.get(0), record.get(1), record.get(2)); - deploy(record.get(0), record.get(1), record.get(2)); - } + if(!serviceList.contains(record.get(0))){ + stage(record.get(0)){ + println "Deploy on going of component: record.get(0)" + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + checkup(record.get(0), record.get(1), record.get(2)); + deploy(record.get(0), record.get(1), record.get(2)); + } + } + serviceList << record.get(0) } } sh "rm ${file.path}" } - // sh "rm -Rf ${deployFolder}" + sh "rm -Rf ${deployFolder}" } } From 922ffd83beae0b433edc2f6b7eae6dc0dafe32cb Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 12:23:08 +0200 Subject: [PATCH 114/118] merge works. add new print --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 8005fa4..46e3980 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -128,7 +128,10 @@ pipeline { } } serviceList << record.get(0) + }else{ + echo "${record.get(0)} already deployed" } + } sh "rm ${file.path}" } From 368b9609a22bb842dfde9e17f425507c63b47bd6 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 12:31:28 +0200 Subject: [PATCH 115/118] fix print --- Jenkinsfile | 57 +++-------------------------------------------------- 1 file changed, 3 insertions(+), 54 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 46e3980..d1b9e4a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,8 +14,6 @@ import jenkins.model.Jenkins; // related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/ def agent_root_folder = '/var/lib/jenkins' -//def agent_deploy_filename = 'deploy.${env.BUILD_NUMBER}.csv' -//def agent_deploy_backup_filename = 'deploy.${env.BUILD_NUMBER}.bck' def deployList def backupList @@ -24,15 +22,6 @@ if (params.deployFile) { deployList = params.deployFile } -//locate the targetHost file -//String targetHostURL = "https://code-repo.d4science.org/gCubeCI/gCubeDeployer/raw/branch/master/open/gcube-${gCube_release_version}.yaml" -//if (verbose) -// println "Querying ${targetHostURL}" -//load the release file -//def text = targetHostURL.toURL().getText() - - - pipeline { agent { label 'ansible' @@ -121,7 +110,7 @@ pipeline { println("Processing record: "+record) if(!serviceList.contains(record.get(0))){ stage(record.get(0)){ - println "Deploy on going of component: record.get(0)" + println "Deploy on going of component: ${record.get(0)}" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { checkup(record.get(0), record.get(1), record.get(2)); deploy(record.get(0), record.get(1), record.get(2)); @@ -129,7 +118,7 @@ pipeline { } serviceList << record.get(0) }else{ - echo "${record.get(0)} already deployed" + echo "${record.get(0)} already deployed. Deployment skipped." } } @@ -310,44 +299,4 @@ def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ echo "deploy file empty" fi ''' -} - -//clean and update the local deploy file -def clean(files){ - sh ''' - echo "parsing ${files}"; - for file in `ls $files`; do - echo "cleaning $file"; - if [ -f $file ]; then - rm $file; - else - echo "file not exist" - fi - done - ''' -} - - -//experimental test -def getComponentsFromCSVDir4(def dirPath){ - sh ''' - echo "parsing $dirPath"; - for entry in `ls $dirPath`; do - INPUT=$entry - OLDIFS=$IFS - IFS=',' - [ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; } - while read artifactid version host - do - echo "Component name : $artifactid" - echo "Component version : $version" - echo "Target host : $host" - done < $INPUT - IFS=$OLDIFS - done - ''' -} - - - - +} \ No newline at end of file From 4508798749e0afd7ebdc57e090415582496ace5f Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 16:01:00 +0200 Subject: [PATCH 116/118] removed echo --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d1b9e4a..1a875bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -101,10 +101,6 @@ pipeline { }else{ def serviceList = [] for (def file : files){ - echo """ - Found: ${file.name} with path ${file.path} - length ${files[0].length} lastModified ${files[0].lastModified} - """ def records = readCSV file: "${file.path}" for (def record : records) { println("Processing record: "+record) From c99b667698b492985b5c1e4b9abe1177870009c2 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 16:04:32 +0200 Subject: [PATCH 117/118] clean unused methods --- Jenkinsfile | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1a875bb..12eb36c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -237,29 +237,6 @@ pipeline { } } - -//parse a csv file formatted in this way: ComponentName,ComponentVersion, ComponentHost -def getComponentsFromCSV(def deployList) { - def components = [] - if (fileExists("${deployList}")) { - echo 'file found' - readFile("${deployList}").split('\n').each { line, count-> - if (line.startsWith('#')) - return - def fields = line.split(',') - components.add([ - name : fields[0], - version : fields[1], - host : fields[2] - ] - ) - } - } else { - echo ' File Not found. Failing.' - } - return components -} - //launch ansible deploy def deploy(String service, String version, String host){ def now = new Date(); @@ -281,18 +258,3 @@ def checkup(String service, String version, String host){ esac """) } -//clean and update the local deploy file -def cleanup(def DEPLOY_FILE, def BACKUP_FILE){ - sh ''' - echo "cleanup $DEPLOY_FILE"; - if [ -f ${DEPLOY_FILE} ]; then - if [ -f ${BACKUP_FILE} ]; then - echo "backup found: ${BACKUP_FILE} going to replace it"; - rm ${BACKUP_FILE}; - fi - mv ${DEPLOY_FILE} ${BACKUP_FILE}; - else - echo "deploy file empty" - fi - ''' -} \ No newline at end of file From 0735ed9edaa3633e4ec4a4ece8bbcc8eeff63c54 Mon Sep 17 00:00:00 2001 From: "roberto.cirillo" Date: Tue, 4 Oct 2022 16:20:42 +0200 Subject: [PATCH 118/118] clean code --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 12eb36c..e1b489a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,8 +92,7 @@ pipeline { steps { echo 'Cron build enabled. Deploy from system ongoing' script { - println "Going to check the deploy file in ${PENDING_DEPLOY_FOLDER}" - def deployFolder="CD-${env.BUILD_NUMBER}"; + def deployFolder="CD-${env.BUILD_NUMBER}"; println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") if (files == null){