#!groovy import org.yaml.snakeyaml.Yaml /** * Deploy components to the D4Science Infrastructure (dev-only) * * Roberto Cirillo (ISTI-CNR) */ // related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/ def agent_root_folder = '/var/lib/jenkins' def verbose = true def jsonConfig def text if ("${params.gCube_release_version}" || "${params.RELEASE_URL}"){ String releaseURL=''; if ("${params.gCube_release_version}"){ //locate the release file releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleases/raw/branch/master/open/gcube-${gCube_release_version}.yaml" }else{ releaseURL = "${params.RELEASE_URL}" } //load the release file text = releaseURL.toURL().getText() //parsing jsonConfig = new Yaml().load(text) if (verbose) println jsonConfig.inspect() echo "Building gCube v. ${jsonConfig.gCube_release.Version}" if (verbose) { echo "Found components:" jsonConfig.gCube_release.Components.each { println it.key } } } def deployList def backupList pipeline { agent { label 'ansible' } options { ansiColor('xterm') } triggers { // every fifteen minutes (perhaps at :07, :22, :37, :52) // cron('H/15 * * * *') // once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM) cron('H H(9-16)/6 * * 1-5') } environment { AGENT_ROOT_FOLDER = "${agent_root_folder}" // folder where all the pending deployment will be added PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/" ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" // deploy file related to the current pipeline job DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv" BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck" // deploy folder that contains all the pending jobs associated to the current pipeline execution DEPLOY_FOLDER= "${WORKSPACE}/CD-${PIPELINE_BUILD_NUMBER}" // artifacts coordinates TRIGGER_JOB= "${params.TRIGGER_JOB}" TRIGGER_VERSION= "${params.TRIGGER_VERSION}" TRIGGER_HOST="${params.TRIGGER_HOST}" // enable the ContinuousDeployment if TRUE TRIGGER_CD="${params.TRIGGER_CD}" // Useful to identify the remote YAML file GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}" //Category name of the components candidates to the remote deployment DEPLOY_CATEGORY="${env.DEPLOY_CATEGORY}" //Job name of the release pipeline RELEASE_JOB='gCubeBuilder' //URL to the YAML file. Used alternatively to RELEASE_JOB RELEASE_URL= "${params.RELEASE_URL}" } parameters { string(name: 'TRIGGER_JOB', defaultValue: '', description: 'Name of the service or job to deploy') string(name: 'TRIGGER_VERSION', defaultValue: '', description: 'service version to deploy') string(name: 'TRIGGER_HOST', defaultValue: '', description: 'Target Host / Host group where deploy the service') booleanParam(name: 'TRIGGER_CD', defaultValue: true, description: 'Set to false to avoid current deploy') string(name: 'gCube_release_version', defaultValue: '', description: 'The number of the gCube release. Leave blank if executed outside gCube release.') string(name: 'report_number', defaultValue: '', description: 'The build report number on Git to pull the notes. Leave blank if executed outside gCube release.') string(name: 'RELEASE_URL', defaultValue: '', description: 'URL to the yaml file. Leave blank if executed outside gCube release.') // string(name: 'report', // defaultValue: '', // description: 'The build report. Leave blank to pull the build report from Git. Leave blank if executed outside gCube release.') } stages { stage('Initialize environment') { steps { sh ''' date=`date`; mkdir -p ${PENDING_DEPLOY_FOLDER} mkdir -p "${DEPLOY_FOLDER}" find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${DEPLOY_FOLDER}" '\'{'\'} '\'+ ''' } } stage('Deploy from system') { when{ anyOf{ allOf{ triggeredBy 'TimerTrigger' environment name: 'IS_SCHEDULED_TEST', value: 'True' } } } steps { echo 'Cron build enabled. Deploy from system ongoing' script { echo "pipeline was triggered by ${params.TRIGGER_JOB}" println("Going to deploy all the pending deployments") def deployFolder="CD-${env.BUILD_NUMBER}"; parseDeployPendingJobs(deployFolder); } } } stage('Nothing to do ') { when{ anyOf{ allOf{ triggeredBy 'TimerTrigger' environment name: 'IS_SCHEDULED_TEST', value: 'False' } } } steps { echo 'Going to sleep' sh ' exit 1; ' } } stage('New pending deploy ') { when{ environment name: 'IS_SCHEDULED_TEST', value: 'True' anyOf{ triggeredBy 'BuildUpstreamCause' triggeredBy 'UpstreamCause' triggeredBy 'UserIdCause' } } steps { sh ''' if [ "${TRIGGER_JOB}" != 'gCubeBuilder' ]; then echo "Cron build enabled. New deploy of ${TRIGGER_JOB} - ${TRIGGER_VERSION} will be added to the pending deploy file" touch $DEPLOY_FILE; if grep -q \"\${TRIGGER_JOB}\" \${DEPLOY_FILE}; then echo "component ${TRIGGER_JOB} already added. Nothing to add." else echo "${TRIGGER_JOB},${TRIGGER_VERSION},${TRIGGER_HOST}" >> ${DEPLOY_FILE} fi fi ''' // the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { sh(""" exit 126; """) } } } stage('Deploy from job ') { when{ environment name: 'IS_SCHEDULED_TEST', value: 'False'; expression { env.TRIGGER_CD.toBoolean() } anyOf{ triggeredBy 'BuildUpstreamCause' triggeredBy 'UpstreamCause' } } steps { script{ if("${TRIGGER_JOB}" != 'gCubeBuilder'){ echo "Cron build disabled. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing" ansiColor("xterm") { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}"); deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}"); } } } } } } stage('Deploy from gCubeRelease ') { when{ anyOf{ triggeredBy 'BuildUpstreamCause' triggeredBy 'UpstreamCause' //added just for testing . To Remove. triggeredBy 'UserIdCause' } // we should improve this conditional check equals(actual: "${params.TRIGGER_JOB}", expected: 'gCubeBuilder') } steps { script{ println("Going to deploy the Release components") def releaseList = [] jsonConfig.gCube_release.Components.each { group_name, component_list -> if("${group_name}" == "${DEPLOY_CATEGORY}"){ // serviceName=component_list?.collect { "${it.name}" } // serviceVersion=component_list?.collect { "${it.version}" } releaseList=component_list.each{ list -> return list +="it.name,it.version" } // println("found ${serviceName} with version: ${serviceVersion}"); println("found ${releaseList}"); releaseList += "${serviceName},${serviceVersion}," } } deployJobs(releaseList); } } } stage('Deploy manually ') { when{ allOf{ environment name: 'IS_SCHEDULED_TEST', value: 'False'; triggeredBy 'UserIdCause' } } steps { echo "Cron build disabled. Pipeline executed Manually. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing" ansiColor("xterm") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}"); } } } } } post { always { script { sh ''' echo 'Sending report' ''' //cat ./${ACTION_DEPLOY_FILE}.csv } } success { echo 'The deploy pipeline worked!' emailext attachLog: true,//attachmentsPattern: "**/${ACTION_DEPLOY_FILE}.csv", to: 'roberto.cirillo@isti.cnr.it', subject: "[Jenkins-gCubeDeployer] Deploy report", body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}. " } failure { echo 'The deploy pipeline has failed' emailext attachLog: true, to: 'roberto.cirillo@isti.cnr.it', subject: "[Jenkins-gCubeDeployer] deploy ${currentBuild.fullDisplayName} failed", body: "Something is wrong with ${env.BUILD_URL}" } } } //launch ansible deploy def deploy(String service, String version, String host){ def now = new Date(); println("Going to deploy the service "+service+" with version: "+version+" on target: "+host); def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;./deployService.sh $service $version $host;", returnStdout: true); sh(""" echo " last exit code \$?"; """) println("Deploy ended with status: "+statusCode); } //Implement a new method in order to check the input parameters def checkup(String service, String version, String host){ sh(""" case "$version" in **SNAPSHOT) echo "version contains SNAPSHOT" ;; **latest) echo "version contains latest" ;; * ) echo "version without SNAPSHOT. EXIT WITHOUT DEPLOY "; exit 1;; esac """) } //launch ansible deploy def deployRelease(String service, String version){ def now = new Date(); println("Going to deploy the service "+service+" with version: "+version+" on preproduction "); def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;./deployPreprodService.sh $service $version ;", returnStdout: true); sh(""" echo " last exit code \$?"; """) println("Deploy ended with status: "+statusCode); } //Implement a new method in order to check the input parameters def checkupRelease(String service, String version){ sh(""" case "$version" in **SNAPSHOT) echo "version contains SNAPSHOT" ; exit 1;; **latest) echo "version contains latest" ;; * ) echo "version without SNAPSHOT. go ahead;; esac """) } //parse all csv files found in the local folder and deploy the components defined inside def deployPendingJobs( def deployFolder){ println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") if (files == null){ println ("Nothing to do"); }else{ def serviceList = [] for (def file : files){ def records = readCSV file: "${file.path}" for (def record : records) { println("Processing record: "+record) if(!serviceList.contains(record.get(0))){ stage(record.get(0)){ println "Deploy on going of component: ${record.get(0)}" catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { checkup(record.get(0), record.get(1), record.get(2)); deploy(record.get(0), record.get(1), record.get(2)); } } serviceList << record.get(0) }else{ echo "${record.get(0)} already deployed. Deployment skipped." } } sh "rm ${file.path}" } sh "rm -Rf ${deployFolder}" } } //parse all csv files found in the local folder and deploy the components defined inside def parseDeployPendingJobs( def deployFolder){ println ("searching files in folder ${deployFolder}"); def files = findFiles(glob: "${deployFolder}/*.csv") if (files == null){ println ("Nothing to do"); }else{ def serviceList = [] for (def file : files){ def records = readCSV file: "${file.path}" for (def record : records) { println("Processing record: "+record) serviceList += "${record.get(0)},${record.get(1)},${record.get(2)}"; } sh "rm ${file.path}" } sh "rm -Rf ${deployFolder}" deployJobs(serviceList.unique()) } } //Deploy jobs from an input list with the following elements: serviceName,serviceVersion,targetHost. def deployJobs(def serviceList){ for (def record : serviceList) { println("Processing record: "+record) service=record.split(","); //check on target host if(service[ 2 ]){ stage(service[0]){ println("Processing deploy: "+service[ 0 ]) catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { checkup(service[0], service[ 1 ], service[2]); deploy(service[0], service[ 1 ], service[2]); } } }else{ //this is a release build stage(service[0]){ println("Processing RELEASE deploy: "+service[ 0 ]) catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { checkupRelease(service[0], service[ 1 ], service[2]); deployRelease(service[0], service[ 1 ], service[2]); } } } } } def parseBuildComponents( def text){ def components = [] "${text}".splitEachLine(',') { columns -> if (columns[0].startsWith('#') || columns[0].startsWith('GroupID')) return components.add([ name : columns[1], version : columns[2], gitRepo : columns[3], commitID: columns[4] ] ) } return components }