gCubeDeployer/Jenkinsfile

#!groovy
import org.yaml.snakeyaml.Yaml
import groovy.json.JsonSlurper

/**
 * Deploy gCube components on D4Science Infrastructure (dev and preprod environments)
 *
 * Roberto Cirillo (ISTI-CNR)
 */

// related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/

def agent_root_folder = '/var/lib/jenkins'
def verbose = true
def yamlConfig
def text
if ("${params.gCube_release_version}" ||  "${params.RELEASE_URL}"){
	String releaseURL='';
	if ("${params.gCube_release_version}"){
	//locate the release file
		releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleases/raw/branch/master/open/gcube-${gCube_release_version}.yaml"
	}else{
	    releaseURL = "${params.RELEASE_URL}"
	}
//load the release file
	text = releaseURL.toURL().getText()
	//parsing
	yamlConfig = new Yaml().load(text)
	if (verbose)
	    println yamlConfig.inspect()
	echo "Building gCube v. ${yamlConfig.gCube_release.Version}"
	if (verbose) {
	    echo "Found components:"
	    yamlConfig.gCube_release.Components.each { println it.key }
	}
}


def deployList
def backupList

pipeline {
    agent {
        label 'ansible'
    }
    options {
        ansiColor('xterm')
    }

    triggers {
//     every fifteen minutes (perhaps at :07, :22, :37, :52)
//       cron('H/15 * * * *')
//     once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM)
       cron('H H(9-16)/6 * * 1-5')
    }
    
    environment {
        AGENT_ROOT_FOLDER = "${agent_root_folder}"
// folder where all the pending deployment will be added
        PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/"
        ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster"
        PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
// deploy file related to the current pipeline job
        DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv"
        BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck"
 // deploy folder that contains all the pending jobs associated to the current pipeline execution
        DEPLOY_FOLDER= "${WORKSPACE}/CD-${PIPELINE_BUILD_NUMBER}"
  // artifacts coordinates
        TRIGGER_JOB= "${params.TRIGGER_JOB}"
        TRIGGER_VERSION= "${params.TRIGGER_VERSION}"
        TRIGGER_HOST="${params.TRIGGER_HOST}"
  // enable the ContinuousDeployment if TRUE
        TRIGGER_CD="${params.TRIGGER_CD}"
 // Useful to identify the remote YAML file
        GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
 //Category name of the components candidates to the remote deployment
        DEPLOY_CATEGORY="${env.DEPLOY_CATEGORY}"
 //Job name of the release pipeline
        RELEASE_JOB='Pipeline-gCubeBuilder'
 //URL to the YAML file. Used alternatively to RELEASE_JOB
        RELEASE_URL= "${params.RELEASE_URL}"
 // username of ReleaseManager. Used for limiting the preproduction deployments
 		RELEASE_MANAGER_USER='roberto.cirillo'
 // name of Smartgears distribution group, defined in the yaml file ( only for release deployments)
 		DISTRIBUTION_CATEGORY='Distribution'
 // name of Portlets group, defined in the yaml file ( only for release deployments)
 		PORTLET_CATEGORY='Portlets' 		
    }

    parameters {
         string(name: 'TRIGGER_JOB',
                 defaultValue: '',
                 description: 'Name of the service or job to deploy')
         string(name: 'TRIGGER_VERSION',
                 defaultValue: '',
                 description: 'service version to deploy')
         string(name: 'TRIGGER_HOST',
                 defaultValue: '',
                 description: 'Target Host / Host group where deploy the service')
         booleanParam(name: 'TRIGGER_CD',
                 defaultValue: true,
                 description: 'Set to false to avoid current deploy')
        string(name: 'gCube_release_version',
                defaultValue: '',
                description: 'The number of the current gCube release. Leave blank if executed outside gCube release.')
        string(name: 'RELEASE_URL',
                defaultValue: '',
                description: 'URL to the yaml file.  Leave blank if executed outside gCube release.')
    }

    stages {
        stage('Initialize environment') {
            steps {
                sh '''
                  date=`date`;
                  mkdir -p ${PENDING_DEPLOY_FOLDER}
                  mkdir -p "${DEPLOY_FOLDER}"
                  find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${DEPLOY_FOLDER}" '\'{'\'}  '\'+
                '''
            }
        }
        stage('Deploy from system') {
            when{
            	anyOf{
	                allOf{
	                    triggeredBy 'TimerTrigger'
	                    environment name: 'IS_SCHEDULED', value: 'True'
	                }
            	}
            }
            steps {
                echo 'Cron build enabled. \033[31;1;4mDeploy from system ongoing\033[0m'
                script {
                	echo "pipeline was triggered by ${params.TRIGGER_JOB}"
            		println("Going to deploy all the pending deployments")
 					def deployFolder="CD-${env.BUILD_NUMBER}";
 					parseDeployPendingJobs(deployFolder);
                }

            }
        }
        stage('Nothing to do ') {
            when{
                allOf{
                    triggeredBy 'TimerTrigger'
                    environment name: 'IS_SCHEDULED', value: 'False'
                }
            }
            steps {
                 echo '\033[31;1;4mGoing to sleep\033[0m'
                 sh ' exit 0; '
            }
        }
        stage('New pending deploy ') {
            when{
                environment name: 'IS_SCHEDULED', value: 'True'
                anyOf{
                    triggeredBy 'BuildUpstreamCause'
                    triggeredBy 'UpstreamCause'
                    triggeredBy 'UserIdCause'
                }
                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
            }
            steps {
                sh '''
                   echo '\033[31;1;4mNew pending deploy/033[0m'
                   echo "Cron build enabled. Adding ${TRIGGER_JOB} - ${TRIGGER_VERSION} to the queue"
                   touch $DEPLOY_FILE;
                   if grep -q \"\${TRIGGER_JOB}\" \${DEPLOY_FILE}; then
                       echo "component ${TRIGGER_JOB} already added. Nothing to add."                      
                   else
                       echo "${TRIGGER_JOB},${TRIGGER_VERSION},${TRIGGER_HOST}" >> ${DEPLOY_FILE}    
                   fi
                '''
          // the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed.
          // Not sure this is the best approach
                catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
                	sh("""
                		exit 126;
                	""")
                }	
            }
        }
        stage('Deploy from job ') {
            when{
                environment name: 'IS_SCHEDULED', value: 'False';
                expression {
                    env.TRIGGER_CD.toBoolean()
            	}
                anyOf{
                    triggeredBy 'BuildUpstreamCause'
                    triggeredBy 'UpstreamCause'
                }
                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
            }
            steps {
            	script{
            	     echo '\033[31;1;4mNew upstream deploy ongoing\033[0m'
	                 echo "Cron build disabled. Component: ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION}"
	                 ansiColor('xterm') {
	                 	 catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
	                         checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
	                         deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
	                     }
	                 }
            	}


            }
        }
        stage('Deploy from gCubeRelease ') {
            when{
                anyOf{
                    triggeredBy 'BuildUpstreamCause'
                    triggeredBy 'UpstreamCause'
        // added just for testing . To Remove.
                    triggeredBy  cause: "UserIdCause", detail: "${RELEASE_MANAGER_USER}"
                }
   		// we should improve this conditional check
                equals(actual: "${params.TRIGGER_JOB}", expected: "${RELEASE_JOB}")
            }
            steps {
            	script{
            		if("${DEPLOY_CATEGORY}"){
            			echo '\033[31;1;4mGoing to deploy gCube Release \033[0m'
                		deployRelease("${DEPLOY_CATEGORY}", yamlConfig);
            		}else println ("CATEGORIES NOT FOUND. ENVIRONMENT NOT SET FOR RELEASE DEPLOYMENT. NOTHING TO DO");
            	}
             }
        }
        stage('Deploy manually ') {
            when{
                allOf{
                    environment name: 'IS_SCHEDULED', value: 'False';
                    triggeredBy 'UserIdCause'
                }
                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
            }
            steps {
                 echo '\033[31;1;4mNew deploy ongoing manually triggered\033[0m'
                 echo "Cron build disabled. Pipeline executed Manually. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
                 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                     deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
                 }
            }
        }
    }
    post {
        always {
            script {
                sh '''
                  echo 'Sending report'
                  '''
            }
        }
        success {
            echo 'The deploy pipeline worked!'

            emailext attachLog: true,//attachmentsPattern: "**/${ACTION_DEPLOY_FILE}.csv",
                    to: 'roberto.cirillo@isti.cnr.it',
                    subject: "[Jenkins-gCubeDeployer] Deploy report",
                    body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}. "
        }
        failure {
            echo 'The deploy pipeline has failed'
            emailext attachLog: true,
                    to: 'roberto.cirillo@isti.cnr.it',
                    subject: "[Jenkins-gCubeDeployer] deploy ${currentBuild.fullDisplayName} failed",
                    body: "Something is wrong with ${env.BUILD_URL}"
        }

    }
}


/* CHECKUP METHODS */

//Implement a new method in order to check the input parameters
def checkup(String service, String version, String host){
    sh("""
        case "$version" in
        **SNAPSHOT) echo "version contains SNAPSHOT" ;;
        **latest) echo "version contains latest" ;;
        *		  ) echo "version without SNAPSHOT. EXIT WITHOUT DEPLOY "; exit 1;;
        esac
    """)
}

//Implement a new method in order to check the input parameters
def checkupRelease(String service, String version){
    sh("""
        case "$version" in
        **SNAPSHOT) echo "version contains SNAPSHOT. EXIT because this is a release build" ; exit 1;;
        **latest) echo "version contains latest" ;;
        *		  ) echo "version without SNAPSHOT. Go ahead";;
        esac
    """)
}


/* DEPLOY METHODS*/

//parse all csv files found in the local folder and deploy the components defined inside
def parseDeployPendingJobs( def deployFolder){
	println ("searching files in folder ${deployFolder}");
	def files = findFiles(glob: "${deployFolder}/*.csv")
	if (files == null){
		println ("Nothing to do");
	}else{
		def serviceList = []
		for (def file : files){
		    def records = readCSV file: "${file.path}"
            for (def record : records) {
                println("Processing record: "+record)
                serviceList += "${record.get(0)},${record.get(1)},${record.get(2)}";
            }
            sh "rm ${file.path}"
        }
        sh "rm -Rf ${deployFolder}"
        deployJobs(serviceList.unique())
	}
}

//Deploy jobs from an input list with the following elements: serviceName,serviceVersion,targetHost. Create a new stage for any job
def deployJobs(def serviceList){
   for (def record : serviceList) {
        service=record.split(",");
		stage(service[0]){
			println("Processing deploy: "+service[ 0 ])
            catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
                checkup(service[0], service[ 1 ], service[2]);
                deploy(service[0], service[ 1 ], service[2]);
            }
        }
   }

}

//Run ansible deploy
def deploy(String service, String version, String host){
    def now = new Date();
    println("Going to deploy the service "+service+" with version: "+version+" on target: "+host);
    def statusCode = sh( script: "cd ${ANSIBLE_ROOT_FOLDER}/CD;./deployService.sh $service $version $host;", returnStdout: true);
    sh("""
        echo " last exit code \$?";
    """)
    println("Deploy ended with status: "+statusCode);
}


/* DEPLOY RELEASE METHODS*/

// collects all the components defined inside one or more yaml categories matching categoryList and invoke the deploy
def deployRelease(def categoryList, def yaml){
	def isPortlet= false
	def releaseList = []
	def portlets = []
	String smartgears
	def categories=categoryList.split(',')
	yaml.gCube_release.Components.each { group_name, component_list ->
		if (categories.contains(group_name)){
			for ( def category : categories){
				if("${group_name}" == "$category"){
					if (category == "${DISTRIBUTION_CATEGORY}"){
						//searching smartgears-distribution (os smartgears-distribution-legacy) version in Distribution component defined inside the yaml file
						def sg= component_list?.collect{ if ("$it.name" == 'smartgears-distribution' || "$it.name" == 'smartgears-distribution-legacy' ){ return "$it.version"; }}
						smartgears=(sg !=null ) ? sg[0] : ''
					}else if (category =="${PORTLET_CATEGORY}"){
						portlets += component_list?.collect{ if ("${it.group_id}" != "null") return  "{\"group_id\": \"$it.group_id\", \"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"} "; else return  "{\"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"}"}
						def jsonContent= "{\"generic_portlets\": "+portlets+"}"
						echo "json: $jsonContent"
						writeJSON file: 'portlets.json', json: jsonContent
						def output = sh(returnStdout: true, returnStdoutTrim: true, script: "mv portlets.json $ANSIBLE_ROOT_FOLDER")
						isPortlet= true
					}else{
						releaseList += component_list?.collect {return "$it.name,$it.version" }
					}
					return;
				}
			}
		}
	}
    deployReleaseJobs(releaseList, smartgears, isPortlet);
}

//Deploy only release components, starting from a servicelist. All the component are passed as: componentName,componentVersion
def deployReleaseJobs(def serviceList, String smartgears, boolean isPortlet){
   if (serviceList || isPortlet){
      println("Going to deploy release components");	
	   for (def record : serviceList) {
	        service=record.split(",");
			stage(service[0]){
				println("Processing RELEASE deploy: service "+service[ 0 ]+" with version "+service[ 1 ]+" and SG "+smartgears)
	            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
	                checkupRelease(service[0], service[ 1 ]);
	                if (smartgears)
	                	deployReleaseJob(service[0], service[ 1 ], smartgears);
	                else
	                	deployReleaseJob(service[0], service[ 1 ], '');
	            }
	        }
	   }
	   if (isPortlet){
	       deployReleaseJob('generic-portlet', '','');
	   }
	   
   }else{
       println("Release components not found. Skip this stage");
       
   }

}

//Run ansible deploy related to a single service
def deployReleaseJob(String service, String version, String smartgears){
    def now = new Date();
    println("Going to deploy the service "+service+" with version: "+version+" on preproduction ");
    def statusCode = sh( script: "cd ${ANSIBLE_ROOT_FOLDER}/CD;git pull;./deployPreprodService.sh $service $version $smartgears ;", returnStdout: true);
    sh("""
        echo " last exit code \$?";
    """)
    println("Deploy ended with status: "+statusCode);
}