gCubeDeployer/Jenkinsfile

437 lines
16 KiB
Groovy

#!groovy
import org.yaml.snakeyaml.Yaml
/**
* Deploy components to the D4Science Infrastructure (dev-only)
*
* Roberto Cirillo (ISTI-CNR)
*/
// related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/
def agent_root_folder = '/var/lib/jenkins'
def verbose = true
def jsonConfig
def text
if ("${params.gCube_release_version}" || "${params.RELEASE_URL}"){
String releaseURL='';
if ("${params.gCube_release_version}"){
//locate the release file
releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleases/raw/branch/master/open/gcube-${gCube_release_version}.yaml"
}else{
releaseURL = "${params.RELEASE_URL}"
}
//load the release file
text = releaseURL.toURL().getText()
//parsing
jsonConfig = new Yaml().load(text)
if (verbose)
println jsonConfig.inspect()
echo "Building gCube v. ${jsonConfig.gCube_release.Version}"
if (verbose) {
echo "Found components:"
jsonConfig.gCube_release.Components.each { println it.key }
}
}
def deployList
def backupList
pipeline {
agent {
label 'ansible'
}
options {
ansiColor('xterm')
}
triggers {
// every fifteen minutes (perhaps at :07, :22, :37, :52)
// cron('H/15 * * * *')
// once in every two hours slot between 9 AM and 5 PM every weekday (perhaps at 10:38 AM, 12:38 PM, 2:38 PM, 4:38 PM)
cron('H H(9-16)/6 * * 1-5')
}
environment {
AGENT_ROOT_FOLDER = "${agent_root_folder}"
// folder where all the pending deployment will be added
PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/"
ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD"
PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
// deploy file related to the current pipeline job
DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv"
BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck"
// deploy folder that contains all the pending jobs associated to the current pipeline execution
DEPLOY_FOLDER= "${WORKSPACE}/CD-${PIPELINE_BUILD_NUMBER}"
// artifacts coordinates
TRIGGER_JOB= "${params.TRIGGER_JOB}"
TRIGGER_VERSION= "${params.TRIGGER_VERSION}"
TRIGGER_HOST="${params.TRIGGER_HOST}"
// enable the ContinuousDeployment if TRUE
TRIGGER_CD="${params.TRIGGER_CD}"
// Useful to identify the remote YAML file
GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
//Category name of the components candidates to the remote deployment
DEPLOY_CATEGORY="${env.DEPLOY_CATEGORY}"
//Job name of the release pipeline
RELEASE_JOB='gCubeBuilder'
//URL to the YAML file. Used alternatively to RELEASE_JOB
RELEASE_URL= "${params.RELEASE_URL}"
}
parameters {
string(name: 'TRIGGER_JOB',
defaultValue: '',
description: 'Name of the service or job to deploy')
string(name: 'TRIGGER_VERSION',
defaultValue: '',
description: 'service version to deploy')
string(name: 'TRIGGER_HOST',
defaultValue: '',
description: 'Target Host / Host group where deploy the service')
booleanParam(name: 'TRIGGER_CD',
defaultValue: true,
description: 'Set to false to avoid current deploy')
string(name: 'gCube_release_version',
defaultValue: '',
description: 'The number of the gCube release. Leave blank if executed outside gCube release.')
string(name: 'report_number',
defaultValue: '',
description: 'The build report number on Git to pull the notes. Leave blank if executed outside gCube release.')
string(name: 'RELEASE_URL',
defaultValue: '',
description: 'URL to the yaml file. Leave blank if executed outside gCube release.')
// string(name: 'report',
// defaultValue: '',
// description: 'The build report. Leave blank to pull the build report from Git. Leave blank if executed outside gCube release.')
}
stages {
stage('Initialize environment') {
steps {
sh '''
date=`date`;
mkdir -p ${PENDING_DEPLOY_FOLDER}
mkdir -p "${DEPLOY_FOLDER}"
find "${PENDING_DEPLOY_FOLDER}" -type f -exec mv --target-directory="${DEPLOY_FOLDER}" '\'{'\'} '\'+
'''
}
}
stage('Deploy from system') {
when{
anyOf{
allOf{
triggeredBy 'TimerTrigger'
environment name: 'IS_SCHEDULED_TEST', value: 'True'
}
}
}
steps {
echo 'Cron build enabled. Deploy from system ongoing'
script {
echo "pipeline was triggered by ${params.TRIGGER_JOB}"
println("Going to deploy all the pending deployments")
def deployFolder="CD-${env.BUILD_NUMBER}";
parseDeployPendingJobs(deployFolder);
}
}
}
stage('Nothing to do ') {
when{
anyOf{
allOf{
triggeredBy 'TimerTrigger'
environment name: 'IS_SCHEDULED_TEST', value: 'False'
}
}
}
steps {
echo 'Going to sleep'
sh ' exit 1; '
}
}
stage('New pending deploy ') {
when{
environment name: 'IS_SCHEDULED_TEST', value: 'True'
anyOf{
triggeredBy 'BuildUpstreamCause'
triggeredBy 'UpstreamCause'
triggeredBy 'UserIdCause'
}
}
steps {
sh '''
if [ "${TRIGGER_JOB}" != 'gCubeBuilder' ]; then
echo "Cron build enabled. New deploy of ${TRIGGER_JOB} - ${TRIGGER_VERSION} will be added to the pending deploy file"
touch $DEPLOY_FILE;
if grep -q \"\${TRIGGER_JOB}\" \${DEPLOY_FILE}; then
echo "component ${TRIGGER_JOB} already added. Nothing to add."
else
echo "${TRIGGER_JOB},${TRIGGER_VERSION},${TRIGGER_HOST}" >> ${DEPLOY_FILE}
fi
fi
'''
// the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed
catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
sh("""
exit 126;
""")
}
}
}
stage('Deploy from job ') {
when{
environment name: 'IS_SCHEDULED_TEST', value: 'False';
expression {
env.TRIGGER_CD.toBoolean()
}
anyOf{
triggeredBy 'BuildUpstreamCause'
triggeredBy 'UpstreamCause'
}
}
steps {
script{
if("${TRIGGER_JOB}" != 'gCubeBuilder'){
echo "Cron build disabled. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
ansiColor("xterm") {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
}
}
}
}
}
}
stage('Deploy from gCubeRelease ') {
when{
anyOf{
triggeredBy 'BuildUpstreamCause'
triggeredBy 'UpstreamCause'
//added just for testing . To Remove.
triggeredBy 'UserIdCause'
}
// we should improve this conditional check
equals(actual: "${params.TRIGGER_JOB}", expected: 'gCubeBuilder')
}
steps {
script{
println("Going to deploy the Release components")
def releaseList = []
jsonConfig.gCube_release.Components.each { group_name, component_list ->
if("${group_name}" == "${DEPLOY_CATEGORY}"){
releaseList += component_list?.collect {return "$it.name,$it.version" }
println("found ${releaseList}");
}
}
deployReleaseJobs(releaseList);
}
}
}
stage('Deploy manually ') {
when{
allOf{
environment name: 'IS_SCHEDULED_TEST', value: 'False';
triggeredBy 'UserIdCause'
}
}
steps {
echo "Cron build disabled. Pipeline executed Manually. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
ansiColor("xterm") {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
}
}
}
}
}
post {
always {
script {
sh '''
echo 'Sending report'
'''
//cat ./${ACTION_DEPLOY_FILE}.csv
}
}
success {
echo 'The deploy pipeline worked!'
emailext attachLog: true,//attachmentsPattern: "**/${ACTION_DEPLOY_FILE}.csv",
to: 'roberto.cirillo@isti.cnr.it',
subject: "[Jenkins-gCubeDeployer] Deploy report",
body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}. "
}
failure {
echo 'The deploy pipeline has failed'
emailext attachLog: true,
to: 'roberto.cirillo@isti.cnr.it',
subject: "[Jenkins-gCubeDeployer] deploy ${currentBuild.fullDisplayName} failed",
body: "Something is wrong with ${env.BUILD_URL}"
}
}
}
//launch ansible deploy
def deploy(String service, String version, String host){
def now = new Date();
println("Going to deploy the service "+service+" with version: "+version+" on target: "+host);
def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;./deployService.sh $service $version $host;", returnStdout: true);
sh("""
echo " last exit code \$?";
""")
println("Deploy ended with status: "+statusCode);
}
//Implement a new method in order to check the input parameters
def checkup(String service, String version, String host){
sh("""
case "$version" in
**SNAPSHOT) echo "version contains SNAPSHOT" ;;
**latest) echo "version contains latest" ;;
* ) echo "version without SNAPSHOT. EXIT WITHOUT DEPLOY "; exit 1;;
esac
""")
}
//launch ansible deploy
def deployRelease(String service, String version){
def now = new Date();
println("Going to deploy the service "+service+" with version: "+version+" on preproduction ");
def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;./deployPreprodService.sh $service $version ;", returnStdout: true);
sh("""
echo " last exit code \$?";
""")
println("Deploy ended with status: "+statusCode);
}
//Implement a new method in order to check the input parameters
def checkupRelease(String service, String version){
sh("""
case "$version" in
**SNAPSHOT) echo "version contains SNAPSHOT. EXIT because this is a release build" ; exit 1;;
**latest) echo "version contains latest" ;;
* ) echo "version without SNAPSHOT. go;;
esac
""")
}
//parse all csv files found in the local folder and deploy the components defined inside
def deployPendingJobs( def deployFolder){
println ("searching files in folder ${deployFolder}");
def files = findFiles(glob: "${deployFolder}/*.csv")
if (files == null){
println ("Nothing to do");
}else{
def serviceList = []
for (def file : files){
def records = readCSV file: "${file.path}"
for (def record : records) {
println("Processing record: "+record)
if(!serviceList.contains(record.get(0))){
stage(record.get(0)){
println "Deploy on going of component: ${record.get(0)}"
catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
checkup(record.get(0), record.get(1), record.get(2));
deploy(record.get(0), record.get(1), record.get(2));
}
}
serviceList << record.get(0)
}else{
echo "${record.get(0)} already deployed. Deployment skipped."
}
}
sh "rm ${file.path}"
}
sh "rm -Rf ${deployFolder}"
}
}
//parse all csv files found in the local folder and deploy the components defined inside
def parseDeployPendingJobs( def deployFolder){
println ("searching files in folder ${deployFolder}");
def files = findFiles(glob: "${deployFolder}/*.csv")
if (files == null){
println ("Nothing to do");
}else{
def serviceList = []
for (def file : files){
def records = readCSV file: "${file.path}"
for (def record : records) {
println("Processing record: "+record)
serviceList += "${record.get(0)},${record.get(1)},${record.get(2)}";
}
sh "rm ${file.path}"
}
sh "rm -Rf ${deployFolder}"
deployJobs(serviceList.unique())
}
}
//Deploy jobs from an input list with the following elements: serviceName,serviceVersion,targetHost.
def deployJobs(def serviceList){
for (def record : serviceList) {
println("Processing record: "+record)
service=record.split(",");
stage(service[0]){
println("Processing deploy: "+service[ 0 ])
catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
checkup(service[0], service[ 1 ], service[2]);
deploy(service[0], service[ 1 ], service[2]);
}
}
}
}
def deployReleaseJobs(def serviceList){
for (def record : serviceList) {
println("Processing record: "+record)
service=record.split(",");
stage(service[0]){
println("Processing RELEASE deploy: "+service[ 0 ])
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
checkupRelease(service[0], service[ 1 ]);
deployRelease(service[0], service[ 1 ]);
}
}
}
}
def parseBuildComponents( def text){
def components = []
"${text}".splitEachLine(',') { columns ->
if (columns[0].startsWith('#') || columns[0].startsWith('GroupID'))
return
components.add([
name : columns[1],
version : columns[2],
gitRepo : columns[3],
commitID: columns[4]
]
)
}
return components
}