gCubeBuilder/Jenkinsfile

#!groovy​
import org.yaml.snakeyaml.Yaml

// set the build options according to the Type of build
def (options, maven_local_repo_path, maven_settings_file, maven_parent_file) = ['', '', '', '']
def agent_root_folder = '/var/lib/jenkins/.m2'
def verbose = true
def resume = params.resume_from
if (params.Type == 'SNAPSHOT-DRY-RUN') {
    echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts"
    options = ''
    maven_local_repo_path = "local-snapshots"
    maven_settings_file = "jenkins-snapshots-dry-run-settings.xml"
    maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
}
if (params.Type == 'SNAPSHOT') {
    echo "Configure Maven for SNAPSHOT artifacts"
    options = ''
    maven_local_repo_path = "local-snapshots"
    maven_settings_file = "jenkins-snapshots-settings.xml"
    maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
}
if (params.Type == 'RELEASE-DRY-RUN') {
    echo "Configure Maven for RELEASE-DRY-RUN artifacts"
    options = ''
    maven_local_repo_path = "local-releases"
    maven_settings_file = "jenkins-releases-dry-run-settings.xml"
    maven_parent_file = "jenkins-releases-dry-run-settings.xml"
}
if (params.Type == 'STAGING') {
    echo "Configure Maven for STAGING artifacts"
    options = ''
    maven_local_repo_path = "local-staging"
    maven_settings_file = "jenkins-staging-settings.xml"
    maven_parent_file = "jenkins-staging-dry-run-settings.xml"
}
if (params.Type == 'RELEASE') {
    echo "Configure Maven for RELEASE artifacts"
    options = ''
    maven_local_repo_path = "local-releases"
    maven_settings_file = "jenkins-releases-settings.xml"
    maven_parent_file = "jenkins-releases-dry-run-settings.xml"
}

echo "Use settings file at ${maven_settings_file}"
echo "Use local repo at ${maven_local_repo_path}"
echo "Release number: ${params.gCube_release_version}"
echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}"
echo "Clean up all local artifacts? ${params.cleanup_local_repo}"
echo "Resume from previous build? ${params.resume_from}"


//locate the release file
String releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeRelease/raw/branch/master/releases/gcube-${gCube_release_version}.yaml"

if (verbose)
    println "Querying ${releaseURL}"

//load the release file
def text = releaseURL.toURL().getText()

//parsing
def jsonConfig = new Yaml().load(text)
if (verbose)
    println jsonConfig.inspect()
assert jsonConfig.gCube_release.Version == params.gCube_release_version: "Release versions do not match!"
echo "Building gCube v. ${jsonConfig.gCube_release.Version}"
if (verbose) {
    echo "Found components:"
    jsonConfig.gCube_release.Components.each { println it.key }
}

def jobs = [:]
if (resume_from) {
    def previous_report_file = "${agent_root_folder}/build_jobs.${resume_from}.csv"
    echo "Previous report file: ${previous_report_file}"
    jobs = parseJobs(previous_report_file)
    if (verbose) {
        for (job in jobs)
            println job
    }
}


pipeline {

    // see https://jenkins.io/doc/book/pipeline/syntax/#agent
    agent {
        label 'CD'
    }

    // see https://jenkins.io/doc/book/pipeline/syntax/#environment
    environment {
        //make the JVM start a bit faster with basic just-in-time compilation of the code only (-XX:*)
        //make maven running in a multi-thread fashion (16 threads, 2 threads on each Core)
        MAVEN_OPTS = "${params.build_options}"
        AGENT_ROOT_FOLDER = "${agent_root_folder}"
        MAVEN_SETTINGS_FILE = "${maven_settings_file}"
        MAVEN_PARENT_FILE = "${maven_parent_file}"
        MAVEN_LOCAL_REPO = "${agent_root_folder}/${maven_local_repo_path}"
        CLEANUP_GCUBE_REPO = "${params.cleanup_gcube_artifacts}"
        REMOVE_LOCAL_REPO = "${params.cleanup_local_repo}"
        GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
        PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
        TYPE = "${params.Type}"
        RESUME = "${params.resume}"
        JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
        PREVIOUS_JOB_REPORT = "${previous_report_file}"
    }

    // see https://jenkins.io/doc/book/pipeline/syntax/#parameters
    parameters {
        choice(name: 'Type',
                choices: ['SNAPSHOT-DRY-RUN', 'SNAPSHOT', 'STAGING', 'RELEASE-DRY-RUN', 'RELEASE'],
                description: 'The type of artifacts the build is expected to generate')

        string(name: 'gCube_release_version',
                defaultValue: 'x.y.z',
                description: 'The number of the gCube release to build. Sample values: 4.14, 4.15, etc.')

        booleanParam(name: 'cleanup_gcube_artifacts',
                defaultValue: true,
                description: 'Wipe out the gcube artifacts from the local maven repository before the builds?')

        booleanParam(name: 'cleanup_local_repo',
                defaultValue: true,
                description: 'Wipe out the local maven repository before the builds?')

         string(name: 'resume_from',
                defaultValue: '',
                description: 'Resume from a previous build identified by the build number.')
    }

    //see https://jenkins.io/doc/book/pipeline/syntax/#stages
    stages {
        stage('initialize') {
            steps {
                sh '''
                        echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}"
                        echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}"
                        if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then
                            echo "Remove gCube artifacts from local repository"
                            rm -rf $MAVEN_LOCAL_REPO/org/gcube
                        fi
                        if [ "$REMOVE_LOCAL_REPO" = "true" ]; then
                            echo "Create a fresh local repository"
                            rm -rf $MAVEN_LOCAL_REPO
                            mkdir -p $MAVEN_LOCAL_REPO
                        fi
                        mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}"
                        cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml"
                        echo "Done with local repository and settings"

                        #build report
                        echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${AGENT_ROOT_FOLDER}/build_commits.csv
                        echo "#Release ${GCUBE_RELEASE_NUMBER}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
                        date=`date`
                        echo "#StartTime ${date}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
                        echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv

                        #job report
                        cat ${PREVIOUS_JOB_REPORT}
                        echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT}
                        echo "#StartTime ${date}" >> ${JOB_REPORT}
                        echo -e "JobName,Status" >> ${JOB_REPORT}
                '''
            }
        }
        // the maven-parent needs to be built (once) at each execution
        stage('build maven-parent') {
            steps {
                script {
                        if ( ("${resume}" == 'true') && (jobs['maven-parent'] == 'SUCCESS') ) {
                            echo "Skipping maven-parent"
                            // propagate success
                            sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                        } else {
                            def gjob =  build(job: 'maven-parent', wait: true, propagate: true,
                                    parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"],
                                                 [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
                                                 [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
                                    ])
                            echo "Job report file: ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                            sh "echo -e maven-parent,${gjob.getResult()}  >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                            echo "Done with maven-parent"
                        }

                }
            }
        }
        stage('build components') {
            steps {
                script {
                    jsonConfig.gCube_release.Components.each { group_name, component_list ->
                        stage("build ${group_name} components") {
                            buildComponents items: component_list?.collect { "${it.name}" },
                                    "${maven_settings_file}", "${maven_local_repo_path}", jobs
                            echo "Done with ${group_name} components"
                        }
                    }
                }
            }
        }
    }

    // post-build actions
    post {
        always {
            script {
                sh '''
                  mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml"
                  mv  ${AGENT_ROOT_FOLDER}/build_commits.csv  ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv
                  cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv .
                  cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv
                  '''
            }
            echo 'The default maven settings have been restored'

        }
        success {
            echo 'The pipeline worked!'
            emailext to: 'jenkinsbuilds@d4science.org',
                    subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} worked",
                    body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
            emailext attachmentsPattern: "**/*.${PIPELINE_BUILD_NUMBER}.csv",
                    to: 'jenkinsreleases@d4science.org',
                    subject: "${TYPE} report for release ${GCUBE_RELEASE_NUMBER} (build #${PIPELINE_BUILD_NUMBER})",
                    body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
        }
        failure {
            echo 'The pipeline has failed'
            emailext attachLog: true,
                    to: 'jenkinsbuilds@d4science.org',
                    subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed",
                    body: "Something is wrong with ${env.BUILD_URL}"
        }

    }
}

def buildComponents(args, maven_settings_file, maven_local_repo_path,  jobs) {
    if (args.items) {
        parallel args.items?.collectEntries { name ->
            ["${name}": {
                if (name && !"NONE".equalsIgnoreCase(name)) {
                    if ( ("${resume}" == 'true') && (jobs["${name}"] == 'SUCCESS') ) {
                            echo "Skipping ${name}"
                            // propagate success
                            sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                    } else {
                        def gjob = build(job: name, wait: true, propagate: true,
                            parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file}"],
                                         [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
                                         [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
                            ])
                        echo "Job report file: ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                        sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
                    }
                }
            }
            ]
        }
    }
}

/**
 Loads the jobs from the given report.
*/
@NonCPS
def parseJobs(job_file) {
    def jobs = [:]
    try {
        new File(job_file).splitEachLine(',') {  columns ->
            if (columns[0].startsWith('#') || columns[0].startsWith('JobName'))
                        return
                jobs["${columns[0]}"] = columns[1]
            }
    } catch(Exception e) {
       println "Previous job report not available"
    }
    jobs;
}