#!groovy​ import org.yaml.snakeyaml.Yaml // set the build options according to the Type of build def (options, maven_local_repo_path, maven_settings_file, maven_parent_file) = ['', '', '', ''] def agent_root_folder = '/var/lib/jenkins/.m2' def verbose = true def resume = params.resume_from if (params.Type == 'SNAPSHOT-DRY-RUN') { echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts" options = '' maven_local_repo_path = "local-snapshots" maven_settings_file = "jenkins-snapshots-dry-run-settings.xml" maven_parent_file = "jenkins-snapshots-dry-run-settings.xml" } if (params.Type == 'SNAPSHOT') { echo "Configure Maven for SNAPSHOT artifacts" options = '' maven_local_repo_path = "local-snapshots" maven_settings_file = "jenkins-snapshots-settings.xml" maven_parent_file = "jenkins-snapshots-dry-run-settings.xml" } if (params.Type == 'RELEASE-DRY-RUN') { echo "Configure Maven for RELEASE-DRY-RUN artifacts" options = '' maven_local_repo_path = "local-releases" maven_settings_file = "jenkins-releases-dry-run-settings.xml" maven_parent_file = "jenkins-releases-dry-run-settings.xml" } if (params.Type == 'STAGING') { echo "Configure Maven for STAGING artifacts" options = '' maven_local_repo_path = "local-staging" maven_settings_file = "jenkins-staging-settings.xml" maven_parent_file = "jenkins-staging-dry-run-settings.xml" } if (params.Type == 'RELEASE') { echo "Configure Maven for RELEASE artifacts" options = '' maven_local_repo_path = "local-releases" maven_settings_file = "jenkins-releases-settings.xml" maven_parent_file = "jenkins-releases-dry-run-settings.xml" } echo "Use settings file at ${maven_settings_file}" echo "Use local repo at ${maven_local_repo_path}" echo "Release number: ${params.gCube_release_version}" echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}" echo "Clean up all local artifacts? ${params.cleanup_local_repo}" echo "Resume from previous build? ${params.resume_from}" //locate the release file String releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeRelease/raw/branch/master/releases/gcube-${gCube_release_version}.yaml" if (verbose) println "Querying ${releaseURL}" //load the release file def text = releaseURL.toURL().getText() //parsing def jsonConfig = new Yaml().load(text) if (verbose) println jsonConfig.inspect() assert jsonConfig.gCube_release.Version == params.gCube_release_version: "Release versions do not match!" echo "Building gCube v. ${jsonConfig.gCube_release.Version}" if (verbose) { echo "Found components:" jsonConfig.gCube_release.Components.each { println it.key } } def jobs = [:] if (resume_from) { def previous_report_file = "${agent_root_folder}/build_jobs.${resume_from}.csv" echo "Previous report file: ${previous_report_file}" jobs = parseJobs(previous_report_file) if (verbose) { for (job in jobs) println job } } pipeline { // see https://jenkins.io/doc/book/pipeline/syntax/#agent agent { label 'CD' } // see https://jenkins.io/doc/book/pipeline/syntax/#environment environment { //make the JVM start a bit faster with basic just-in-time compilation of the code only (-XX:*) //make maven running in a multi-thread fashion (16 threads, 2 threads on each Core) MAVEN_OPTS = "${params.build_options}" AGENT_ROOT_FOLDER = "${agent_root_folder}" MAVEN_SETTINGS_FILE = "${maven_settings_file}" MAVEN_PARENT_FILE = "${maven_parent_file}" MAVEN_LOCAL_REPO = "${agent_root_folder}/${maven_local_repo_path}" CLEANUP_GCUBE_REPO = "${params.cleanup_gcube_artifacts}" REMOVE_LOCAL_REPO = "${params.cleanup_local_repo}" GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" TYPE = "${params.Type}" RESUME = "${params.resume}" JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" PREVIOUS_JOB_REPORT = "${previous_report_file}" } // see https://jenkins.io/doc/book/pipeline/syntax/#parameters parameters { choice(name: 'Type', choices: ['SNAPSHOT-DRY-RUN', 'SNAPSHOT', 'STAGING', 'RELEASE-DRY-RUN', 'RELEASE'], description: 'The type of artifacts the build is expected to generate') string(name: 'gCube_release_version', defaultValue: 'x.y.z', description: 'The number of the gCube release to build. Sample values: 4.14, 4.15, etc.') booleanParam(name: 'cleanup_gcube_artifacts', defaultValue: true, description: 'Wipe out the gcube artifacts from the local maven repository before the builds?') booleanParam(name: 'cleanup_local_repo', defaultValue: true, description: 'Wipe out the local maven repository before the builds?') string(name: 'resume_from', defaultValue: '', description: 'Resume from a previous build identified by the build number.') } //see https://jenkins.io/doc/book/pipeline/syntax/#stages stages { stage('initialize') { steps { sh ''' echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}" echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}" if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then echo "Remove gCube artifacts from local repository" rm -rf $MAVEN_LOCAL_REPO/org/gcube fi if [ "$REMOVE_LOCAL_REPO" = "true" ]; then echo "Create a fresh local repository" rm -rf $MAVEN_LOCAL_REPO mkdir -p $MAVEN_LOCAL_REPO fi mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml" echo "Done with local repository and settings" #build report echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${AGENT_ROOT_FOLDER}/build_commits.csv echo "#Release ${GCUBE_RELEASE_NUMBER}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv date=`date` echo "#StartTime ${date}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv #job report cat ${PREVIOUS_JOB_REPORT} echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT} echo "#StartTime ${date}" >> ${JOB_REPORT} echo -e "JobName,Status" >> ${JOB_REPORT} ''' } } // the maven-parent needs to be built (once) at each execution stage('build maven-parent') { steps { script { if ( ("${resume}" == 'true') && (jobs['maven-parent'] == 'SUCCESS') ) { echo "Skipping maven-parent" // propagate success sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" } else { def gjob = build(job: 'maven-parent', wait: true, propagate: true, parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"], [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"], [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']] ]) echo "Job report file: ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" sh "echo -e maven-parent,${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" echo "Done with maven-parent" } } } } stage('build components') { steps { script { jsonConfig.gCube_release.Components.each { group_name, component_list -> stage("build ${group_name} components") { buildComponents items: component_list?.collect { "${it.name}" }, "${maven_settings_file}", "${maven_local_repo_path}", jobs echo "Done with ${group_name} components" } } } } } } // post-build actions post { always { script { sh ''' mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml" mv ${AGENT_ROOT_FOLDER}/build_commits.csv ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv . cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv ''' } echo 'The default maven settings have been restored' } success { echo 'The pipeline worked!' emailext to: 'jenkinsbuilds@d4science.org', subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} worked", body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}" emailext attachmentsPattern: "**/*.${PIPELINE_BUILD_NUMBER}.csv", to: 'jenkinsreleases@d4science.org', subject: "${TYPE} report for release ${GCUBE_RELEASE_NUMBER} (build #${PIPELINE_BUILD_NUMBER})", body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}" } failure { echo 'The pipeline has failed' emailext attachLog: true, to: 'jenkinsbuilds@d4science.org', subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed", body: "Something is wrong with ${env.BUILD_URL}" } } } def buildComponents(args, maven_settings_file, maven_local_repo_path, jobs) { if (args.items) { parallel args.items?.collectEntries { name -> ["${name}": { if (name && !"NONE".equalsIgnoreCase(name)) { if ( ("${resume}" == 'true') && (jobs["${name}"] == 'SUCCESS') ) { echo "Skipping ${name}" // propagate success sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" } else { def gjob = build(job: name, wait: true, propagate: true, parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file}"], [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"], [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']] ]) echo "Job report file: ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" } } } ] } } } /** Loads the jobs from the given report. */ @NonCPS def parseJobs(job_file) { def jobs = [:] try { new File(job_file).splitEachLine(',') { columns -> if (columns[0].startsWith('#') || columns[0].startsWith('JobName')) return jobs["${columns[0]}"] = columns[1] } } catch(Exception e) { println "Previous job report not available" } jobs; }