gCubeBuilder/Jenkinsfile

301 lines
13 KiB
Groovy
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!groovy
import org.yaml.snakeyaml.Yaml
// set the build options according to the Type of build
def (options, maven_local_repo_path, maven_settings_file, maven_parent_file) = ['', '', '', '']
def agent_root_folder = '/var/lib/jenkins/.m2'
def verbose = true
def resume = params.resume_from
if (params.Type == 'SNAPSHOT-DRY-RUN') {
echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts"
options = ''
maven_local_repo_path = "local-snapshots"
maven_settings_file = "jenkins-snapshots-dry-run-settings.xml"
maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
}
if (params.Type == 'SNAPSHOT') {
echo "Configure Maven for SNAPSHOT artifacts"
options = ''
maven_local_repo_path = "local-snapshots"
maven_settings_file = "jenkins-snapshots-settings.xml"
maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
}
if (params.Type == 'RELEASE-DRY-RUN') {
echo "Configure Maven for RELEASE-DRY-RUN artifacts"
options = ''
maven_local_repo_path = "local-releases"
maven_settings_file = "jenkins-releases-dry-run-settings.xml"
maven_parent_file = "jenkins-releases-dry-run-settings.xml"
}
if (params.Type == 'STAGING') {
echo "Configure Maven for STAGING artifacts"
options = ''
maven_local_repo_path = "local-staging"
maven_settings_file = "jenkins-staging-settings.xml"
maven_parent_file = "jenkins-staging-dry-run-settings.xml"
}
if (params.Type == 'RELEASE') {
echo "Configure Maven for RELEASE artifacts"
options = ''
maven_local_repo_path = "local-releases"
maven_settings_file = "jenkins-releases-settings.xml"
maven_parent_file = "jenkins-releases-dry-run-settings.xml"
}
echo "Use settings file at ${maven_settings_file}"
echo "Use local repo at ${maven_local_repo_path}"
echo "Release number: ${params.gCube_release_version}"
echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}"
echo "Clean up all local artifacts? ${params.cleanup_local_repo}"
echo "Resume from previous build? ${params.resume_from}"
//locate the release file
String releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeRelease/raw/branch/master/releases/gcube-${gCube_release_version}.yaml"
if (verbose)
println "Querying ${releaseURL}"
//load the release file
def text = releaseURL.toURL().getText()
//parsing
def jsonConfig = new Yaml().load(text)
if (verbose)
println jsonConfig.inspect()
assert jsonConfig.gCube_release.Version == params.gCube_release_version: "Release versions do not match!"
echo "Building gCube v. ${jsonConfig.gCube_release.Version}"
if (verbose) {
echo "Found components:"
jsonConfig.gCube_release.Components.each { println it.key }
}
def jobs = [:]
def previous_report_content = ''
def previous_report_file = "${agent_root_folder}/build_jobs.${resume}.csv"
def previous_commit_file = "${agent_root_folder}/build_commits.${resume}.csv"
pipeline {
// see https://jenkins.io/doc/book/pipeline/syntax/#agent
agent {
label 'CD'
}
// see https://jenkins.io/doc/book/pipeline/syntax/#environment
environment {
//make the JVM start a bit faster with basic just-in-time compilation of the code only (-XX:*)
//make maven running in a multi-thread fashion (16 threads, 2 threads on each Core)
MAVEN_OPTS = "${params.build_options}"
AGENT_ROOT_FOLDER = "${agent_root_folder}"
MAVEN_SETTINGS_FILE = "${maven_settings_file}"
MAVEN_PARENT_FILE = "${maven_parent_file}"
MAVEN_LOCAL_REPO = "${agent_root_folder}/${maven_local_repo_path}"
CLEANUP_GCUBE_REPO = "${params.cleanup_gcube_artifacts}"
REMOVE_LOCAL_REPO = "${params.cleanup_local_repo}"
GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
RESUME_FROM = "${resume}"
TYPE = "${params.Type}"
JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
PREVIOUS_JOB_REPORT = "${previous_report_file}"
PREVIOUS_COMMIT_REPORT = "${previous_commit_file}"
}
// see https://jenkins.io/doc/book/pipeline/syntax/#parameters
parameters {
choice(name: 'Type',
choices: ['SNAPSHOT-DRY-RUN', 'SNAPSHOT', 'STAGING', 'RELEASE-DRY-RUN', 'RELEASE'],
description: 'The type of artifacts the build is expected to generate')
string(name: 'gCube_release_version',
defaultValue: 'x.y.z',
description: 'The number of the gCube release to build. Sample values: 4.14, 4.15, etc.')
booleanParam(name: 'cleanup_gcube_artifacts',
defaultValue: true,
description: 'Wipe out the gcube artifacts from the local maven repository before the builds?')
booleanParam(name: 'cleanup_local_repo',
defaultValue: true,
description: 'Wipe out the local maven repository before the builds?')
string(name: 'resume_from',
defaultValue: '',
description: 'Resume from a previous build identified by the build number.')
}
//see https://jenkins.io/doc/book/pipeline/syntax/#stages
stages {
stage('initialize reports') {
steps {
script {
if (resume) {
try {
content = readFile("${previous_report_file}")
println "JOB REPORT CONTENT: ${content}"
jobs = parseJobs(content)
sh "rm ${AGENT_ROOT_FOLDER}/build_commits.csv || true"
sh "cp ${PREVIOUS_COMMIT_REPORT} ${AGENT_ROOT_FOLDER}/build_commits.csv"
} catch (Exception e) {
println "Previous job report not available or not parsable"
}
}
}
sh '''
# job report
echo "RESUME_FROM: ${RESUME_FROM}"
echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT}
echo "#StartTime ${date}" >> ${JOB_REPORT}
echo -e "JobName,Status" >> ${JOB_REPORT}
# create build report if not resumed
if [ ! -f ${AGENT_ROOT_FOLDER}/build_commits.csv ]; then
#build report
echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${AGENT_ROOT_FOLDER}/build_commits.csv
echo "#Release ${GCUBE_RELEASE_NUMBER}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
date=`date`
echo "#StartTime ${date}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
fi
'''
}
}
stage('initialize local repository') {
steps {
script {
if (!resume) {
sh '''
echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}"
echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}"
if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then
echo "Remove gCube artifacts from local repository"
rm -rf $MAVEN_LOCAL_REPO/org/gcube
fi
if [ "$REMOVE_LOCAL_REPO" = "true" ]; then
echo "Create a fresh local repository"
rm -rf $MAVEN_LOCAL_REPO
mkdir -p $MAVEN_LOCAL_REPO
fi
'''
}
}
sh '''
mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}"
cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml"
echo "Done with local repository and settings"
'''
}
}
// the maven-parent needs to be built (once) at each execution
stage('build maven-parent') {
steps {
script {
if (jobs['maven-parent'] == 'SUCCESS') {
echo "Skipping maven-parent"
// propagate success
sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
} else {
def gjob = build(job: 'maven-parent', wait: true, propagate: true,
parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"],
[$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
[$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
])
sh "echo -e maven-parent,${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
echo "Done with maven-parent"
}
}
}
}
stage('build components') {
steps {
script {
jsonConfig.gCube_release.Components.each { group_name, component_list ->
stage("build ${group_name} components") {
buildComponents items: component_list?.collect { "${it.name}" },
"${maven_settings_file}", "${maven_local_repo_path}", jobs
echo "Done with ${group_name} components"
}
}
}
}
}
}
// post-build actions
post {
always {
script {
sh '''
mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml"
mv ${AGENT_ROOT_FOLDER}/build_commits.csv ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv
cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv .
cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv
'''
}
echo 'The default maven settings have been restored'
}
success {
echo 'The pipeline worked!'
emailext to: 'jenkinsbuilds@d4science.org',
subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} worked",
body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
emailext attachmentsPattern: "**/*.${PIPELINE_BUILD_NUMBER}.csv",
to: 'jenkinsreleases@d4science.org',
subject: "${TYPE} report for release ${GCUBE_RELEASE_NUMBER} (build #${PIPELINE_BUILD_NUMBER})",
body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
}
failure {
echo 'The pipeline has failed'
emailext attachLog: true,
to: 'jenkinsbuilds@d4science.org',
subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed",
body: "Something is wrong with ${env.BUILD_URL}"
}
}
}
def buildComponents(args, maven_settings_file, maven_local_repo_path, jobs) {
if (args.items) {
parallel args.items?.collectEntries { name ->
["${name}": {
if (name && !"NONE".equalsIgnoreCase(name)) {
if ( (jobs["${name}"] == 'SUCCESS') ) {
echo "Skipping ${name}"
// propagate success
sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
} else {
def gjob = build(job: name, wait: true, propagate: true,
parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file}"],
[$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
[$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
])
sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
}
}
}
]
}
}
}
/**
Loads the jobs from the given report.
*/
@NonCPS
def parseJobs(content) {
def jobs = [:]
for (String line : content.split('\n')) {
if (!line.startsWith('#') && !line.startsWith('JobName')) {
def columns = line.split(',')
jobs["${columns[0]}"] = columns[1]
}
}
jobs;
}