338 lines
15 KiB
Groovy
338 lines
15 KiB
Groovy
#!groovy
|
|
import org.yaml.snakeyaml.Yaml
|
|
|
|
// set the build options according to the Type of build
|
|
def (options, maven_local_repo_path, maven_settings_file, maven_activation_property) = ['', '', '', '']
|
|
def agent_root_folder = '/var/lib/jenkins/.m2'
|
|
def verbose = true
|
|
def resume = params.resume_from
|
|
if (params.Type == 'SNAPSHOT-DRY-RUN') {
|
|
echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts"
|
|
options = ''
|
|
maven_local_repo_path = "local-snapshots"
|
|
maven_settings_file = "gcube-settings.xml"
|
|
maven_activation_property="-Pdry-run";
|
|
}
|
|
if (params.Type == 'SNAPSHOT') {
|
|
echo "Configure Maven for SNAPSHOT artifacts"
|
|
options = ''
|
|
maven_local_repo_path = "local-snapshots"
|
|
maven_settings_file = "gcube-settings.xml"
|
|
maven_activation_property='';
|
|
}
|
|
if (params.Type == 'RELEASE-DRY-RUN') {
|
|
echo "Configure Maven for RELEASE-DRY-RUN artifacts"
|
|
options = ''
|
|
maven_local_repo_path = "local-releases"
|
|
maven_settings_file = "jenkins-releases-settings.xml"
|
|
maven_activation_property="-Pdry-run -Djenkins-releases -DRelease";
|
|
}
|
|
if (params.Type == 'STAGING') {
|
|
echo "Configure Maven for STAGING artifacts"
|
|
options = ''
|
|
maven_local_repo_path = "local-staging"
|
|
maven_settings_file = "jenkins-staging-settings.xml"
|
|
maven_activation_property="-Djenkins-staging -DRelease";
|
|
}
|
|
if (params.Type == 'RELEASE') {
|
|
echo "Configure Maven for RELEASE artifacts"
|
|
options = ''
|
|
maven_local_repo_path = "local-releases"
|
|
maven_settings_file = "jenkins-releases-settings.xml"
|
|
maven_activation_property="-Djenkins-releases -DRelease";
|
|
}
|
|
|
|
echo "Use settings file at ${maven_settings_file} with the following activation property: ${maven_activation_property}"
|
|
echo "Use local repo at ${maven_local_repo_path}"
|
|
echo "Release number: ${params.gCube_release_version}"
|
|
echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}"
|
|
echo "Clean up all local artifacts? ${params.cleanup_local_repo}"
|
|
echo "Resume from previous build? ${params.resume_from}"
|
|
|
|
|
|
//locate the release file
|
|
String releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleases/raw/branch/master/open/gcube-${gCube_release_version}.yaml"
|
|
|
|
if (verbose)
|
|
println "Querying ${releaseURL}"
|
|
|
|
//load the release file
|
|
def text = releaseURL.toURL().getText()
|
|
|
|
//parsing
|
|
def jsonConfig = new Yaml().load(text)
|
|
if (verbose)
|
|
println jsonConfig.inspect()
|
|
assert jsonConfig.gCube_release.Version == params.gCube_release_version: "Release versions do not match!"
|
|
echo "Building gCube v. ${jsonConfig.gCube_release.Version}"
|
|
if (verbose) {
|
|
echo "Found components:"
|
|
jsonConfig.gCube_release.Components.each { println it.key }
|
|
}
|
|
|
|
def jobs = [:]
|
|
def previous_report_content = ''
|
|
def previous_report_file = "${agent_root_folder}/build_jobs.${resume}.csv"
|
|
def previous_commit_file = "${agent_root_folder}/build_commits.${resume}.csv"
|
|
|
|
|
|
pipeline {
|
|
|
|
// see https://jenkins.io/doc/book/pipeline/syntax/#agent
|
|
agent {
|
|
label 'CD'
|
|
}
|
|
|
|
// see https://jenkins.io/doc/book/pipeline/syntax/#environment
|
|
environment {
|
|
//make the JVM start a bit faster with basic just-in-time compilation of the code only (-XX:*)
|
|
//make maven running in a multi-thread fashion (16 threads, 2 threads on each Core)
|
|
MAVEN_OPTS = "${params.build_options}"
|
|
AGENT_ROOT_FOLDER = "${agent_root_folder}"
|
|
MAVEN_SETTINGS_FILE = "${maven_settings_file}"
|
|
MAVEN_ACTIVATION_PROPERTY = "${maven_activation_property}"
|
|
MAVEN_LOCAL_REPO = "${agent_root_folder}/${maven_local_repo_path}"
|
|
CLEANUP_GCUBE_REPO = "${params.cleanup_gcube_artifacts}"
|
|
REMOVE_LOCAL_REPO = "${params.cleanup_local_repo}"
|
|
GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
|
|
PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
|
|
RESUME_FROM = "${resume}"
|
|
TYPE = "${params.Type}"
|
|
JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
|
|
PREVIOUS_JOB_REPORT = "${previous_report_file}"
|
|
PREVIOUS_COMMIT_REPORT = "${previous_commit_file}"
|
|
}
|
|
|
|
// see https://jenkins.io/doc/book/pipeline/syntax/#parameters
|
|
parameters {
|
|
choice(name: 'Type',
|
|
choices: ['SNAPSHOT-DRY-RUN', 'SNAPSHOT', 'STAGING', 'RELEASE-DRY-RUN', 'RELEASE'],
|
|
description: 'The type of artifacts the build is expected to generate')
|
|
|
|
string(name: 'gCube_release_version',
|
|
defaultValue: 'x.y.z',
|
|
description: 'The number of the gCube release to build. Sample values: 4.14, 4.15, etc.')
|
|
|
|
booleanParam(name: 'cleanup_gcube_artifacts',
|
|
defaultValue: true,
|
|
description: 'Wipe out the gcube artifacts from the local maven repository before the builds?')
|
|
|
|
booleanParam(name: 'cleanup_local_repo',
|
|
defaultValue: true,
|
|
description: 'Wipe out the local maven repository before the builds?')
|
|
|
|
string(name: 'resume_from',
|
|
defaultValue: '',
|
|
description: 'Resume from a previous build identified by the build number.')
|
|
|
|
booleanParam(name: 'ContinuousDeployment',
|
|
defaultValue: false,
|
|
description: 'If true, the gCubeDeployer pipeline will be triggered')
|
|
}
|
|
|
|
//see https://jenkins.io/doc/book/pipeline/syntax/#stages
|
|
stages {
|
|
stage('initialize reports') {
|
|
steps {
|
|
script {
|
|
if (resume) {
|
|
try {
|
|
content = readFile("${previous_report_file}")
|
|
println "JOB REPORT CONTENT: ${content}"
|
|
jobs = parseJobs(content)
|
|
sh "rm ${AGENT_ROOT_FOLDER}/build_commits.csv || true"
|
|
sh "cp ${PREVIOUS_COMMIT_REPORT} ${AGENT_ROOT_FOLDER}/build_commits.csv"
|
|
} catch (Exception e) {
|
|
println "Previous job report not available or not parsable"
|
|
}
|
|
}
|
|
}
|
|
|
|
sh '''
|
|
# job report
|
|
echo "RESUME_FROM: ${RESUME_FROM}"
|
|
echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT}
|
|
echo "#StartTime ${date}" >> ${JOB_REPORT}
|
|
echo -e "JobName,Status" >> ${JOB_REPORT}
|
|
|
|
# create build report if not resumed
|
|
if [ ! -f ${AGENT_ROOT_FOLDER}/build_commits.csv ]; then
|
|
#build report
|
|
echo "#Build ${PIPELINE_BUILD_NUMBER},,,,,,," > ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
echo "#Release ${GCUBE_RELEASE_NUMBER},,,,,,," >> ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
date=`date`
|
|
echo "#StartTime ${date},,,,,,," >> ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
fi
|
|
'''
|
|
}
|
|
}
|
|
stage('initialize local repository') {
|
|
steps {
|
|
script {
|
|
if (!resume) {
|
|
sh '''
|
|
echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}"
|
|
echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}"
|
|
if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then
|
|
echo "Remove gCube artifacts from local repository"
|
|
rm -rf $MAVEN_LOCAL_REPO/org/gcube
|
|
fi
|
|
if [ "$REMOVE_LOCAL_REPO" = "true" ]; then
|
|
echo "Create a fresh local repository"
|
|
rm -rf $MAVEN_LOCAL_REPO
|
|
mkdir -p $MAVEN_LOCAL_REPO
|
|
fi
|
|
'''
|
|
}
|
|
}
|
|
sh '''
|
|
mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}"
|
|
cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml"
|
|
echo "Done with local repository and settings"
|
|
'''
|
|
}
|
|
}
|
|
|
|
// the maven-parent needs to be built (once) at each execution
|
|
stage('build maven-parent') {
|
|
steps {
|
|
script {
|
|
if (jobs['maven-parent'] == 'SUCCESS') {
|
|
echo "Skipping maven-parent"
|
|
// propagate success
|
|
sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
|
|
} else {
|
|
def gjob = build(job: 'maven-parent', wait: true, propagate: true,
|
|
parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file} -Pdry-run"],
|
|
[$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
|
|
[$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
|
|
])
|
|
sh "echo -e maven-parent,${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
|
|
echo "Done with maven-parent"
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
stage('build components') {
|
|
steps {
|
|
script {
|
|
jsonConfig.gCube_release.Components.each { group_name, component_list ->
|
|
stage("build ${group_name} components") {
|
|
buildComponents items: component_list?.collect { "${it.name}" },
|
|
"${maven_settings_file}", "${maven_activation_property}", "${maven_local_repo_path}", jobs
|
|
echo "Done with ${group_name} components"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('generate release notes') {
|
|
steps {
|
|
script {
|
|
sh '''
|
|
ls -lrt ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
cat ${AGENT_ROOT_FOLDER}/build_commits.csv
|
|
'''
|
|
def report_text = sh(script: 'cat ${AGENT_ROOT_FOLDER}/build_commits.csv', returnStdout: true)?.trim()
|
|
|
|
def pjob = build(job: 'Pipeline-gCubeReleaseNotes', wait: true, propagate: false,
|
|
parameters: [[$class: 'StringParameterValue', name: 'report', value: "${report_text}"],
|
|
[$class: 'StringParameterValue', name: 'report_number', value: "${env.BUILD_NUMBER}"],
|
|
[$class: 'StringParameterValue', name: 'gCube_release_version', value: "${params.gCube_release_version}"],
|
|
[$class: 'BooleanParameterValue', name: 'use_report_commits', value: true]]
|
|
)
|
|
}
|
|
}
|
|
}
|
|
stage('Deploy components') {
|
|
when {
|
|
expression { params.ContinuousDeployment.toBoolean() }
|
|
}
|
|
steps {
|
|
script {
|
|
def pjob = build(job: 'gCubeDeployer', wait: true, propagate: true,
|
|
parameters: [[$class: 'StringParameterValue', name: 'gCube_release_version', value: "${params.gCube_release_version}"],
|
|
[$class: 'StringParameterValue', name: 'TRIGGER_JOB', value: "${JOB_NAME}"]]
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
// post-build actions
|
|
post {
|
|
always {
|
|
script {
|
|
sh '''
|
|
mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml"
|
|
mv ${AGENT_ROOT_FOLDER}/build_commits.csv ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv
|
|
cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv .
|
|
cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv
|
|
'''
|
|
}
|
|
echo 'The default maven settings have been restored'
|
|
|
|
}
|
|
success {
|
|
echo 'The pipeline worked!'
|
|
emailext to: 'jenkinsbuilds@d4science.org',
|
|
subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} worked",
|
|
body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
|
|
emailext attachmentsPattern: "**/*.${PIPELINE_BUILD_NUMBER}.csv",
|
|
to: 'jenkinsreleases@d4science.org',
|
|
subject: "${TYPE} report for release ${GCUBE_RELEASE_NUMBER} (build #${PIPELINE_BUILD_NUMBER})",
|
|
body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
|
|
}
|
|
failure {
|
|
echo 'The pipeline has failed'
|
|
emailext attachLog: true,
|
|
to: 'jenkinsbuilds@d4science.org',
|
|
subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed",
|
|
body: "Something is wrong with ${env.BUILD_URL}"
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
def buildComponents(args, maven_settings_file, maven_activation_property, maven_local_repo_path, jobs) {
|
|
if (args.items) {
|
|
parallel args.items?.collectEntries { name ->
|
|
["${name}": {
|
|
if (name && !"NONE".equalsIgnoreCase(name)) {
|
|
if ( (jobs["${name}"] == 'SUCCESS') ) {
|
|
echo "Skipping ${name}"
|
|
// propagate success
|
|
sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
|
|
} else {
|
|
def gjob = build(job: name, wait: true, propagate: true,
|
|
parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file} ${maven_activation_property}"],
|
|
[$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
|
|
[$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
|
|
])
|
|
sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
Loads the jobs from the given report.
|
|
*/
|
|
@NonCPS
|
|
def parseJobs(content) {
|
|
def jobs = [:]
|
|
for (String line : content.split('\n')) {
|
|
if (!line.startsWith('#') && !line.startsWith('JobName')) {
|
|
def columns = line.split(',')
|
|
jobs["${columns[0]}"] = columns[1]
|
|
}
|
|
}
|
|
jobs;
|
|
}
|