diff --git a/Jenkinsfile b/Jenkinsfile index 4ae5cee..46ec417 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ import org.yaml.snakeyaml.Yaml def (options, maven_local_repo_path, maven_settings_file, maven_parent_file) = ['', '', '', ''] def agent_root_folder = '/var/lib/jenkins/.m2' def verbose = true - +def resume = params.resume_from if (params.Type == 'SNAPSHOT-DRY-RUN') { echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts" options = '' @@ -47,6 +47,7 @@ echo "Use local repo at ${maven_local_repo_path}" echo "Release number: ${params.gCube_release_version}" echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}" echo "Clean up all local artifacts? ${params.cleanup_local_repo}" +echo "Resume from previous build? ${params.resume_from}" //locate the release file @@ -69,6 +70,12 @@ if (verbose) { jsonConfig.gCube_release.Components.each { println it.key } } +def jobs = [:] +def previous_report_content = '' +def previous_report_file = "${agent_root_folder}/build_jobs.${resume}.csv" +def previous_commit_file = "${agent_root_folder}/build_commits.${resume}.csv" + + pipeline { // see https://jenkins.io/doc/book/pipeline/syntax/#agent @@ -90,6 +97,10 @@ pipeline { GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}" PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}" TYPE = "${params.Type}" + RESUME = "${params.resume}" + JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" + PREVIOUS_JOB_REPORT = "${previous_report_file}" + PREVIOUS_COMMIT_REPORT = "${previous_commit_file}" } // see https://jenkins.io/doc/book/pipeline/syntax/#parameters @@ -109,44 +120,84 @@ pipeline { booleanParam(name: 'cleanup_local_repo', defaultValue: true, description: 'Wipe out the local maven repository before the builds?') + + string(name: 'resume_from', + defaultValue: '', + description: 'Resume from a previous build identified by the build number.') } //see https://jenkins.io/doc/book/pipeline/syntax/#stages stages { - stage('initialize') { + stage('initialize reports') { + steps { + script { + if (resume) { + content = readFile("${previous_report_file}") + println "JOB REPORT CONTENT: ${content}" + jobs = parseJobs(content) + sh "rm ${AGENT_ROOT_FOLDER}/build_commits.csv || true" + sh "cp ${PREVIOUS_COMMIT_REPORT} ${AGENT_ROOT_FOLDER}/build_commits.csv" + } + } + + sh ''' + # job report + echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT} + echo "#StartTime ${date}" >> ${JOB_REPORT} + echo -e "JobName,Status" >> ${JOB_REPORT} + + # create build report if not resumed + if [ ! -f ${AGENT_ROOT_FOLDER}/build_commits.csv ]; then + #build report + echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${AGENT_ROOT_FOLDER}/build_commits.csv + echo "#Release ${GCUBE_RELEASE_NUMBER}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv + date=`date` + echo "#StartTime ${date}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv + echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv + fi + ''' + } + } + stage('initialize local repository') { steps { sh ''' - echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}" - echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}" - if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then - echo "Remove gCube artifacts from local repository" - rm -rf $MAVEN_LOCAL_REPO/org/gcube - fi - if [ "$REMOVE_LOCAL_REPO" = "true" ]; then - echo "Create a fresh local repository" - rm -rf $MAVEN_LOCAL_REPO - mkdir -p $MAVEN_LOCAL_REPO - fi - mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" - cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml" - echo "Done with local repository and settings" - echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${AGENT_ROOT_FOLDER}/build_commits.csv - echo "#Release ${GCUBE_RELEASE_NUMBER}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv - date=`date` - echo "#StartTime ${date}" >> ${AGENT_ROOT_FOLDER}/build_commits.csv - echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv + echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}" + echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}" + if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then + echo "Remove gCube artifacts from local repository" + rm -rf $MAVEN_LOCAL_REPO/org/gcube + fi + if [ "$REMOVE_LOCAL_REPO" = "true" ]; then + echo "Create a fresh local repository" + rm -rf $MAVEN_LOCAL_REPO + mkdir -p $MAVEN_LOCAL_REPO + fi + mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" + cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml" + echo "Done with local repository and settings" + ''' } } // the maven-parent needs to be built (once) at each execution stage('build maven-parent') { steps { - echo build(job: 'maven-parent', wait: true, - parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"], - [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"], - [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']] - ]).result - echo "Done with maven-parent" + script { + if (jobs['maven-parent'] == 'SUCCESS') { + echo "Skipping maven-parent" + // propagate success + sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" + } else { + def gjob = build(job: 'maven-parent', wait: true, propagate: true, + parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"], + [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"], + [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']] + ]) + sh "echo -e maven-parent,${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" + echo "Done with maven-parent" + } + + } } } stage('build components') { @@ -155,7 +206,7 @@ pipeline { jsonConfig.gCube_release.Components.each { group_name, component_list -> stage("build ${group_name} components") { buildComponents items: component_list?.collect { "${it.name}" }, - "${maven_settings_file}", "${maven_local_repo_path}" + "${maven_settings_file}", "${maven_local_repo_path}", jobs echo "Done with ${group_name} components" } } @@ -172,6 +223,7 @@ pipeline { mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml" mv ${AGENT_ROOT_FOLDER}/build_commits.csv ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv . + cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv ''' } echo 'The default maven settings have been restored' @@ -198,18 +250,45 @@ pipeline { } } -def buildComponents(args, maven_settings_file, maven_local_repo_path) { +def buildComponents(args, maven_settings_file, maven_local_repo_path, jobs) { if (args.items) { parallel args.items?.collectEntries { name -> ["${name}": { - if (name && !"NONE".equalsIgnoreCase(name)) - build(job: name, + if (name && !"NONE".equalsIgnoreCase(name)) { + if ( (jobs["${name}"] == 'SUCCESS') ) { + echo "Skipping ${name}" + // propagate success + sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" + } else { + def gjob = build(job: name, wait: true, propagate: true, parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file}"], [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"], [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']] ]) + sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv" + } + } } ] } } +} + +/** + Loads the jobs from the given report. +*/ +@NonCPS +def parseJobs(content) { + def jobs = [:] + try { + for (String line : content.split('\n')) { + if (!line.startsWith('#') && !line.startsWith('JobName')) { + def columns = line.split(',') + jobs["${columns[0]}"] = columns[1] + } + } + } catch(Exception e) { + println "Previous job report not available or not parsable" + } + jobs; } \ No newline at end of file