You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

320 lines
14KB

  1. #!groovy​
  2. import org.yaml.snakeyaml.Yaml
  3. // set the build options according to the Type of build
  4. def (options, maven_local_repo_path, maven_settings_file, maven_parent_file) = ['', '', '', '']
  5. def agent_root_folder = '/var/lib/jenkins/.m2'
  6. def verbose = true
  7. def resume = params.resume_from
  8. if (params.Type == 'SNAPSHOT-DRY-RUN') {
  9. echo "Configure Maven for SNAPSHOT-DRY-RUN artifacts"
  10. options = ''
  11. maven_local_repo_path = "local-snapshots"
  12. maven_settings_file = "jenkins-snapshots-dry-run-settings.xml"
  13. maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
  14. }
  15. if (params.Type == 'SNAPSHOT') {
  16. echo "Configure Maven for SNAPSHOT artifacts"
  17. options = ''
  18. maven_local_repo_path = "local-snapshots"
  19. maven_settings_file = "jenkins-snapshots-settings.xml"
  20. maven_parent_file = "jenkins-snapshots-dry-run-settings.xml"
  21. }
  22. if (params.Type == 'RELEASE-DRY-RUN') {
  23. echo "Configure Maven for RELEASE-DRY-RUN artifacts"
  24. options = ''
  25. maven_local_repo_path = "local-releases"
  26. maven_settings_file = "jenkins-releases-dry-run-settings.xml"
  27. maven_parent_file = "jenkins-releases-dry-run-settings.xml"
  28. }
  29. if (params.Type == 'STAGING') {
  30. echo "Configure Maven for STAGING artifacts"
  31. options = ''
  32. maven_local_repo_path = "local-staging"
  33. maven_settings_file = "jenkins-staging-settings.xml"
  34. maven_parent_file = "jenkins-staging-dry-run-settings.xml"
  35. }
  36. if (params.Type == 'RELEASE') {
  37. echo "Configure Maven for RELEASE artifacts"
  38. options = ''
  39. maven_local_repo_path = "local-releases"
  40. maven_settings_file = "jenkins-releases-settings.xml"
  41. maven_parent_file = "jenkins-releases-dry-run-settings.xml"
  42. }
  43. echo "Use settings file at ${maven_settings_file}"
  44. echo "Use local repo at ${maven_local_repo_path}"
  45. echo "Release number: ${params.gCube_release_version}"
  46. echo "Clean up gcube local artifacts? ${params.cleanup_gcube_artifacts}"
  47. echo "Clean up all local artifacts? ${params.cleanup_local_repo}"
  48. echo "Resume from previous build? ${params.resume_from}"
  49. //locate the release file
  50. String releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleaseConfigs/raw/branch/master/open/gcube-${gCube_release_version}.yaml"
  51. if (verbose)
  52. println "Querying ${releaseURL}"
  53. //load the release file
  54. def text = releaseURL.toURL().getText()
  55. //parsing
  56. def jsonConfig = new Yaml().load(text)
  57. if (verbose)
  58. println jsonConfig.inspect()
  59. assert jsonConfig.gCube_release.Version == params.gCube_release_version: "Release versions do not match!"
  60. echo "Building gCube v. ${jsonConfig.gCube_release.Version}"
  61. if (verbose) {
  62. echo "Found components:"
  63. jsonConfig.gCube_release.Components.each { println it.key }
  64. }
  65. def jobs = [:]
  66. def previous_report_content = ''
  67. def previous_report_file = "${agent_root_folder}/build_jobs.${resume}.csv"
  68. def previous_commit_file = "${agent_root_folder}/build_commits.${resume}.csv"
  69. pipeline {
  70. // see https://jenkins.io/doc/book/pipeline/syntax/#agent
  71. agent {
  72. label 'CD'
  73. }
  74. // see https://jenkins.io/doc/book/pipeline/syntax/#environment
  75. environment {
  76. //make the JVM start a bit faster with basic just-in-time compilation of the code only (-XX:*)
  77. //make maven running in a multi-thread fashion (16 threads, 2 threads on each Core)
  78. MAVEN_OPTS = "${params.build_options}"
  79. AGENT_ROOT_FOLDER = "${agent_root_folder}"
  80. MAVEN_SETTINGS_FILE = "${maven_settings_file}"
  81. MAVEN_PARENT_FILE = "${maven_parent_file}"
  82. MAVEN_LOCAL_REPO = "${agent_root_folder}/${maven_local_repo_path}"
  83. CLEANUP_GCUBE_REPO = "${params.cleanup_gcube_artifacts}"
  84. REMOVE_LOCAL_REPO = "${params.cleanup_local_repo}"
  85. GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
  86. PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
  87. RESUME_FROM = "${resume}"
  88. TYPE = "${params.Type}"
  89. JOB_REPORT = "${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
  90. PREVIOUS_JOB_REPORT = "${previous_report_file}"
  91. PREVIOUS_COMMIT_REPORT = "${previous_commit_file}"
  92. }
  93. // see https://jenkins.io/doc/book/pipeline/syntax/#parameters
  94. parameters {
  95. choice(name: 'Type',
  96. choices: ['SNAPSHOT-DRY-RUN', 'SNAPSHOT', 'STAGING', 'RELEASE-DRY-RUN', 'RELEASE'],
  97. description: 'The type of artifacts the build is expected to generate')
  98. string(name: 'gCube_release_version',
  99. defaultValue: 'x.y.z',
  100. description: 'The number of the gCube release to build. Sample values: 4.14, 4.15, etc.')
  101. booleanParam(name: 'cleanup_gcube_artifacts',
  102. defaultValue: true,
  103. description: 'Wipe out the gcube artifacts from the local maven repository before the builds?')
  104. booleanParam(name: 'cleanup_local_repo',
  105. defaultValue: true,
  106. description: 'Wipe out the local maven repository before the builds?')
  107. string(name: 'resume_from',
  108. defaultValue: '',
  109. description: 'Resume from a previous build identified by the build number.')
  110. }
  111. //see https://jenkins.io/doc/book/pipeline/syntax/#stages
  112. stages {
  113. stage('initialize reports') {
  114. steps {
  115. script {
  116. if (resume) {
  117. try {
  118. content = readFile("${previous_report_file}")
  119. println "JOB REPORT CONTENT: ${content}"
  120. jobs = parseJobs(content)
  121. sh "rm ${AGENT_ROOT_FOLDER}/build_commits.csv || true"
  122. sh "cp ${PREVIOUS_COMMIT_REPORT} ${AGENT_ROOT_FOLDER}/build_commits.csv"
  123. } catch (Exception e) {
  124. println "Previous job report not available or not parsable"
  125. }
  126. }
  127. }
  128. sh '''
  129. # job report
  130. echo "RESUME_FROM: ${RESUME_FROM}"
  131. echo "#Build ${PIPELINE_BUILD_NUMBER}" > ${JOB_REPORT}
  132. echo "#StartTime ${date}" >> ${JOB_REPORT}
  133. echo -e "JobName,Status" >> ${JOB_REPORT}
  134. # create build report if not resumed
  135. if [ ! -f ${AGENT_ROOT_FOLDER}/build_commits.csv ]; then
  136. #build report
  137. echo "#Build ${PIPELINE_BUILD_NUMBER},,,,,,," > ${AGENT_ROOT_FOLDER}/build_commits.csv
  138. echo "#Release ${GCUBE_RELEASE_NUMBER},,,,,,," >> ${AGENT_ROOT_FOLDER}/build_commits.csv
  139. date=`date`
  140. echo "#StartTime ${date},,,,,,," >> ${AGENT_ROOT_FOLDER}/build_commits.csv
  141. echo -e "GroupID,ArtifactID,Version,SCM URL,Build Number,Distribution URL,Filename,Packaging" >> ${AGENT_ROOT_FOLDER}/build_commits.csv
  142. fi
  143. '''
  144. }
  145. }
  146. stage('initialize local repository') {
  147. steps {
  148. script {
  149. if (!resume) {
  150. sh '''
  151. echo "REMOVE_LOCAL_REPO: ${REMOVE_LOCAL_REPO}"
  152. echo "CLEANUP_GCUBE_REPO: ${CLEANUP_GCUBE_REPO}"
  153. if [ "$CLEANUP_GCUBE_REPO" = "true" ]; then
  154. echo "Remove gCube artifacts from local repository"
  155. rm -rf $MAVEN_LOCAL_REPO/org/gcube
  156. fi
  157. if [ "$REMOVE_LOCAL_REPO" = "true" ]; then
  158. echo "Create a fresh local repository"
  159. rm -rf $MAVEN_LOCAL_REPO
  160. mkdir -p $MAVEN_LOCAL_REPO
  161. fi
  162. '''
  163. }
  164. }
  165. sh '''
  166. mv "${AGENT_ROOT_FOLDER}/settings.xml" "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}"
  167. cp "${AGENT_ROOT_FOLDER}/${MAVEN_SETTINGS_FILE}" "${AGENT_ROOT_FOLDER}/settings.xml"
  168. echo "Done with local repository and settings"
  169. '''
  170. }
  171. }
  172. // the maven-parent needs to be built (once) at each execution
  173. stage('build maven-parent') {
  174. steps {
  175. script {
  176. if (jobs['maven-parent'] == 'SUCCESS') {
  177. echo "Skipping maven-parent"
  178. // propagate success
  179. sh "echo -e maven-parent,SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
  180. } else {
  181. def gjob = build(job: 'maven-parent', wait: true, propagate: true,
  182. parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_parent_file}"],
  183. [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
  184. [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
  185. ])
  186. sh "echo -e maven-parent,${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
  187. echo "Done with maven-parent"
  188. }
  189. }
  190. }
  191. }
  192. stage('build components') {
  193. steps {
  194. script {
  195. jsonConfig.gCube_release.Components.each { group_name, component_list ->
  196. stage("build ${group_name} components") {
  197. buildComponents items: component_list?.collect { "${it.name}" },
  198. "${maven_settings_file}", "${maven_local_repo_path}", jobs
  199. echo "Done with ${group_name} components"
  200. }
  201. }
  202. }
  203. }
  204. }
  205. stage('generate release notes') {
  206. steps {
  207. script {
  208. sh '''
  209. ls -lrt ${AGENT_ROOT_FOLDER}/build_commits.csv
  210. cat ${AGENT_ROOT_FOLDER}/build_commits.csv
  211. '''
  212. def report_text = sh(script: 'cat ${AGENT_ROOT_FOLDER}/build_commits.csv', returnStdout: true)?.trim()
  213. def pjob = build(job: 'gCube-ReleaseNotes', wait: true, propagate: true,
  214. parameters: [[$class: 'StringParameterValue', name: 'report', value: "${report_text}"],
  215. [$class: 'StringParameterValue', name: 'report_number', value: "${env.BUILD_NUMBER}"],
  216. [$class: 'StringParameterValue', name: 'gCube_release_version', value: "${params.gCube_release_version}"],
  217. [$class: 'BooleanParameterValue', name: 'use_report_commits', value: true]]
  218. )
  219. }
  220. }
  221. }
  222. }
  223. // post-build actions
  224. post {
  225. always {
  226. script {
  227. sh '''
  228. mv "${AGENT_ROOT_FOLDER}/settings.${PIPELINE_BUILD_NUMBER}" "${AGENT_ROOT_FOLDER}/settings.xml"
  229. mv ${AGENT_ROOT_FOLDER}/build_commits.csv ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv
  230. cp ${AGENT_ROOT_FOLDER}/build_commits.${PIPELINE_BUILD_NUMBER}.csv .
  231. cat ${AGENT_ROOT_FOLDER}/build_jobs.${PIPELINE_BUILD_NUMBER}.csv
  232. '''
  233. }
  234. echo 'The default maven settings have been restored'
  235. }
  236. success {
  237. echo 'The pipeline worked!'
  238. emailext to: 'jenkinsbuilds@d4science.org',
  239. subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} worked",
  240. body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
  241. emailext attachmentsPattern: "**/*.${PIPELINE_BUILD_NUMBER}.csv",
  242. to: 'jenkinsreleases@d4science.org',
  243. subject: "${TYPE} report for release ${GCUBE_RELEASE_NUMBER} (build #${PIPELINE_BUILD_NUMBER})",
  244. body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
  245. }
  246. failure {
  247. echo 'The pipeline has failed'
  248. emailext attachLog: true,
  249. to: 'jenkinsbuilds@d4science.org',
  250. subject: "[Jenkins build D4S] build ${currentBuild.fullDisplayName} failed",
  251. body: "Something is wrong with ${env.BUILD_URL}"
  252. }
  253. }
  254. }
  255. def buildComponents(args, maven_settings_file, maven_local_repo_path, jobs) {
  256. if (args.items) {
  257. parallel args.items?.collectEntries { name ->
  258. ["${name}": {
  259. if (name && !"NONE".equalsIgnoreCase(name)) {
  260. if ( (jobs["${name}"] == 'SUCCESS') ) {
  261. echo "Skipping ${name}"
  262. // propagate success
  263. sh "echo -e ${name},SUCCESS >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
  264. } else {
  265. def gjob = build(job: name, wait: true, propagate: true,
  266. parameters: [[$class: 'StringParameterValue', name: 'gcube_settings', value: "${maven_settings_file}"],
  267. [$class: 'StringParameterValue', name: 'local_repo', value: "${maven_local_repo_path}"],
  268. [$class: 'LabelParameterValue', name: 'exec_label', label: "CD", nodeEligibility: [$class: 'AllNodeEligibility']]
  269. ])
  270. sh "echo -e ${name},${gjob.getResult()} >> ${agent_root_folder}/build_jobs.${env.BUILD_NUMBER}.csv"
  271. }
  272. }
  273. }
  274. ]
  275. }
  276. }
  277. }
  278. /**
  279. Loads the jobs from the given report.
  280. */
  281. @NonCPS
  282. def parseJobs(content) {
  283. def jobs = [:]
  284. for (String line : content.split('\n')) {
  285. if (!line.startsWith('#') && !line.startsWith('JobName')) {
  286. def columns = line.split(',')
  287. jobs["${columns[0]}"] = columns[1]
  288. }
  289. }
  290. jobs;
  291. }