Update Jenkinsfile

change email address to jenkinsbuilds@d4science.org

Jenkinsfile

@@ -1,12 +1,9 @@
 #!groovy
-import groovy.io.FileType
+import org.yaml.snakeyaml.Yaml
-import static groovy.io.FileType.FILES
+import groovy.json.JsonSlurper
-import jenkins.model.*
-import hudson.FilePath;
-import jenkins.model.Jenkins;
 
 /**
- * Deploy components to the D4Science Infrastructure (dev-only)
+ * Deploy gCube components on D4Science Infrastructure (dev and preprod environments)
  *
  * Roberto Cirillo (ISTI-CNR)
  */
@@ -14,13 +11,33 @@ import jenkins.model.Jenkins;
 // related jenkins job: https://jenkins.d4science.org/job/gCubeDeployer/
 
 def agent_root_folder = '/var/lib/jenkins'
+def verbose = true
+def yamlConfig
+def text
+if ("${params.gCube_release_version}" ||  "${params.RELEASE_URL}"){
+	String releaseURL='';
+	if ("${params.gCube_release_version}"){
+	//locate the release file
+		releaseURL = "https://code-repo.d4science.org/gCubeCI/gCubeReleases/raw/branch/master/open/gcube-${gCube_release_version}.yaml"
+	}else{
+	    releaseURL = "${params.RELEASE_URL}"
+	}
+//load the release file
+	text = releaseURL.toURL().getText()
+	//parsing
+	yamlConfig = new Yaml().load(text)
+	if (verbose)
+	    println yamlConfig.inspect()
+	echo "Building gCube v. ${yamlConfig.gCube_release.Version}"
+	if (verbose) {
+	    echo "Found components:"
+	    yamlConfig.gCube_release.Components.each { println it.key }
+	}
+}
+
 
 def deployList
 def backupList
-if (params.deployFile) {
-    println "Using custom deploy file"
-    deployList = params.deployFile
-}
 
 pipeline {
     agent {
@@ -39,16 +56,37 @@ pipeline {
     
     environment {
         AGENT_ROOT_FOLDER = "${agent_root_folder}"
+// folder where all the pending deployment will be added
         PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/"
-        CD_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD"
+        ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-smartgears-services"
         PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
+// deploy file related to the current pipeline job
         DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv"
         BACKUP_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.bck"
+ // deploy folder that contains all the pending jobs associated to the current pipeline execution
         DEPLOY_FOLDER= "${WORKSPACE}/CD-${PIPELINE_BUILD_NUMBER}"
+  // artifacts coordinates
         TRIGGER_JOB= "${params.TRIGGER_JOB}"
         TRIGGER_VERSION= "${params.TRIGGER_VERSION}"
         TRIGGER_HOST="${params.TRIGGER_HOST}"
+  // enable the ContinuousDeployment if TRUE
         TRIGGER_CD="${params.TRIGGER_CD}"
+  // generic parameter that can be passed to the ansible job
+         TRIGGER_FOO="${params.TRIGGER_FOO}"
+ // Useful to identify the remote YAML file
+        GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
+ //Category name of the components candidates to the remote deployment
+        DEPLOY_CATEGORY="${env.DEPLOY_CATEGORY}"
+ //Job name of the release pipeline
+        RELEASE_JOB='Pipeline-gCubeBuilder'
+ //URL to the YAML file. Used alternatively to RELEASE_JOB
+        RELEASE_URL= "${params.RELEASE_URL}"
+ // username of ReleaseManager. Used for limiting the preproduction deployments
+ 		RELEASE_MANAGER_USER='giancarlo.panichi'
+ // name of Smartgears distribution group, defined in the yaml file ( only for release deployments)
+ 		DISTRIBUTION_CATEGORY='Distribution'
+ // name of Portlets group, defined in the yaml file ( only for release deployments)
+ 		PORTLET_CATEGORY='Portlets' 		
     }
 
     parameters {
@@ -64,10 +102,15 @@ pipeline {
          booleanParam(name: 'TRIGGER_CD',
                  defaultValue: true,
                  description: 'Set to false to avoid current deploy')
+        string(name: 'TRIGGER_FOO',
+                 defaultValue: '',
+                 description: 'Additional parameter to pass to the ansible job. Leave blank if not used')
         string(name: 'gCube_release_version',
                 defaultValue: '',
-                description: 'Leave blank if executed outside gCube release')
+                description: 'The number of the current gCube release. Leave blank if executed outside gCube release.')
-
+        string(name: 'RELEASE_URL',
+                defaultValue: '',
+                description: 'URL to the yaml file.  Leave blank if executed outside gCube release.')
     }
 
     stages {
@@ -83,34 +126,34 @@ pipeline {
         }
         stage('Deploy from system') {
             when{
-                allOf{
+            	anyOf{
-                    triggeredBy 'TimerTrigger'
+	                allOf{
-          // maybe we can add a new condition in order to consider the manual execution of this pipeline
+	                    triggeredBy 'TimerTrigger'
-                    environment name: 'IS_SCHEDULED', value: 'True'
+	                    environment name: 'IS_SCHEDULED', value: 'True'
-                }
+	                }
+            	}
             }
             steps {
-                echo 'Cron build enabled. Deploy from system ongoing'
+                echo 'Cron build enabled. \033[31;1;4mDeploy from system ongoing\033[0m'
                 script {
+                	echo "pipeline was triggered by ${params.TRIGGER_JOB}"
+            		println("Going to deploy all the pending deployments")
  					def deployFolder="CD-${env.BUILD_NUMBER}";
- 					deployPendingJobs(deployFolder);
+ 					parseDeployPendingJobs(deployFolder);
-
                 }
 
             }
         }
         stage('Nothing to do ') {
             when{
-                anyOf{
+                allOf{
-                    allOf{
+                    triggeredBy 'TimerTrigger'
-                        triggeredBy 'TimerTrigger'
+                    environment name: 'IS_SCHEDULED', value: 'False'
-                        environment name: 'IS_SCHEDULED', value: 'False'
-                    }
                 }
-
             }
             steps {
-                 echo 'Do Nothing: cron build disabled'
+                 echo '\033[31;1;4mGoing to sleep\033[0m'
+                 sh ' exit 0; '
             }
         }
         stage('New pending deploy ') {
@@ -121,10 +164,12 @@ pipeline {
                     triggeredBy 'UpstreamCause'
                     triggeredBy 'UserIdCause'
                 }
+                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
             }
             steps {
                 sh '''
-                   echo "Cron build enabled. New deploy of ${TRIGGER_JOB} - ${TRIGGER_VERSION} will be added to the deploy file"
+                   echo '\033[31;1;4mNew pending deploy/033[0m'
+                   echo "Cron build enabled. Adding ${TRIGGER_JOB} - ${TRIGGER_VERSION} to the queue"
                    touch $DEPLOY_FILE;
                    if grep -q \"\${TRIGGER_JOB}\" \${DEPLOY_FILE}; then
                        echo "component ${TRIGGER_JOB} already added. Nothing to add."                      
@@ -132,7 +177,8 @@ pipeline {
                        echo "${TRIGGER_JOB},${TRIGGER_VERSION},${TRIGGER_HOST}" >> ${DEPLOY_FILE}    
                    fi
                 '''
-          // the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed      
+          // the following catch give always an error in order to return a feedback UNSTABLE to the final user since the deploy is still not performed.
+          // Not sure this is the best approach
                 catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
                 	sh("""
                 		exit 126;
@@ -150,34 +196,62 @@ pipeline {
                     triggeredBy 'BuildUpstreamCause'
                     triggeredBy 'UpstreamCause'
                 }
+                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
             }
             steps {
-                 echo "Cron build disabled. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
+            	script{
-                 ansiColor("xterm") {
+            	     echo '\033[31;1;4mNew upstream deploy ongoing\033[0m'
-                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+	                 echo "Cron build disabled. Component: ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION}"
-                         checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
+	                 ansiColor('xterm') {
-                         deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
+    	                 catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
-                     }
+                             checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
-                 }
+                         }
+                         if (currentBuild.result == "UNSTABLE") {
+                             return;
+                         }
+                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                             deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}", "${params.TRIGGER_FOO}");
+                         }
+	                 }
+            	}
+
 
             }
         }
+        stage('Deploy from gCubeRelease ') {
+            when{
+                anyOf{
+                    triggeredBy 'BuildUpstreamCause'
+                    triggeredBy 'UpstreamCause'
+        // added just for testing . To Remove.
+                    triggeredBy  cause: "UserIdCause", detail: "${RELEASE_MANAGER_USER}"
+                }
+   		// we should improve this conditional check
+                equals(actual: "${params.TRIGGER_JOB}", expected: "${RELEASE_JOB}")
+            }
+            steps {
+            	script{
+            		if("${DEPLOY_CATEGORY}"){
+            			echo '\033[31;1;4mGoing to deploy gCube Release \033[0m'
+                		deployRelease("${DEPLOY_CATEGORY}", yamlConfig);
+            		}else println ("CATEGORIES NOT FOUND. ENVIRONMENT NOT SET FOR RELEASE DEPLOYMENT. NOTHING TO DO");
+            	}
+             }
+        }
         stage('Deploy manually ') {
             when{
                 allOf{
                     environment name: 'IS_SCHEDULED', value: 'False';
                     triggeredBy 'UserIdCause'
                 }
+                expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
             }
             steps {
+                 echo '\033[31;1;4mNew deploy ongoing manually triggered\033[0m'
                  echo "Cron build disabled. Pipeline executed Manually. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
-                 ansiColor("xterm") {
+                 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                     deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}", "${params.TRIGGER_FOO}");
-                         checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
-                         deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
-                     }
                  }
-
             }
         }
     }
@@ -187,21 +261,20 @@ pipeline {
                 sh '''
                   echo 'Sending report'
                   '''
-                  //cat ./${ACTION_DEPLOY_FILE}.csv
             }
         }
         success {
             echo 'The deploy pipeline worked!'
 
             emailext attachLog: true,//attachmentsPattern: "**/${ACTION_DEPLOY_FILE}.csv",
-                    to: 'roberto.cirillo@isti.cnr.it',
+                    to: 'jenkinsbuilds@d4science.org',
                     subject: "[Jenkins-gCubeDeployer] Deploy report",
                     body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}. "
         }
         failure {
             echo 'The deploy pipeline has failed'
             emailext attachLog: true,
-                    to: 'roberto.cirillo@isti.cnr.it',
+                    to: 'jenkinsbuilds@d4science.org',
                     subject: "[Jenkins-gCubeDeployer] deploy ${currentBuild.fullDisplayName} failed",
                     body: "Something is wrong with ${env.BUILD_URL}"
         }
@@ -209,16 +282,8 @@ pipeline {
     }
 }
 
-//launch ansible deploy
+
-def deploy(String service, String version, String host){
+/* CHECKUP METHODS */
-    def now = new Date();
-    println("Going to deploy the service "+service+" with version: "+version+" on target: "+host);
-    def statusCode = sh( script: "cd $CD_ROOT_FOLDER;./deployService.sh $service $version $host;", returnStdout: true);
-    sh("""
-        echo " last exit code \$?";
-    """)
-    println("Deploy ended with status: "+statusCode);
-}
 
 //Implement a new method in order to check the input parameters
 def checkup(String service, String version, String host){
@@ -231,9 +296,22 @@ def checkup(String service, String version, String host){
     """)
 }
 
+//Implement a new method in order to check the input parameters
+def checkupRelease(String service, String version){
+    sh("""
+        case "$version" in
+        **SNAPSHOT) echo "version contains SNAPSHOT. EXIT because this is a release build" ; exit 1;;
+        **latest) echo "version contains latest" ;;
+        *		  ) echo "version without SNAPSHOT. Go ahead";;
+        esac
+    """)
+}
+
+
+/* DEPLOY METHODS*/
 
 //parse all csv files found in the local folder and deploy the components defined inside
-def deployPendingJobs( def deployFolder){
+def parseDeployPendingJobs( def deployFolder){
 	println ("searching files in folder ${deployFolder}");
 	def files = findFiles(glob: "${deployFolder}/*.csv")
 	if (files == null){
@@ -244,23 +322,143 @@ def deployPendingJobs( def deployFolder){
 		    def records = readCSV file: "${file.path}"
             for (def record : records) {
                 println("Processing record: "+record)
-                if(!serviceList.contains(record.get(0))){
+                serviceList += "${record.get(0)},${record.get(1)},${record.get(2)}";
-                    stage(record.get(0)){
-                        println "Deploy on going of component: ${record.get(0)}"
-                        catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
-                            checkup(record.get(0), record.get(1), record.get(2));
-                            deploy(record.get(0), record.get(1), record.get(2));
-                        }
-                    }
-                    serviceList << record.get(0)
-                }else{
-                    echo "${record.get(0)} already deployed. Deployment skipped."
-                }
-
             }
             sh "rm ${file.path}"
         }
         sh "rm -Rf ${deployFolder}"
+        deployJobs(serviceList.unique())
 	}
 }
 
+//Deploy jobs from an input list with the following elements: serviceName,serviceVersion,targetHost. Create a new stage for any job
+def deployJobs(def serviceList){
+   for (def record : serviceList) {
+        service=record.split(",");
+		stage(service[0]){
+			println("Processing deploy: "+service[ 0 ])
+            catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
+                checkup(service[0], service[ 1 ], service[2]);
+           }
+            if (currentBuild.result == "UNSTABLE") {
+                return;
+            }
+            catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                deploy(service[0], service[ 1 ], service[2]);
+           }
+        }
+   }
+
+}
+
+//Run ansible deploy
+def deploy(String service, String version, String host, String foo){
+    def now = new Date();
+    println("Going to deploy the service "+service+" with version: "+version+" on target: "+host+" with argument: "+foo);
+    dir("${ANSIBLE_ROOT_FOLDER}/CD/"){
+        sh("""
+            git pull;
+        """)
+        def statusCode = sh( script: "./deployService.sh $service $version $host $foo;", returnStatus: true);
+        sh("""
+            echo " last exit code \$?";
+        """)
+        println("Deploy statusCode value: "+statusCode);
+        if (statusCode != 0){
+        	currentBuild.result = 'FAILURE'
+        	sh("""
+        		exit $statusCode;
+          	""")
+        }
+    }
+
+}
+
+
+/* DEPLOY RELEASE METHODS*/
+
+// collects all the components defined inside one or more yaml categories matching categoryList and invoke the deploy
+def deployRelease(def categoryList, def yaml){
+	def isPortlet= false
+	def releaseList = []
+	def portlets = []
+	String smartgears
+	def categories=categoryList.split(',')
+	yaml.gCube_release.Components.each { group_name, component_list ->
+		if (categories.contains(group_name)){
+			for ( def category : categories){
+				if("${group_name}" == "$category"){
+					if (category == "${DISTRIBUTION_CATEGORY}"){
+						//searching smartgears-distribution (os smartgears-distribution-legacy) version in Distribution component defined inside the yaml file
+						def sg= component_list?.collect{ if ("$it.name" == 'smartgears-distribution' || "$it.name" == 'smartgears-distribution-legacy' ){ return "$it.version"; }}
+						smartgears=(sg !=null ) ? sg[0] : ''
+					}else if (category =="${PORTLET_CATEGORY}"){
+						portlets += component_list?.collect{ if ("${it.group_id}" != "null") return  "{\"group_id\": \"$it.group_id\", \"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"} "; else return  "{\"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"}"}
+						def jsonContent= "{\"generic_portlets\": "+portlets+"}"
+						echo "json: $jsonContent"
+						writeJSON file: 'portlets.json', json: jsonContent
+						def output = sh(returnStdout: true, returnStdoutTrim: true, script: "mv portlets.json $ANSIBLE_ROOT_FOLDER")
+						isPortlet= true
+					}else{
+						releaseList += component_list?.collect {return "$it.name,$it.version" }
+					}
+					return;
+				}
+			}
+		}
+	}
+    deployReleaseJobs(releaseList, smartgears, isPortlet);
+}
+
+//Deploy only release components, starting from a servicelist. All the component are passed as: componentName,componentVersion
+def deployReleaseJobs(def serviceList, String smartgears, boolean isPortlet){
+   if (serviceList || isPortlet){
+      println("Going to deploy release components");	
+	   for (def record : serviceList) {
+	        service=record.split(",");
+			stage(service[0]){
+				println("Processing RELEASE deploy: service "+service[ 0 ]+" with version "+service[ 1 ]+" and SG "+smartgears)
+	            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+	                checkupRelease(service[0], service[ 1 ]);
+	                if (smartgears)
+	                	deployReleaseJob(service[0], service[ 1 ], smartgears);
+	                else
+	                	deployReleaseJob(service[0], service[ 1 ], '');
+	            }
+	        }
+	   }
+	   if (isPortlet){
+	   		stage("portlets"){
+	        	deployReleaseJob('generic-portlet', '','');
+	        }
+	   }
+	   
+   }else{
+       println("Release components not found. Skip this stage");
+       
+   }
+
+}
+
+//Run ansible deploy related to a single service
+def deployReleaseJob(String service, String version, String smartgears){
+    def now = new Date();
+    println("Going to deploy "+service+" "+version+" on preproduction ");
+    dir("${ANSIBLE_ROOT_FOLDER}/CD/"){
+        sh("""
+            git pull;
+        """)
+        def statusCode = sh( script: "./deployPreprodService.sh $service $version $smartgears ;", returnStatus: true);
+        sh("""
+            echo " last exit code \$?";
+        """)
+        println("Deploy ended with status: "+statusCode);
+        if (statusCode != 0){
+     	    currentBuild.result = 'FAILURE'
+       	    sh("""
+       		    exit $statusCode;
+       	    """)
+        }
+    }
+
+}

README.md

@@ -9,16 +9,42 @@ Jenkins Pipeline script to manage one or more gCube components deployment
 * [Pipeline: Basic Steps](https://plugins.jenkins.io/workflow-basic-steps) 
 * [Email Extension](https://plugins.jenkins.io/email-ext) (to send emails with attachments)
 * Jenkins agent(s) configured with Ansible ver. 2.9.27 or newer
-* One or more Jenkins agents labeled as 'CD'
+* One or more Jenkins agents labeled as 'ansible'
 
-## Expected parameters
+## Expected usage
+* ContinuousDeployment(CD): configured as post step build on jenkinsjob
+* Scheduled by system: In this case all the pending deployments (CD) will be managed by a temporized trigger 
+* Manually from Jenkins GUI
 
+## Expected environment variables
+The variables below must be specified as environment variables on the agent where the deployments will be executed:
+* IS_SCHEDULED (True | False): if true all the deployment requests will be performed periodically as specified by cron
+* ENVIRONMENT: ( DEV | PREPROD | PROD) Indicates the target environment where will be deployed all the artifacts
+* DEPLOY_CATEGORY: (Related to release build) one or more Component group (separated by commas) on yaml file where search the components to deploy
+* CD: (Continuous Deployment: True | False) if True the remote deploy will be performed
 
 ## References 
 * [Jenkins Pipeline](https://jenkins.io/doc/book/pipeline/) 
 * [Pipeline Syntax](https://jenkins.io/doc/book/pipeline/syntax/)
 
-## Wiki doc
+## Examples 
+* When configured as post step build on jenkinsjob, there are some variables to pass as triggered parameters. All the values are dinamically set as shown below:
+
+[[images/gCubeDeployer-JenkinsJob.png]]
+
+* The pipeline can run manually. This option is conceived for debug purpose:
+
+[[images/gCubeDeployer-manualRun.png]]
+
+* Follows a stage view example when the pipeline was scheduled by system, but there were any action to do:
+
+[[images/gCubeDeployer-stageview.png]]
+
+
+
+## Flow Chart
+
+[[images/FlowChartgCubeDeployer.jpeg]]
 
 
 ## License