Update Jenkinsfile

change email address to jenkinsbuilds@d4science.org

Jenkinsfile

@@ -1,8 +1,9 @@
 #!groovy
 import org.yaml.snakeyaml.Yaml
+import groovy.json.JsonSlurper
 
 /**
- * Deploy gCube components on D4Science Infrastructure (dev-only)
+ * Deploy gCube components on D4Science Infrastructure (dev and preprod environments)
  *
  * Roberto Cirillo (ISTI-CNR)
  */
@@ -57,7 +58,7 @@ pipeline {
         AGENT_ROOT_FOLDER = "${agent_root_folder}"
 // folder where all the pending deployment will be added
         PENDING_DEPLOY_FOLDER="${agent_root_folder}/CD/"
-        ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-ghn-cluster/CD"
+        ANSIBLE_ROOT_FOLDER = "${agent_root_folder}/ansible-repos/ansible-playbooks/d4science-smartgears-services"
         PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
 // deploy file related to the current pipeline job
         DEPLOY_FILE = "${PENDING_DEPLOY_FOLDER}deploy.${PIPELINE_BUILD_NUMBER}.csv"
@@ -70,18 +71,22 @@ pipeline {
         TRIGGER_HOST="${params.TRIGGER_HOST}"
   // enable the ContinuousDeployment if TRUE
         TRIGGER_CD="${params.TRIGGER_CD}"
+  // generic parameter that can be passed to the ansible job
+         TRIGGER_FOO="${params.TRIGGER_FOO}"
  // Useful to identify the remote YAML file
         GCUBE_RELEASE_NUMBER = "${params.gCube_release_version}"
  //Category name of the components candidates to the remote deployment
         DEPLOY_CATEGORY="${env.DEPLOY_CATEGORY}"
  //Job name of the release pipeline
-        RELEASE_JOB='gCubeBuilder'
+        RELEASE_JOB='Pipeline-gCubeBuilder'
  //URL to the YAML file. Used alternatively to RELEASE_JOB
         RELEASE_URL= "${params.RELEASE_URL}"
  // username of ReleaseManager. Used for limiting the preproduction deployments
- 		RELEASE_MANAGER_USER='roberto.cirillo'
+ 		RELEASE_MANAGER_USER='giancarlo.panichi'
  // name of Smartgears distribution group, defined in the yaml file ( only for release deployments)
  		DISTRIBUTION_CATEGORY='Distribution'
+ // name of Portlets group, defined in the yaml file ( only for release deployments)
+ 		PORTLET_CATEGORY='Portlets' 		
     }
 
     parameters {
@@ -97,12 +102,12 @@ pipeline {
          booleanParam(name: 'TRIGGER_CD',
                  defaultValue: true,
                  description: 'Set to false to avoid current deploy')
+        string(name: 'TRIGGER_FOO',
+                 defaultValue: '',
+                 description: 'Additional parameter to pass to the ansible job. Leave blank if not used')
         string(name: 'gCube_release_version',
                 defaultValue: '',
-                description: 'The number of the gCube release. Leave blank if executed outside gCube release.')
+                description: 'The number of the current gCube release. Leave blank if executed outside gCube release.')
-        string(name: 'report_number',
-                defaultValue: '',
-                description: 'The build report number on Git to pull the notes. Leave blank if executed outside gCube release.')
         string(name: 'RELEASE_URL',
                 defaultValue: '',
                 description: 'URL to the yaml file.  Leave blank if executed outside gCube release.')
@@ -124,12 +129,12 @@ pipeline {
             	anyOf{
 	                allOf{
 	                    triggeredBy 'TimerTrigger'
-	                    environment name: 'IS_SCHEDULED_TEST', value: 'True'
+	                    environment name: 'IS_SCHEDULED', value: 'True'
 	                }
             	}
             }
             steps {
-                echo 'Cron build enabled. Deploy from system ongoing'
+                echo 'Cron build enabled. \033[31;1;4mDeploy from system ongoing\033[0m'
                 script {
                 	echo "pipeline was triggered by ${params.TRIGGER_JOB}"
             		println("Going to deploy all the pending deployments")
@@ -141,22 +146,19 @@ pipeline {
         }
         stage('Nothing to do ') {
             when{
-                anyOf{
+                allOf{
-                    allOf{
+                    triggeredBy 'TimerTrigger'
-                        triggeredBy 'TimerTrigger'
+                    environment name: 'IS_SCHEDULED', value: 'False'
-                        environment name: 'IS_SCHEDULED_TEST', value: 'False'
-                    }
                 }
-
             }
             steps {
-                 echo 'Going to sleep'
+                 echo '\033[31;1;4mGoing to sleep\033[0m'
-                 sh ' exit 1; ' 
+                 sh ' exit 0; '
             }
         }
         stage('New pending deploy ') {
             when{
-                environment name: 'IS_SCHEDULED_TEST', value: 'True'
+                environment name: 'IS_SCHEDULED', value: 'True'
                 anyOf{
                     triggeredBy 'BuildUpstreamCause'
                     triggeredBy 'UpstreamCause'
@@ -166,7 +168,8 @@ pipeline {
             }
             steps {
                 sh '''
-                   echo "Cron build enabled. New deploy of ${TRIGGER_JOB} - ${TRIGGER_VERSION} will be added to the pending deploy file"
+                   echo '\033[31;1;4mNew pending deploy/033[0m'
+                   echo "Cron build enabled. Adding ${TRIGGER_JOB} - ${TRIGGER_VERSION} to the queue"
                    touch $DEPLOY_FILE;
                    if grep -q \"\${TRIGGER_JOB}\" \${DEPLOY_FILE}; then
                        echo "component ${TRIGGER_JOB} already added. Nothing to add."                      
@@ -185,7 +188,7 @@ pipeline {
         }
         stage('Deploy from job ') {
             when{
-                environment name: 'IS_SCHEDULED_TEST', value: 'False';
+                environment name: 'IS_SCHEDULED', value: 'False';
                 expression {
                     env.TRIGGER_CD.toBoolean()
             	}
@@ -197,12 +200,18 @@ pipeline {
             }
             steps {
             	script{
-	                 echo "Cron build disabled. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
+            	     echo '\033[31;1;4mNew upstream deploy ongoing\033[0m'
-	                 ansiColor("xterm") {
+	                 echo "Cron build disabled. Component: ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION}"
-	                 	 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+	                 ansiColor('xterm') {
-	                         checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
+    	                 catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
-	                         deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
+                             checkup("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
-	                     }
+                         }
+                         if (currentBuild.result == "UNSTABLE") {
+                             return;
+                         }
+                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                             deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}", "${params.TRIGGER_FOO}");
+                         }
 	                 }
             	}
 
@@ -214,37 +223,35 @@ pipeline {
                 anyOf{
                     triggeredBy 'BuildUpstreamCause'
                     triggeredBy 'UpstreamCause'
-               //added just for testing . To Remove.     
+        // added just for testing . To Remove.
                     triggeredBy  cause: "UserIdCause", detail: "${RELEASE_MANAGER_USER}"
                 }
-   // we should improve this conditional check             
+   		// we should improve this conditional check
                 equals(actual: "${params.TRIGGER_JOB}", expected: "${RELEASE_JOB}")
             }
             steps {
             	script{
             		if("${DEPLOY_CATEGORY}"){
-                		println("Going to deploy the Release components")
+            			echo '\033[31;1;4mGoing to deploy gCube Release \033[0m'
                 		deployRelease("${DEPLOY_CATEGORY}", yamlConfig);
-            		}else { println ("CATEGORIES NOT FOUND ON YAML FILE. SKIP TO NEXT STAGE");}
+            		}else println ("CATEGORIES NOT FOUND. ENVIRONMENT NOT SET FOR RELEASE DEPLOYMENT. NOTHING TO DO");
             	}
              }
         }
         stage('Deploy manually ') {
             when{
                 allOf{
-                    environment name: 'IS_SCHEDULED_TEST', value: 'False';
+                    environment name: 'IS_SCHEDULED', value: 'False';
                     triggeredBy 'UserIdCause'
                 }
                 expression { params.TRIGGER_JOB != "${RELEASE_JOB}" }
             }
             steps {
+                 echo '\033[31;1;4mNew deploy ongoing manually triggered\033[0m'
                  echo "Cron build disabled. Pipeline executed Manually. New deploy of ${params.TRIGGER_JOB} - ${params.TRIGGER_VERSION} ongoing"
-                 ansiColor("xterm") {
+                 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                     deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}", "${params.TRIGGER_FOO}");
-                         deploy("${params.TRIGGER_JOB}", "${params.TRIGGER_VERSION}", "${params.TRIGGER_HOST}");
-                     }
                  }
-
             }
         }
     }
@@ -260,30 +267,23 @@ pipeline {
             echo 'The deploy pipeline worked!'
 
             emailext attachLog: true,//attachmentsPattern: "**/${ACTION_DEPLOY_FILE}.csv",
-                    to: 'roberto.cirillo@isti.cnr.it',
+                    to: 'jenkinsbuilds@d4science.org',
                     subject: "[Jenkins-gCubeDeployer] Deploy report",
                     body: "${currentBuild.fullDisplayName}. Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}. "
         }
         failure {
             echo 'The deploy pipeline has failed'
             emailext attachLog: true,
-                    to: 'roberto.cirillo@isti.cnr.it',
+                    to: 'jenkinsbuilds@d4science.org',
                     subject: "[Jenkins-gCubeDeployer] deploy ${currentBuild.fullDisplayName} failed",
                     body: "Something is wrong with ${env.BUILD_URL}"
         }
 
     }
 }
-//launch ansible deploy
+
-def deploy(String service, String version, String host){
+
-    def now = new Date();
+/* CHECKUP METHODS */
-    println("Going to deploy the service "+service+" with version: "+version+" on target: "+host);
-    def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;git pull;./deployService.sh $service $version $host;", returnStdout: true);
-    sh("""
-        echo " last exit code \$?";
-    """)
-    println("Deploy ended with status: "+statusCode);
-}
 
 //Implement a new method in order to check the input parameters
 def checkup(String service, String version, String host){
@@ -296,17 +296,6 @@ def checkup(String service, String version, String host){
     """)
 }
 
-//launch ansible deploy
-def deployRelease(String service, String version, String smartgears){
-    def now = new Date();
-    println("Going to deploy the service "+service+" with version: "+version+" on preproduction ");
-    def statusCode = sh( script: "cd $ANSIBLE_ROOT_FOLDER;git pull;./deployPreprodService.sh $service $version $smartgears ;", returnStdout: true);
-    sh("""
-        echo " last exit code \$?";
-    """)
-    println("Deploy ended with status: "+statusCode);
-}
-
 //Implement a new method in order to check the input parameters
 def checkupRelease(String service, String version){
     sh("""
@@ -317,6 +306,10 @@ def checkupRelease(String service, String version){
         esac
     """)
 }
+
+
+/* DEPLOY METHODS*/
+
 //parse all csv files found in the local folder and deploy the components defined inside
 def parseDeployPendingJobs( def deployFolder){
 	println ("searching files in folder ${deployFolder}");
@@ -337,6 +330,7 @@ def parseDeployPendingJobs( def deployFolder){
         deployJobs(serviceList.unique())
 	}
 }
+
 //Deploy jobs from an input list with the following elements: serviceName,serviceVersion,targetHost. Create a new stage for any job
 def deployJobs(def serviceList){
    for (def record : serviceList) {
@@ -345,31 +339,49 @@ def deployJobs(def serviceList){
 			println("Processing deploy: "+service[ 0 ])
             catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
                 checkup(service[0], service[ 1 ], service[2]);
-                deploy(service[0], service[ 1 ], service[2]);
+           }
+            if (currentBuild.result == "UNSTABLE") {
+                return;
             }
+            catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                deploy(service[0], service[ 1 ], service[2]);
+           }
         }
    }
 
 }
-//Deploy only release components. All the component are passed as: componentName,componentVersion
+
-def deployReleaseJobs(def serviceList, String smartgears){
+//Run ansible deploy
-   for (def record : serviceList) {
+def deploy(String service, String version, String host, String foo){
-        service=record.split(",");
+    def now = new Date();
-		stage(service[0]){
+    println("Going to deploy the service "+service+" with version: "+version+" on target: "+host+" with argument: "+foo);
-			println("Processing RELEASE deploy: service "+service[ 0 ]+" with version "+service[ 1 ]+" and SG "+smartgears)
+    dir("${ANSIBLE_ROOT_FOLDER}/CD/"){
-            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+        sh("""
-                checkupRelease(service[0], service[ 1 ]);
+            git pull;
-                if (smartgears)
+        """)
-                	deployRelease(service[0], service[ 1 ], smartgears);
+        def statusCode = sh( script: "./deployService.sh $service $version $host $foo;", returnStatus: true);
-                else
+        sh("""
-                	deployRelease(service[0], service[ 1 ], null);	
+            echo " last exit code \$?";
-            }
+        """)
+        println("Deploy statusCode value: "+statusCode);
+        if (statusCode != 0){
+        	currentBuild.result = 'FAILURE'
+        	sh("""
+        		exit $statusCode;
+          	""")
         }
-   }
+    }
+
 }
-// collects all the components defined inside one or more yaml categories matching categoryList
+
+
+/* DEPLOY RELEASE METHODS*/
+
+// collects all the components defined inside one or more yaml categories matching categoryList and invoke the deploy
 def deployRelease(def categoryList, def yaml){
+	def isPortlet= false
 	def releaseList = []
+	def portlets = []
 	String smartgears
 	def categories=categoryList.split(',')
 	yaml.gCube_release.Components.each { group_name, component_list ->
@@ -377,9 +389,16 @@ def deployRelease(def categoryList, def yaml){
 			for ( def category : categories){
 				if("${group_name}" == "$category"){
 					if (category == "${DISTRIBUTION_CATEGORY}"){
-						//searching smartgears-distribution version in Distribution component of defined inside the yaml file
+						//searching smartgears-distribution (os smartgears-distribution-legacy) version in Distribution component defined inside the yaml file
-						def sg= component_list?.collect{ if ("$it.name" == 'smartgears-distribution'){ return "$it.version"; }}
+						def sg= component_list?.collect{ if ("$it.name" == 'smartgears-distribution' || "$it.name" == 'smartgears-distribution-legacy' ){ return "$it.version"; }}
-						smartgears=(sg !=null ) ? sg[0] : null
+						smartgears=(sg !=null ) ? sg[0] : ''
+					}else if (category =="${PORTLET_CATEGORY}"){
+						portlets += component_list?.collect{ if ("${it.group_id}" != "null") return  "{\"group_id\": \"$it.group_id\", \"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"} "; else return  "{\"name\": \"$it.name\", \"version\": \"$it.version\", \"extension\": \"war\"}"}
+						def jsonContent= "{\"generic_portlets\": "+portlets+"}"
+						echo "json: $jsonContent"
+						writeJSON file: 'portlets.json', json: jsonContent
+						def output = sh(returnStdout: true, returnStdoutTrim: true, script: "mv portlets.json $ANSIBLE_ROOT_FOLDER")
+						isPortlet= true
 					}else{
 						releaseList += component_list?.collect {return "$it.name,$it.version" }
 					}
@@ -388,5 +407,58 @@ def deployRelease(def categoryList, def yaml){
 			}
 		}
 	}
-    deployReleaseJobs(releaseList, smartgears);                                    
+    deployReleaseJobs(releaseList, smartgears, isPortlet);
+}
+
+//Deploy only release components, starting from a servicelist. All the component are passed as: componentName,componentVersion
+def deployReleaseJobs(def serviceList, String smartgears, boolean isPortlet){
+   if (serviceList || isPortlet){
+      println("Going to deploy release components");	
+	   for (def record : serviceList) {
+	        service=record.split(",");
+			stage(service[0]){
+				println("Processing RELEASE deploy: service "+service[ 0 ]+" with version "+service[ 1 ]+" and SG "+smartgears)
+	            catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+	                checkupRelease(service[0], service[ 1 ]);
+	                if (smartgears)
+	                	deployReleaseJob(service[0], service[ 1 ], smartgears);
+	                else
+	                	deployReleaseJob(service[0], service[ 1 ], '');
+	            }
+	        }
+	   }
+	   if (isPortlet){
+	   		stage("portlets"){
+	        	deployReleaseJob('generic-portlet', '','');
+	        }
+	   }
+	   
+   }else{
+       println("Release components not found. Skip this stage");
+       
+   }
+
+}
+
+//Run ansible deploy related to a single service
+def deployReleaseJob(String service, String version, String smartgears){
+    def now = new Date();
+    println("Going to deploy "+service+" "+version+" on preproduction ");
+    dir("${ANSIBLE_ROOT_FOLDER}/CD/"){
+        sh("""
+            git pull;
+        """)
+        def statusCode = sh( script: "./deployPreprodService.sh $service $version $smartgears ;", returnStatus: true);
+        sh("""
+            echo " last exit code \$?";
+        """)
+        println("Deploy ended with status: "+statusCode);
+        if (statusCode != 0){
+     	    currentBuild.result = 'FAILURE'
+       	    sh("""
+       		    exit $statusCode;
+       	    """)
+        }
+    }
+
 }

README.md

@@ -9,20 +9,42 @@ Jenkins Pipeline script to manage one or more gCube components deployment
 * [Pipeline: Basic Steps](https://plugins.jenkins.io/workflow-basic-steps) 
 * [Email Extension](https://plugins.jenkins.io/email-ext) (to send emails with attachments)
 * Jenkins agent(s) configured with Ansible ver. 2.9.27 or newer
-* One or more Jenkins agents labeled as 'CD'
+* One or more Jenkins agents labeled as 'ansible'
 
 ## Expected usage
 * ContinuousDeployment(CD): configured as post step build on jenkinsjob
 * Scheduled by system: In this case all the pending deployments (CD) will be managed by a temporized trigger 
 * Manually from Jenkins GUI
 
-## Expected usage
+## Expected environment variables
+The variables below must be specified as environment variables on the agent where the deployments will be executed:
+* IS_SCHEDULED (True | False): if true all the deployment requests will be performed periodically as specified by cron
+* ENVIRONMENT: ( DEV | PREPROD | PROD) Indicates the target environment where will be deployed all the artifacts
+* DEPLOY_CATEGORY: (Related to release build) one or more Component group (separated by commas) on yaml file where search the components to deploy
+* CD: (Continuous Deployment: True | False) if True the remote deploy will be performed
 
 ## References 
 * [Jenkins Pipeline](https://jenkins.io/doc/book/pipeline/) 
 * [Pipeline Syntax](https://jenkins.io/doc/book/pipeline/syntax/)
 
-## Wiki doc
+## Examples 
+* When configured as post step build on jenkinsjob, there are some variables to pass as triggered parameters. All the values are dinamically set as shown below:
+
+[[images/gCubeDeployer-JenkinsJob.png]]
+
+* The pipeline can run manually. This option is conceived for debug purpose:
+
+[[images/gCubeDeployer-manualRun.png]]
+
+* Follows a stage view example when the pipeline was scheduled by system, but there were any action to do:
+
+[[images/gCubeDeployer-stageview.png]]
+
+
+
+## Flow Chart
+
+[[images/FlowChartgCubeDeployer.jpeg]]
 
 
 ## License