gCubeDepsWalker/Jenkinsfile

def agent_root_folder = '/var/lib/jenkins/.m2'
def projects2artifacts = [:]
def artifacts2projects = [:]
def modules2deps = [:]
def alreadyInTheTree = []
def inputProject = params.jenkins_project.trim()
def report = [:]

report['project'] = inputProject

pipeline {
    agent {
        label 'CD'
    }
    
    environment {
        AGENT_ROOT_FOLDER = "${agent_root_folder}"
        PIPELINE_BUILD_NUMBER = "${env.BUILD_NUMBER}"
        WALKER_NOTES = "${agent_root_folder}/walker_notes.${inputProject}.md"
    }
    
    
    parameters {
        string(name: 'jenkins_project',
                defaultValue: '',
                description: 'The name of the Jenkins project to analyze.')
    }
    stages {

        stage('walking projects') {
            steps {
                script {
                    Jenkins.get().getAllItems(TopLevelItem.class).each { p ->
                        projects2artifacts[p.name] = []
                        p.getAllJobs().each { j -> projects2artifacts[p.name] << j.name; artifacts2projects[j.name] = p.name }
                    }
                    println "FOUND ${projects2artifacts.size()} projects"
                    //projects2artifacts.each { k,v -> println ("PROJECT ${k} BUILDS ${v}") }
                }
            }
        }

        stage('walking maven modules') {
            steps {
                script {
                    // get all the maven modules and their dependencies
                    Jenkins.get().getAllItems(hudson.maven.MavenModule.class).each { m ->
                        modules2deps[m.name] = []
                        m.getDependencies().each { d -> modules2deps[m.name] << "${d.groupId}:${d.artifactId}" }
                    }
                    println "FOUND ${modules2deps.size()} modules"
                    //modules2deps.each { k,v -> println ("MODULE ${k} DEPENDS on ${v}") }
                }
            }
        }

        stage('analyze downstream projects') {
            steps {
                script {
                    // println "PROJECT ${inputProject} BUILDS ${projects2artifacts[inputProject]} artifacts"
                    // first, let's find out what components depend on the project's artifacts (i.e. downstream dependencies)
                    report['downstream_modules'] = [:]
                    report['downstream_projects'] = []
                    for ( level in 1..50 ) report['downstream_modules']["L${level}"] = [:] // can't initialize with withDefault closure in jenkins
                    projects2artifacts[inputProject].each { a ->
                         if (a.split(':').length > 1) { //skip the parent (only groupId)
                            report = analyzeDependency(modules2deps, alreadyInTheTree, artifacts2projects, report, a, 1)
                         }
                    }
                }
            }
        }

        stage('print report') {
            steps {
                script {
                    printReport(report)
//                    saveReport(report)
                }
            }
        }
    }
        // post-build actions
    post {
        always {
            script {
                sh '''
                  cp $WALKER_NOTES ./walker_notes.${PIPELINE_BUILD_NUMBER}.md
                  cat ./walker_notes.${PIPELINE_BUILD_NUMBER}.md
                  '''
            }
        }
        success {
            echo 'The dependencies walker pipeline worked!'
            emailext to: 'jenkinsbuilds@d4science.org',
                    subject: "[Jenkins WalkerPipeline D4S] build ${currentBuild.fullDisplayName} worked",
                    body: "Build time: ${currentBuild.durationString}. See ${env.BUILD_URL}"
            emailext attachmentsPattern: "**/walker_notes.${PIPELINE_BUILD_NUMBER}.md",
                    to: 'jenkinsreleases@d4science.org',
                    subject: "Dependency Report for gCube component ${inputProject} (build #${PIPELINE_BUILD_NUMBER})",
                    body: "Downstream dependencies of ${inputProject}. See ${env.BUILD_URL}"
        }
        failure {
            echo 'The dependencies walker pipeline has failed'
            emailext attachLog: true,
                    to: 'jenkinsbuilds@d4science.org',
                    subject: "[JenkinsDependenciesWalker D4S] build ${currentBuild.fullDisplayName} failed for component  ${inputProject}",
                    body: "Something is wrong with ${env.BUILD_URL}"
        }
    }

}
// look for modules that use this artifact
def findDownstreamDependencies(modules2deps, artifact) {
    def downdeps = []
    //println "Looking for users of ${artifact}"
    modules2deps.each { k, v -> if (v.contains("${artifact}")) downdeps << k }
    return downdeps
}

//build the report of the given dependency, go recursive on its dependencies
def analyzeDependency(modules2deps, alreadyInTheTree, artifacts2projects, report, artifact, depth) {
    def level = "L${depth}"
    def downdeps = findDownstreamDependencies(modules2deps, artifact)
    def skipTree = alreadyInTheTree.contains(artifact)
    if (!skipTree) {
        report['downstream_modules'][level][artifact] = ['dependencies': [], 'projects': []]

        // get all downstream dependencies
        report['downstream_modules'][level][artifact]['dependencies'].addAll(downdeps)
        //println "${artifact} is used by ${report['downstream_modules'][level][artifact]['dependencies']}"

        // add the project that builds the artifact
        report['downstream_modules'][level][artifact]['projects'] << artifacts2projects[artifact]
        report['downstream_projects'] << artifacts2projects[artifact]
        alreadyInTheTree << artifact
    }

    // then we look for the projects that build the downstream dependencies and finally we go recursive
    if (downdeps.size() > 0) {
        def nextDepth = ++depth
        downdeps.each { d ->
            if (!skipTree) {
                report['downstream_modules'][level][artifact]['projects'] << artifacts2projects[d]
                report['downstream_projects'] << artifacts2projects[d]
            }
            // go recursive
            analyzeDependency(modules2deps, alreadyInTheTree, artifacts2projects, report, d, nextDepth)
        }
    }
    report
}

// print the final report
def printReport(report) {
    def text = ''
    def indent = '\t'
    text += "Dependency Report for ${report['project']}"
    text += "\n\n"
    text += "|--Project Maven Modules\n"
    report['downstream_modules'].each { depth, artifacts ->
        artifacts.each { name, data ->
            text += "${indent}|--Module: ${name}\n"
            text += "${indent * 2}|--Deepest Dependency Level: ${depth}\n"
            text += "${indent * 2}|--Used by (Maven Modules)\n"
            data['dependencies'].each { d ->
                text += "${indent * 3}|--${d}"
                text += "\n"
            }
            text += "${indent * 2}|--Referred by (Jenkins Projects)\n"
            data['projects'].each { p ->
                text += "${indent * 3}|--${p}"
                text += "\n"
            }
        }
    }

    text += "\n\n"
    text += "|--All Downstream Projects\n"
    report['downstream_projects'].unique().sort()
    report['downstream_projects'].each { p ->
        text += "${indent}|--${p}"
        text += "\n"
    }
    println text
    sh("""
        printf "%s" $text > $WALKER_NOTES
    """)    
    
}

// print the final report
def saveReport(report) {
    def text = ''
    def indent = '\t'
    sh("""
	    echo "Dependency Report for ${report['project']}" > $WALKER_NOTES
	    echo  "\n\n" >>   $WALKER_NOTES
	    echo "|--Project Maven Modules " >> $WALKER_NOTES
	""")    
	report['downstream_modules'].each { depth, artifacts ->
	    artifacts.each { name, data ->
	        sh("""
	            echo "${indent}|--Module: ${name}\n" >>   $WALKER_NOTES
	            echo "${indent * 2}|--Deepest Dependency Level: ${depth}\n" >>   $WALKER_NOTES
	            echo "${indent * 2}|--Used by (Maven Modules)\n" >>   $WALKER_NOTES
	        """)    
	        data['dependencies'].each { d ->
	            sh("""
	                echo "${indent * 3}|--${d}" >>   $WALKER_NOTES
	                echo "\n"  >>   $WALKER_NOTES
	            """)    
	        }
	        sh("""
	            echo "${indent * 2}|--Referred by (Jenkins Projects)\n" >>   $WALKER_NOTES
	        """)    
	        data['projects'].each { p ->
	            sh("""
	                echo "${indent * 3}|--${p}" >>   $WALKER_NOTES
	                echo  "\n" >>   $WALKER_NOTES
	            """)    
	        }
	    }
	}
	sh("""
	    echo "\n\n"  >>   $WALKER_NOTES
	    echo "|--All Downstream Projects\n" >>   $WALKER_NOTES
	""")    
	report['downstream_projects'].unique().sort()
	report['downstream_projects'].each { p ->
	    sh("""
	         echo "${indent}|--${p}" >> $WALKER_NOTES
	         echo "\n" >> $WALKER_NOTES
	     """)    
	}
 
}




// debug job
def printJob(job) {
    println("fullname ${job.fullName}")
    println("name ${job.name}")
    println("display name ${job.displayName}")
    job.getAllJobs().each { j -> println("dep: ${j.name}") }
}