Setup the jenkins agent in kubernetes cluster use pods

Supplement

  • When you upgrade the jenkins.war in docker container, commit to another tag the Entrypoint and CMD is lost
  • So can commit specific the Entrypoint and CMD
1docker commit --change='ENTRYPOINT [/usr/bin/tini", "--", "/usr/local/bin/jenkins.sh"]' --change='CMD [/usr/local/bin/jenkins.sh"]' <container_id> new_image_name

Basic settings

  • Add Global credentials the Secret is kubernetes login token
  • Add new cloud configure in Manage Jenkins –> Clouds
  • Config the Kubernetes Clouds websocket connection, use internal jenkins URL, specific the Pod Labels, Pod Retention: On Failure, Max connections, Seconds to wait for pod
  • Add Pod template settings
    • Specific Namespace, Usage: Only build jobs with label expressions matching this node
    • Container Template:
      • jnlp: jenkins/inbound-agent:latest,
      • Working directory: /home/jenkins/agent
      • Command to run: /usr/local/bin/jenkins-agent
    • You can specific Volumes to mount cache etc. NPM, Maven, Nuget, PIP
    • You can config more for pod template, merge the raw yaml to tolerations
       1 apiVersion: v1
       2 kind: Pod
       3 spec:
       4 affinity: 
       5     priorityClassName: normal
       6     nodeAffinity:    
       7         requiredDuringSchedulingIgnoredDuringExecution:
       8         nodeSelectorTerms:
       9         - matchExpressions:                    
      10             - key: node-group
      11             operator: In
      12             values:
      13             - spotInstances
      14 tolerations:
      15 - key: "devops"
      16     operator: "Equal"
      17     value: "cicd"
      18     effect: "NoSchedule"
      

Write the pipeline use podtemplate

 1node("podTemplateName") {
 2    
 3    def rawData = "${params.DATA}"
 4    def mapData = readJSON text: rawData
 5    def AutoDeployRepoMap = [
 6        "https://local.test.git": "xxxxxxxx"
 7    ]
 8
 9    container("podContainerName") {
10        withCredentials([string(credentialsId: 'task-token', variable: 'TOKEN')]) {
11            def git_url = mapData['project']['git_http_url']
12            if(AutoDeployRepoMap.containsKey(git_url)) {
13                deployId = AutoDeployRepoMap["${git_url}"]                    
14                response = sh(script: "curl -X PUT -H \"Authorization: Bearer ${TOKEN}\"  https://local.test.dev/api/v1/app/${deployId}/deploy?branch=${tag}", returnStdout: true)   
15                echo "response: ${response}"
16            }
17        }
18    }
19
20}

Or inherit podtemplate to change the configure

 1def CONTAINERS = [
 2    containerTemplate(args: '', command: 'sh', image: "${params.SONAR_SCAN_BASE_IMAGE}", livenessProbe: containerLivenessProbe(execArgs: '', failureThreshold: 0, initialDelaySeconds: 0, periodSeconds: 0, successThreshold: 0, timeoutSeconds: 0), name: 'sonarscan', resourceLimitCpu: '2000m', resourceLimitEphemeralStorage: '', resourceLimitMemory: '4Gi', resourceRequestCpu: '500m', resourceRequestEphemeralStorage: '', resourceRequestMemory: '500Mi', ttyEnabled: true, workingDir: '/home/jenkins/agent')
 3]
 4def GIT_COMMIT_EMAIL = ""
 5
 6if(params.CODE_LANGUAGE == 'java' && params.LANGUAGE_VERSION == '8') {
 7    CONTAINERS << containerTemplate(args: '', command: 'sh', image: "local.docker.dev/sonarsource/sonar-scanner-cli:4.7.0-java8", livenessProbe: containerLivenessProbe(execArgs: '', failureThreshold: 0, initialDelaySeconds: 0, periodSeconds: 0, successThreshold: 0, timeoutSeconds: 0), name: 'compile', resourceLimitCpu: '2000m', resourceLimitEphemeralStorage: '', resourceLimitMemory: '4Gi', resourceRequestCpu: '500m', resourceRequestEphemeralStorage: '', resourceRequestMemory: '500Mi', ttyEnabled: true, workingDir: '/home/jenkins/agent')
 8}
 9
10podTemplate(cloud: 'k8s-cluster', containers: CONTAINERS, inheritFrom: 'podTemplateName', label: "${params.PROJECT_NAME}", name: "${params.PROJECT_NAME}", namespace: 'cicd') {
11
12    def buildResult = 'Success'
13    node("${params.PROJECT_NAME}") {
14        try{
15            stage "Pull source code from ${params.PROJECT_NAME}"
16            dir("source_code") {
17                git branch: "${params.BRANCH_NAME}", url: "${params.REPO_URL}", credentialsId: "git_token"
18                GIT_COMMIT_EMAIL = sh (
19                        script: 'git --no-pager show -s --format=\'%ae\'',
20                        returnStdout: true
21                ).trim().replaceAll(/@.*/, "@local.test.dev")
22                echo "=====>${GIT_COMMIT_EMAIL}<======="
23                sh "ls -alh"
24            }
25        }catch(Exception e) {
26            buildResult = "Failed"
27        }finally{
28            echo 'finally'
29        }
30    }
31}
32        

Dynamic change the shared library in script console

 1#!groovy
 2
 3// imports
 4import hudson.scm.SCM
 5import jenkins.model.Jenkins
 6import jenkins.plugins.git.GitSCMSource
 7import org.jenkinsci.plugins.workflow.libs.*
 8import org.jenkinsci.plugins.workflow.libs.LibraryConfiguration
 9import org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever
10
11// parameters
12def globalLibrariesParameters = [
13  branch:               "master",
14  credentialId:         "global-shared-library-key",
15  implicit:             false,
16  name:                 "Your Global Shared Library name here",
17  repository:           "[email protected]:your-company/your-repo.git"
18  path: "/shared-library"
19]
20
21// define global library
22GitSCMSource gitSCMSource = new GitSCMSource(
23  "global-shared-library",
24  globalLibrariesParameters.repository,
25  globalLibrariesParameters.credentialId,
26  "*",
27  "",
28  false
29)
30
31// define retriever
32SCMSourceRetriever sCMSourceRetriever = new SCMSourceRetriever(gitSCMSource)
33sCMSourceRetriever.setLibraryPath(globalLibrariesParameters.path)
34// get Jenkins instance
35Jenkins jenkins = Jenkins.getInstance()
36
37// get Jenkins Global Libraries
38def globalLibraries = jenkins.getDescriptor("org.jenkinsci.plugins.workflow.libs.GlobalLibraries")
39
40// define new library configuration
41LibraryConfiguration libraryConfiguration = new LibraryConfiguration(globalLibrariesParameters.name, sCMSourceRetriever)
42libraryConfiguration.setDefaultVersion(globalLibrariesParameters.branch)
43libraryConfiguration.setImplicit(globalLibrariesParameters.implicit)
44
45// set new Jenkins Global Library
46globalLibraries.get().setLibraries([libraryConfiguration])
47
48// save current Jenkins state to disk
49jenkins.save()

Reference