Modify Jenkins deployment script to make more stable
1. Remove the SR-IOV Plugin Installation
> Since our cni-config is stable,
we only need to install the plugin config once,
if we delete them and install again, it may cause built-in multus cni crash.
2. Add waituntil on both BBU and SPGW-U
> Although I upgrade the ixgbe driver version in our cluster,
but it still have the probability to crash the node.
So I would prefer to use the wait loop for doing the cable trick.
If the POD works, then this waituntil loop will be ignored.
Change-Id: Ia767865c0b8efe9e93645fe062c4e6bd09c96617
diff --git a/Jenkinsfile-mcord-local-build b/Jenkinsfile-mcord-local-build
index 85f2de4..609080f 100644
--- a/Jenkinsfile-mcord-local-build
+++ b/Jenkinsfile-mcord-local-build
@@ -20,7 +20,6 @@
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-repo-tools"
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-charts"
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
- //sh returnStdout: true, script: "cd ${configBaseDir}; git fetch https://gerrit.opencord.org/pod-configs refs/changes/02/12902/1 && git checkout FETCH_HEAD; cd -"
deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
}
stage('Clean up') {
@@ -44,22 +43,22 @@
}
// In Multi-cluster scenario, we only need to install SR-IOV on data plane cluster
- if ( ! params.installEpcControlPlane ) {
- timeout(5) {
- dir ("helm-charts") {
- stage("Cleanup SR-IOV Network Device Plugin Daemonset") {
- sh returnStdout: true, script: """
- export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
- kubectl delete -f mcord/cni-config/05-sriov-device-plugin.yaml || true
- kubectl delete -f mcord/cni-config/04-sriov-device-plugin-configmap.yaml || true
- kubectl delete -f mcord/cni-config/03-network-definition.yaml || true
- kubectl delete -f mcord/cni-config/02-network-crd.yaml || true
- kubectl delete -f mcord/cni-config/01-cni-service-account.yaml || true
- """
- }
- }
- }
- }
+ // if ( ! params.installEpcControlPlane ) {
+ // timeout(5) {
+ // dir ("helm-charts") {
+ // stage("Cleanup SR-IOV Network Device Plugin Daemonset") {
+ // sh returnStdout: true, script: """
+ // export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ // kubectl delete -f mcord/cni-config/05-sriov-device-plugin.yaml || true
+ // kubectl delete -f mcord/cni-config/04-sriov-device-plugin-configmap.yaml || true
+ // kubectl delete -f mcord/cni-config/03-network-definition.yaml || true
+ // kubectl delete -f mcord/cni-config/02-network-crd.yaml || true
+ // kubectl delete -f mcord/cni-config/01-cni-service-account.yaml || true
+ // """
+ // }
+ // }
+ // }
+ // }
timeout(5) {
waitUntil {
@@ -221,10 +220,10 @@
}
}
- sh returnStdout: true, script: """
- export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
- kubectl apply -f mcord/cni-config/
- """
+ // sh returnStdout: true, script: """
+ // export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ // kubectl apply -f mcord/cni-config/
+ // """
// Because of SR-IOV daemonset will restart the kubelet and containerd, so we need to wait tiller up.
timeout(1) {
@@ -256,6 +255,16 @@
export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
helm install -n mcord-data-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-data-plane
"""
+
+ timeout (10) {
+ waitUntil {
+ mcord_dataplane_running = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl -n epc get pods | grep ngic-dp | grep -i running | grep 1/1 | wc -l
+ """
+ return mcord_dataplane_running.toInteger() == 1
+ }
+ }
}
stage("Install M-CORD BBU Services") {
@@ -263,6 +272,16 @@
export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
helm install -n mcord-bbu --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-bbu
"""
+
+ timeout (10) {
+ waitUntil {
+ mcord_bbu_running = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl -n epc get pods | grep accelleran | grep -i running | grep 3/3 | wc -l
+ """
+ return mcord_bbu_running.toInteger() == 1
+ }
+ }
}
// if ( params.installEpcControlPlane ) {
diff --git a/Jenkinsfile-mcord-remote-build b/Jenkinsfile-mcord-remote-build
index 102691e..5673a80 100644
--- a/Jenkinsfile-mcord-remote-build
+++ b/Jenkinsfile-mcord-remote-build
@@ -20,6 +20,7 @@
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-repo-tools"
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-charts"
sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
+
deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
}
stage('Clean up') {
@@ -80,8 +81,6 @@
}
}
-
-
stage("Install M-CORD Control Plane Services") {
sh returnStdout: true, script: """
export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
@@ -118,6 +117,7 @@
currentBuild.result = 'SUCCESS'
} catch (err) {
+ println err.message
currentBuild.result = 'FAILURE'
step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
}