Merge "[VOL-1961] Add new stage to test the failure of etcd"
diff --git a/jjb/pipeline/voltha-system-test-bbsim.groovy b/jjb/pipeline/voltha-system-test-bbsim.groovy
index ba61757..68f96d7 100644
--- a/jjb/pipeline/voltha-system-test-bbsim.groovy
+++ b/jjb/pipeline/voltha-system-test-bbsim.groovy
@@ -16,6 +16,40 @@
// uses kind-voltha to deploy voltha-2.X
// uses bbsim to simulate OLT/ONUs
+
+ def logKubernetes(prefix) {
+ sh """
+ set +e
+ cd kind-voltha/
+ cp install-minimal.log $WORKSPACE/${prefix}_instsall-minimal.log
+ kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
+ kubectl get nodes -o wide
+ kubectl get pods -o wide
+ kubectl get pods -n voltha -o wide
+ ## get default pod logs
+ for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
+ do
+ if [[ \$pod == *"onos"* && \$pod != *"onos-service"* ]]; then
+ kubectl logs \$pod onos> $WORKSPACE/${prefix}_\$pod.log;
+ else
+ kubectl logs \$pod> $WORKSPACE/${prefix}_\$pod.log;
+ fi
+ done
+ ## get voltha pod logs
+ for pod in \$(kubectl get pods --no-headers -n voltha | awk '{print \$1}');
+ do
+ if [[ \$pod == *"-api-"* ]]; then
+ kubectl logs \$pod arouter -n voltha > $WORKSPACE/${prefix}_\$pod.log;
+ elif [[ \$pod == "bbsim-"* ]]; then
+ kubectl logs \$pod -n voltha -p > $WORKSPACE/${prefix}_\$pod.log;
+ else
+ kubectl logs \$pod -n voltha > $WORKSPACE/${prefix}_\$pod.log;
+ fi
+ done
+ """
+ }
+
+
pipeline {
/* no label, executor is determined by JJB */
@@ -23,7 +57,7 @@
label "${params.buildNode}"
}
options {
- timeout(time: 40, unit: 'MINUTES')
+ timeout(time: 80, unit: 'MINUTES')
}
environment {
KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
@@ -41,7 +75,6 @@
ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs"
}
stages {
-
stage('Download kind-voltha') {
steps {
sh """
@@ -65,67 +98,79 @@
sh '''
rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
git clone https://gerrit.opencord.org/voltha-system-tests
- make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
+ make ROBOT_DEBUG_LOG_OPT="-l sanity_log.html -r sanity_result.html -o sanity_result.xml" -C $WORKSPACE/voltha-system-tests ${makeTarget}
'''
}
}
+ stage('Log the kubernetes for sanity-test') {
+ steps {
+ logKubernetes('sanity_test')
+ }
+ }
//Remove this stage once https://jira.opencord.org/browse/VOL-1977 be resolved
- stage('Deploy Voltha Again') {
+ stage('Deploy Voltha Again for Functional Tests') {
steps {
sh """
pushd kind-voltha/
- ./voltha down
- ./voltha up
+ WAIT_ON_DOWN=yes DEPLOY_K8S=no ./voltha down
+ DEPLOY_K8S=no ./voltha up
popd
"""
}
}
+
stage('Kubernetes Functional Tests') {
steps {
sh '''
- rm -rf $WORKSPACE/RobotLogs; mkdir -p $WORKSPACE/RobotLogs
- make -C $WORKSPACE/voltha-system-tests system-scale-test || true
+ make ROBOT_DEBUG_LOG_OPT="-l functional_log.html -r functional_result.html -o functional_output.xml" -C $WORKSPACE/voltha-system-tests system-scale-test
'''
}
}
+
+ stage('Log the kubernetes for functional-test') {
+ steps {
+ logKubernetes('functional')
+ }
+ }
+
+ //Remove this stage once https://jira.opencord.org/browse/VOL-1977 be resolved
+ stage('Deploy Voltha Again for Failure Scenario Tests') {
+ steps {
+ sh """
+ pushd kind-voltha/
+ WAIT_ON_DOWN=yes DEPLOY_K8S=no ./voltha down
+ DEPLOY_K8S=no ./voltha up
+ popd
+ """
+ }
+ }
+
+ stage('Kubernetes Failure Scenario Tests') {
+ steps {
+ sh '''
+ make ROBOT_DEBUG_LOG_OPT="-l failure_log.html -r failure_result.html -o failure_output.xml" -C $WORKSPACE/voltha-system-tests failure-test
+ '''
+ }
+ }
+
+ stage('Log the kubernetes for failure scenario test') {
+ steps {
+ logKubernetes('failure')
+ }
+ }
+
}
post {
+ failure {
+ logKubernetes('last')
+ }
+ aborted {
+ logKubernetes('last')
+ }
always {
- sh '''
- set +e
- cd kind-voltha/
- cp install-minimal.log $WORKSPACE/
- kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
- kubectl get nodes -o wide
- kubectl get pods -o wide
- kubectl get pods -n voltha -o wide
- ## get default pod logs
- for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
- do
- if [[ \$pod == *"onos"* && \$pod != *"onos-service"* ]]; then
- kubectl logs \$pod onos> $WORKSPACE/\$pod.log;
- else
- kubectl logs \$pod> $WORKSPACE/\$pod.log;
- fi
- done
- ## get voltha pod logs
- for pod in \$(kubectl get pods --no-headers -n voltha | awk '{print \$1}');
- do
- if [[ \$pod == *"-api-"* ]]; then
- kubectl logs \$pod arouter -n voltha > $WORKSPACE/\$pod.log;
- elif [[ \$pod == "bbsim-"* ]]; then
- kubectl logs \$pod -n voltha -p > $WORKSPACE/\$pod.log;
- else
- kubectl logs \$pod -n voltha > $WORKSPACE/\$pod.log;
- fi
- done
- ## clean up node
- WAIT_ON_DOWN=y ./voltha down
- cd $WORKSPACE/
- rm -rf kind-voltha/ voltha-system-tests/ || true
- '''
+
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
logFileName: 'RobotLogs/*log*.html',