VOL-2453 Use 'kail' tool to collect, combine all container logs
Change-Id: I2e489f005378bb8c70d03b9f93b449cda37fe76d
diff --git a/jjb/pipeline/voltha-go-multi-tests.groovy b/jjb/pipeline/voltha-go-multi-tests.groovy
index a562fd5..5caea0f 100644
--- a/jjb/pipeline/voltha-go-multi-tests.groovy
+++ b/jjb/pipeline/voltha-go-multi-tests.groovy
@@ -37,7 +37,7 @@
DEPLOY_K8S="y"
VOLTHA_LOG_LEVEL="DEBUG"
CONFIG_SADIS="n"
- EXTRA_HELM_FLAGS="${params.extraHelmFlags}"
+ EXTRA_HELM_FLAGS="--set log_agent.enabled=False ${params.extraHelmFlags}"
ROBOT_MISC_ARGS="${params.extraRobotArgs} -d $WORKSPACE/RobotLogs"
}
stages {
@@ -73,6 +73,8 @@
steps {
sh """
cd kind-voltha/
+ JUST_K8S=y ./voltha up
+ kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
./voltha up
"""
}
@@ -86,10 +88,6 @@
kubectl -n voltha exec $(kubectl -n voltha get pods -lapp=etcd -o=name) -- sh -c "ETCDCTL_API=3 etcdctl del --prefix $1"
}
- wait_for_state_change() {
- timeout 60 bash -c "until voltctl device list -f Type=$2,AdminState=$3,OperStatus=$4,ConnectStatus=$5 -q | wc -l | grep -q 1; do echo Waiting for $1 to change states; voltctl device list; echo; sleep 5; done"
- }
-
mkdir -p $WORKSPACE/RobotLogs
git clone https://gerrit.opencord.org/voltha-system-tests
cd kind-voltha
@@ -121,31 +119,37 @@
always {
sh '''
set +e
- cd kind-voltha/
- cp install-minimal.log $WORKSPACE/
+ cp $WORKSPACE/kind-voltha/install-minimal.log $WORKSPACE/
kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
kubectl get nodes -o wide
kubectl get pods -o wide
kubectl get pods -n voltha -o wide
- ## get default pod logs
- for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
- do
- if [[ \$pod == *"onos"* && \$pod != *"onos-service"* ]]; then
- kubectl logs \$pod onos> $WORKSPACE/\$pod.log;
- else
- kubectl logs \$pod> $WORKSPACE/\$pod.log;
- fi
- done
- ## get voltha pod logs
- for pod in \$(kubectl get pods --no-headers -n voltha | awk '{print \$1}');
- do
- if [[ \$pod == *"-api-"* ]]; then
- kubectl logs \$pod arouter -n voltha > $WORKSPACE/\$pod.log;
- else
- kubectl logs \$pod -n voltha > $WORKSPACE/\$pod.log;
- fi
- done
+
+ sync
+ pkill kail || true
+
+ ## Pull out errors from log files
+ extract_errors_go() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+ echo
+ }
+
+ extract_errors_python() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+ echo
+ }
+
+ extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
+ extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
+ extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
+ extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+
## shut down voltha
+ cd $WORKSPACE/kind-voltha/
WAIT_ON_DOWN=y ./voltha down
'''
step([$class: 'RobotPublisher',