blob: 306a4268fc6c067717d27b34139871335f5b3bd5 [file] [log] [blame]
Andy Bavier61c5b2a2019-11-12 12:08:19 -07001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA built from patchset on a physical pod and run e2e test
16// uses kind-voltha to deploy voltha-2.X
17
Andy Baviere06dcff2020-02-06 11:36:40 -070018// Need this so that deployment_config has global scope when it's read later
19deployment_config = null
20localDeploymentConfigFile = null
21localKindVolthaValuesFile = null
22localSadisConfigFile = null
23
24// The pipeline assumes these variables are always defined
Zack Williams03ebb272020-03-27 09:42:33 -070025if ( params.manualBranch != "" ) {
Andy Baviere06dcff2020-02-06 11:36:40 -070026 GERRIT_EVENT_COMMENT_TEXT = ""
27 GERRIT_PROJECT = ""
Zack Williams03ebb272020-03-27 09:42:33 -070028 GERRIT_BRANCH = "${params.manualBranch}"
Andy Baviere06dcff2020-02-06 11:36:40 -070029 GERRIT_CHANGE_NUMBER = ""
30 GERRIT_PATCHSET_NUMBER = ""
Andy Bavier61c5b2a2019-11-12 12:08:19 -070031}
32
33pipeline {
34
35 /* no label, executor is determined by JJB */
36 agent {
37 label "${params.buildNode}"
38 }
39 options {
Andy Bavier880d8862020-06-28 12:03:35 -070040 timeout(time: 120, unit: 'MINUTES')
Andy Bavier61c5b2a2019-11-12 12:08:19 -070041 }
42
43 environment {
44 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
45 VOLTCONFIG="$HOME/.volt/config-minimal"
Andy Bavierb5c8caf2020-04-06 14:12:07 -070046 PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Matteo Scandolo4d1a12f2020-12-07 16:17:12 -080047 NAME="minimal"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070048 FANCY=0
49 //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
50 ONOS_SSH_PORT=30115
51 ONOS_API_PORT=30120
52 }
53
54 stages {
55 stage ('Initialize') {
56 steps {
hwchiu1f0b0fe2019-12-06 11:45:23 -080057 sh returnStdout: false, script: """
Andy Bavierb5c8caf2020-04-06 14:12:07 -070058 test -e $WORKSPACE/voltha/kind-voltha/voltha && cd $WORKSPACE/voltha/kind-voltha && ./voltha down
Andy Bavier61c5b2a2019-11-12 12:08:19 -070059 cd $WORKSPACE
60 rm -rf $WORKSPACE/*
61 """
62 script {
63 if (env.configRepo && ! env.localConfigDir) {
64 env.localConfigDir = "$WORKSPACE"
hwchiu1f0b0fe2019-12-06 11:45:23 -080065 sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070066 }
67 localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
68 localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
69 localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070070 }
71 }
72 }
73
Andy Bavier06d816c2019-11-19 11:40:48 -070074 stage('Repo') {
75 steps {
76 checkout(changelog: true,
77 poll: false,
78 scm: [$class: 'RepoScm',
79 manifestRepositoryUrl: "${params.manifestUrl}",
Zack Williams03ebb272020-03-27 09:42:33 -070080 manifestBranch: "${params.branch}",
Andy Bavier06d816c2019-11-19 11:40:48 -070081 currentBranch: true,
82 destinationDir: 'voltha',
83 forceSync: true,
84 resetFirst: true,
85 quiet: true,
86 jobs: 4,
87 showAllChanges: true]
88 )
89 }
90 }
91
Andy Bavier61c5b2a2019-11-12 12:08:19 -070092 stage('Get Patch') {
93 when {
Andy Bavierfc3ca142020-03-27 13:44:25 -070094 expression { params.manualBranch == "" }
Andy Bavier61c5b2a2019-11-12 12:08:19 -070095 }
96 steps {
Andy Bavier61c5b2a2019-11-12 12:08:19 -070097 sh returnStdout: false, script: """
98 cd voltha
Andy Bavier5b523532020-02-24 15:31:25 -070099 repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700100 """
101 }
102 }
103
Andy Bavier9a376602019-11-13 11:53:37 -0700104 stage('Check config files') {
105 steps {
Andy Bavier85ab0942019-11-13 14:43:25 -0700106 script {
107 try {
108 deployment_config = readYaml file: "${localDeploymentConfigFile}"
109 } catch (err) {
110 echo "Error reading ${localDeploymentConfigFile}"
111 throw err
112 }
113 sh returnStdout: false, script: """
114 if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
115 if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
116 """
Andy Bavier9a376602019-11-13 11:53:37 -0700117 }
Andy Bavier9a376602019-11-13 11:53:37 -0700118 }
119 }
120
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700121 stage('Create KinD Cluster') {
122 steps {
123 sh returnStdout: false, script: """
Zack Williams03ebb272020-03-27 09:42:33 -0700124 if [ "${branch}" != "master" ]; then
125 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700126 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
Zack Williams03ebb272020-03-27 09:42:33 -0700127 else
128 echo "on master, using default settings for kind-voltha"
129 fi
130
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700131 cd $WORKSPACE/voltha/kind-voltha/
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700132 JUST_K8S=y ./voltha up
133 """
134 }
135 }
136
137 stage('Build and Push Images') {
138 when {
Andy Bavierfc3ca142020-03-27 13:44:25 -0700139 expression { params.manualBranch == "" }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700140 }
141 steps {
142 sh returnStdout: false, script: """
Zack Williams03ebb272020-03-27 09:42:33 -0700143
144 if [ "${branch}" != "master" ]; then
145 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700146 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
Zack Williams03ebb272020-03-27 09:42:33 -0700147 else
148 echo "on master, using default settings for kind-voltha"
149 fi
150
Andy Bavierc1cb94b2020-06-04 16:39:32 -0700151 if ! [[ "${gerritProject}" =~ ^(voltha-system-tests|kind-voltha|voltha-helm-charts)\$ ]]; then
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700152 make -C $WORKSPACE/voltha/${gerritProject} DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
153 docker images | grep citest
154 for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}")
155 do
156 echo "Pushing \$image to nodes"
Matteo Scandolo6fede0e2020-12-09 17:19:09 -0800157 kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700158 docker rmi \$image:citest \$image:latest || true
159 done
160 fi
161 """
162 }
163 }
164
165 stage('Deploy Voltha') {
166 environment {
Andy Bavier1f530f52020-04-14 15:57:33 -0700167 WITH_SIM_ADAPTERS="no"
168 WITH_RADIUS="yes"
169 DEPLOY_K8S="no"
170 VOLTHA_LOG_LEVEL="DEBUG"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700171 }
172 steps {
173 script {
Andy Baviere06dcff2020-02-06 11:36:40 -0700174 sh returnStdout: false, script: """
Zack Williams03ebb272020-03-27 09:42:33 -0700175 if [ "${branch}" != "master" ]; then
176 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700177 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
Zack Williams03ebb272020-03-27 09:42:33 -0700178 else
179 echo "on master, using default settings for kind-voltha"
180 fi
181
182 export EXTRA_HELM_FLAGS+='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700183
Andy Baviere06dcff2020-02-06 11:36:40 -0700184 IMAGES=""
185 if [ "${gerritProject}" = "voltha-go" ]; then
186 IMAGES="rw_core ro_core "
187 elif [ "${gerritProject}" = "ofagent-py" ]; then
188 IMAGES="ofagent "
189 elif [ "${gerritProject}" = "voltha-onos" ]; then
190 IMAGES="onos "
191 elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
192 IMAGES="adapter_open_olt "
193 elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
194 IMAGES="adapter_open_onu "
195 elif [ "${gerritProject}" = "voltha-api-server" ]; then
196 IMAGES="afrouter afrouterd "
197 else
198 echo "No images to push"
199 fi
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700200
Andy Baviere06dcff2020-02-06 11:36:40 -0700201 for I in \$IMAGES
202 do
203 EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
204 done
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700205
Andy Bavierc1cb94b2020-06-04 16:39:32 -0700206 if [ "${gerritProject}" = "voltha-helm-charts" ]; then
207 export CHART_PATH=$WORKSPACE/voltha/voltha-helm-charts
208 export VOLTHA_CHART=\$CHART_PATH/voltha
209 export VOLTHA_ADAPTER_OPEN_OLT_CHART=\$CHART_PATH/voltha-adapter-openolt
210 export VOLTHA_ADAPTER_OPEN_ONU_CHART=\$CHART_PATH/voltha-adapter-openonu
211 helm dep update \$VOLTHA_CHART
212 helm dep update \$VOLTHA_ADAPTER_OPEN_OLT_CHART
213 helm dep update \$VOLTHA_ADAPTER_OPEN_ONU_CHART
214 fi
215
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700216 cd $WORKSPACE/voltha/kind-voltha/
Andy Baviere06dcff2020-02-06 11:36:40 -0700217 echo \$EXTRA_HELM_FLAGS
218 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
219 ./voltha up
Andy Bavier1f530f52020-04-14 15:57:33 -0700220
Andy Baviercf7a5812020-04-24 15:06:30 -0700221 set +e
222
Andy Bavier1f530f52020-04-14 15:57:33 -0700223 # Remove noise from voltha-core logs
Andy Baviercf7a5812020-04-24 15:06:30 -0700224 voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
225 voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
Andy Bavier1f530f52020-04-14 15:57:33 -0700226 # Remove noise from openolt logs
Andy Baviercf7a5812020-04-24 15:06:30 -0700227 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
228 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
229 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
Andy Baviere06dcff2020-02-06 11:36:40 -0700230 """
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700231 }
232 }
233 }
234
Scott Bakerfb0284c2019-12-12 17:22:38 -0800235 stage('Deploy Kafka Dump Chart') {
236 steps {
237 script {
238 sh returnStdout: false, script: """
239 helm repo add cord https://charts.opencord.org
240 helm repo update
Andy Bavierb0f4e002020-07-23 09:37:23 -0700241 if helm version -c --short|grep v2 -q; then
242 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
243 else
244 helm install voltha-kafka-dump cord/voltha-kafka-dump
245 fi
Scott Bakerfb0284c2019-12-12 17:22:38 -0800246 """
247 }
248 }
249 }
250
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700251 stage('Push Tech-Profile') {
252 when {
253 expression { params.profile != "Default" }
254 }
255 steps {
256 sh returnStdout: false, script: """
257 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
258 kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
Andy Bavier6ebf3182019-11-21 13:47:21 -0700259 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700260 """
261 }
262 }
263
264 stage('Push Sadis-config') {
265 steps {
266 sh returnStdout: false, script: """
Suchitra Vemuri63b71742020-08-11 16:54:57 -0700267 ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115
268 ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts
269 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay"
270 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa"
271 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt"
272 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.FlowObjectiveManager"
273 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700274 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
275 """
276 }
277 }
278
279 stage('Reinstall OLT software') {
280 when {
281 expression { params.reinstallOlt }
282 }
283 steps {
284 script {
285 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800286 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700287 waitUntil {
288 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
289 return olt_sw_present.toInteger() == 0
290 }
Andy Bavier893d2442020-05-21 12:41:17 -0700291 if ( params.branch == 'voltha-2.3' ) {
292 oltDebVersion = oltDebVersionVoltha23
293 } else {
294 oltDebVersion = oltDebVersionMaster
295 }
hwchiu1f0b0fe2019-12-06 11:45:23 -0800296 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700297 waitUntil {
298 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
299 return olt_sw_present.toInteger() == 1
300 }
301 if ( olt.fortygig ) {
302 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
hwchiu1f0b0fe2019-12-06 11:45:23 -0800303 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700304 }
305 }
306 }
307 }
308 }
309
310 stage('Restart OLT processes') {
311 steps {
312 script {
313 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800314 sh returnStdout: false, script: """
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700315 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
Andy Bavier5bb88bc2020-06-18 14:51:21 -0700316 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; reboot'
317 sleep 120
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700318 """
319 waitUntil {
320 onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
321 return onu_discovered.toInteger() > 0
322 }
323 }
324 }
325 }
326 }
327
328 stage('Run E2E Tests') {
329 environment {
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700330 ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
Andy Bavierd3be2bf2020-02-07 17:02:21 -0700331 ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700332 ROBOT_FILE="Voltha_PODTests.robot"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700333 }
334 steps {
335 sh returnStdout: false, script: """
336 cd voltha
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700337 mkdir -p $WORKSPACE/RobotLogs
Andy Baviere06dcff2020-02-06 11:36:40 -0700338
339 # If the Gerrit comment contains a line with "functional tests" then run the full
340 # functional test suite. This covers tests tagged either 'sanity' or 'functional'.
341 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
342 REGEX="functional tests"
343 if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
344 ROBOT_MISC_ARGS+="-i functional"
345 fi
Andy Bavierc8ee4322020-08-18 16:37:38 -0700346 # Likewise for dataplane tests
347 REGEX="dataplane tests"
348 if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
349 ROBOT_MISC_ARGS+="-i dataplane"
350 fi
Andy Baviere06dcff2020-02-06 11:36:40 -0700351
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700352 make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700353 """
354 }
355 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800356
357 stage('After-Test Delay') {
Andy Bavier4af02722020-01-15 10:24:24 -0700358 when {
Andy Bavierfc3ca142020-03-27 13:44:25 -0700359 expression { params.manualBranch == "" }
Andy Bavier4af02722020-01-15 10:24:24 -0700360 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800361 steps {
362 sh returnStdout: false, script: """
363 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
364 REGEX="hardware test with delay\$"
365 [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
366 """
367 }
368 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700369 }
370
371 post {
372 always {
Andy Bavier4af02722020-01-15 10:24:24 -0700373 sh returnStdout: false, script: '''
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700374 set +e
Andy Bavierb5c8caf2020-04-06 14:12:07 -0700375 cp $WORKSPACE/voltha/kind-voltha/install-minimal.log $WORKSPACE/
Matteo Scandolo97b12572020-04-13 12:44:46 -0700376 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
377 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700378 kubectl get nodes -o wide
379 kubectl get pods -o wide
380 kubectl get pods -n voltha -o wide
Andy Bavier4af02722020-01-15 10:24:24 -0700381
382 sync
383 pkill kail || true
384
385 ## Pull out errors from log files
386 extract_errors_go() {
387 echo
388 echo "Error summary for $1:"
389 grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
390 echo
391 }
392
393 extract_errors_python() {
394 echo
395 echo "Error summary for $1:"
396 grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
397 echo
398 }
399
400 extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
401 extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
402 extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
403 extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
404
Andy Bavier2178d102020-02-06 16:22:11 -0700405 gzip $WORKSPACE/onos-voltha-combined.log
406
Scott Bakerfb0284c2019-12-12 17:22:38 -0800407 ## collect events, the chart should be running by now
408 kubectl get pods | grep -i voltha-kafka-dump | grep -i running
Andy Bavier4af02722020-01-15 10:24:24 -0700409 if [[ $? == 0 ]]; then
Scott Bakerfb0284c2019-12-12 17:22:38 -0800410 kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
411 fi
Andy Bavier4af02722020-01-15 10:24:24 -0700412 '''
Andy Bavier6ebf3182019-11-21 13:47:21 -0700413 script {
414 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800415 sh returnStdout: false, script: """
Andy Bavier827560b2020-06-05 09:49:32 -0700416 until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log
417 do
418 echo "Fetching openolt.log log failed, retrying..."
419 sleep 10
420 done
Andy Bavier6ebf3182019-11-21 13:47:21 -0700421 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
Andy Bavier827560b2020-06-05 09:49:32 -0700422 until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log
423 do
424 echo "Fetching dev_mgmt_daemon.log failed, retrying..."
425 sleep 10
426 done
Andy Bavierbecaa8e2020-01-08 11:49:57 -0700427 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
Andy Bavier6ebf3182019-11-21 13:47:21 -0700428 """
429 }
430 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700431 step([$class: 'RobotPublisher',
432 disableArchiveOutput: false,
433 logFileName: 'RobotLogs/log*.html',
434 otherFiles: '',
435 outputFileName: 'RobotLogs/output*.xml',
436 outputPath: '.',
Andy Baviere19b67b2020-02-14 11:29:50 -0700437 passThreshold: 100,
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700438 reportFileName: 'RobotLogs/report*.html',
439 unstableThreshold: 0]);
Andy Bavier2178d102020-02-06 16:22:11 -0700440 archiveArtifacts artifacts: '*.log,*.gz'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700441 }
442 }
443}