blob: 916fe18b74c8c721e3550db6b2ad2c5a28d6483a [file] [log] [blame]
Andy Bavierd83bf3a2020-08-13 14:55:07 -07001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA built from patchset on a physical pod and run e2e test
16// uses kind-voltha to deploy voltha-2.X
17
18// Need this so that deployment_config has global scope when it's read later
19deployment_config = null
20localDeploymentConfigFile = null
21localKindVolthaValuesFile = null
22localSadisConfigFile = null
23
24// The pipeline assumes these variables are always defined
25if ( params.manualBranch != "" ) {
26 GERRIT_EVENT_COMMENT_TEXT = ""
27 GERRIT_PROJECT = ""
28 GERRIT_BRANCH = "${params.manualBranch}"
29 GERRIT_CHANGE_NUMBER = ""
30 GERRIT_PATCHSET_NUMBER = ""
31}
32
33pipeline {
34
35 /* no label, executor is determined by JJB */
36 agent {
37 label "${params.buildNode}"
38 }
39 options {
40 timeout(time: 120, unit: 'MINUTES')
41 }
42
43 environment {
44 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
45 VOLTCONFIG="$HOME/.volt/config-minimal"
46 PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
47 TYPE="minimal"
48 FANCY=0
49 //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
50 ONOS_SSH_PORT=30115
51 ONOS_API_PORT=30120
52 }
53
54 stages {
55 stage ('Initialize') {
56 steps {
57 sh returnStdout: false, script: """
58 test -e $WORKSPACE/voltha/kind-voltha/voltha && cd $WORKSPACE/voltha/kind-voltha && ./voltha down
59 cd $WORKSPACE
60 rm -rf $WORKSPACE/*
61 """
62 script {
63 if (env.configRepo && ! env.localConfigDir) {
64 env.localConfigDir = "$WORKSPACE"
65 sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
66 }
67 localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
68 localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
69 localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
70 }
71 }
72 }
73
74 stage('Repo') {
75 steps {
76 checkout(changelog: true,
77 poll: false,
78 scm: [$class: 'RepoScm',
79 manifestRepositoryUrl: "${params.manifestUrl}",
80 manifestBranch: "${params.branch}",
81 currentBranch: true,
82 destinationDir: 'voltha',
83 forceSync: true,
84 resetFirst: true,
85 quiet: true,
86 jobs: 4,
87 showAllChanges: true]
88 )
89 }
90 }
91
92 stage('Get Patch') {
93 when {
94 expression { params.manualBranch == "" }
95 }
96 steps {
97 sh returnStdout: false, script: """
98 cd voltha
99 repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
100 """
101 }
102 }
103
104 stage('Check config files') {
105 steps {
106 script {
107 try {
108 deployment_config = readYaml file: "${localDeploymentConfigFile}"
109 } catch (err) {
110 echo "Error reading ${localDeploymentConfigFile}"
111 throw err
112 }
113 sh returnStdout: false, script: """
114 if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
115 if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
116 """
117 }
118 }
119 }
120
121 stage('Create KinD Cluster') {
122 steps {
123 sh returnStdout: false, script: """
124 if [ "${branch}" != "master" ]; then
125 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
126 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
127 else
128 echo "on master, using default settings for kind-voltha"
129 fi
130
131 cd $WORKSPACE/voltha/kind-voltha/
132 JUST_K8S=y ./voltha up
133 """
134 }
135 }
136
137 stage('Build and Push Images') {
138 when {
139 expression { params.manualBranch == "" }
140 }
141 steps {
142 sh returnStdout: false, script: """
143
144 if [ "${branch}" != "master" ]; then
145 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
146 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
147 else
148 echo "on master, using default settings for kind-voltha"
149 fi
150
151 if ! [[ "${gerritProject}" =~ ^(voltha-system-tests|kind-voltha|voltha-helm-charts)\$ ]]; then
152 make -C $WORKSPACE/voltha/${gerritProject} DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
153 docker images | grep citest
154 for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}")
155 do
156 echo "Pushing \$image to nodes"
157 kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2
158 docker rmi \$image:citest \$image:latest || true
159 done
160 fi
161 """
162 }
163 }
164
165 stage('Deploy Voltha') {
166 environment {
167 WITH_RADIUS="no"
168 WITH_EAPOL="no"
169 WITH_DHCP="no"
170 WITH_IGMP="no"
171 CONFIG_SADIS="no"
172 WITH_SIM_ADAPTERS="no"
173 DEPLOY_K8S="no"
174 VOLTHA_LOG_LEVEL="DEBUG"
175 }
176 steps {
177 script {
178 sh returnStdout: false, script: """
179 if [ "${branch}" != "master" ]; then
180 echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
181 source "$WORKSPACE/voltha/kind-voltha/releases/${branch}"
182 else
183 echo "on master, using default settings for kind-voltha"
184 fi
185
186 export EXTRA_HELM_FLAGS+='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
187
188 IMAGES=""
189 if [ "${gerritProject}" = "voltha-go" ]; then
190 IMAGES="rw_core ro_core "
191 elif [ "${gerritProject}" = "ofagent-py" ]; then
192 IMAGES="ofagent "
193 elif [ "${gerritProject}" = "voltha-onos" ]; then
194 IMAGES="onos "
195 elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
196 IMAGES="adapter_open_olt "
197 elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
198 IMAGES="adapter_open_onu "
199 elif [ "${gerritProject}" = "voltha-api-server" ]; then
200 IMAGES="afrouter afrouterd "
201 else
202 echo "No images to push"
203 fi
204
205 for I in \$IMAGES
206 do
207 EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
208 done
209
210 if [ "${gerritProject}" = "voltha-helm-charts" ]; then
211 export CHART_PATH=$WORKSPACE/voltha/voltha-helm-charts
212 export VOLTHA_CHART=\$CHART_PATH/voltha
213 export VOLTHA_ADAPTER_OPEN_OLT_CHART=\$CHART_PATH/voltha-adapter-openolt
214 export VOLTHA_ADAPTER_OPEN_ONU_CHART=\$CHART_PATH/voltha-adapter-openonu
215 helm dep update \$VOLTHA_CHART
216 helm dep update \$VOLTHA_ADAPTER_OPEN_OLT_CHART
217 helm dep update \$VOLTHA_ADAPTER_OPEN_ONU_CHART
218 fi
219
220 cd $WORKSPACE/voltha/kind-voltha/
221 echo \$EXTRA_HELM_FLAGS
222 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
223 ./voltha up
224
225 set +e
226
227 # Remove noise from voltha-core logs
228 voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
229 voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
230 # Remove noise from openolt logs
231 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
232 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
233 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
234 """
235 }
236 }
237 }
238
239 stage('Deploy Kafka Dump Chart') {
240 steps {
241 script {
242 sh returnStdout: false, script: """
243 helm repo add cord https://charts.opencord.org
244 helm repo update
245 if helm version -c --short|grep v2 -q; then
246 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
247 else
248 helm install voltha-kafka-dump cord/voltha-kafka-dump
249 fi
250 """
251 }
252 }
253 }
254
255 stage('Push Tech-Profile') {
256 when {
257 expression { params.profile != "Default" }
258 }
259 steps {
260 sh returnStdout: false, script: """
261 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
262 kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
263 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
264 """
265 }
266 }
267
268 stage('Push Sadis-config') {
269 steps {
270 sh returnStdout: false, script: """
271 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
272 """
273 }
274 }
275
276 stage('Reinstall OLT software') {
277 when {
278 expression { params.reinstallOlt }
279 }
280 steps {
281 script {
282 deployment_config.olts.each { olt ->
283 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
284 waitUntil {
285 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
286 return olt_sw_present.toInteger() == 0
287 }
288 if ( params.branch == 'voltha-2.3' ) {
289 oltDebVersion = oltDebVersionVoltha23
290 } else {
291 oltDebVersion = oltDebVersionMaster
292 }
293 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
294 waitUntil {
295 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
296 return olt_sw_present.toInteger() == 1
297 }
298 if ( olt.fortygig ) {
299 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
300 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
301 }
302 }
303 }
304 }
305 }
306
307 stage('Restart OLT processes') {
308 steps {
309 script {
310 deployment_config.olts.each { olt ->
311 sh returnStdout: false, script: """
312 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
313 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; reboot'
314 sleep 120
315 """
316 waitUntil {
317 onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
318 return onu_discovered.toInteger() > 0
319 }
320 }
321 }
322 }
323 }
324
325 stage('Run E2E Tests') {
326 environment {
327 ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
328 ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
329 ROBOT_FILE="Voltha_DT_PODTests.robot"
330 }
331 steps {
332 sh returnStdout: false, script: """
333 cd voltha
334 mkdir -p $WORKSPACE/RobotLogs
335
336 # If the Gerrit comment contains a line with "functional tests" then run the full
337 # functional test suite. This covers tests tagged either 'sanity' or 'functional'.
338 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
339 REGEX="functional tests"
340 if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
341 ROBOT_MISC_ARGS+="-i functionalDt"
342 fi
343
344 make -C $WORKSPACE/voltha/voltha-system-tests voltha-dt-test || true
345 """
346 }
347 }
348
349 stage('After-Test Delay') {
350 when {
351 expression { params.manualBranch == "" }
352 }
353 steps {
354 sh returnStdout: false, script: """
355 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
356 REGEX="hardware test with delay\$"
357 [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
358 """
359 }
360 }
361 }
362
363 post {
364 always {
365 sh returnStdout: false, script: '''
366 set +e
367 cp $WORKSPACE/voltha/kind-voltha/install-minimal.log $WORKSPACE/
368 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
369 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
370 kubectl get nodes -o wide
371 kubectl get pods -o wide
372 kubectl get pods -n voltha -o wide
373
374 sync
375 pkill kail || true
376
377 ## Pull out errors from log files
378 extract_errors_go() {
379 echo
380 echo "Error summary for $1:"
381 grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
382 echo
383 }
384
385 extract_errors_python() {
386 echo
387 echo "Error summary for $1:"
388 grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
389 echo
390 }
391
392 extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
393 extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
394 extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
395 extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
396
397 gzip $WORKSPACE/onos-voltha-combined.log
398
399 ## collect events, the chart should be running by now
400 kubectl get pods | grep -i voltha-kafka-dump | grep -i running
401 if [[ $? == 0 ]]; then
402 kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
403 fi
404 '''
405 script {
406 deployment_config.olts.each { olt ->
407 sh returnStdout: false, script: """
408 until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log
409 do
410 echo "Fetching openolt.log log failed, retrying..."
411 sleep 10
412 done
413 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
414 until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log
415 do
416 echo "Fetching dev_mgmt_daemon.log failed, retrying..."
417 sleep 10
418 done
419 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
420 """
421 }
422 }
423 step([$class: 'RobotPublisher',
424 disableArchiveOutput: false,
425 logFileName: 'RobotLogs/log*.html',
426 otherFiles: '',
427 outputFileName: 'RobotLogs/output*.xml',
428 outputPath: '.',
429 passThreshold: 100,
430 reportFileName: 'RobotLogs/report*.html',
431 unstableThreshold: 0]);
432 archiveArtifacts artifacts: '*.log,*.gz'
433 }
434 }
435}