blob: 0769878d6f72ff62f6d79a363760115524c0ee17 [file] [log] [blame]
Andy Bavier61c5b2a2019-11-12 12:08:19 -07001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA built from patchset on a physical pod and run e2e test
16// uses kind-voltha to deploy voltha-2.X
17
Andy Baviere06dcff2020-02-06 11:36:40 -070018// Need this so that deployment_config has global scope when it's read later
19deployment_config = null
20localDeploymentConfigFile = null
21localKindVolthaValuesFile = null
22localSadisConfigFile = null
23
24// The pipeline assumes these variables are always defined
25if ( ! params.withPatchset ) {
26 GERRIT_EVENT_COMMENT_TEXT = ""
27 GERRIT_PROJECT = ""
28 GERRIT_CHANGE_NUMBER = ""
29 GERRIT_PATCHSET_NUMBER = ""
Andy Bavier61c5b2a2019-11-12 12:08:19 -070030}
31
32pipeline {
33
34 /* no label, executor is determined by JJB */
35 agent {
36 label "${params.buildNode}"
37 }
38 options {
39 timeout(time: 60, unit: 'MINUTES')
40 }
41
42 environment {
43 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
44 VOLTCONFIG="$HOME/.volt/config-minimal"
Andy Bavier6ba9f9c2019-12-12 17:54:53 -070045 PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070046 TYPE="minimal"
47 FANCY=0
48 //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
49 ONOS_SSH_PORT=30115
50 ONOS_API_PORT=30120
51 }
52
53 stages {
54 stage ('Initialize') {
55 steps {
hwchiu1f0b0fe2019-12-06 11:45:23 -080056 sh returnStdout: false, script: """
Andy Bavier61c5b2a2019-11-12 12:08:19 -070057 test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
58 cd $WORKSPACE
59 rm -rf $WORKSPACE/*
60 """
61 script {
62 if (env.configRepo && ! env.localConfigDir) {
63 env.localConfigDir = "$WORKSPACE"
hwchiu1f0b0fe2019-12-06 11:45:23 -080064 sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070065 }
66 localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
67 localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
68 localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070069 }
70 }
71 }
72
Andy Bavier06d816c2019-11-19 11:40:48 -070073 stage('Repo') {
74 steps {
75 checkout(changelog: true,
76 poll: false,
77 scm: [$class: 'RepoScm',
78 manifestRepositoryUrl: "${params.manifestUrl}",
79 manifestBranch: "${params.manifestBranch}",
80 currentBranch: true,
81 destinationDir: 'voltha',
82 forceSync: true,
83 resetFirst: true,
84 quiet: true,
85 jobs: 4,
86 showAllChanges: true]
87 )
88 }
89 }
90
Andy Bavier61c5b2a2019-11-12 12:08:19 -070091 stage('Get Patch') {
92 when {
93 expression { params.withPatchset }
94 }
95 steps {
Andy Bavier61c5b2a2019-11-12 12:08:19 -070096 sh returnStdout: false, script: """
97 cd voltha
98 PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifest.xml)
99 repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
100 """
101 }
102 }
103
Andy Bavier9a376602019-11-13 11:53:37 -0700104 stage('Check config files') {
105 steps {
Andy Bavier85ab0942019-11-13 14:43:25 -0700106 script {
107 try {
108 deployment_config = readYaml file: "${localDeploymentConfigFile}"
109 } catch (err) {
110 echo "Error reading ${localDeploymentConfigFile}"
111 throw err
112 }
113 sh returnStdout: false, script: """
114 if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
115 if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
116 """
Andy Bavier9a376602019-11-13 11:53:37 -0700117 }
Andy Bavier9a376602019-11-13 11:53:37 -0700118 }
119 }
120
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700121 stage('Create KinD Cluster') {
122 steps {
123 sh returnStdout: false, script: """
124 git clone https://github.com/ciena/kind-voltha.git
125 cd kind-voltha/
126 JUST_K8S=y ./voltha up
127 """
128 }
129 }
130
131 stage('Build and Push Images') {
132 when {
133 expression { params.withPatchset }
134 }
135 steps {
136 sh returnStdout: false, script: """
137 if ! [[ "${gerritProject}" =~ ^(voltha-system-tests)\$ ]]; then
138 make -C $WORKSPACE/voltha/${gerritProject} DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
139 docker images | grep citest
140 for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}")
141 do
142 echo "Pushing \$image to nodes"
143 kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2
144 docker rmi \$image:citest \$image:latest || true
145 done
146 fi
147 """
148 }
149 }
150
151 stage('Deploy Voltha') {
152 environment {
153 WITH_SIM_ADAPTERS="n"
154 WITH_RADIUS="y"
155 DEPLOY_K8S="n"
156 VOLTHA_LOG_LEVEL="debug"
157 }
158 steps {
159 script {
Andy Baviere06dcff2020-02-06 11:36:40 -0700160 sh returnStdout: false, script: """
161 export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700162
Andy Baviere06dcff2020-02-06 11:36:40 -0700163 IMAGES=""
164 if [ "${gerritProject}" = "voltha-go" ]; then
165 IMAGES="rw_core ro_core "
166 elif [ "${gerritProject}" = "ofagent-py" ]; then
167 IMAGES="ofagent "
168 elif [ "${gerritProject}" = "voltha-onos" ]; then
169 IMAGES="onos "
170 elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
171 IMAGES="adapter_open_olt "
172 elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
173 IMAGES="adapter_open_onu "
174 elif [ "${gerritProject}" = "voltha-api-server" ]; then
175 IMAGES="afrouter afrouterd "
176 else
177 echo "No images to push"
178 fi
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700179
Andy Baviere06dcff2020-02-06 11:36:40 -0700180 for I in \$IMAGES
181 do
182 EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
183 done
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700184
Andy Baviere06dcff2020-02-06 11:36:40 -0700185 cd $WORKSPACE/kind-voltha/
186 echo \$EXTRA_HELM_FLAGS
187 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
188 ./voltha up
189 """
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700190 }
191 }
192 }
193
Scott Bakerfb0284c2019-12-12 17:22:38 -0800194 stage('Deploy Kafka Dump Chart') {
195 steps {
196 script {
197 sh returnStdout: false, script: """
198 helm repo add cord https://charts.opencord.org
199 helm repo update
200 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
201 """
202 }
203 }
204 }
205
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700206 stage('Push Tech-Profile') {
207 when {
208 expression { params.profile != "Default" }
209 }
210 steps {
211 sh returnStdout: false, script: """
212 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
213 kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
Andy Bavier6ebf3182019-11-21 13:47:21 -0700214 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700215 """
216 }
217 }
218
219 stage('Push Sadis-config') {
220 steps {
221 sh returnStdout: false, script: """
222 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
223 """
224 }
225 }
226
227 stage('Reinstall OLT software') {
228 when {
229 expression { params.reinstallOlt }
230 }
231 steps {
232 script {
233 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800234 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true"
235 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true"
236 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700237 waitUntil {
238 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
239 return olt_sw_present.toInteger() == 0
240 }
hwchiu1f0b0fe2019-12-06 11:45:23 -0800241 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700242 waitUntil {
243 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
244 return olt_sw_present.toInteger() == 1
245 }
246 if ( olt.fortygig ) {
247 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
hwchiu1f0b0fe2019-12-06 11:45:23 -0800248 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700249 }
250 }
251 }
252 }
253 }
254
255 stage('Restart OLT processes') {
256 steps {
257 script {
258 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800259 sh returnStdout: false, script: """
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700260 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700261 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true
Andy Bavierb502a372019-11-22 01:05:04 +0000262 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true
Andy Bavierfc4dcc42019-11-18 15:33:52 -0700263 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log'
Andy Bavierbecaa8e2020-01-08 11:49:57 -0700264 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/dev_mgmt_daemon.log'
Andy Bavierb502a372019-11-22 01:05:04 +0000265 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service dev_mgmt_daemon start &'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700266 sleep 5
267 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt start &'
268 """
269 waitUntil {
270 onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
271 return onu_discovered.toInteger() > 0
272 }
273 }
274 }
275 }
276 }
277
278 stage('Run E2E Tests') {
279 environment {
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700280 ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
Andy Bavierd3be2bf2020-02-07 17:02:21 -0700281 ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700282 ROBOT_FILE="Voltha_PODTests.robot"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700283 }
284 steps {
285 sh returnStdout: false, script: """
286 cd voltha
287 git clone -b ${branch} ${cordRepoUrl}/cord-tester
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700288 mkdir -p $WORKSPACE/RobotLogs
Andy Baviere06dcff2020-02-06 11:36:40 -0700289
290 # If the Gerrit comment contains a line with "functional tests" then run the full
291 # functional test suite. This covers tests tagged either 'sanity' or 'functional'.
292 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
293 REGEX="functional tests"
294 if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
295 ROBOT_MISC_ARGS+="-i functional"
296 fi
297
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700298 make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700299 """
300 }
301 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800302
303 stage('After-Test Delay') {
Andy Bavier4af02722020-01-15 10:24:24 -0700304 when {
305 expression { params.withPatchset }
306 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800307 steps {
308 sh returnStdout: false, script: """
309 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
310 REGEX="hardware test with delay\$"
311 [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
312 """
313 }
314 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700315 }
316
317 post {
318 always {
Andy Bavier4af02722020-01-15 10:24:24 -0700319 sh returnStdout: false, script: '''
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700320 set +e
321 cp kind-voltha/install-minimal.log $WORKSPACE/
322 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
323 kubectl get nodes -o wide
324 kubectl get pods -o wide
325 kubectl get pods -n voltha -o wide
Andy Bavier4af02722020-01-15 10:24:24 -0700326
327 sync
328 pkill kail || true
329
330 ## Pull out errors from log files
331 extract_errors_go() {
332 echo
333 echo "Error summary for $1:"
334 grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
335 echo
336 }
337
338 extract_errors_python() {
339 echo
340 echo "Error summary for $1:"
341 grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
342 echo
343 }
344
345 extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
346 extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
347 extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
348 extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
349
Andy Bavier2178d102020-02-06 16:22:11 -0700350 gzip $WORKSPACE/onos-voltha-combined.log
351
Scott Bakerfb0284c2019-12-12 17:22:38 -0800352 ## collect events, the chart should be running by now
353 kubectl get pods | grep -i voltha-kafka-dump | grep -i running
Andy Bavier4af02722020-01-15 10:24:24 -0700354 if [[ $? == 0 ]]; then
Scott Bakerfb0284c2019-12-12 17:22:38 -0800355 kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
356 fi
Andy Bavier4af02722020-01-15 10:24:24 -0700357 '''
Andy Bavier6ebf3182019-11-21 13:47:21 -0700358 script {
359 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800360 sh returnStdout: false, script: """
Andy Bavier6ebf3182019-11-21 13:47:21 -0700361 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log || true
362 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
Andy Bavierbecaa8e2020-01-08 11:49:57 -0700363 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log || true
364 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
Andy Bavier6ebf3182019-11-21 13:47:21 -0700365 """
366 }
367 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700368 step([$class: 'RobotPublisher',
369 disableArchiveOutput: false,
370 logFileName: 'RobotLogs/log*.html',
371 otherFiles: '',
372 outputFileName: 'RobotLogs/output*.xml',
373 outputPath: '.',
374 passThreshold: 80,
375 reportFileName: 'RobotLogs/report*.html',
376 unstableThreshold: 0]);
Andy Bavier2178d102020-02-06 16:22:11 -0700377 archiveArtifacts artifacts: '*.log,*.gz'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700378 }
379 }
380}