blob: ebc5174f89fa281d40433346079610a98478499f [file] [log] [blame]
Andy Bavier61c5b2a2019-11-12 12:08:19 -07001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA built from patchset on a physical pod and run e2e test
16// uses kind-voltha to deploy voltha-2.X
17
18node {
19 // Need this so that deployment_config has global scope when it's read later
20 deployment_config = null
21 localDeploymentConfigFile = null
22 localKindVolthaValuesFile = null
23 localSadisConfigFile = null
24}
25
26pipeline {
27
28 /* no label, executor is determined by JJB */
29 agent {
30 label "${params.buildNode}"
31 }
32 options {
33 timeout(time: 60, unit: 'MINUTES')
34 }
35
36 environment {
37 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
38 VOLTCONFIG="$HOME/.volt/config-minimal"
Andy Bavier6ba9f9c2019-12-12 17:54:53 -070039 PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070040 TYPE="minimal"
41 FANCY=0
42 //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
43 ONOS_SSH_PORT=30115
44 ONOS_API_PORT=30120
45 }
46
47 stages {
48 stage ('Initialize') {
49 steps {
hwchiu1f0b0fe2019-12-06 11:45:23 -080050 sh returnStdout: false, script: """
Andy Bavier61c5b2a2019-11-12 12:08:19 -070051 test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
52 cd $WORKSPACE
53 rm -rf $WORKSPACE/*
54 """
55 script {
56 if (env.configRepo && ! env.localConfigDir) {
57 env.localConfigDir = "$WORKSPACE"
hwchiu1f0b0fe2019-12-06 11:45:23 -080058 sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070059 }
60 localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
61 localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
62 localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
Andy Bavier61c5b2a2019-11-12 12:08:19 -070063 }
64 }
65 }
66
Andy Bavier06d816c2019-11-19 11:40:48 -070067 stage('Repo') {
68 steps {
69 checkout(changelog: true,
70 poll: false,
71 scm: [$class: 'RepoScm',
72 manifestRepositoryUrl: "${params.manifestUrl}",
73 manifestBranch: "${params.manifestBranch}",
74 currentBranch: true,
75 destinationDir: 'voltha',
76 forceSync: true,
77 resetFirst: true,
78 quiet: true,
79 jobs: 4,
80 showAllChanges: true]
81 )
82 }
83 }
84
Andy Bavier61c5b2a2019-11-12 12:08:19 -070085 stage('Get Patch') {
86 when {
87 expression { params.withPatchset }
88 }
89 steps {
Andy Bavier61c5b2a2019-11-12 12:08:19 -070090 sh returnStdout: false, script: """
91 cd voltha
92 PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifest.xml)
93 repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
94 """
95 }
96 }
97
Andy Bavier9a376602019-11-13 11:53:37 -070098 stage('Check config files') {
99 steps {
Andy Bavier85ab0942019-11-13 14:43:25 -0700100 script {
101 try {
102 deployment_config = readYaml file: "${localDeploymentConfigFile}"
103 } catch (err) {
104 echo "Error reading ${localDeploymentConfigFile}"
105 throw err
106 }
107 sh returnStdout: false, script: """
108 if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
109 if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
110 """
Andy Bavier9a376602019-11-13 11:53:37 -0700111 }
Andy Bavier9a376602019-11-13 11:53:37 -0700112 }
113 }
114
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700115 stage('Create KinD Cluster') {
116 steps {
117 sh returnStdout: false, script: """
118 git clone https://github.com/ciena/kind-voltha.git
119 cd kind-voltha/
120 JUST_K8S=y ./voltha up
121 """
122 }
123 }
124
125 stage('Build and Push Images') {
126 when {
127 expression { params.withPatchset }
128 }
129 steps {
130 sh returnStdout: false, script: """
131 if ! [[ "${gerritProject}" =~ ^(voltha-system-tests)\$ ]]; then
132 make -C $WORKSPACE/voltha/${gerritProject} DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
133 docker images | grep citest
134 for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}")
135 do
136 echo "Pushing \$image to nodes"
137 kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2
138 docker rmi \$image:citest \$image:latest || true
139 done
140 fi
141 """
142 }
143 }
144
145 stage('Deploy Voltha') {
146 environment {
147 WITH_SIM_ADAPTERS="n"
148 WITH_RADIUS="y"
149 DEPLOY_K8S="n"
150 VOLTHA_LOG_LEVEL="debug"
151 }
152 steps {
153 script {
154 if ( params.withPatchset ) {
155 sh returnStdout: false, script: """
Andy Bavier4af02722020-01-15 10:24:24 -0700156 export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700157
158 IMAGES=""
159 if [ "${gerritProject}" = "voltha-go" ]; then
Zack Williamse65b1ee2019-12-09 15:57:16 -0700160 IMAGES="rw_core ro_core "
161 elif [ "${gerritProject}" = "ofagent-py" ]; then
162 IMAGES="ofagent "
Matteo Scandolo70800a32020-01-06 11:55:27 -0800163 elif [ "${gerritProject}" = "voltha-onos" ]; then
164 IMAGES="onos "
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700165 elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
166 IMAGES="adapter_open_olt "
167 elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
168 IMAGES="adapter_open_onu "
169 elif [ "${gerritProject}" = "voltha-api-server" ]; then
170 IMAGES="afrouter afrouterd "
171 else
172 echo "No images to push"
173 fi
174
175 for I in \$IMAGES
176 do
177 EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
178 done
179
180 cd $WORKSPACE/kind-voltha/
181 echo \$EXTRA_HELM_FLAGS
Andy Bavier4af02722020-01-15 10:24:24 -0700182 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700183 ./voltha up
184 """
185 } else {
186 sh returnStdout: false, script: """
Andy Bavier4af02722020-01-15 10:24:24 -0700187 export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700188 cd $WORKSPACE/kind-voltha/
189 echo \$EXTRA_HELM_FLAGS
Andy Bavier4af02722020-01-15 10:24:24 -0700190 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700191 ./voltha up
192 """
193 }
194 }
195 }
196 }
197
Scott Bakerfb0284c2019-12-12 17:22:38 -0800198 stage('Deploy Kafka Dump Chart') {
199 steps {
200 script {
201 sh returnStdout: false, script: """
202 helm repo add cord https://charts.opencord.org
203 helm repo update
204 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
205 """
206 }
207 }
208 }
209
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700210 stage('Push Tech-Profile') {
211 when {
212 expression { params.profile != "Default" }
213 }
214 steps {
215 sh returnStdout: false, script: """
216 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
217 kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
Andy Bavier6ebf3182019-11-21 13:47:21 -0700218 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700219 """
220 }
221 }
222
223 stage('Push Sadis-config') {
224 steps {
225 sh returnStdout: false, script: """
226 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
227 """
228 }
229 }
230
231 stage('Reinstall OLT software') {
232 when {
233 expression { params.reinstallOlt }
234 }
235 steps {
236 script {
237 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800238 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true"
239 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true"
240 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700241 waitUntil {
242 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
243 return olt_sw_present.toInteger() == 0
244 }
hwchiu1f0b0fe2019-12-06 11:45:23 -0800245 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700246 waitUntil {
247 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
248 return olt_sw_present.toInteger() == 1
249 }
250 if ( olt.fortygig ) {
251 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
hwchiu1f0b0fe2019-12-06 11:45:23 -0800252 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700253 }
254 }
255 }
256 }
257 }
258
259 stage('Restart OLT processes') {
260 steps {
261 script {
262 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800263 sh returnStdout: false, script: """
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700264 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700265 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true
Andy Bavierb502a372019-11-22 01:05:04 +0000266 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true
Andy Bavierfc4dcc42019-11-18 15:33:52 -0700267 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log'
Andy Bavierbecaa8e2020-01-08 11:49:57 -0700268 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/dev_mgmt_daemon.log'
Andy Bavierb502a372019-11-22 01:05:04 +0000269 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service dev_mgmt_daemon start &'
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700270 sleep 5
271 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt start &'
272 """
273 waitUntil {
274 onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
275 return onu_discovered.toInteger() > 0
276 }
277 }
278 }
279 }
280 }
281
282 stage('Run E2E Tests') {
283 environment {
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700284 ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
Andy Bavier96c428f2019-12-17 11:27:45 -0700285 ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs"
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700286 ROBOT_FILE="Voltha_PODTests.robot"
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700287 }
288 steps {
289 sh returnStdout: false, script: """
290 cd voltha
291 git clone -b ${branch} ${cordRepoUrl}/cord-tester
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700292 mkdir -p $WORKSPACE/RobotLogs
Andy Bavierbda4c6b2019-11-12 17:06:22 -0700293 make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700294 """
295 }
296 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800297
298 stage('After-Test Delay') {
Andy Bavier4af02722020-01-15 10:24:24 -0700299 when {
300 expression { params.withPatchset }
301 }
Scott Bakerda31d7b2020-01-08 16:35:52 -0800302 steps {
303 sh returnStdout: false, script: """
304 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
305 REGEX="hardware test with delay\$"
306 [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
307 """
308 }
309 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700310 }
311
312 post {
313 always {
Andy Bavier4af02722020-01-15 10:24:24 -0700314 sh returnStdout: false, script: '''
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700315 set +e
316 cp kind-voltha/install-minimal.log $WORKSPACE/
317 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
318 kubectl get nodes -o wide
319 kubectl get pods -o wide
320 kubectl get pods -n voltha -o wide
Andy Bavier4af02722020-01-15 10:24:24 -0700321
322 sync
323 pkill kail || true
324
325 ## Pull out errors from log files
326 extract_errors_go() {
327 echo
328 echo "Error summary for $1:"
329 grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
330 echo
331 }
332
333 extract_errors_python() {
334 echo
335 echo "Error summary for $1:"
336 grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
337 echo
338 }
339
340 extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
341 extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
342 extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
343 extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
344
Scott Bakerfb0284c2019-12-12 17:22:38 -0800345 ## collect events, the chart should be running by now
346 kubectl get pods | grep -i voltha-kafka-dump | grep -i running
Andy Bavier4af02722020-01-15 10:24:24 -0700347 if [[ $? == 0 ]]; then
Scott Bakerfb0284c2019-12-12 17:22:38 -0800348 kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
349 fi
Andy Bavier4af02722020-01-15 10:24:24 -0700350 '''
Andy Bavier6ebf3182019-11-21 13:47:21 -0700351 script {
352 deployment_config.olts.each { olt ->
hwchiu1f0b0fe2019-12-06 11:45:23 -0800353 sh returnStdout: false, script: """
Andy Bavier6ebf3182019-11-21 13:47:21 -0700354 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log || true
355 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
Andy Bavierbecaa8e2020-01-08 11:49:57 -0700356 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log || true
357 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
Andy Bavier6ebf3182019-11-21 13:47:21 -0700358 """
359 }
360 }
Andy Bavier61c5b2a2019-11-12 12:08:19 -0700361 step([$class: 'RobotPublisher',
362 disableArchiveOutput: false,
363 logFileName: 'RobotLogs/log*.html',
364 otherFiles: '',
365 outputFileName: 'RobotLogs/output*.xml',
366 outputPath: '.',
367 passThreshold: 80,
368 reportFileName: 'RobotLogs/report*.html',
369 unstableThreshold: 0]);
370 archiveArtifacts artifacts: '*.log'
371 }
372 }
373}