blob: 6c183ff746d048fd427200a18565f83f7d931f9a [file] [log] [blame]
Scott Bakerff887712020-04-29 13:39:41 -07001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA built from patchset on a physical pod and run e2e test
16// uses kind-voltha to deploy voltha-2.X
17
18// Need this so that deployment_config has global scope when it's read later
19deployment_config = null
20localDeploymentConfigFile = null
21localKindVolthaValuesFile = null
22localSadisConfigFile = null
23
Scott Bakerff887712020-04-29 13:39:41 -070024pipeline {
25
26 /* no label, executor is determined by JJB */
27 agent {
28 label "${params.buildNode}"
29 }
30 options {
31 timeout(time: 90, unit: 'MINUTES')
32 }
33
34 environment {
35 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
36 VOLTCONFIG="$HOME/.volt/config-minimal"
Scott Baker1ace6d72020-05-01 14:54:41 -070037 PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Scott Bakerff887712020-04-29 13:39:41 -070038 TYPE="minimal"
39 FANCY=0
40 //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
41 ONOS_SSH_PORT=30115
42 ONOS_API_PORT=30120
43 }
44
45 stages {
46 stage ('Initialize') {
47 steps {
48 sh returnStdout: false, script: """
Scott Baker1ace6d72020-05-01 14:54:41 -070049 test -e $WORKSPACE/voltha/kind-voltha/voltha && cd $WORKSPACE/voltha/kind-voltha && ./voltha down
Scott Bakerff887712020-04-29 13:39:41 -070050 cd $WORKSPACE
51 rm -rf $WORKSPACE/*
52 """
53 script {
54 if (env.configRepo && ! env.localConfigDir) {
55 env.localConfigDir = "$WORKSPACE"
56 sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
57 }
58 localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
59 localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
60 localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
61 }
62 }
63 }
64
65 stage('Repo') {
66 steps {
67 checkout(changelog: true,
68 poll: false,
69 scm: [$class: 'RepoScm',
70 manifestRepositoryUrl: "${params.manifestUrl}",
71 manifestBranch: "${params.manifestBranch}",
72 currentBranch: true,
73 destinationDir: 'voltha',
74 forceSync: true,
75 resetFirst: true,
76 quiet: true,
77 jobs: 4,
78 showAllChanges: true]
79 )
80 }
81 }
82
Scott Baker1ace6d72020-05-01 14:54:41 -070083 stage('Patch') {
Scott Bakerff887712020-04-29 13:39:41 -070084 steps {
Scott Baker1ace6d72020-05-01 14:54:41 -070085 sh """
86 pushd $WORKSPACE/
Scott Baker1ace6d72020-05-01 14:54:41 -070087 git clone https://gerrit.opencord.org/${gerritProject}
88 cd "${gerritProject}"
Scott Bakerf21f8b82020-05-05 17:33:13 -070089 if [[ ! -z "${gerritRefSpec}" ]]; then
90 git fetch https://gerrit.opencord.org/${gerritProject} "${gerritRefSpec}" && git checkout FETCH_HEAD
91 fi
Scott Baker1ace6d72020-05-01 14:54:41 -070092 popd
93 """
Scott Bakerff887712020-04-29 13:39:41 -070094 }
95 }
96
97 stage('Check config files') {
98 steps {
99 script {
100 try {
101 deployment_config = readYaml file: "${localDeploymentConfigFile}"
102 } catch (err) {
103 echo "Error reading ${localDeploymentConfigFile}"
104 throw err
105 }
106 sh returnStdout: false, script: """
107 if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
108 if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
109 """
110 }
111 }
112 }
113
Scott Baker1ace6d72020-05-01 14:54:41 -0700114 stage('Build olt addr list') {
115 steps {
116 script {
117 sh """
118 echo "ADDR_LIST:" > /tmp/robot_vars.yaml
119 """
120 deployment_config.olts.each { olt ->
121 sh """
122 echo " - ${olt.ip}:8888" >> /tmp/robot_vars.yaml
123 """
124 }
125 sh """
126 cat /tmp/robot_vars.yaml
127 """
128 }
129 }
130 }
131
Scott Bakerff887712020-04-29 13:39:41 -0700132 stage('Create KinD Cluster') {
133 steps {
134 sh returnStdout: false, script: """
Scott Baker1ace6d72020-05-01 14:54:41 -0700135 cd $WORKSPACE/voltha/kind-voltha/
Scott Bakerff887712020-04-29 13:39:41 -0700136 JUST_K8S=y ./voltha up
137 """
138 }
139 }
140
Scott Baker1ace6d72020-05-01 14:54:41 -0700141 stage('Build Redfish Importer Image') {
Scott Bakerff887712020-04-29 13:39:41 -0700142 steps {
Scott Baker1ace6d72020-05-01 14:54:41 -0700143 sh """
144 make -C $WORKSPACE/device-management/\$1 DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build-importer
145 """
146 }
147 }
148
149 stage('Build demo_test Image') {
150 steps {
151 sh """
152 make -C $WORKSPACE/device-management/\$1/demo_test DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
153 """
154 }
155 }
156
157 stage('Build mock-redfish-server Image') {
158 steps {
159 sh """
160 make -C $WORKSPACE/device-management/\$1/mock-redfish-server DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
161 """
162 }
163 }
164
165 stage('Push Images') {
166 steps {
167 sh '''
168 docker images | grep citest
169 for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
170 '''
Scott Bakerff887712020-04-29 13:39:41 -0700171 }
172 }
173
174 stage('Deploy Voltha') {
175 environment {
Scott Baker1ace6d72020-05-01 14:54:41 -0700176 WITH_SIM_ADAPTERS="no"
177 WITH_RADIUS="yes"
178 DEPLOY_K8S="no"
179 VOLTHA_LOG_LEVEL="DEBUG"
Scott Bakerff887712020-04-29 13:39:41 -0700180 }
181 steps {
182 script {
183 sh returnStdout: false, script: """
184 export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
185
Scott Baker1ace6d72020-05-01 14:54:41 -0700186 cd $WORKSPACE/voltha/kind-voltha/
Scott Bakerff887712020-04-29 13:39:41 -0700187 echo \$EXTRA_HELM_FLAGS
188 kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
189 ./voltha up
190 """
191 }
192 }
193 }
194
195 stage('Deploy Kafka Dump Chart') {
196 steps {
197 script {
198 sh returnStdout: false, script: """
199 helm repo add cord https://charts.opencord.org
200 helm repo update
Scott Bakerf21f8b82020-05-05 17:33:13 -0700201 helm del --purge voltha-kafka-dump || true
Scott Bakerff887712020-04-29 13:39:41 -0700202 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
203 """
204 }
205 }
206 }
207
208 stage('Push Tech-Profile') {
209 when {
210 expression { params.profile != "Default" }
211 }
212 steps {
213 sh returnStdout: false, script: """
214 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
215 kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
216 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
217 """
218 }
219 }
220
221 stage('Push Sadis-config') {
222 steps {
223 sh returnStdout: false, script: """
224 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
225 """
226 }
227 }
228
229 stage('Reinstall OLT software') {
230 when {
231 expression { params.reinstallOlt }
232 }
233 steps {
234 script {
235 deployment_config.olts.each { olt ->
236 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true"
237 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true"
238 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
239 waitUntil {
240 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
241 return olt_sw_present.toInteger() == 0
242 }
243 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
244 waitUntil {
245 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
246 return olt_sw_present.toInteger() == 1
247 }
248 if ( olt.fortygig ) {
249 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
250 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
251 }
252 }
253 }
254 }
255 }
256
257 stage('Restart OLT processes') {
258 steps {
259 script {
260 deployment_config.olts.each { olt ->
261 sh returnStdout: false, script: """
262 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
263 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true
264 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true
265 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log'
266 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/dev_mgmt_daemon.log'
267 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service dev_mgmt_daemon start &'
268 sleep 5
269 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt start &'
Scott Baker1ace6d72020-05-01 14:54:41 -0700270 # restart redfish server
271 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service psme stop' || true
272 sleep 10
273 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service psme start'
274 sleep 10
275 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'ps auxw | grep -i psme'
Scott Bakerff887712020-04-29 13:39:41 -0700276 """
Scott Baker1ace6d72020-05-01 14:54:41 -0700277 // Note: ONU Discovery wait loop removed as it was not necessary
Scott Bakerff887712020-04-29 13:39:41 -0700278 }
279 }
280 }
281 }
282
283 stage('Run E2E Tests') {
Scott Bakerff887712020-04-29 13:39:41 -0700284 steps {
Scott Baker1ace6d72020-05-01 14:54:41 -0700285 sh '''
286 mkdir -p $WORKSPACE/RobotLogs
287 # tell the kubernetes script to use images tagged citest and pullPolicy:Never
288 sed -i 's/master/citest/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
289 sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
290 # passing a list to robot framework on the command line is hard, so put the vars in a file
291 make -C $WORKSPACE/device-management ROBOT_EXTRA_ARGS="-V /tmp/robot_vars.yaml" functional-physical-test-single || true
292 '''
Scott Bakerff887712020-04-29 13:39:41 -0700293 }
294 }
295
296 stage('After-Test Delay') {
297 when {
298 expression { params.withPatchset }
299 }
300 steps {
301 sh returnStdout: false, script: """
302 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
303 REGEX="hardware test with delay\$"
Scott Bakerf21f8b82020-05-05 17:33:13 -0700304 [[ "${gerritEventCommentText}" =~ \$REGEX ]] && sleep 10m || true
Scott Bakerff887712020-04-29 13:39:41 -0700305 """
306 }
307 }
308 }
309
310 post {
311 always {
312 sh returnStdout: false, script: '''
313 set +e
314 cp kind-voltha/install-minimal.log $WORKSPACE/
315 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
316 kubectl get nodes -o wide
317 kubectl get pods -o wide
318 kubectl get pods -n voltha -o wide
319
320 sync
321 pkill kail || true
322
323 ## Pull out errors from log files
324 extract_errors_go() {
325 echo
326 echo "Error summary for $1:"
327 grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
328 echo
329 }
330
331 extract_errors_python() {
332 echo
333 echo "Error summary for $1:"
334 grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
335 echo
336 }
337
338 extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
339 extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
340 extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
341 extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
342
343 gzip $WORKSPACE/onos-voltha-combined.log
344
345 ## collect events, the chart should be running by now
346 kubectl get pods | grep -i voltha-kafka-dump | grep -i running
347 if [[ $? == 0 ]]; then
348 kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
349 fi
350 '''
351 script {
352 deployment_config.olts.each { olt ->
353 sh returnStdout: false, script: """
354 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log || true
355 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
356 sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log || true
357 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
358 """
359 }
360 }
361 step([$class: 'RobotPublisher',
362 disableArchiveOutput: false,
Scott Baker1ace6d72020-05-01 14:54:41 -0700363 logFileName: 'device-management/demo_test/functional_test/log*.html',
Scott Bakerff887712020-04-29 13:39:41 -0700364 otherFiles: '',
Scott Baker1ace6d72020-05-01 14:54:41 -0700365 outputFileName: 'device-management/demo_test/functional_test/output*.xml',
Scott Bakerff887712020-04-29 13:39:41 -0700366 outputPath: '.',
367 passThreshold: 100,
Scott Baker1ace6d72020-05-01 14:54:41 -0700368 reportFileName: 'device-management/demo_test/functional_test/report*.html',
Scott Bakerff887712020-04-29 13:39:41 -0700369 unstableThreshold: 0]);
370 archiveArtifacts artifacts: '*.log,*.gz'
371 }
372 }
373}