blob: df4ad02327f77c8fb5e97ff2fdbb82ecf604dc8a [file] [log] [blame]
Joey Armstrong56fdfec2024-03-01 13:43:36 -05001// -----------------------------------------------------------------------
Eric Ball5d8ca532025-01-23 17:31:45 -08002// Copyright 2021-2025 Open Networking Foundation Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08003//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
Joey Armstrong56fdfec2024-03-01 13:43:36 -050015// -----------------------------------------------------------------------
16// SPDX-FileCopyrightText: 2021-2024 Open Networking Foundation Contributors
17// SPDX-License-Identifier: Apache-2.0
18// -----------------------------------------------------------------------
Hardik Windlassec9341b2021-06-07 11:58:29 +000019// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080020// uses bbsim to simulate OLT/ONUs
Joey Armstrong56fdfec2024-03-01 13:43:36 -050021// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -080022
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040023// [TODO] Update library() to the latest DSL syntax supported by jenkins
Matteo Scandoloa156b572021-02-04 11:52:18 -080024library identifier: 'cord-jenkins-libraries@master',
25 retriever: modernSCM([
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040026 $class: 'GitSCMSource',
27 remote: 'https://gerrit.opencord.org/ci-management.git'
Matteo Scandoloa156b572021-02-04 11:52:18 -080028])
29
Joey Armstronge5aae1c2023-07-24 14:11:20 -040030//------------------//
31//---] GLOBAL [---//
32//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040033String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040034
35// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040036// Intent: Return branch name for the script. A hardcoded value is used
37// as a guarantee release jobs are running in an expected sandbox.
Joey Armstrongf076c312023-08-01 17:17:10 -040038// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040039String branchName() {
Eric Ball5d8ca532025-01-23 17:31:45 -080040 def validBranches = ["master", "voltha-2.13"]
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040041
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040042 // "${branch}" is assigned by jenkins
Eric Ball5d8ca532025-01-23 17:31:45 -080043 if (!validBranches.contains(branch)) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040044 String err = [
45 'ERROR: Detected invalid branch',
Roger Luethi92354bf2023-10-05 09:22:19 +020046 "(expected=[${br}] != found=[${branch}])"
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040047 ].join(' ')
48 throw new Exception(err) // groovylint-disable-line ThrowException
49 }
50
Eric Ball5d8ca532025-01-23 17:31:45 -080051 return (branch)
Joey Armstrongf076c312023-08-01 17:17:10 -040052}
Hardik Windlassec9341b2021-06-07 11:58:29 +000053
Joey Armstronge5aae1c2023-07-24 14:11:20 -040054// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040055// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040056// Jenkins will re-write the call stack for serialization.S
57// -----------------------------------------------------------------------
58// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040059// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040060String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040061 String branchName = branchName()
Joey Armstrong06a68372023-07-24 16:37:16 -040062 String src = [
63 'ci-management',
64 'jjb',
65 'pipeline',
66 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040067 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040068 'bbsim-tests.groovy'
69 ].join('/')
70
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040071 String name = [src, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040072 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040073}
74
75// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040076// Intent: Log progress message
77// -----------------------------------------------------------------------
78void enter(String name) {
79 // Announce ourselves for log usability
80 String iam = getIam(name)
81 println("${iam}: ENTER")
82 return
83}
84
85// -----------------------------------------------------------------------
86// Intent: Log progress message
87// -----------------------------------------------------------------------
88void leave(String name) {
89 // Announce ourselves for log usability
90 String iam = getIam(name)
91 println("${iam}: LEAVE")
92 return
93}
94
95// -----------------------------------------------------------------------
Joey Armstrongf0232762024-02-11 17:23:04 -050096// Intent: Display a message with visibility for logging
97// -----------------------------------------------------------------------
98String banner(String message) {
99 String iam = getIam('banner')
100
101 println("""
102
103** -----------------------------------------------------------------------
104** IAM: $iam
105** ${message}
106** -----------------------------------------------------------------------
107""")
108 return
109}
110
111// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400112// Intent: Determine if working on a release branch.
113// Note: Conditional is legacy, should also check for *-dev or *-pre
114// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400115Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400116 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -0400117 // if branchName in modifiers
118 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400119}
120
121// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400122// Intent: Terminate orphaned port-forward from different namespaces
123// -----------------------------------------------------------------------
124void cleanupPortForward() {
125 enter('cleanupPortForward')
126
127 Map pkpfArgs =\
128 [
129 'banner' : true, // display banner for logging
130 'show_procs' : true, // display procs under consideration
131 'filler' : true // fix conditional trailing comma
132 ]
133
134 // 'kubectl.*port-forward'
135 pkill_port_forward('port-forward', pkpfArgs)
136 leave('cleanupPortForward')
137 return
138}
139
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400140// find . \( -name 'log*.html' -o -name 'output*.xml' -o -name 'report*.html' \) -p
141// -----------------------------------------------------------------------
142// Intent: Display contents of the logs directory
143// -----------------------------------------------------------------------
144// [TODO]
145// o where-4-art-thou logs directory ?
146// o Replace find {logfile} command with /bin/ls {logdir} when found.
147// Individual logs may be missing due to failure, show what is available.
148// -----------------------------------------------------------------------
149void findPublishedLogs() {
150 String iam = 'findPublishedLogs'
151
152 enter(iam)
153 sh(label : iam,
154 script : """
155find . -name 'output.xml' -print
156""")
157 leave(iam)
158 return
159}
160
161// -----------------------------------------------------------------------
162// Intent: Terminate kail-startup process launched earlier
163// -----------------------------------------------------------------------
164// :param caller: Name of parent calling function (debug context)
165// :type caller: String, optional
166// :returns: none
167// :rtype: void
168// -----------------------------------------------------------------------
169void killKailStartup(String caller='') {
170 String iam = "killKailStartup (caller=$caller)"
171
172 enter(iam)
173 sh(label : 'Terminate kail-startup',
174 script : """
175if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
176 pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
177fi
178""")
179 leave(iam)
180 return
181}
182
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400183// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400184// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400185// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400186void execute_test\
187(
188 String testTarget, // functional-single-kind-dt
189 String workflow, // dt
190 String testLogging, // 'True'
191 Boolean teardown, // true
Guru4dc382f2025-01-10 18:14:58 +0530192 String testSpecificHelmFlags='',
Gurub7a354a2025-01-10 19:05:06 +0530193 Boolean vgcEnabled
Joey Armstrong0251e962023-08-25 20:51:31 -0400194) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400195 String infraNamespace = 'default'
196 String volthaNamespace = 'voltha'
197 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500198
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400199 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400200 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400201 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400202 stage('Cleanup') {
203 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400204 timeout(15) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400205 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400206 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400207 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400208 } // timeout
209
210 timeout(5) {
211 script {
212 enter('Cleanup')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400213 cleanupPortForward()
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400214 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400215 } // script
216 } // timeout
217 } // teardown
218 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500219
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400220 // -----------------------------------------------------------------------
221 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400222 stage('Deploy common infrastructure') {
223 script {
224 local dashargs = [
225 'kpi_exporter.enabled=false',
226 'dashboards.xos=false',
227 'dashboards.onos=false',
228 'dashboards.aaa=false',
229 'dashboards.voltha=false',
230 ].join(',')
231
232 local promargs = [
233 'prometheus.alertmanager.enabled=false',
234 'prometheus.pushgateway.enabled=false',
235 ].join(',')
236
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400237 sh(label : 'Deploy common infrastructure',
238 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000239 helm repo add onf https://charts.opencord.org
240 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400241
242 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000243 if [ ${withMonitoring} = true ] ; then
244 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400245 --set ${promargs} \
246 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000247 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400248 """)
249 } // script
250 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500251
Joey Armstrong0251e962023-08-25 20:51:31 -0400252 // -----------------------------------------------------------------------
253 // [TODO] Check onos_log output
254 // -----------------------------------------------------------------------
255 stage('Deploy Voltha') {
256 if (teardown) {
257 timeout(10) {
258 script {
259 String iam = getIam('Deploy Voltha')
260 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000261
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400262 sh(label : 'Launch kail-startup',
263 script : """
264mkdir -p "$logsDir"
265touch "$onosLog"
Joey Armstrongf404b642023-08-04 14:39:13 -0400266
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400267_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
268""")
269
270 // if we're downloading a voltha-helm-charts patch,
271 // install from a local copy of the charts
Joey Armstrong0251e962023-08-25 20:51:31 -0400272 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400273
Joey Armstrong0251e962023-08-25 20:51:31 -0400274 if (volthaHelmChartsChange != ''
275 || gerritProject == 'voltha-helm-charts'
276 || isReleaseBranch(branch) // branch != 'master'
277 ) {
278 localCharts = true
279 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000280
Joey Armstrong0251e962023-08-25 20:51:31 -0400281 String branchName = branchName()
282 Boolean isRelease = isReleaseBranch(branch)
283 println([
284 " ** localCharts=${localCharts}",
285 "branchName=${branchName}",
286 "branch=${branch}",
287 "branch=isReleaseBranch=${isRelease}",
288 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800289
Joey Armstrong0251e962023-08-25 20:51:31 -0400290 // -----------------------------------------------------------------------
291 // Rewrite localHelmFlags using array join, moving code around and
292 // refactoring into standalone functions
293 // -----------------------------------------------------------------------
294 // NOTE temporary workaround expose ONOS node ports
295 // -----------------------------------------------------------------------
gstc79131a2025-01-13 11:19:12 +0530296 String localHelmFlags = ""
297 if (!vgcEnabled) {
298 localHelmFlags = [
299 extraHelmFlags.trim(),
300 "--set global.log_level=${logLevel.toUpperCase()}",
301 '--set onos-classic.onosSshPort=30115',
302 '--set onos-classic.onosApiPort=30120',
303 '--set onos-classic.onosOfPort=31653',
304 '--set onos-classic.individualOpenFlowNodePorts=true',
305 testSpecificHelmFlags
306 ].join(' ')
307 } else {
308 localHelmFlags = [
309 extraHelmFlags.trim(),
310 testSpecificHelmFlags
311 ].join(' ')
312 }
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400313
Joey Armstrong0251e962023-08-25 20:51:31 -0400314 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000315
Joey Armstrong0251e962023-08-25 20:51:31 -0400316 if (gerritProject != '') {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400317 localHelmFlags += getVolthaImageFlags(gerritProject)
Joey Armstrong0251e962023-08-25 20:51:31 -0400318 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800319
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400320 enter('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400321 volthaDeploy([
322 infraNamespace: infraNamespace,
323 volthaNamespace: volthaNamespace,
324 workflow: workflow.toLowerCase(),
325 withMacLearning: enableMacLearning.toBoolean(),
326 extraHelmFlags: localHelmFlags,
327 localCharts: localCharts,
328 bbsimReplica: olts.toInteger(),
329 dockerRegistry: registry,
Gurub7a354a2025-01-10 19:05:06 +0530330 vgcEnabled: vgcEnabled,
Joey Armstrong0251e962023-08-25 20:51:31 -0400331 ])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400332 leave('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400333 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400334
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400335 // Display spawned procs
336 script {
337 enter('bbsim-tests::pgrep_port_forward::0')
338 pgrep_port_forward('port-forw')
339 leave('bbsim-tests::pgrep_port_forward::0')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400340
341 killKailStartup('Deploy Voltha')
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400342 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400343
Joey Armstrong0251e962023-08-25 20:51:31 -0400344 // -----------------------------------------------------------------------
345 // Bundle onos-voltha / kail logs
346 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400347 sh(
348 label : 'Bundle logs: onos-voltha-startup-combined',
349 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400350cat <<EOM
351
352** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400353** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400354** -----------------------------------------------------------------------
355EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400356
357pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
358gzip -k onos-voltha-startup-combined.log
359rm onos-voltha-startup-combined.log
360popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
Joey Armstrong0251e962023-08-25 20:51:31 -0400361 """)
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400362 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400363
Joey Armstrong0251e962023-08-25 20:51:31 -0400364 // -----------------------------------------------------------------------
365 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400366 sh(label : 'while-true-port-forward',
Roger Luethicebf79a2023-09-28 09:41:45 +0200367 script : """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000368 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
369 bbsimDmiPortFwd=50075
370 for i in {0..${olts.toInteger() - 1}}; do
371 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
372 ((bbsimDmiPortFwd++))
373 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000374 if [ ${withMonitoring} = true ] ; then
375 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
376 fi
gst2ff418d2025-01-13 13:23:05 +0530377 if [ ${vgcEnabled} = true ] ; then
gstc79131a2025-01-13 11:19:12 +0530378 JENKINS_NODE_COOKIE="dontKillMe" _TAG="vgc-Port-Forward-Enable" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-go-controller 8181:8181; done"&
379 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400380# ps aux | grep port-forward
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400381""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400382 // ---------------------------------
383 // Sanity check port-forward spawned
384 // ---------------------------------
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400385 script {
386 enter('bbsim-tests::pgrep_port_forward::1')
387 pgrep_port_forward('port-forw')
388 leave('bbsim-tests::pgrep_port_forward::1')
389 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500390
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400391 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400392 script {
393 enter('setOnosLogLevels')
394 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400395 onosNamespace: infraNamespace,
396 apps: [
397 'org.opencord.dhcpl2relay',
398 'org.opencord.olt',
399 'org.opencord.aaa',
400 'org.opencord.maclearner',
401 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
402 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
403 ],
404 logLevel: logLevel
405 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400406 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400407 } // script
408 } // if (teardown)
409 } // stage('Deploy Voltha')
410
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400411 // -----------------------------------------------------------------------
412 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400413 stage("Run test ${testTarget} on workflow ${workflow}") {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400414 sh(
415 label : 'Monitor using mem_consumption.py',
416 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400417echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400418
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400419if [ ${withMonitoring} = true ] ; then
420 cat <<EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400421
422** -----------------------------------------------------------------------
423** Monitoring memory usage with mem_consumption.py
424** -----------------------------------------------------------------------
425EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400426 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
427 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400428
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400429 echo '** Installing python virtualenv'
430 make venv-activate-patched
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400431
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400432 # Collect initial memory consumption
433 set +u && source .venv/bin/activate && set -u
434 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
435fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400436
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400437echo -e '** Monitor memory consumption: LEAVE\n'
438""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400439
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400440 sh(
441 label : "make testTarget=[${testTarget}]",
442 script : """
443echo -e "\n** make testTarget=[${testTarget}]"
444mkdir -p ${logsDir}
445export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
446ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
447export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800448
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400449make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
450""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400451
452 getPodsInfo("${logsDir}")
453
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400454 // [TODO] make conditional, bundle when logs are available
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400455 sh(
456 label : 'Gather robot Framework logs',
457 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400458echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400459
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400460# set +e
461# collect logs collected in the Robot Framework StartLogging keyword
462cd "${logsDir}"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400463
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400464echo "** Available logs:"
465/bin/ls -l "$logsDir"
466echo
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400467
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400468echo '** Bundle combined log'
469gzip *-combined.log || true
470rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400471
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400472echo -e '** Gather robot Framework logs: LEAVE\n'
473""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400474
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400475 // -----------------------------------------------------------------------
476 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400477 sh(
478 label : 'Monitor pod-mem-consumption',
479 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400480echo -e '** Monitor pod-mem-consumption: ENTER'
481if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400482 cat <<EOM
483
484** -----------------------------------------------------------------------
485** Monitoring pod-memory-consumption using mem_consumption.py
486** -----------------------------------------------------------------------
487EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400488
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400489cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400490
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400491echo '** Installing python virtualenv'
492make venv-activate-patched
493
494# Collect memory consumption of voltha pods once all the tests are complete
495set +u && source .venv/bin/activate && set -u
496python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
497fi
498echo -e '** Monitor pod-mem-consumption: LEAVE\n'
499""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400500 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400501
502 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400503} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800504
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400505// -----------------------------------------------------------------------
506// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400507void collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400508 script {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400509 enter("exitStatus=${exitStatus}")
Joey Armstrongf0232762024-02-11 17:23:04 -0500510 banner('collectArtifacts')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400511 }
Joey Armstrongdaa1f0a2024-04-03 18:07:59 -0400512
Joey Armstrongcd419122023-08-07 14:56:39 -0400513 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400514
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400515 sh(label : 'kubectl logs > voltha.log',
516 script : """
517kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
518 > $WORKSPACE/${exitStatus}/voltha.log
519""")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400520
Joey Armstrongcd419122023-08-07 14:56:39 -0400521 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400522
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400523 script { killKailStartup('collectArtifacts') }
524 script { findPublishedLogs() }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400525
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400526 enter('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400527 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400528 disableArchiveOutput: false,
529 logFileName: '**/*/log*.html',
530 otherFiles: '',
531 outputFileName: '**/*/output*.xml',
532 outputPath: '.',
533 passThreshold: 100,
534 reportFileName: '**/*/report*.html',
535 unstableThreshold: 0,
536 onlyCritical: true])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400537 leave('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400538
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400539 leave("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400540 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000541}
542
Joey Armstrongb65ada32023-08-03 12:50:20 -0400543// -----------------------------------------------------------------------
544// Intent: main
545// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800546pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400547 /* no label, executor is determined by JJB */
548 agent {
549 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800550 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400551
Joey Armstrong0251e962023-08-25 20:51:31 -0400552 options {
553 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800554 }
Joey Armstronged161f72023-04-11 13:16:59 -0400555
Joey Armstrong0251e962023-08-25 20:51:31 -0400556 environment {
557 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
558 VOLTCONFIG = "$HOME/.volt/config"
559 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
560 DIAGS_PROFILE = 'VOLTHA_PROFILE'
561 SSHPASS = 'karaf'
562 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400563
Joey Armstrong0251e962023-08-25 20:51:31 -0400564 stages {
Joey Armstrongf3922212024-07-22 17:11:02 -0400565
Joey Armstrong0251e962023-08-25 20:51:31 -0400566 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400567 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500568 enter('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400569 getVolthaCode([
570 branch: "${branch}",
571 gerritProject: "${gerritProject}",
572 gerritRefspec: "${gerritRefspec}",
573 volthaSystemTestsChange: "${volthaSystemTestsChange}",
574 volthaHelmChartsChange: "${volthaHelmChartsChange}",
575 ])
Joey Armstrongdef9c402024-01-30 18:05:29 -0500576 leave('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400577 }
578 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400579
Joey Armstrong0251e962023-08-25 20:51:31 -0400580 stage('Build patch v1.1') {
581 // build the patch only if gerritProject is specified
582 when {
583 expression { return !gerritProject.isEmpty() }
584 }
585
586 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500587 enter('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400588 // NOTE that the correct patch has already been checked out
589 // during the getVolthaCode step
590 buildVolthaComponent("${gerritProject}")
Joey Armstrongdef9c402024-01-30 18:05:29 -0500591 leave('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400592 }
593 }
594
595 // -----------------------------------------------------------------------
596 // -----------------------------------------------------------------------
597 stage('Install Kail')
598 {
599 steps
600 {
601 script
602 {
603 String cmd = [
604 'make',
605 '--no-print-directory',
606 '-C', "$WORKSPACE/voltha-system-tests",
607 "KAIL_PATH=\"$WORKSPACE/bin\"",
608 'kail',
609 ].join(' ')
610
611 println(" ** Running: ${cmd}")
612 sh("${cmd}")
613 } // script
614 } // steps
615 } // stage
616
617 // -----------------------------------------------------------------------
618 // -----------------------------------------------------------------------
Joey Armstrongf3922212024-07-22 17:11:02 -0400619 stage('Install kubectl')
620 {
621 steps
622 {
623 script
624 {
625 String cmd = [
626 'make',
627 '--no-print-directory',
628 '-C', "$WORKSPACE",
629 "KUBECTL_PATH=\"$WORKSPACE/bin\"",
630 'kubectl',
631 ].join(' ')
632
633 println(" ** Running: ${cmd}")
634 sh("${cmd}")
635 } // script
636 } // steps
637 } // stage
638
639 // -----------------------------------------------------------------------
640 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400641 stage('Install Tools') {
642 steps {
643 script {
644 String branchName = branchName()
645 String iam = getIam('Install Tools')
646
647 println("${iam}: ENTER (branch=$branch)")
648 installKind(branch) // needed early by stage(Cleanup)
649 println("${iam}: LEAVE (branch=$branch)")
650 } // script
651 } // steps
652 } // stage
653
654 // -----------------------------------------------------------------------
655 // -----------------------------------------------------------------------
656 stage('Create K8s Cluster') {
657 steps {
658 script {
659 def clusterExists = sh(
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400660 label : 'Create K8s Cluster',
Joey Armstrong0251e962023-08-25 20:51:31 -0400661 returnStdout: true,
662 script: """kind get clusters | grep "${clusterName}" | wc -l""")
663
664 if (clusterExists.trim() == '0') {
665 createKubernetesCluster([nodes: 3, name: clusterName])
666 }
667 } // script
668 } // steps
669 } // stage('Create K8s Cluster')
670
671 // -----------------------------------------------------------------------
672 // -----------------------------------------------------------------------
673 stage('Replace voltctl') {
674 // if the project is voltctl, override the downloaded one with the built one
675 when {
676 expression { return gerritProject == 'voltctl' }
677 }
678
679 // Hmmmm(?) where did the voltctl download happen ?
680 // Likely Makefile but would be helpful to document here.
681 steps {
682 script {
683 String iam = getIam('Replace voltctl')
684
685 println("${iam} Running: installVoltctl($branch)")
686 println("${iam}: ENTER")
687 installVoltctl("$branch")
688 println("${iam}: LEAVE")
689 } // script
690 } // step
691 } // stage
692
693 // -----------------------------------------------------------------------
694 // -----------------------------------------------------------------------
695 stage('Load image in kind nodes')
696 {
697 when {
698 expression { return !gerritProject.isEmpty() }
699 }
700 steps {
701 loadToKind()
702 } // steps
703 } // stage
704
705 // -----------------------------------------------------------------------
706 // [TODO] verify testing output
707 // -----------------------------------------------------------------------
708 stage('Parse and execute tests')
709 {
710 steps {
711 script {
712 // Announce ourselves for log usability
713 enter('Parse and execute tests')
714
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400715 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400716 println("** [DEBUG]: tests=$tests")
717
718 // Display expected tests for times when output goes dark
719 tests.eachWithIndex { test, idx ->
720 String target = test['target']
721 println("** test[${idx}]: ${target}\n")
722 }
723
724 println('''
725** -----------------------------------------------------------------------
726** NOTE: For odd/silent job failures verify a few details
727** - All tests mentioned in the tests-to-run index were logged.
728** - Test suites display ENTER/LEAVE mesasge pairs.
729** - Processing terminated prematurely when LEAVE strings are missing.
730** -----------------------------------------------------------------------
731''')
732 tests.eachWithIndex { test, idx ->
733 println "** readYaml test suite[$idx]) test=[${test}]"
734
Joey Armstrongf404b642023-08-04 14:39:13 -0400735 String target = test['target']
736 String workflow = test['workflow']
737 String flags = test['flags']
738 Boolean teardown = test['teardown'].toBoolean()
739 Boolean logging = test['logging'].toBoolean()
740 String testLogging = (logging) ? 'True' : 'False'
Guru876a28c2025-01-10 22:59:56 +0530741 Boolean vgcEnabled = test['vgcEnabled'] != null ? test['vgcEnabled'].toBoolean() : false
Joey Armstrongf404b642023-08-04 14:39:13 -0400742
Joey Armstrong0251e962023-08-25 20:51:31 -0400743 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400744** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400745** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400746** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400747""")
748
Joey Armstrong0251e962023-08-25 20:51:31 -0400749 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400750 enter("execute_test (target=$target)")
Guru4dc382f2025-01-10 18:14:58 +0530751 execute_test(target, workflow, testLogging, teardown, flags, vgcEnabled)
Joey Armstrong0251e962023-08-25 20:51:31 -0400752 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400753 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400754 catch (Exception err) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400755 String iamexc = getIam(test)
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400756 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400757 }
758 finally {
759 leave("execute_test (target=$target)")
760 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400761 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400762 // Premature exit if this message is not logged
763 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400764 } // script
765 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400766 } // stage
767 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400768
769 post
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400770 { // https://www.jenkins.io/doc/book/pipeline/syntax/#post
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400771 aborted {
772 collectArtifacts('aborted')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400773 }
774 failure {
775 collectArtifacts('failed')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400776 }
777 always {
778 collectArtifacts('always')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400779 }
780 cleanup {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400781 script { cleanupPortForward() }
782 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800783 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400784} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400785
Joey Armstrong664c55a2023-08-28 14:22:33 -0400786// [EOF]