blob: 398e32d8a68186e94179538ce2de8caa455eccd9 [file] [log] [blame]
Joey Armstrong56fdfec2024-03-01 13:43:36 -05001// -----------------------------------------------------------------------
2// Copyright 2021-2024 Open Networking Foundation Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08003//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
Joey Armstrong56fdfec2024-03-01 13:43:36 -050015// -----------------------------------------------------------------------
16// SPDX-FileCopyrightText: 2021-2024 Open Networking Foundation Contributors
17// SPDX-License-Identifier: Apache-2.0
18// -----------------------------------------------------------------------
Hardik Windlassec9341b2021-06-07 11:58:29 +000019// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080020// uses bbsim to simulate OLT/ONUs
Joey Armstrong56fdfec2024-03-01 13:43:36 -050021// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -080022
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040023// [TODO] Update library() to the latest DSL syntax supported by jenkins
Matteo Scandoloa156b572021-02-04 11:52:18 -080024library identifier: 'cord-jenkins-libraries@master',
25 retriever: modernSCM([
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040026 $class: 'GitSCMSource',
27 remote: 'https://gerrit.opencord.org/ci-management.git'
Matteo Scandoloa156b572021-02-04 11:52:18 -080028])
29
Joey Armstronge5aae1c2023-07-24 14:11:20 -040030//------------------//
31//---] GLOBAL [---//
32//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040033String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040034
35// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040036// Intent: Return branch name for the script. A hardcoded value is used
37// as a guarantee release jobs are running in an expected sandbox.
Joey Armstrongf076c312023-08-01 17:17:10 -040038// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040039String branchName() {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040040 String br = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040041
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040042 // "${branch}" is assigned by jenkins
43 if (br != branch) {
44 String err = [
45 'ERROR: Detected invalid branch',
Roger Luethi92354bf2023-10-05 09:22:19 +020046 "(expected=[${br}] != found=[${branch}])"
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040047 ].join(' ')
48 throw new Exception(err) // groovylint-disable-line ThrowException
49 }
50
51 return (br)
Joey Armstrongf076c312023-08-01 17:17:10 -040052}
Hardik Windlassec9341b2021-06-07 11:58:29 +000053
Joey Armstronge5aae1c2023-07-24 14:11:20 -040054// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040055// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040056// Jenkins will re-write the call stack for serialization.S
57// -----------------------------------------------------------------------
58// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040059// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040060String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040061 String branchName = branchName()
Joey Armstrong06a68372023-07-24 16:37:16 -040062 String src = [
63 'ci-management',
64 'jjb',
65 'pipeline',
66 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040067 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040068 'bbsim-tests.groovy'
69 ].join('/')
70
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040071 String name = [src, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040072 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040073}
74
75// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040076// Intent: Log progress message
77// -----------------------------------------------------------------------
78void enter(String name) {
79 // Announce ourselves for log usability
80 String iam = getIam(name)
81 println("${iam}: ENTER")
82 return
83}
84
85// -----------------------------------------------------------------------
86// Intent: Log progress message
87// -----------------------------------------------------------------------
88void leave(String name) {
89 // Announce ourselves for log usability
90 String iam = getIam(name)
91 println("${iam}: LEAVE")
92 return
93}
94
95// -----------------------------------------------------------------------
Joey Armstrongf0232762024-02-11 17:23:04 -050096// Intent: Display a message with visibility for logging
97// -----------------------------------------------------------------------
98String banner(String message) {
99 String iam = getIam('banner')
100
101 println("""
102
103** -----------------------------------------------------------------------
104** IAM: $iam
105** ${message}
106** -----------------------------------------------------------------------
107""")
108 return
109}
110
111// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400112// Intent: Determine if working on a release branch.
113// Note: Conditional is legacy, should also check for *-dev or *-pre
114// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400115Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400116 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -0400117 // if branchName in modifiers
118 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400119}
120
121// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400122// Intent: Terminate orphaned port-forward from different namespaces
123// -----------------------------------------------------------------------
124void cleanupPortForward() {
125 enter('cleanupPortForward')
126
127 Map pkpfArgs =\
128 [
129 'banner' : true, // display banner for logging
130 'show_procs' : true, // display procs under consideration
131 'filler' : true // fix conditional trailing comma
132 ]
133
134 // 'kubectl.*port-forward'
135 pkill_port_forward('port-forward', pkpfArgs)
136 leave('cleanupPortForward')
137 return
138}
139
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400140// find . \( -name 'log*.html' -o -name 'output*.xml' -o -name 'report*.html' \) -p
141// -----------------------------------------------------------------------
142// Intent: Display contents of the logs directory
143// -----------------------------------------------------------------------
144// [TODO]
145// o where-4-art-thou logs directory ?
146// o Replace find {logfile} command with /bin/ls {logdir} when found.
147// Individual logs may be missing due to failure, show what is available.
148// -----------------------------------------------------------------------
149void findPublishedLogs() {
150 String iam = 'findPublishedLogs'
151
152 enter(iam)
153 sh(label : iam,
154 script : """
155find . -name 'output.xml' -print
156""")
157 leave(iam)
158 return
159}
160
161// -----------------------------------------------------------------------
162// Intent: Terminate kail-startup process launched earlier
163// -----------------------------------------------------------------------
164// :param caller: Name of parent calling function (debug context)
165// :type caller: String, optional
166// :returns: none
167// :rtype: void
168// -----------------------------------------------------------------------
169void killKailStartup(String caller='') {
170 String iam = "killKailStartup (caller=$caller)"
171
172 enter(iam)
173 sh(label : 'Terminate kail-startup',
174 script : """
175if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
176 pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
177fi
178""")
179 leave(iam)
180 return
181}
182
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400183// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400184// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400185// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400186void execute_test\
187(
188 String testTarget, // functional-single-kind-dt
189 String workflow, // dt
190 String testLogging, // 'True'
191 Boolean teardown, // true
192 String testSpecificHelmFlags=''
193) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400194 String infraNamespace = 'default'
195 String volthaNamespace = 'voltha'
196 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500197
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400198 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400199 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400200 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400201 stage('Cleanup') {
202 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400203 timeout(15) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400204 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400205 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400206 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400207 } // timeout
208
209 timeout(5) {
210 script {
211 enter('Cleanup')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400212 cleanupPortForward()
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400213 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400214 } // script
215 } // timeout
216 } // teardown
217 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500218
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400219 // -----------------------------------------------------------------------
220 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400221 stage('Deploy common infrastructure') {
222 script {
223 local dashargs = [
224 'kpi_exporter.enabled=false',
225 'dashboards.xos=false',
226 'dashboards.onos=false',
227 'dashboards.aaa=false',
228 'dashboards.voltha=false',
229 ].join(',')
230
231 local promargs = [
232 'prometheus.alertmanager.enabled=false',
233 'prometheus.pushgateway.enabled=false',
234 ].join(',')
235
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400236 sh(label : 'Deploy common infrastructure',
237 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000238 helm repo add onf https://charts.opencord.org
239 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400240
241 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000242 if [ ${withMonitoring} = true ] ; then
243 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400244 --set ${promargs} \
245 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000246 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400247 """)
248 } // script
249 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500250
Joey Armstrong0251e962023-08-25 20:51:31 -0400251 // -----------------------------------------------------------------------
252 // [TODO] Check onos_log output
253 // -----------------------------------------------------------------------
254 stage('Deploy Voltha') {
255 if (teardown) {
256 timeout(10) {
257 script {
258 String iam = getIam('Deploy Voltha')
259 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000260
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400261 sh(label : 'Launch kail-startup',
262 script : """
263mkdir -p "$logsDir"
264touch "$onosLog"
Joey Armstrongf404b642023-08-04 14:39:13 -0400265
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400266_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
267""")
268
269 // if we're downloading a voltha-helm-charts patch,
270 // install from a local copy of the charts
Joey Armstrong0251e962023-08-25 20:51:31 -0400271 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400272
Joey Armstrong0251e962023-08-25 20:51:31 -0400273 if (volthaHelmChartsChange != ''
274 || gerritProject == 'voltha-helm-charts'
275 || isReleaseBranch(branch) // branch != 'master'
276 ) {
277 localCharts = true
278 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000279
Joey Armstrong0251e962023-08-25 20:51:31 -0400280 String branchName = branchName()
281 Boolean isRelease = isReleaseBranch(branch)
282 println([
283 " ** localCharts=${localCharts}",
284 "branchName=${branchName}",
285 "branch=${branch}",
286 "branch=isReleaseBranch=${isRelease}",
287 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800288
Joey Armstrong0251e962023-08-25 20:51:31 -0400289 // -----------------------------------------------------------------------
290 // Rewrite localHelmFlags using array join, moving code around and
291 // refactoring into standalone functions
292 // -----------------------------------------------------------------------
293 // NOTE temporary workaround expose ONOS node ports
294 // -----------------------------------------------------------------------
295 String localHelmFlags = [
296 extraHelmFlags.trim(),
297 "--set global.log_level=${logLevel.toUpperCase()}",
298 '--set onos-classic.onosSshPort=30115',
299 '--set onos-classic.onosApiPort=30120',
300 '--set onos-classic.onosOfPort=31653',
301 '--set onos-classic.individualOpenFlowNodePorts=true',
302 testSpecificHelmFlags
303 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400304
Joey Armstrong0251e962023-08-25 20:51:31 -0400305 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000306
Joey Armstrong0251e962023-08-25 20:51:31 -0400307 if (gerritProject != '') {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400308 localHelmFlags += getVolthaImageFlags(gerritProject)
Joey Armstrong0251e962023-08-25 20:51:31 -0400309 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800310
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400311 enter('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400312 volthaDeploy([
313 infraNamespace: infraNamespace,
314 volthaNamespace: volthaNamespace,
315 workflow: workflow.toLowerCase(),
316 withMacLearning: enableMacLearning.toBoolean(),
317 extraHelmFlags: localHelmFlags,
318 localCharts: localCharts,
319 bbsimReplica: olts.toInteger(),
320 dockerRegistry: registry,
321 ])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400322 leave('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400323 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400324
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400325 // Display spawned procs
326 script {
327 enter('bbsim-tests::pgrep_port_forward::0')
328 pgrep_port_forward('port-forw')
329 leave('bbsim-tests::pgrep_port_forward::0')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400330
331 killKailStartup('Deploy Voltha')
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400332 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400333
Joey Armstrong0251e962023-08-25 20:51:31 -0400334 // -----------------------------------------------------------------------
335 // Bundle onos-voltha / kail logs
336 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400337 sh(
338 label : 'Bundle logs: onos-voltha-startup-combined',
339 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400340cat <<EOM
341
342** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400343** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400344** -----------------------------------------------------------------------
345EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400346
347pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
348gzip -k onos-voltha-startup-combined.log
349rm onos-voltha-startup-combined.log
350popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
Joey Armstrong0251e962023-08-25 20:51:31 -0400351 """)
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400352 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400353
Joey Armstrong0251e962023-08-25 20:51:31 -0400354 // -----------------------------------------------------------------------
355 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400356 sh(label : 'while-true-port-forward',
Roger Luethicebf79a2023-09-28 09:41:45 +0200357 script : """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000358 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
359 bbsimDmiPortFwd=50075
360 for i in {0..${olts.toInteger() - 1}}; do
361 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
362 ((bbsimDmiPortFwd++))
363 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000364 if [ ${withMonitoring} = true ] ; then
365 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
366 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400367# ps aux | grep port-forward
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400368""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400369 // ---------------------------------
370 // Sanity check port-forward spawned
371 // ---------------------------------
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400372 script {
373 enter('bbsim-tests::pgrep_port_forward::1')
374 pgrep_port_forward('port-forw')
375 leave('bbsim-tests::pgrep_port_forward::1')
376 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500377
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400378 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400379 script {
380 enter('setOnosLogLevels')
381 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400382 onosNamespace: infraNamespace,
383 apps: [
384 'org.opencord.dhcpl2relay',
385 'org.opencord.olt',
386 'org.opencord.aaa',
387 'org.opencord.maclearner',
388 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
389 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
390 ],
391 logLevel: logLevel
392 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400393 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400394 } // script
395 } // if (teardown)
396 } // stage('Deploy Voltha')
397
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400398 // -----------------------------------------------------------------------
399 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400400 stage("Run test ${testTarget} on workflow ${workflow}") {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400401 sh(
402 label : 'Monitor using mem_consumption.py',
403 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400404echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400405
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400406if [ ${withMonitoring} = true ] ; then
407 cat <<EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400408
409** -----------------------------------------------------------------------
410** Monitoring memory usage with mem_consumption.py
411** -----------------------------------------------------------------------
412EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400413 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
414 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400415
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400416 echo '** Installing python virtualenv'
417 make venv-activate-patched
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400418
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400419 # Collect initial memory consumption
420 set +u && source .venv/bin/activate && set -u
421 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
422fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400423
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400424echo -e '** Monitor memory consumption: LEAVE\n'
425""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400426
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400427 sh(
428 label : "make testTarget=[${testTarget}]",
429 script : """
430echo -e "\n** make testTarget=[${testTarget}]"
431mkdir -p ${logsDir}
432export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
433ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
434export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800435
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400436make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
437""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400438
439 getPodsInfo("${logsDir}")
440
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400441 // [TODO] make conditional, bundle when logs are available
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400442 sh(
443 label : 'Gather robot Framework logs',
444 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400445echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400446
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400447# set +e
448# collect logs collected in the Robot Framework StartLogging keyword
449cd "${logsDir}"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400450
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400451echo "** Available logs:"
452/bin/ls -l "$logsDir"
453echo
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400454
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400455echo '** Bundle combined log'
456gzip *-combined.log || true
457rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400458
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400459echo -e '** Gather robot Framework logs: LEAVE\n'
460""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400461
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400462 // -----------------------------------------------------------------------
463 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400464 sh(
465 label : 'Monitor pod-mem-consumption',
466 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400467echo -e '** Monitor pod-mem-consumption: ENTER'
468if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400469 cat <<EOM
470
471** -----------------------------------------------------------------------
472** Monitoring pod-memory-consumption using mem_consumption.py
473** -----------------------------------------------------------------------
474EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400475
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400476cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400477
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400478echo '** Installing python virtualenv'
479make venv-activate-patched
480
481# Collect memory consumption of voltha pods once all the tests are complete
482set +u && source .venv/bin/activate && set -u
483python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
484fi
485echo -e '** Monitor pod-mem-consumption: LEAVE\n'
486""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400487 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400488
489 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400490} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800491
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400492// -----------------------------------------------------------------------
493// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400494void collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400495 script {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400496 enter("exitStatus=${exitStatus}")
Joey Armstrongf0232762024-02-11 17:23:04 -0500497 banner('collectArtifacts')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400498 }
Joey Armstrongdaa1f0a2024-04-03 18:07:59 -0400499
Joey Armstrongcd419122023-08-07 14:56:39 -0400500 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400501
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400502 sh(label : 'kubectl logs > voltha.log',
503 script : """
504kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
505 > $WORKSPACE/${exitStatus}/voltha.log
506""")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400507
Joey Armstrongcd419122023-08-07 14:56:39 -0400508 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400509
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400510 script { killKailStartup('collectArtifacts') }
511 script { findPublishedLogs() }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400512
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400513 enter('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400514 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400515 disableArchiveOutput: false,
516 logFileName: '**/*/log*.html',
517 otherFiles: '',
518 outputFileName: '**/*/output*.xml',
519 outputPath: '.',
520 passThreshold: 100,
521 reportFileName: '**/*/report*.html',
522 unstableThreshold: 0,
523 onlyCritical: true])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400524 leave('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400525
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400526 leave("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400527 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000528}
529
Joey Armstrongb65ada32023-08-03 12:50:20 -0400530// -----------------------------------------------------------------------
531// Intent: main
532// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800533pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400534 /* no label, executor is determined by JJB */
535 agent {
536 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800537 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400538
Joey Armstrong0251e962023-08-25 20:51:31 -0400539 options {
540 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800541 }
Joey Armstronged161f72023-04-11 13:16:59 -0400542
Joey Armstrong0251e962023-08-25 20:51:31 -0400543 environment {
544 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
545 VOLTCONFIG = "$HOME/.volt/config"
546 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
547 DIAGS_PROFILE = 'VOLTHA_PROFILE'
548 SSHPASS = 'karaf'
549 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400550
Joey Armstrong0251e962023-08-25 20:51:31 -0400551 stages {
Joey Armstrongf3922212024-07-22 17:11:02 -0400552
Joey Armstrong0251e962023-08-25 20:51:31 -0400553 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400554 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500555 enter('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400556 getVolthaCode([
557 branch: "${branch}",
558 gerritProject: "${gerritProject}",
559 gerritRefspec: "${gerritRefspec}",
560 volthaSystemTestsChange: "${volthaSystemTestsChange}",
561 volthaHelmChartsChange: "${volthaHelmChartsChange}",
562 ])
Joey Armstrongdef9c402024-01-30 18:05:29 -0500563 leave('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400564 }
565 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400566
Joey Armstrong0251e962023-08-25 20:51:31 -0400567 stage('Build patch v1.1') {
568 // build the patch only if gerritProject is specified
569 when {
570 expression { return !gerritProject.isEmpty() }
571 }
572
573 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500574 enter('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400575 // NOTE that the correct patch has already been checked out
576 // during the getVolthaCode step
577 buildVolthaComponent("${gerritProject}")
Joey Armstrongdef9c402024-01-30 18:05:29 -0500578 leave('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400579 }
580 }
581
582 // -----------------------------------------------------------------------
583 // -----------------------------------------------------------------------
584 stage('Install Kail')
585 {
586 steps
587 {
588 script
589 {
590 String cmd = [
591 'make',
592 '--no-print-directory',
593 '-C', "$WORKSPACE/voltha-system-tests",
594 "KAIL_PATH=\"$WORKSPACE/bin\"",
595 'kail',
596 ].join(' ')
597
598 println(" ** Running: ${cmd}")
599 sh("${cmd}")
600 } // script
601 } // steps
602 } // stage
603
604 // -----------------------------------------------------------------------
605 // -----------------------------------------------------------------------
Joey Armstrongf3922212024-07-22 17:11:02 -0400606 stage('Install kubectl')
607 {
608 steps
609 {
610 script
611 {
612 String cmd = [
613 'make',
614 '--no-print-directory',
615 '-C', "$WORKSPACE",
616 "KUBECTL_PATH=\"$WORKSPACE/bin\"",
617 'kubectl',
618 ].join(' ')
619
620 println(" ** Running: ${cmd}")
621 sh("${cmd}")
622 } // script
623 } // steps
624 } // stage
625
626 // -----------------------------------------------------------------------
627 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400628 stage('Install Tools') {
629 steps {
630 script {
631 String branchName = branchName()
632 String iam = getIam('Install Tools')
633
634 println("${iam}: ENTER (branch=$branch)")
635 installKind(branch) // needed early by stage(Cleanup)
636 println("${iam}: LEAVE (branch=$branch)")
637 } // script
638 } // steps
639 } // stage
640
641 // -----------------------------------------------------------------------
642 // -----------------------------------------------------------------------
643 stage('Create K8s Cluster') {
644 steps {
645 script {
646 def clusterExists = sh(
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400647 label : 'Create K8s Cluster',
Joey Armstrong0251e962023-08-25 20:51:31 -0400648 returnStdout: true,
649 script: """kind get clusters | grep "${clusterName}" | wc -l""")
650
651 if (clusterExists.trim() == '0') {
652 createKubernetesCluster([nodes: 3, name: clusterName])
653 }
654 } // script
655 } // steps
656 } // stage('Create K8s Cluster')
657
658 // -----------------------------------------------------------------------
659 // -----------------------------------------------------------------------
660 stage('Replace voltctl') {
661 // if the project is voltctl, override the downloaded one with the built one
662 when {
663 expression { return gerritProject == 'voltctl' }
664 }
665
666 // Hmmmm(?) where did the voltctl download happen ?
667 // Likely Makefile but would be helpful to document here.
668 steps {
669 script {
670 String iam = getIam('Replace voltctl')
671
672 println("${iam} Running: installVoltctl($branch)")
673 println("${iam}: ENTER")
674 installVoltctl("$branch")
675 println("${iam}: LEAVE")
676 } // script
677 } // step
678 } // stage
679
680 // -----------------------------------------------------------------------
681 // -----------------------------------------------------------------------
682 stage('Load image in kind nodes')
683 {
684 when {
685 expression { return !gerritProject.isEmpty() }
686 }
687 steps {
688 loadToKind()
689 } // steps
690 } // stage
691
692 // -----------------------------------------------------------------------
693 // [TODO] verify testing output
694 // -----------------------------------------------------------------------
695 stage('Parse and execute tests')
696 {
697 steps {
698 script {
699 // Announce ourselves for log usability
700 enter('Parse and execute tests')
701
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400702 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400703 println("** [DEBUG]: tests=$tests")
704
705 // Display expected tests for times when output goes dark
706 tests.eachWithIndex { test, idx ->
707 String target = test['target']
708 println("** test[${idx}]: ${target}\n")
709 }
710
711 println('''
712** -----------------------------------------------------------------------
713** NOTE: For odd/silent job failures verify a few details
714** - All tests mentioned in the tests-to-run index were logged.
715** - Test suites display ENTER/LEAVE mesasge pairs.
716** - Processing terminated prematurely when LEAVE strings are missing.
717** -----------------------------------------------------------------------
718''')
719 tests.eachWithIndex { test, idx ->
720 println "** readYaml test suite[$idx]) test=[${test}]"
721
Joey Armstrongf404b642023-08-04 14:39:13 -0400722 String target = test['target']
723 String workflow = test['workflow']
724 String flags = test['flags']
725 Boolean teardown = test['teardown'].toBoolean()
726 Boolean logging = test['logging'].toBoolean()
727 String testLogging = (logging) ? 'True' : 'False'
728
Joey Armstrong0251e962023-08-25 20:51:31 -0400729 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400730** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400731** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400732** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400733""")
734
Joey Armstrong0251e962023-08-25 20:51:31 -0400735 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400736 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400737 execute_test(target, workflow, testLogging, teardown, flags)
738 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400739 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400740 catch (Exception err) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400741 String iamexc = getIam(test)
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400742 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400743 }
744 finally {
745 leave("execute_test (target=$target)")
746 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400747 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400748 // Premature exit if this message is not logged
749 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400750 } // script
751 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400752 } // stage
753 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400754
755 post
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400756 { // https://www.jenkins.io/doc/book/pipeline/syntax/#post
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400757 aborted {
758 collectArtifacts('aborted')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400759 }
760 failure {
761 collectArtifacts('failed')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400762 }
763 always {
764 collectArtifacts('always')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400765 }
766 cleanup {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400767 script { cleanupPortForward() }
768 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800769 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400770} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400771
Joey Armstrong664c55a2023-08-28 14:22:33 -0400772// [EOF]