blob: 4268ecdfbedb4ebe4aac0a701ee08e607b03b920 [file] [log] [blame]
Joey Armstrongabd73c22024-01-31 21:17:51 -05001// -*- groovy -*-
2// -----------------------------------------------------------------------
3// Copyright 2021-2024 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08004//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16
Hardik Windlassec9341b2021-06-07 11:58:29 +000017// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080018// uses bbsim to simulate OLT/ONUs
19
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040020// [TODO] Update library() to the latest DSL syntax supported by jenkins
Joey Armstrongabd73c22024-01-31 21:17:51 -050021// -----------------------------------------------------------------------
22
23// Should this be branch=relase ?
Matteo Scandoloa156b572021-02-04 11:52:18 -080024library identifier: 'cord-jenkins-libraries@master',
25 retriever: modernSCM([
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040026 $class: 'GitSCMSource',
27 remote: 'https://gerrit.opencord.org/ci-management.git'
Matteo Scandoloa156b572021-02-04 11:52:18 -080028])
29
Joey Armstronge5aae1c2023-07-24 14:11:20 -040030//------------------//
31//---] GLOBAL [---//
32//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040033String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040034
35// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040036// Intent: Return branch name for the script. A hardcoded value is used
37// as a guarantee release jobs are running in an expected sandbox.
Joey Armstrongf076c312023-08-01 17:17:10 -040038// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040039String branchName() {
Joey Armstrongabd73c22024-01-31 21:17:51 -050040 String br = 'voltha-2.12'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040041
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040042 // "${branch}" is assigned by jenkins
43 if (br != branch) {
44 String err = [
45 'ERROR: Detected invalid branch',
Roger Luethi92354bf2023-10-05 09:22:19 +020046 "(expected=[${br}] != found=[${branch}])"
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040047 ].join(' ')
48 throw new Exception(err) // groovylint-disable-line ThrowException
49 }
50
51 return (br)
Joey Armstrongf076c312023-08-01 17:17:10 -040052}
Hardik Windlassec9341b2021-06-07 11:58:29 +000053
Joey Armstronge5aae1c2023-07-24 14:11:20 -040054// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040055// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040056// Jenkins will re-write the call stack for serialization.S
57// -----------------------------------------------------------------------
58// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040059// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040060String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040061 String branchName = branchName()
Joey Armstrong06a68372023-07-24 16:37:16 -040062 String src = [
63 'ci-management',
64 'jjb',
65 'pipeline',
66 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040067 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040068 'bbsim-tests.groovy'
69 ].join('/')
70
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040071 String name = [src, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040072 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040073}
74
75// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040076// Intent: Log progress message
77// -----------------------------------------------------------------------
78void enter(String name) {
79 // Announce ourselves for log usability
80 String iam = getIam(name)
81 println("${iam}: ENTER")
82 return
83}
84
85// -----------------------------------------------------------------------
86// Intent: Log progress message
87// -----------------------------------------------------------------------
88void leave(String name) {
89 // Announce ourselves for log usability
90 String iam = getIam(name)
91 println("${iam}: LEAVE")
92 return
93}
94
95// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -040096// Intent: Determine if working on a release branch.
97// Note: Conditional is legacy, should also check for *-dev or *-pre
98// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040099Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400100 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -0400101 // if branchName in modifiers
102 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400103}
104
105// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400106// Intent: Terminate orphaned port-forward from different namespaces
107// -----------------------------------------------------------------------
108void cleanupPortForward() {
109 enter('cleanupPortForward')
110
111 Map pkpfArgs =\
112 [
113 'banner' : true, // display banner for logging
114 'show_procs' : true, // display procs under consideration
115 'filler' : true // fix conditional trailing comma
116 ]
117
118 // 'kubectl.*port-forward'
119 pkill_port_forward('port-forward', pkpfArgs)
120 leave('cleanupPortForward')
121 return
122}
123
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400124// find . \( -name 'log*.html' -o -name 'output*.xml' -o -name 'report*.html' \) -p
125// -----------------------------------------------------------------------
126// Intent: Display contents of the logs directory
127// -----------------------------------------------------------------------
128// [TODO]
129// o where-4-art-thou logs directory ?
130// o Replace find {logfile} command with /bin/ls {logdir} when found.
131// Individual logs may be missing due to failure, show what is available.
132// -----------------------------------------------------------------------
133void findPublishedLogs() {
134 String iam = 'findPublishedLogs'
135
136 enter(iam)
137 sh(label : iam,
138 script : """
139find . -name 'output.xml' -print
140""")
141 leave(iam)
142 return
143}
144
145// -----------------------------------------------------------------------
146// Intent: Terminate kail-startup process launched earlier
147// -----------------------------------------------------------------------
148// :param caller: Name of parent calling function (debug context)
149// :type caller: String, optional
150// :returns: none
151// :rtype: void
152// -----------------------------------------------------------------------
153void killKailStartup(String caller='') {
154 String iam = "killKailStartup (caller=$caller)"
155
156 enter(iam)
157 sh(label : 'Terminate kail-startup',
158 script : """
159if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
160 pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
161fi
162""")
163 leave(iam)
164 return
165}
166
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400167// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400168// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400169// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400170void execute_test\
171(
172 String testTarget, // functional-single-kind-dt
173 String workflow, // dt
174 String testLogging, // 'True'
175 Boolean teardown, // true
176 String testSpecificHelmFlags=''
177) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400178 String infraNamespace = 'default'
179 String volthaNamespace = 'voltha'
180 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500181
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400182 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400183 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400184 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400185 stage('Cleanup') {
186 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400187 timeout(15) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400188 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400189 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400190 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400191 } // timeout
192
193 timeout(5) {
194 script {
195 enter('Cleanup')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400196 cleanupPortForward()
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400197 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400198 } // script
199 } // timeout
200 } // teardown
201 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500202
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400203 // -----------------------------------------------------------------------
204 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400205 stage('Deploy common infrastructure') {
206 script {
207 local dashargs = [
208 'kpi_exporter.enabled=false',
209 'dashboards.xos=false',
210 'dashboards.onos=false',
211 'dashboards.aaa=false',
212 'dashboards.voltha=false',
213 ].join(',')
214
215 local promargs = [
216 'prometheus.alertmanager.enabled=false',
217 'prometheus.pushgateway.enabled=false',
218 ].join(',')
219
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400220 sh(label : 'Deploy common infrastructure',
221 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000222 helm repo add onf https://charts.opencord.org
223 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400224
225 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000226 if [ ${withMonitoring} = true ] ; then
227 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400228 --set ${promargs} \
229 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000230 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400231 """)
232 } // script
233 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500234
Joey Armstrong0251e962023-08-25 20:51:31 -0400235 // -----------------------------------------------------------------------
236 // [TODO] Check onos_log output
237 // -----------------------------------------------------------------------
238 stage('Deploy Voltha') {
239 if (teardown) {
240 timeout(10) {
241 script {
242 String iam = getIam('Deploy Voltha')
243 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000244
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400245 sh(label : 'Launch kail-startup',
246 script : """
247mkdir -p "$logsDir"
248touch "$onosLog"
Joey Armstrongf404b642023-08-04 14:39:13 -0400249
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400250_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
251""")
252
253 // if we're downloading a voltha-helm-charts patch,
254 // install from a local copy of the charts
Joey Armstrong0251e962023-08-25 20:51:31 -0400255 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400256
Joey Armstrong0251e962023-08-25 20:51:31 -0400257 if (volthaHelmChartsChange != ''
258 || gerritProject == 'voltha-helm-charts'
259 || isReleaseBranch(branch) // branch != 'master'
260 ) {
261 localCharts = true
262 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000263
Joey Armstrong0251e962023-08-25 20:51:31 -0400264 String branchName = branchName()
265 Boolean isRelease = isReleaseBranch(branch)
266 println([
267 " ** localCharts=${localCharts}",
268 "branchName=${branchName}",
269 "branch=${branch}",
270 "branch=isReleaseBranch=${isRelease}",
271 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800272
Joey Armstrong0251e962023-08-25 20:51:31 -0400273 // -----------------------------------------------------------------------
274 // Rewrite localHelmFlags using array join, moving code around and
275 // refactoring into standalone functions
276 // -----------------------------------------------------------------------
277 // NOTE temporary workaround expose ONOS node ports
278 // -----------------------------------------------------------------------
279 String localHelmFlags = [
280 extraHelmFlags.trim(),
281 "--set global.log_level=${logLevel.toUpperCase()}",
282 '--set onos-classic.onosSshPort=30115',
283 '--set onos-classic.onosApiPort=30120',
284 '--set onos-classic.onosOfPort=31653',
285 '--set onos-classic.individualOpenFlowNodePorts=true',
286 testSpecificHelmFlags
287 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400288
Joey Armstrong0251e962023-08-25 20:51:31 -0400289 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000290
Joey Armstrong0251e962023-08-25 20:51:31 -0400291 if (gerritProject != '') {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400292 localHelmFlags += getVolthaImageFlags(gerritProject)
Joey Armstrong0251e962023-08-25 20:51:31 -0400293 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800294
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400295 enter('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400296 volthaDeploy([
297 infraNamespace: infraNamespace,
298 volthaNamespace: volthaNamespace,
299 workflow: workflow.toLowerCase(),
300 withMacLearning: enableMacLearning.toBoolean(),
301 extraHelmFlags: localHelmFlags,
302 localCharts: localCharts,
303 bbsimReplica: olts.toInteger(),
304 dockerRegistry: registry,
305 ])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400306 leave('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400307 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400308
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400309 // Display spawned procs
310 script {
311 enter('bbsim-tests::pgrep_port_forward::0')
312 pgrep_port_forward('port-forw')
313 leave('bbsim-tests::pgrep_port_forward::0')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400314
315 killKailStartup('Deploy Voltha')
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400316 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400317
Joey Armstrong0251e962023-08-25 20:51:31 -0400318 // -----------------------------------------------------------------------
319 // Bundle onos-voltha / kail logs
320 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400321 sh(
322 label : 'Bundle logs: onos-voltha-startup-combined',
323 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400324cat <<EOM
325
326** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400327** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400328** -----------------------------------------------------------------------
329EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400330
331pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
332gzip -k onos-voltha-startup-combined.log
333rm onos-voltha-startup-combined.log
334popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
Joey Armstrong0251e962023-08-25 20:51:31 -0400335 """)
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400336 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400337
Joey Armstrong0251e962023-08-25 20:51:31 -0400338 // -----------------------------------------------------------------------
339 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400340 sh(label : 'while-true-port-forward',
Roger Luethicebf79a2023-09-28 09:41:45 +0200341 script : """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000342 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
343 bbsimDmiPortFwd=50075
344 for i in {0..${olts.toInteger() - 1}}; do
345 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
346 ((bbsimDmiPortFwd++))
347 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000348 if [ ${withMonitoring} = true ] ; then
349 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
350 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400351# ps aux | grep port-forward
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400352""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400353 // ---------------------------------
354 // Sanity check port-forward spawned
355 // ---------------------------------
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400356 script {
357 enter('bbsim-tests::pgrep_port_forward::1')
358 pgrep_port_forward('port-forw')
359 leave('bbsim-tests::pgrep_port_forward::1')
360 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500361
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400362 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400363 script {
364 enter('setOnosLogLevels')
365 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400366 onosNamespace: infraNamespace,
367 apps: [
368 'org.opencord.dhcpl2relay',
369 'org.opencord.olt',
370 'org.opencord.aaa',
371 'org.opencord.maclearner',
372 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
373 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
374 ],
375 logLevel: logLevel
376 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400377 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400378 } // script
379 } // if (teardown)
380 } // stage('Deploy Voltha')
381
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400382 // -----------------------------------------------------------------------
383 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400384 stage("Run test ${testTarget} on workflow ${workflow}") {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400385 sh(
386 label : 'Monitor using mem_consumption.py',
387 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400388echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400389
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400390if [ ${withMonitoring} = true ] ; then
391 cat <<EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400392
393** -----------------------------------------------------------------------
394** Monitoring memory usage with mem_consumption.py
395** -----------------------------------------------------------------------
396EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400397 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
398 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400399
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400400 echo '** Installing python virtualenv'
401 make venv-activate-patched
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400402
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400403 # Collect initial memory consumption
404 set +u && source .venv/bin/activate && set -u
405 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
406fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400407
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400408echo -e '** Monitor memory consumption: LEAVE\n'
409""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400410
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400411 sh(
412 label : "make testTarget=[${testTarget}]",
413 script : """
414echo -e "\n** make testTarget=[${testTarget}]"
415mkdir -p ${logsDir}
416export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
417ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
418export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800419
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400420make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
421""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400422
423 getPodsInfo("${logsDir}")
424
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400425 // [TODO] make conditional, bundle when logs are available
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400426 sh(
427 label : 'Gather robot Framework logs',
428 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400429echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400430
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400431# set +e
432# collect logs collected in the Robot Framework StartLogging keyword
433cd "${logsDir}"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400434
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400435echo "** Available logs:"
436/bin/ls -l "$logsDir"
437echo
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400438
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400439echo '** Bundle combined log'
440gzip *-combined.log || true
441rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400442
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400443echo -e '** Gather robot Framework logs: LEAVE\n'
444""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400445
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400446 // -----------------------------------------------------------------------
447 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400448 sh(
449 label : 'Monitor pod-mem-consumption',
450 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400451echo -e '** Monitor pod-mem-consumption: ENTER'
452if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400453 cat <<EOM
454
455** -----------------------------------------------------------------------
456** Monitoring pod-memory-consumption using mem_consumption.py
457** -----------------------------------------------------------------------
458EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400459
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400460cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400461
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400462echo '** Installing python virtualenv'
463make venv-activate-patched
464
465# Collect memory consumption of voltha pods once all the tests are complete
466set +u && source .venv/bin/activate && set -u
467python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
468fi
469echo -e '** Monitor pod-mem-consumption: LEAVE\n'
470""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400471 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400472
473 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400474} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800475
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400476// -----------------------------------------------------------------------
477// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400478void collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400479 script {
480 String iam = getIam('collectArtifacts')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400481 enter("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400482
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400483 println("""
Joey Armstrong97a8b882023-08-02 16:08:52 -0400484
485** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400486** IAM: $iam
Joey Armstrong97a8b882023-08-02 16:08:52 -0400487** collectArtifacts
488** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400489""")
490 }
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400491
Joey Armstrongcd419122023-08-07 14:56:39 -0400492 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400493
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400494 sh(label : 'kubectl logs > voltha.log',
495 script : """
496kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
497 > $WORKSPACE/${exitStatus}/voltha.log
498""")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400499
Joey Armstrongcd419122023-08-07 14:56:39 -0400500 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400501
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400502 script { killKailStartup('collectArtifacts') }
503 script { findPublishedLogs() }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400504
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400505 enter('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400506 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400507 disableArchiveOutput: false,
508 logFileName: '**/*/log*.html',
509 otherFiles: '',
510 outputFileName: '**/*/output*.xml',
511 outputPath: '.',
512 passThreshold: 100,
513 reportFileName: '**/*/report*.html',
514 unstableThreshold: 0,
515 onlyCritical: true])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400516 leave('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400517
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400518 leave("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400519 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000520}
521
Joey Armstrongb65ada32023-08-03 12:50:20 -0400522// -----------------------------------------------------------------------
523// Intent: main
524// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800525pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400526 /* no label, executor is determined by JJB */
527 agent {
528 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800529 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400530
Joey Armstrong0251e962023-08-25 20:51:31 -0400531 options {
532 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800533 }
Joey Armstronged161f72023-04-11 13:16:59 -0400534
Joey Armstrong0251e962023-08-25 20:51:31 -0400535 environment {
536 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
537 VOLTCONFIG = "$HOME/.volt/config"
538 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
539 DIAGS_PROFILE = 'VOLTHA_PROFILE'
540 SSHPASS = 'karaf'
541 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400542
Joey Armstrong0251e962023-08-25 20:51:31 -0400543 stages {
544 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400545 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500546 enter('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400547 getVolthaCode([
548 branch: "${branch}",
549 gerritProject: "${gerritProject}",
550 gerritRefspec: "${gerritRefspec}",
551 volthaSystemTestsChange: "${volthaSystemTestsChange}",
552 volthaHelmChartsChange: "${volthaHelmChartsChange}",
553 ])
Joey Armstrongdef9c402024-01-30 18:05:29 -0500554 leave('getVolthaCode')
Joey Armstrong0251e962023-08-25 20:51:31 -0400555 }
556 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400557
Joey Armstrong0251e962023-08-25 20:51:31 -0400558 stage('Build patch v1.1') {
559 // build the patch only if gerritProject is specified
560 when {
561 expression { return !gerritProject.isEmpty() }
562 }
563
564 steps {
Joey Armstrongdef9c402024-01-30 18:05:29 -0500565 enter('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400566 // NOTE that the correct patch has already been checked out
567 // during the getVolthaCode step
568 buildVolthaComponent("${gerritProject}")
Joey Armstrongdef9c402024-01-30 18:05:29 -0500569 leave('buildVolthaComponent')
Joey Armstrong0251e962023-08-25 20:51:31 -0400570 }
571 }
572
573 // -----------------------------------------------------------------------
574 // -----------------------------------------------------------------------
575 stage('Install Kail')
576 {
577 steps
578 {
579 script
580 {
581 String cmd = [
582 'make',
583 '--no-print-directory',
584 '-C', "$WORKSPACE/voltha-system-tests",
585 "KAIL_PATH=\"$WORKSPACE/bin\"",
586 'kail',
587 ].join(' ')
588
589 println(" ** Running: ${cmd}")
590 sh("${cmd}")
591 } // script
592 } // steps
593 } // stage
594
595 // -----------------------------------------------------------------------
596 // -----------------------------------------------------------------------
597 stage('Install Tools') {
598 steps {
599 script {
600 String branchName = branchName()
601 String iam = getIam('Install Tools')
602
603 println("${iam}: ENTER (branch=$branch)")
604 installKind(branch) // needed early by stage(Cleanup)
605 println("${iam}: LEAVE (branch=$branch)")
606 } // script
607 } // steps
608 } // stage
609
610 // -----------------------------------------------------------------------
611 // -----------------------------------------------------------------------
612 stage('Create K8s Cluster') {
613 steps {
614 script {
615 def clusterExists = sh(
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400616 label : 'Create K8s Cluster',
Joey Armstrong0251e962023-08-25 20:51:31 -0400617 returnStdout: true,
618 script: """kind get clusters | grep "${clusterName}" | wc -l""")
619
620 if (clusterExists.trim() == '0') {
621 createKubernetesCluster([nodes: 3, name: clusterName])
622 }
623 } // script
624 } // steps
625 } // stage('Create K8s Cluster')
626
627 // -----------------------------------------------------------------------
628 // -----------------------------------------------------------------------
629 stage('Replace voltctl') {
630 // if the project is voltctl, override the downloaded one with the built one
631 when {
632 expression { return gerritProject == 'voltctl' }
633 }
634
635 // Hmmmm(?) where did the voltctl download happen ?
636 // Likely Makefile but would be helpful to document here.
637 steps {
638 script {
639 String iam = getIam('Replace voltctl')
640
641 println("${iam} Running: installVoltctl($branch)")
642 println("${iam}: ENTER")
643 installVoltctl("$branch")
644 println("${iam}: LEAVE")
645 } // script
646 } // step
647 } // stage
648
649 // -----------------------------------------------------------------------
650 // -----------------------------------------------------------------------
651 stage('Load image in kind nodes')
652 {
653 when {
654 expression { return !gerritProject.isEmpty() }
655 }
656 steps {
657 loadToKind()
658 } // steps
659 } // stage
660
661 // -----------------------------------------------------------------------
662 // [TODO] verify testing output
663 // -----------------------------------------------------------------------
664 stage('Parse and execute tests')
665 {
666 steps {
667 script {
668 // Announce ourselves for log usability
669 enter('Parse and execute tests')
670
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400671 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400672 println("** [DEBUG]: tests=$tests")
673
674 // Display expected tests for times when output goes dark
675 tests.eachWithIndex { test, idx ->
676 String target = test['target']
677 println("** test[${idx}]: ${target}\n")
678 }
679
680 println('''
681** -----------------------------------------------------------------------
682** NOTE: For odd/silent job failures verify a few details
683** - All tests mentioned in the tests-to-run index were logged.
684** - Test suites display ENTER/LEAVE mesasge pairs.
685** - Processing terminated prematurely when LEAVE strings are missing.
686** -----------------------------------------------------------------------
687''')
688 tests.eachWithIndex { test, idx ->
689 println "** readYaml test suite[$idx]) test=[${test}]"
690
Joey Armstrongf404b642023-08-04 14:39:13 -0400691 String target = test['target']
692 String workflow = test['workflow']
693 String flags = test['flags']
694 Boolean teardown = test['teardown'].toBoolean()
695 Boolean logging = test['logging'].toBoolean()
696 String testLogging = (logging) ? 'True' : 'False'
697
Joey Armstrong0251e962023-08-25 20:51:31 -0400698 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400699** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400700** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400701** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400702""")
703
Joey Armstrong0251e962023-08-25 20:51:31 -0400704 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400705 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400706 execute_test(target, workflow, testLogging, teardown, flags)
707 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400708 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400709 catch (Exception err) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400710 String iamexc = getIam(test)
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400711 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400712 }
713 finally {
714 leave("execute_test (target=$target)")
715 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400716 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400717 // Premature exit if this message is not logged
718 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400719 } // script
720 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400721 } // stage
722 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400723
724 post
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400725 { // https://www.jenkins.io/doc/book/pipeline/syntax/#post
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400726 aborted {
727 collectArtifacts('aborted')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400728 }
729 failure {
730 collectArtifacts('failed')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400731 }
732 always {
733 collectArtifacts('always')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400734 }
735 cleanup {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400736 script { cleanupPortForward() }
737 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800738 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400739} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400740
Joey Armstrong664c55a2023-08-28 14:22:33 -0400741// [EOF]