blob: 991ff5b417f032563a734b04248d934330acd60a [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040018// [TODO] Update library() to the latest DSL syntax supported by jenkins
Matteo Scandoloa156b572021-02-04 11:52:18 -080019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040021 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
Matteo Scandoloa156b572021-02-04 11:52:18 -080023])
24
Joey Armstronge5aae1c2023-07-24 14:11:20 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040028String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040029
30// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040031// Intent: Return branch name for the script. A hardcoded value is used
32// as a guarantee release jobs are running in an expected sandbox.
Joey Armstrongf076c312023-08-01 17:17:10 -040033// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040034String branchName() {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040035 String br = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040036
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040037 // "${branch}" is assigned by jenkins
38 if (br != branch) {
39 String err = [
40 'ERROR: Detected invalid branch',
Roger Luethi92354bf2023-10-05 09:22:19 +020041 "(expected=[${br}] != found=[${branch}])"
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040042 ].join(' ')
43 throw new Exception(err) // groovylint-disable-line ThrowException
44 }
45
46 return (br)
Joey Armstrongf076c312023-08-01 17:17:10 -040047}
Hardik Windlassec9341b2021-06-07 11:58:29 +000048
Joey Armstronge5aae1c2023-07-24 14:11:20 -040049// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040050// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040051// Jenkins will re-write the call stack for serialization.S
52// -----------------------------------------------------------------------
53// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040054// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040055String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040056 String branchName = branchName()
Joey Armstrong06a68372023-07-24 16:37:16 -040057 String src = [
58 'ci-management',
59 'jjb',
60 'pipeline',
61 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040062 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040063 'bbsim-tests.groovy'
64 ].join('/')
65
Joey Armstrongbccfa4b2023-09-27 17:34:22 -040066 String name = [src, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040067 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040068}
69
70// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040071// Intent: Log progress message
72// -----------------------------------------------------------------------
73void enter(String name) {
74 // Announce ourselves for log usability
75 String iam = getIam(name)
76 println("${iam}: ENTER")
77 return
78}
79
80// -----------------------------------------------------------------------
81// Intent: Log progress message
82// -----------------------------------------------------------------------
83void leave(String name) {
84 // Announce ourselves for log usability
85 String iam = getIam(name)
86 println("${iam}: LEAVE")
87 return
88}
89
90// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -040091// Intent: Determine if working on a release branch.
92// Note: Conditional is legacy, should also check for *-dev or *-pre
93// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040094Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -040095 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -040096 // if branchName in modifiers
97 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -040098}
99
100// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400101// Intent: Terminate orphaned port-forward from different namespaces
102// -----------------------------------------------------------------------
103void cleanupPortForward() {
104 enter('cleanupPortForward')
105
106 Map pkpfArgs =\
107 [
108 'banner' : true, // display banner for logging
109 'show_procs' : true, // display procs under consideration
110 'filler' : true // fix conditional trailing comma
111 ]
112
113 // 'kubectl.*port-forward'
114 pkill_port_forward('port-forward', pkpfArgs)
115 leave('cleanupPortForward')
116 return
117}
118
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400119// find . \( -name 'log*.html' -o -name 'output*.xml' -o -name 'report*.html' \) -p
120// -----------------------------------------------------------------------
121// Intent: Display contents of the logs directory
122// -----------------------------------------------------------------------
123// [TODO]
124// o where-4-art-thou logs directory ?
125// o Replace find {logfile} command with /bin/ls {logdir} when found.
126// Individual logs may be missing due to failure, show what is available.
127// -----------------------------------------------------------------------
128void findPublishedLogs() {
129 String iam = 'findPublishedLogs'
130
131 enter(iam)
132 sh(label : iam,
133 script : """
134find . -name 'output.xml' -print
135""")
136 leave(iam)
137 return
138}
139
140// -----------------------------------------------------------------------
141// Intent: Terminate kail-startup process launched earlier
142// -----------------------------------------------------------------------
143// :param caller: Name of parent calling function (debug context)
144// :type caller: String, optional
145// :returns: none
146// :rtype: void
147// -----------------------------------------------------------------------
148void killKailStartup(String caller='') {
149 String iam = "killKailStartup (caller=$caller)"
150
151 enter(iam)
152 sh(label : 'Terminate kail-startup',
153 script : """
154if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
155 pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
156fi
157""")
158 leave(iam)
159 return
160}
161
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400162// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400163// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400164// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400165void execute_test\
166(
167 String testTarget, // functional-single-kind-dt
168 String workflow, // dt
169 String testLogging, // 'True'
170 Boolean teardown, // true
171 String testSpecificHelmFlags=''
172) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400173 String infraNamespace = 'default'
174 String volthaNamespace = 'voltha'
175 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500176
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400177 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400178 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400179 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400180 stage('Cleanup') {
181 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400182 timeout(15) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400183 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400184 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400185 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400186 } // timeout
187
188 timeout(5) {
189 script {
190 enter('Cleanup')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400191 cleanupPortForward()
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400192 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400193 } // script
194 } // timeout
195 } // teardown
196 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500197
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400198 // -----------------------------------------------------------------------
199 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400200 stage('Deploy common infrastructure') {
201 script {
202 local dashargs = [
203 'kpi_exporter.enabled=false',
204 'dashboards.xos=false',
205 'dashboards.onos=false',
206 'dashboards.aaa=false',
207 'dashboards.voltha=false',
208 ].join(',')
209
210 local promargs = [
211 'prometheus.alertmanager.enabled=false',
212 'prometheus.pushgateway.enabled=false',
213 ].join(',')
214
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400215 sh(label : 'Deploy common infrastructure',
216 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000217 helm repo add onf https://charts.opencord.org
218 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400219
220 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000221 if [ ${withMonitoring} = true ] ; then
222 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400223 --set ${promargs} \
224 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000225 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400226 """)
227 } // script
228 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500229
Joey Armstrong0251e962023-08-25 20:51:31 -0400230 // -----------------------------------------------------------------------
231 // [TODO] Check onos_log output
232 // -----------------------------------------------------------------------
233 stage('Deploy Voltha') {
234 if (teardown) {
235 timeout(10) {
236 script {
237 String iam = getIam('Deploy Voltha')
238 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000239
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400240 sh(label : 'Launch kail-startup',
241 script : """
242mkdir -p "$logsDir"
243touch "$onosLog"
Joey Armstrongf404b642023-08-04 14:39:13 -0400244
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400245_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
246""")
247
248 // if we're downloading a voltha-helm-charts patch,
249 // install from a local copy of the charts
Joey Armstrong0251e962023-08-25 20:51:31 -0400250 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400251
Joey Armstrong0251e962023-08-25 20:51:31 -0400252 if (volthaHelmChartsChange != ''
253 || gerritProject == 'voltha-helm-charts'
254 || isReleaseBranch(branch) // branch != 'master'
255 ) {
256 localCharts = true
257 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000258
Joey Armstrong0251e962023-08-25 20:51:31 -0400259 String branchName = branchName()
260 Boolean isRelease = isReleaseBranch(branch)
261 println([
262 " ** localCharts=${localCharts}",
263 "branchName=${branchName}",
264 "branch=${branch}",
265 "branch=isReleaseBranch=${isRelease}",
266 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800267
Joey Armstrong0251e962023-08-25 20:51:31 -0400268 // -----------------------------------------------------------------------
269 // Rewrite localHelmFlags using array join, moving code around and
270 // refactoring into standalone functions
271 // -----------------------------------------------------------------------
272 // NOTE temporary workaround expose ONOS node ports
273 // -----------------------------------------------------------------------
274 String localHelmFlags = [
275 extraHelmFlags.trim(),
276 "--set global.log_level=${logLevel.toUpperCase()}",
277 '--set onos-classic.onosSshPort=30115',
278 '--set onos-classic.onosApiPort=30120',
279 '--set onos-classic.onosOfPort=31653',
280 '--set onos-classic.individualOpenFlowNodePorts=true',
281 testSpecificHelmFlags
282 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400283
Joey Armstrong0251e962023-08-25 20:51:31 -0400284 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000285
Joey Armstrong0251e962023-08-25 20:51:31 -0400286 if (gerritProject != '') {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400287 localHelmFlags += getVolthaImageFlags(gerritProject)
Joey Armstrong0251e962023-08-25 20:51:31 -0400288 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800289
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400290 enter('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400291 volthaDeploy([
292 infraNamespace: infraNamespace,
293 volthaNamespace: volthaNamespace,
294 workflow: workflow.toLowerCase(),
295 withMacLearning: enableMacLearning.toBoolean(),
296 extraHelmFlags: localHelmFlags,
297 localCharts: localCharts,
298 bbsimReplica: olts.toInteger(),
299 dockerRegistry: registry,
300 ])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400301 leave('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400302 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400303
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400304 // Display spawned procs
305 script {
306 enter('bbsim-tests::pgrep_port_forward::0')
307 pgrep_port_forward('port-forw')
308 leave('bbsim-tests::pgrep_port_forward::0')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400309
310 killKailStartup('Deploy Voltha')
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400311 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400312
Joey Armstrong0251e962023-08-25 20:51:31 -0400313 // -----------------------------------------------------------------------
314 // Bundle onos-voltha / kail logs
315 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400316 sh(
317 label : 'Bundle logs: onos-voltha-startup-combined',
318 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400319cat <<EOM
320
321** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400322** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400323** -----------------------------------------------------------------------
324EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400325
326pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
327gzip -k onos-voltha-startup-combined.log
328rm onos-voltha-startup-combined.log
329popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
Joey Armstrong0251e962023-08-25 20:51:31 -0400330 """)
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400331 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400332
Joey Armstrong0251e962023-08-25 20:51:31 -0400333 // -----------------------------------------------------------------------
334 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400335 sh(label : 'while-true-port-forward',
Roger Luethicebf79a2023-09-28 09:41:45 +0200336 script : """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000337 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
338 bbsimDmiPortFwd=50075
339 for i in {0..${olts.toInteger() - 1}}; do
340 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
341 ((bbsimDmiPortFwd++))
342 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000343 if [ ${withMonitoring} = true ] ; then
344 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
345 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400346# ps aux | grep port-forward
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400347""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400348 // ---------------------------------
349 // Sanity check port-forward spawned
350 // ---------------------------------
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400351 script {
352 enter('bbsim-tests::pgrep_port_forward::1')
353 pgrep_port_forward('port-forw')
354 leave('bbsim-tests::pgrep_port_forward::1')
355 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500356
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400357 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400358 script {
359 enter('setOnosLogLevels')
360 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400361 onosNamespace: infraNamespace,
362 apps: [
363 'org.opencord.dhcpl2relay',
364 'org.opencord.olt',
365 'org.opencord.aaa',
366 'org.opencord.maclearner',
367 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
368 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
369 ],
370 logLevel: logLevel
371 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400372 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400373 } // script
374 } // if (teardown)
375 } // stage('Deploy Voltha')
376
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400377 // -----------------------------------------------------------------------
378 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400379 stage("Run test ${testTarget} on workflow ${workflow}") {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400380 sh(
381 label : 'Monitor using mem_consumption.py',
382 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400383echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400384
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400385if [ ${withMonitoring} = true ] ; then
386 cat <<EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400387
388** -----------------------------------------------------------------------
389** Monitoring memory usage with mem_consumption.py
390** -----------------------------------------------------------------------
391EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400392 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
393 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400394
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400395 echo '** Installing python virtualenv'
396 make venv-activate-patched
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400397
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400398 # Collect initial memory consumption
399 set +u && source .venv/bin/activate && set -u
400 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
401fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400402
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400403echo -e '** Monitor memory consumption: LEAVE\n'
404""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400405
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400406 sh(
407 label : "make testTarget=[${testTarget}]",
408 script : """
409echo -e "\n** make testTarget=[${testTarget}]"
410mkdir -p ${logsDir}
411export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
412ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
413export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800414
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400415make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
416""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400417
418 getPodsInfo("${logsDir}")
419
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400420 // [TODO] make conditional, bundle when logs are available
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400421 sh(
422 label : 'Gather robot Framework logs',
423 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400424echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400425
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400426# set +e
427# collect logs collected in the Robot Framework StartLogging keyword
428cd "${logsDir}"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400429
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400430echo "** Available logs:"
431/bin/ls -l "$logsDir"
432echo
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400433
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400434echo '** Bundle combined log'
435gzip *-combined.log || true
436rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400437
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400438echo -e '** Gather robot Framework logs: LEAVE\n'
439""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400440
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400441 // -----------------------------------------------------------------------
442 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400443 sh(
444 label : 'Monitor pod-mem-consumption',
445 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400446echo -e '** Monitor pod-mem-consumption: ENTER'
447if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400448 cat <<EOM
449
450** -----------------------------------------------------------------------
451** Monitoring pod-memory-consumption using mem_consumption.py
452** -----------------------------------------------------------------------
453EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400454
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400455cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400456
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400457echo '** Installing python virtualenv'
458make venv-activate-patched
459
460# Collect memory consumption of voltha pods once all the tests are complete
461set +u && source .venv/bin/activate && set -u
462python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
463fi
464echo -e '** Monitor pod-mem-consumption: LEAVE\n'
465""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400466 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400467
468 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400469} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800470
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400471// -----------------------------------------------------------------------
472// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400473void collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400474 script {
475 String iam = getIam('collectArtifacts')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400476 enter("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400477
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400478 println("""
Joey Armstrong97a8b882023-08-02 16:08:52 -0400479
480** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400481** IAM: $iam
Joey Armstrong97a8b882023-08-02 16:08:52 -0400482** collectArtifacts
483** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400484""")
485 }
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400486
Joey Armstrongcd419122023-08-07 14:56:39 -0400487 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400488
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400489 sh(label : 'kubectl logs > voltha.log',
490 script : """
491kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
492 > $WORKSPACE/${exitStatus}/voltha.log
493""")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400494
Joey Armstrongcd419122023-08-07 14:56:39 -0400495 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400496
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400497 script { killKailStartup('collectArtifacts') }
498 script { findPublishedLogs() }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400499
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400500 enter('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400501 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400502 disableArchiveOutput: false,
503 logFileName: '**/*/log*.html',
504 otherFiles: '',
505 outputFileName: '**/*/output*.xml',
506 outputPath: '.',
507 passThreshold: 100,
508 reportFileName: '**/*/report*.html',
509 unstableThreshold: 0,
510 onlyCritical: true])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400511 leave('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400512
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400513 leave("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400514 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000515}
516
Joey Armstrongb65ada32023-08-03 12:50:20 -0400517// -----------------------------------------------------------------------
518// Intent: main
519// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800520pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400521 /* no label, executor is determined by JJB */
522 agent {
523 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800524 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400525
Joey Armstrong0251e962023-08-25 20:51:31 -0400526 options {
527 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800528 }
Joey Armstronged161f72023-04-11 13:16:59 -0400529
Joey Armstrong0251e962023-08-25 20:51:31 -0400530 environment {
531 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
532 VOLTCONFIG = "$HOME/.volt/config"
533 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
534 DIAGS_PROFILE = 'VOLTHA_PROFILE'
535 SSHPASS = 'karaf'
536 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400537
Joey Armstrong0251e962023-08-25 20:51:31 -0400538 stages {
539 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400540 steps {
Joey Armstrong0251e962023-08-25 20:51:31 -0400541 getVolthaCode([
542 branch: "${branch}",
543 gerritProject: "${gerritProject}",
544 gerritRefspec: "${gerritRefspec}",
545 volthaSystemTestsChange: "${volthaSystemTestsChange}",
546 volthaHelmChartsChange: "${volthaHelmChartsChange}",
547 ])
548 }
549 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400550
Joey Armstrong0251e962023-08-25 20:51:31 -0400551 stage('Build patch v1.1') {
552 // build the patch only if gerritProject is specified
553 when {
554 expression { return !gerritProject.isEmpty() }
555 }
556
557 steps {
558 // NOTE that the correct patch has already been checked out
559 // during the getVolthaCode step
560 buildVolthaComponent("${gerritProject}")
561 }
562 }
563
564 // -----------------------------------------------------------------------
565 // -----------------------------------------------------------------------
566 stage('Install Kail')
567 {
568 steps
569 {
570 script
571 {
572 String cmd = [
573 'make',
574 '--no-print-directory',
575 '-C', "$WORKSPACE/voltha-system-tests",
576 "KAIL_PATH=\"$WORKSPACE/bin\"",
577 'kail',
578 ].join(' ')
579
580 println(" ** Running: ${cmd}")
581 sh("${cmd}")
582 } // script
583 } // steps
584 } // stage
585
586 // -----------------------------------------------------------------------
587 // -----------------------------------------------------------------------
588 stage('Install Tools') {
589 steps {
590 script {
591 String branchName = branchName()
592 String iam = getIam('Install Tools')
593
594 println("${iam}: ENTER (branch=$branch)")
595 installKind(branch) // needed early by stage(Cleanup)
596 println("${iam}: LEAVE (branch=$branch)")
597 } // script
598 } // steps
599 } // stage
600
601 // -----------------------------------------------------------------------
602 // -----------------------------------------------------------------------
603 stage('Create K8s Cluster') {
604 steps {
605 script {
606 def clusterExists = sh(
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400607 label : 'Create K8s Cluster',
Joey Armstrong0251e962023-08-25 20:51:31 -0400608 returnStdout: true,
609 script: """kind get clusters | grep "${clusterName}" | wc -l""")
610
611 if (clusterExists.trim() == '0') {
612 createKubernetesCluster([nodes: 3, name: clusterName])
613 }
614 } // script
615 } // steps
616 } // stage('Create K8s Cluster')
617
618 // -----------------------------------------------------------------------
619 // -----------------------------------------------------------------------
620 stage('Replace voltctl') {
621 // if the project is voltctl, override the downloaded one with the built one
622 when {
623 expression { return gerritProject == 'voltctl' }
624 }
625
626 // Hmmmm(?) where did the voltctl download happen ?
627 // Likely Makefile but would be helpful to document here.
628 steps {
629 script {
630 String iam = getIam('Replace voltctl')
631
632 println("${iam} Running: installVoltctl($branch)")
633 println("${iam}: ENTER")
634 installVoltctl("$branch")
635 println("${iam}: LEAVE")
636 } // script
637 } // step
638 } // stage
639
640 // -----------------------------------------------------------------------
641 // -----------------------------------------------------------------------
642 stage('Load image in kind nodes')
643 {
644 when {
645 expression { return !gerritProject.isEmpty() }
646 }
647 steps {
648 loadToKind()
649 } // steps
650 } // stage
651
652 // -----------------------------------------------------------------------
653 // [TODO] verify testing output
654 // -----------------------------------------------------------------------
655 stage('Parse and execute tests')
656 {
657 steps {
658 script {
659 // Announce ourselves for log usability
660 enter('Parse and execute tests')
661
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400662 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400663 println("** [DEBUG]: tests=$tests")
664
665 // Display expected tests for times when output goes dark
666 tests.eachWithIndex { test, idx ->
667 String target = test['target']
668 println("** test[${idx}]: ${target}\n")
669 }
670
671 println('''
672** -----------------------------------------------------------------------
673** NOTE: For odd/silent job failures verify a few details
674** - All tests mentioned in the tests-to-run index were logged.
675** - Test suites display ENTER/LEAVE mesasge pairs.
676** - Processing terminated prematurely when LEAVE strings are missing.
677** -----------------------------------------------------------------------
678''')
679 tests.eachWithIndex { test, idx ->
680 println "** readYaml test suite[$idx]) test=[${test}]"
681
Joey Armstrongf404b642023-08-04 14:39:13 -0400682 String target = test['target']
683 String workflow = test['workflow']
684 String flags = test['flags']
685 Boolean teardown = test['teardown'].toBoolean()
686 Boolean logging = test['logging'].toBoolean()
687 String testLogging = (logging) ? 'True' : 'False'
688
Joey Armstrong0251e962023-08-25 20:51:31 -0400689 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400690** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400691** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400692** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400693""")
694
Joey Armstrong0251e962023-08-25 20:51:31 -0400695 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400696 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400697 execute_test(target, workflow, testLogging, teardown, flags)
698 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400699 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400700 catch (Exception err) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400701 String iamexc = getIam(test)
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400702 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400703 }
704 finally {
705 leave("execute_test (target=$target)")
706 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400707 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400708 // Premature exit if this message is not logged
709 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400710 } // script
711 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400712 } // stage
713 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400714
715 post
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400716 { // https://www.jenkins.io/doc/book/pipeline/syntax/#post
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400717 aborted {
718 collectArtifacts('aborted')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400719 }
720 failure {
721 collectArtifacts('failed')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400722 }
723 always {
724 collectArtifacts('always')
Joey Armstrongbccfa4b2023-09-27 17:34:22 -0400725 }
726 cleanup {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400727 script { cleanupPortForward() }
728 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800729 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400730} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400731
Joey Armstrong664c55a2023-08-28 14:22:33 -0400732// [EOF]