blob: 2941e2cb371d4b891b37f0fb6c97f11e86b2cfe5 [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Joey Armstronge5aae1c2023-07-24 14:11:20 -040018// [TODO] Update syntax below to the latest supported
Matteo Scandoloa156b572021-02-04 11:52:18 -080019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040021 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
Matteo Scandoloa156b572021-02-04 11:52:18 -080023])
24
Joey Armstronge5aae1c2023-07-24 14:11:20 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040028String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040029
30// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040031// Intent: Return branch name for the script. A hardcoded value is used
32// as a guarantee release jobs are running in an expected sandbox.
Joey Armstrongf076c312023-08-01 17:17:10 -040033// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040034String branchName() {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040035 String br = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040036
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040037 // "${branch}" is assigned by jenkins
38 if (br != branch) {
39 String err = [
40 'ERROR: Detected invalid branch',
41 '(expected=[$br] != found=[$branch])'
42 ].join(' ')
43 throw new Exception(err) // groovylint-disable-line ThrowException
44 }
45
46 return (br)
Joey Armstrongf076c312023-08-01 17:17:10 -040047}
Hardik Windlassec9341b2021-06-07 11:58:29 +000048
Joey Armstronge5aae1c2023-07-24 14:11:20 -040049// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040050// Intent: Difficult at times to determine when pipeline jobs have
51// regenerated. Hardcode a version string that can be assigned
52// per-script to be sure latest repository changes are being used.
53// -----------------------------------------------------------------------
54String pipelineVer() {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -040055 String version = 'ff337d86399e107cd417793454c4bbd398855d31'
Joey Armstrong0251e962023-08-25 20:51:31 -040056 return(version)
57}
58
59// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040060// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040061// Jenkins will re-write the call stack for serialization.S
62// -----------------------------------------------------------------------
63// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040064// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040065String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040066 String branchName = branchName()
Joey Armstrong0251e962023-08-25 20:51:31 -040067 String version = pipelineVer()
Joey Armstrong06a68372023-07-24 16:37:16 -040068 String src = [
69 'ci-management',
70 'jjb',
71 'pipeline',
72 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040073 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040074 'bbsim-tests.groovy'
75 ].join('/')
76
Joey Armstrong0251e962023-08-25 20:51:31 -040077 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040078 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040079}
80
81// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040082// Intent: Log progress message
83// -----------------------------------------------------------------------
84void enter(String name) {
85 // Announce ourselves for log usability
86 String iam = getIam(name)
87 println("${iam}: ENTER")
88 return
89}
90
91// -----------------------------------------------------------------------
92// Intent: Log progress message
93// -----------------------------------------------------------------------
94void leave(String name) {
95 // Announce ourselves for log usability
96 String iam = getIam(name)
97 println("${iam}: LEAVE")
98 return
99}
100
101// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400102// Intent: Determine if working on a release branch.
103// Note: Conditional is legacy, should also check for *-dev or *-pre
104// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400105Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400106 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -0400107 // if branchName in modifiers
108 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400109}
110
111// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400112// Intent: Terminate orphaned port-forward from different namespaces
113// -----------------------------------------------------------------------
114void cleanupPortForward() {
115 enter('cleanupPortForward')
116
117 Map pkpfArgs =\
118 [
119 'banner' : true, // display banner for logging
120 'show_procs' : true, // display procs under consideration
121 'filler' : true // fix conditional trailing comma
122 ]
123
124 // 'kubectl.*port-forward'
125 pkill_port_forward('port-forward', pkpfArgs)
126 leave('cleanupPortForward')
127 return
128}
129
130// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400131// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400132// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400133void execute_test\
134(
135 String testTarget, // functional-single-kind-dt
136 String workflow, // dt
137 String testLogging, // 'True'
138 Boolean teardown, // true
139 String testSpecificHelmFlags=''
140) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400141 String infraNamespace = 'default'
142 String volthaNamespace = 'voltha'
143 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500144
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400145 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400146 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400147 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400148 stage('Cleanup') {
149 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400150 timeout(15) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400151 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400152 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400153 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400154 } // timeout
155
156 timeout(5) {
157 script {
158 enter('Cleanup')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400159 cleanupPortForward()
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400160 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400161 } // script
162 } // timeout
163 } // teardown
164 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500165
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400166 // -----------------------------------------------------------------------
167 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400168 stage('Deploy common infrastructure') {
169 script {
170 local dashargs = [
171 'kpi_exporter.enabled=false',
172 'dashboards.xos=false',
173 'dashboards.onos=false',
174 'dashboards.aaa=false',
175 'dashboards.voltha=false',
176 ].join(',')
177
178 local promargs = [
179 'prometheus.alertmanager.enabled=false',
180 'prometheus.pushgateway.enabled=false',
181 ].join(',')
182
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400183 sh(label : 'Deploy common infrastructure',
184 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000185 helm repo add onf https://charts.opencord.org
186 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400187
188 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000189 if [ ${withMonitoring} = true ] ; then
190 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400191 --set ${promargs} \
192 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000193 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400194 """)
195 } // script
196 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500197
Joey Armstrong0251e962023-08-25 20:51:31 -0400198 // -----------------------------------------------------------------------
199 // [TODO] Check onos_log output
200 // -----------------------------------------------------------------------
201 stage('Deploy Voltha') {
202 if (teardown) {
203 timeout(10) {
204 script {
205 String iam = getIam('Deploy Voltha')
206 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000207
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400208 sh(label : 'Launch kail-startup',
209 script : """
210mkdir -p "$logsDir"
211touch "$onosLog"
Joey Armstrongf404b642023-08-04 14:39:13 -0400212
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400213_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > "$onosLog" &
214""")
215
216 // if we're downloading a voltha-helm-charts patch,
217 // install from a local copy of the charts
Joey Armstrong0251e962023-08-25 20:51:31 -0400218 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400219
Joey Armstrong0251e962023-08-25 20:51:31 -0400220 if (volthaHelmChartsChange != ''
221 || gerritProject == 'voltha-helm-charts'
222 || isReleaseBranch(branch) // branch != 'master'
223 ) {
224 localCharts = true
225 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000226
Joey Armstrong0251e962023-08-25 20:51:31 -0400227 String branchName = branchName()
228 Boolean isRelease = isReleaseBranch(branch)
229 println([
230 " ** localCharts=${localCharts}",
231 "branchName=${branchName}",
232 "branch=${branch}",
233 "branch=isReleaseBranch=${isRelease}",
234 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800235
Joey Armstrong0251e962023-08-25 20:51:31 -0400236 // -----------------------------------------------------------------------
237 // Rewrite localHelmFlags using array join, moving code around and
238 // refactoring into standalone functions
239 // -----------------------------------------------------------------------
240 // NOTE temporary workaround expose ONOS node ports
241 // -----------------------------------------------------------------------
242 String localHelmFlags = [
243 extraHelmFlags.trim(),
244 "--set global.log_level=${logLevel.toUpperCase()}",
245 '--set onos-classic.onosSshPort=30115',
246 '--set onos-classic.onosApiPort=30120',
247 '--set onos-classic.onosOfPort=31653',
248 '--set onos-classic.individualOpenFlowNodePorts=true',
249 testSpecificHelmFlags
250 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400251
Joey Armstrong0251e962023-08-25 20:51:31 -0400252 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000253
Joey Armstrong0251e962023-08-25 20:51:31 -0400254 if (gerritProject != '') {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400255 localHelmFlags += getVolthaImageFlags(gerritProject)
Joey Armstrong0251e962023-08-25 20:51:31 -0400256 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800257
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400258 enter('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400259 volthaDeploy([
260 infraNamespace: infraNamespace,
261 volthaNamespace: volthaNamespace,
262 workflow: workflow.toLowerCase(),
263 withMacLearning: enableMacLearning.toBoolean(),
264 extraHelmFlags: localHelmFlags,
265 localCharts: localCharts,
266 bbsimReplica: olts.toInteger(),
267 dockerRegistry: registry,
268 ])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400269 leave('volthaDeploy')
Joey Armstrong0251e962023-08-25 20:51:31 -0400270 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400271
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400272 // Display spawned procs
273 script {
274 enter('bbsim-tests::pgrep_port_forward::0')
275 pgrep_port_forward('port-forw')
276 leave('bbsim-tests::pgrep_port_forward::0')
277 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400278
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400279 sh(label : 'Terminate kail-startup',
280 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400281if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400282 pkill --uid \$(id -u) --echo --list-full --full '_TAG=kail-startup'
Joey Armstrong12a8c832023-08-28 16:50:24 -0400283fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400284""")
285
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400286 sh(label : 'Lingering kail-startup check',
287 script : """
288pgrep --uid \$(id -u) --list-full --full 'kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400289""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400290
291 // -----------------------------------------------------------------------
292 // Bundle onos-voltha / kail logs
293 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400294 sh(
295 label : 'Bundle logs: onos-voltha-startup-combined',
296 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400297cat <<EOM
298
299** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400300** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400301** -----------------------------------------------------------------------
302EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400303
304pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
305gzip -k onos-voltha-startup-combined.log
306rm onos-voltha-startup-combined.log
307popd || { echo "ERROR: popd $logsDir failed"; exit 1; }
Joey Armstrong0251e962023-08-25 20:51:31 -0400308 """)
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400309 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400310
Joey Armstrong0251e962023-08-25 20:51:31 -0400311 // -----------------------------------------------------------------------
312 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400313 sh(label : 'while-true-port-forward',
Roger Luethicebf79a2023-09-28 09:41:45 +0200314 script : """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000315 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
316 bbsimDmiPortFwd=50075
317 for i in {0..${olts.toInteger() - 1}}; do
318 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
319 ((bbsimDmiPortFwd++))
320 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000321 if [ ${withMonitoring} = true ] ; then
322 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
323 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400324# ps aux | grep port-forward
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400325""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400326 // ---------------------------------
327 // Sanity check port-forward spawned
328 // ---------------------------------
Joey Armstrong1b7cc792023-09-26 14:19:21 -0400329 script {
330 enter('bbsim-tests::pgrep_port_forward::1')
331 pgrep_port_forward('port-forw')
332 leave('bbsim-tests::pgrep_port_forward::1')
333 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500334
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400335 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400336 script {
337 enter('setOnosLogLevels')
338 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400339 onosNamespace: infraNamespace,
340 apps: [
341 'org.opencord.dhcpl2relay',
342 'org.opencord.olt',
343 'org.opencord.aaa',
344 'org.opencord.maclearner',
345 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
346 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
347 ],
348 logLevel: logLevel
349 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400350 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400351 } // script
352 } // if (teardown)
353 } // stage('Deploy Voltha')
354
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400355 // -----------------------------------------------------------------------
356 // -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400357 stage("Run test ${testTarget} on workflow ${workflow}") {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400358 sh(
359 label : 'Monitor using mem_consumption.py',
360 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400361echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400362
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400363if [ ${withMonitoring} = true ] ; then
364 cat <<EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400365
366** -----------------------------------------------------------------------
367** Monitoring memory usage with mem_consumption.py
368** -----------------------------------------------------------------------
369EOM
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400370 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
371 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400372
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400373 echo '** Installing python virtualenv'
374 make venv-activate-patched
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400375
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400376 # Collect initial memory consumption
377 set +u && source .venv/bin/activate && set -u
378 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
379fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400380
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400381echo -e '** Monitor memory consumption: LEAVE\n'
382""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400383
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400384 sh(
385 label : "make testTarget=[${testTarget}]",
386 script : """
387echo -e "\n** make testTarget=[${testTarget}]"
388mkdir -p ${logsDir}
389export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
390ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
391export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800392
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400393make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
394""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400395
396 getPodsInfo("${logsDir}")
397
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400398 // [TODO] make conditional, bundle when logs are available
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400399 sh(
400 label : 'Gather robot Framework logs',
401 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400402echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400403
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400404# set +e
405# collect logs collected in the Robot Framework StartLogging keyword
406cd "${logsDir}"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400407
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400408echo "** Available logs:"
409/bin/ls -l "$logsDir"
410echo
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400411
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400412echo '** Bundle combined log'
413gzip *-combined.log || true
414rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400415
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400416echo -e '** Gather robot Framework logs: LEAVE\n'
417""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400418
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400419 // -----------------------------------------------------------------------
420 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400421 sh(
422 label : 'Monitor pod-mem-consumption',
423 script : """
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400424echo -e '** Monitor pod-mem-consumption: ENTER'
425if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400426 cat <<EOM
427
428** -----------------------------------------------------------------------
429** Monitoring pod-memory-consumption using mem_consumption.py
430** -----------------------------------------------------------------------
431EOM
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400432
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400433cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400434
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400435echo '** Installing python virtualenv'
436make venv-activate-patched
437
438# Collect memory consumption of voltha pods once all the tests are complete
439set +u && source .venv/bin/activate && set -u
440python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
441fi
442echo -e '** Monitor pod-mem-consumption: LEAVE\n'
443""")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400444 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400445
446 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400447} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800448
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400449// -----------------------------------------------------------------------
450// -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400451void collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400452 script {
453 String iam = getIam('collectArtifacts')
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400454 enter("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400455
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400456 println("""
Joey Armstrong97a8b882023-08-02 16:08:52 -0400457
458** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400459** IAM: $iam
Joey Armstrong97a8b882023-08-02 16:08:52 -0400460** collectArtifacts
461** -----------------------------------------------------------------------
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400462""")
463 }
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400464
Joey Armstrongcd419122023-08-07 14:56:39 -0400465 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400466
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400467 sh(label : 'kubectl logs > voltha.log',
468 script : """
469kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
470 > $WORKSPACE/${exitStatus}/voltha.log
471""")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400472
Joey Armstrongcd419122023-08-07 14:56:39 -0400473 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400474
Joey Armstrong0251e962023-08-25 20:51:31 -0400475 script {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400476 enter('pkill _TAG=kail-startup')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400477 sh(label : 'pgrep_proc - kill-pre',
478 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400479pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400480""")
481 sh(label : 'pkill_proc - kail',
482 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400483if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then
484 pkill --uid "\$(id -u)" --echo --full 'kail'
485fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400486""")
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400487 leave('pkill _TAG=kail-startup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400488 }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400489
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400490 enter('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400491 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400492 disableArchiveOutput: false,
493 logFileName: '**/*/log*.html',
494 otherFiles: '',
495 outputFileName: '**/*/output*.xml',
496 outputPath: '.',
497 passThreshold: 100,
498 reportFileName: '**/*/report*.html',
499 unstableThreshold: 0,
500 onlyCritical: true])
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400501 leave('RobotPublisher')
Joey Armstrongcd419122023-08-07 14:56:39 -0400502
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400503 leave("exitStatus=${exitStatus}")
Joey Armstrongcd419122023-08-07 14:56:39 -0400504 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000505}
506
Joey Armstrongb65ada32023-08-03 12:50:20 -0400507// -----------------------------------------------------------------------
508// Intent: main
509// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800510pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400511 /* no label, executor is determined by JJB */
512 agent {
513 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800514 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400515
Joey Armstrong0251e962023-08-25 20:51:31 -0400516 options {
517 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800518 }
Joey Armstronged161f72023-04-11 13:16:59 -0400519
Joey Armstrong0251e962023-08-25 20:51:31 -0400520 environment {
521 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
522 VOLTCONFIG = "$HOME/.volt/config"
523 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
524 DIAGS_PROFILE = 'VOLTHA_PROFILE'
525 SSHPASS = 'karaf'
526 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400527
Joey Armstrong0251e962023-08-25 20:51:31 -0400528 stages {
529 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400530 steps {
Joey Armstrong0251e962023-08-25 20:51:31 -0400531 getVolthaCode([
532 branch: "${branch}",
533 gerritProject: "${gerritProject}",
534 gerritRefspec: "${gerritRefspec}",
535 volthaSystemTestsChange: "${volthaSystemTestsChange}",
536 volthaHelmChartsChange: "${volthaHelmChartsChange}",
537 ])
538 }
539 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400540
Joey Armstrong0251e962023-08-25 20:51:31 -0400541 stage('Build patch v1.1') {
542 // build the patch only if gerritProject is specified
543 when {
544 expression { return !gerritProject.isEmpty() }
545 }
546
547 steps {
548 // NOTE that the correct patch has already been checked out
549 // during the getVolthaCode step
550 buildVolthaComponent("${gerritProject}")
551 }
552 }
553
554 // -----------------------------------------------------------------------
555 // -----------------------------------------------------------------------
556 stage('Install Kail')
557 {
558 steps
559 {
560 script
561 {
562 String cmd = [
563 'make',
564 '--no-print-directory',
565 '-C', "$WORKSPACE/voltha-system-tests",
566 "KAIL_PATH=\"$WORKSPACE/bin\"",
567 'kail',
568 ].join(' ')
569
570 println(" ** Running: ${cmd}")
571 sh("${cmd}")
572 } // script
573 } // steps
574 } // stage
575
576 // -----------------------------------------------------------------------
577 // -----------------------------------------------------------------------
578 stage('Install Tools') {
579 steps {
580 script {
581 String branchName = branchName()
582 String iam = getIam('Install Tools')
583
584 println("${iam}: ENTER (branch=$branch)")
585 installKind(branch) // needed early by stage(Cleanup)
586 println("${iam}: LEAVE (branch=$branch)")
587 } // script
588 } // steps
589 } // stage
590
591 // -----------------------------------------------------------------------
592 // -----------------------------------------------------------------------
593 stage('Create K8s Cluster') {
594 steps {
595 script {
596 def clusterExists = sh(
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400597 label : 'Create K8s Cluster',
Joey Armstrong0251e962023-08-25 20:51:31 -0400598 returnStdout: true,
599 script: """kind get clusters | grep "${clusterName}" | wc -l""")
600
601 if (clusterExists.trim() == '0') {
602 createKubernetesCluster([nodes: 3, name: clusterName])
603 }
604 } // script
605 } // steps
606 } // stage('Create K8s Cluster')
607
608 // -----------------------------------------------------------------------
609 // -----------------------------------------------------------------------
610 stage('Replace voltctl') {
611 // if the project is voltctl, override the downloaded one with the built one
612 when {
613 expression { return gerritProject == 'voltctl' }
614 }
615
616 // Hmmmm(?) where did the voltctl download happen ?
617 // Likely Makefile but would be helpful to document here.
618 steps {
619 script {
620 String iam = getIam('Replace voltctl')
621
622 println("${iam} Running: installVoltctl($branch)")
623 println("${iam}: ENTER")
624 installVoltctl("$branch")
625 println("${iam}: LEAVE")
626 } // script
627 } // step
628 } // stage
629
630 // -----------------------------------------------------------------------
631 // -----------------------------------------------------------------------
632 stage('Load image in kind nodes')
633 {
634 when {
635 expression { return !gerritProject.isEmpty() }
636 }
637 steps {
638 loadToKind()
639 } // steps
640 } // stage
641
642 // -----------------------------------------------------------------------
643 // [TODO] verify testing output
644 // -----------------------------------------------------------------------
645 stage('Parse and execute tests')
646 {
647 steps {
648 script {
649 // Announce ourselves for log usability
650 enter('Parse and execute tests')
651
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400652 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400653 println("** [DEBUG]: tests=$tests")
654
655 // Display expected tests for times when output goes dark
656 tests.eachWithIndex { test, idx ->
657 String target = test['target']
658 println("** test[${idx}]: ${target}\n")
659 }
660
661 println('''
662** -----------------------------------------------------------------------
663** NOTE: For odd/silent job failures verify a few details
664** - All tests mentioned in the tests-to-run index were logged.
665** - Test suites display ENTER/LEAVE mesasge pairs.
666** - Processing terminated prematurely when LEAVE strings are missing.
667** -----------------------------------------------------------------------
668''')
669 tests.eachWithIndex { test, idx ->
670 println "** readYaml test suite[$idx]) test=[${test}]"
671
Joey Armstrongf404b642023-08-04 14:39:13 -0400672 String target = test['target']
673 String workflow = test['workflow']
674 String flags = test['flags']
675 Boolean teardown = test['teardown'].toBoolean()
676 Boolean logging = test['logging'].toBoolean()
677 String testLogging = (logging) ? 'True' : 'False'
678
Joey Armstrong0251e962023-08-25 20:51:31 -0400679 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400680** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400681** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400682** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400683""")
684
Joey Armstrong0251e962023-08-25 20:51:31 -0400685 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400686 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400687 execute_test(target, workflow, testLogging, teardown, flags)
688 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400689 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400690 catch (Exception err) {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400691 String iamexc = getIam(test)
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400692 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400693 }
694 finally {
695 leave("execute_test (target=$target)")
696 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400697 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400698 // Premature exit if this message is not logged
699 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400700 } // script
701 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400702 } // stage
703 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400704
705 post
706 {
Joey Armstrong9f66e3f2023-09-20 16:29:29 -0400707 aborted {
708 collectArtifacts('aborted')
709 script { cleanupPortForward() }
710 }
711 failure {
712 collectArtifacts('failed')
713 script { cleanupPortForward() }
714 }
715 always {
716 collectArtifacts('always')
717 script { cleanupPortForward() }
718 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800719 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400720} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400721
Joey Armstrong664c55a2023-08-28 14:22:33 -0400722// [EOF]