blob: c465ab33d9c36e58a98e783d1a30d417e3c08362 [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Joey Armstronge5aae1c2023-07-24 14:11:20 -040018// [TODO] Update syntax below to the latest supported
Matteo Scandoloa156b572021-02-04 11:52:18 -080019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
21 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
23])
24
Joey Armstronge5aae1c2023-07-24 14:11:20 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040028String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040029
30// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040031// Intent:
Joey Armstrongf076c312023-08-01 17:17:10 -040032// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040033String branchName() {
Joey Armstrongf076c312023-08-01 17:17:10 -040034 String name = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040035
36 // [TODO] Sanity check the target branch
37 // if (name != jenkins.branch) { fatal }
Joey Armstrongf076c312023-08-01 17:17:10 -040038 return(name)
39}
Hardik Windlassec9341b2021-06-07 11:58:29 +000040
Joey Armstronge5aae1c2023-07-24 14:11:20 -040041// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040042// Intent: Difficult at times to determine when pipeline jobs have
43// regenerated. Hardcode a version string that can be assigned
44// per-script to be sure latest repository changes are being used.
45// -----------------------------------------------------------------------
46String pipelineVer() {
Joey Armstrongdeae1372023-08-28 19:17:45 -040047 String version = 'e9725b199be034061750c6e9c4c3ac46cf2d0f68'
Joey Armstrong0251e962023-08-25 20:51:31 -040048 return(version)
49}
50
51// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040052// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040053// Jenkins will re-write the call stack for serialization.S
54// -----------------------------------------------------------------------
55// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040056// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040057String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040058 String branchName = branchName()
Joey Armstrong0251e962023-08-25 20:51:31 -040059 String version = pipelineVer()
Joey Armstrong06a68372023-07-24 16:37:16 -040060 String src = [
61 'ci-management',
62 'jjb',
63 'pipeline',
64 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040065 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040066 'bbsim-tests.groovy'
67 ].join('/')
68
Joey Armstrong0251e962023-08-25 20:51:31 -040069 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040070 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040071}
72
73// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040074// Intent: Log progress message
75// -----------------------------------------------------------------------
76void enter(String name) {
77 // Announce ourselves for log usability
78 String iam = getIam(name)
79 println("${iam}: ENTER")
80 return
81}
82
83// -----------------------------------------------------------------------
84// Intent: Log progress message
85// -----------------------------------------------------------------------
86void leave(String name) {
87 // Announce ourselves for log usability
88 String iam = getIam(name)
89 println("${iam}: LEAVE")
90 return
91}
92
93// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -040094// Intent: Determine if working on a release branch.
95// Note: Conditional is legacy, should also check for *-dev or *-pre
96// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040097Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -040098 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -040099 // if branchName in modifiers
100 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400101}
102
103// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400104// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400105// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400106void execute_test\
107(
108 String testTarget, // functional-single-kind-dt
109 String workflow, // dt
110 String testLogging, // 'True'
111 Boolean teardown, // true
112 String testSpecificHelmFlags=''
113) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400114 String infraNamespace = 'default'
115 String volthaNamespace = 'voltha'
116 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500117
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400118 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400119 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400120 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400121 stage('Cleanup') {
122 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400123 timeout(15) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400124 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400125 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400126 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400127 } // timeout
128
129 timeout(5) {
130 script {
131 enter('Cleanup')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400132
Joey Armstrong0251e962023-08-25 20:51:31 -0400133 // remove orphaned port-forward from different namespaces
Joey Armstrong664c55a2023-08-28 14:22:33 -0400134 String proc = 'kubectl.*port-forward' // was 'port-forw'
135 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400136 pgrep_proc(proc)
137 pkill_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400138 pgrep_proc(proc) // todo: fatal unless (proc count==0)
139 */
Joey Armstrongebc18022023-08-26 13:20:49 -0400140
Joey Armstrong664c55a2023-08-28 14:22:33 -0400141 sh(label : 'pgrep_proc - kill-pre',
142 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400143pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400144""")
145
146 sh(label : 'pkill_proc - kubectl.*port-forward',
147 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400148if [[ \$(pgrep --count 'kubectl.*port-forward') -gt 0 ]]; then
149 pkill --uid "\$(id -u)" --echo --full 'kubectl.*port-forward'
150fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400151""")
152
153 sh(label : 'pgrep_proc - kill-post',
154 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400155pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400156""")
157
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400158 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400159 } // script
160 } // timeout
161 } // teardown
162 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500163
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400164 // -----------------------------------------------------------------------
165 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400166 stage('Deploy common infrastructure') {
167 script {
168 local dashargs = [
169 'kpi_exporter.enabled=false',
170 'dashboards.xos=false',
171 'dashboards.onos=false',
172 'dashboards.aaa=false',
173 'dashboards.voltha=false',
174 ].join(',')
175
176 local promargs = [
177 'prometheus.alertmanager.enabled=false',
178 'prometheus.pushgateway.enabled=false',
179 ].join(',')
180
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400181 sh(label : 'Deploy common infrastructure',
182 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000183 helm repo add onf https://charts.opencord.org
184 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400185
186 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000187 if [ ${withMonitoring} = true ] ; then
188 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400189 --set ${promargs} \
190 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000191 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400192 """)
193 } // script
194 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500195
Joey Armstrong0251e962023-08-25 20:51:31 -0400196 // -----------------------------------------------------------------------
197 // [TODO] Check onos_log output
198 // -----------------------------------------------------------------------
199 stage('Deploy Voltha') {
200 if (teardown) {
201 timeout(10) {
202 script {
203 String iam = getIam('Deploy Voltha')
204 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
205 sh("""
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700206 mkdir -p ${logsDir}
Joey Armstrong0251e962023-08-25 20:51:31 -0400207 touch "$onosLog"
208 echo "** kail-startup ENTER: \$(date)" > "$onosLog"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000209
Joey Armstrong0251e962023-08-25 20:51:31 -0400210 # Intermixed output (tee -a &) may get conflusing but let(s) see
211 # what messages are logged during startup.
212 # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
213 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
214 """)
Joey Armstrongf404b642023-08-04 14:39:13 -0400215
Joey Armstrong0251e962023-08-25 20:51:31 -0400216 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
217 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400218
Joey Armstrong0251e962023-08-25 20:51:31 -0400219 if (volthaHelmChartsChange != ''
220 || gerritProject == 'voltha-helm-charts'
221 || isReleaseBranch(branch) // branch != 'master'
222 ) {
223 localCharts = true
224 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000225
Joey Armstrong0251e962023-08-25 20:51:31 -0400226 String branchName = branchName()
227 Boolean isRelease = isReleaseBranch(branch)
228 println([
229 " ** localCharts=${localCharts}",
230 "branchName=${branchName}",
231 "branch=${branch}",
232 "branch=isReleaseBranch=${isRelease}",
233 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800234
Joey Armstrong0251e962023-08-25 20:51:31 -0400235 // -----------------------------------------------------------------------
236 // Rewrite localHelmFlags using array join, moving code around and
237 // refactoring into standalone functions
238 // -----------------------------------------------------------------------
239 // NOTE temporary workaround expose ONOS node ports
240 // -----------------------------------------------------------------------
241 String localHelmFlags = [
242 extraHelmFlags.trim(),
243 "--set global.log_level=${logLevel.toUpperCase()}",
244 '--set onos-classic.onosSshPort=30115',
245 '--set onos-classic.onosApiPort=30120',
246 '--set onos-classic.onosOfPort=31653',
247 '--set onos-classic.individualOpenFlowNodePorts=true',
248 testSpecificHelmFlags
249 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400250
Joey Armstrong0251e962023-08-25 20:51:31 -0400251 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000252
Joey Armstrong0251e962023-08-25 20:51:31 -0400253 if (gerritProject != '') {
254 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
255 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800256
Joey Armstrong0251e962023-08-25 20:51:31 -0400257 println("** ${iam}: ENTER")
258 volthaDeploy([
259 infraNamespace: infraNamespace,
260 volthaNamespace: volthaNamespace,
261 workflow: workflow.toLowerCase(),
262 withMacLearning: enableMacLearning.toBoolean(),
263 extraHelmFlags: localHelmFlags,
264 localCharts: localCharts,
265 bbsimReplica: olts.toInteger(),
266 dockerRegistry: registry,
267 ])
268 println("** ${iam}: LEAVE")
269 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400270
Joey Armstrong0251e962023-08-25 20:51:31 -0400271 // -----------------------------------------------------------------------
272 // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
273 // Why not simply use a pid file, capture _TAG=kail-startup above
274 // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
275 // -----------------------------------------------------------------------
276 script {
277 String proc = '_TAG=kail-startup'
278
279 println("${iam}: ENTER")
280 println("${iam}: Shutdown process $proc")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400281 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400282 pgrep_proc(proc)
283 pkill_proc(proc)
284 pgrep_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400285 */
286
287 sh(label : 'pgrep_proc - kill-pre',
288 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400289pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400290""")
291
292 sh(label : 'pkill_proc - _TAG=kail-startup',
293 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400294if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
295 pkill --uid "\$(id -u)" --echo --full '_TAG=kail-startup'
296fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400297""")
298
299 sh(label : 'pgrep_proc - kill-post',
300 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400301pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400302""")
303
Joey Armstrong0251e962023-08-25 20:51:31 -0400304 println("${iam}: LEAVE")
305 }
306
307 // -----------------------------------------------------------------------
308 // Bundle onos-voltha / kail logs
309 // -----------------------------------------------------------------------
310 sh("""
311cat <<EOM
312
313** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400314** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400315** -----------------------------------------------------------------------
316EOM
317 pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000318 gzip -k onos-voltha-startup-combined.log
319 rm onos-voltha-startup-combined.log
Joey Armstrong0251e962023-08-25 20:51:31 -0400320 popd
321 """)
322 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400323
Joey Armstrong0251e962023-08-25 20:51:31 -0400324 // -----------------------------------------------------------------------
325 // -----------------------------------------------------------------------
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400326 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000327 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
328 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
329 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
330 bbsimDmiPortFwd=50075
331 for i in {0..${olts.toInteger() - 1}}; do
332 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
333 ((bbsimDmiPortFwd++))
334 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000335 if [ ${withMonitoring} = true ] ; then
336 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
337 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400338# ps aux | grep port-forward
Hardik Windlassec9341b2021-06-07 11:58:29 +0000339 """
Joey Armstrong0251e962023-08-25 20:51:31 -0400340 // ---------------------------------
341 // Sanity check port-forward spawned
342 // ---------------------------------
343 script {
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400344 enter('Display port-forward procs')
Joey Armstrongebc18022023-08-26 13:20:49 -0400345 // String proc = 'kubectl.*port-forward' // was 'port-forward'
Joey Armstrong0251e962023-08-25 20:51:31 -0400346 String proc = 'port-forward'
347 println("Display spawned ${proc}")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400348 sh(label : 'pgrep_proc - check',
349 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400350pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400351""")
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400352 leave('Display port-forward procs')
Joey Armstrong0251e962023-08-25 20:51:31 -0400353 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500354
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400355 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400356 script {
357 enter('setOnosLogLevels')
358 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400359 onosNamespace: infraNamespace,
360 apps: [
361 'org.opencord.dhcpl2relay',
362 'org.opencord.olt',
363 'org.opencord.aaa',
364 'org.opencord.maclearner',
365 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
366 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
367 ],
368 logLevel: logLevel
369 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400370 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400371 } // script
372 } // if (teardown)
373 } // stage('Deploy Voltha')
374
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400375 // -----------------------------------------------------------------------
376 // -----------------------------------------------------------------------
Joey Armstrongfd896522023-08-04 13:30:48 -0400377 stage("Run test ${testTarget} on workflow ${workflow}")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400378 {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400379 sh(
380 label : 'Monitor using mem_consumption.py',
381 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400382 echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400383
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000384 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400385 cat <<EOM
386
387** -----------------------------------------------------------------------
388** Monitoring memory usage with mem_consumption.py
389** -----------------------------------------------------------------------
390EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500391 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
392 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400393
394 echo '** Installing python virtualenv'
395 make venv-activate-patched
396
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400397 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000398 # Collect initial memory consumption
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400399 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000400 fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400401
402 echo -e '** Monitor memory consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400403 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400404
405 sh """
Joey Armstrong54dec092023-08-03 18:21:38 -0400406 echo -e "\n** make testTarget=[${testTarget}]"
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700407 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000408 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700409 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000410 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800411
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400412 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Hardik Windlass72947302021-06-14 10:36:57 +0000413 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400414
415 getPodsInfo("${logsDir}")
416
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400417 sh(
418 label : 'Gather robot Framework logs',
419 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400420 echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400421
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400422 # set +e
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000423 # collect logs collected in the Robot Framework StartLogging keyword
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400424 cd "${logsDir}"
425
426 echo "** Available logs:"
427 /bin/ls -l "${logsDir}"
428 echo
429
430 readarray -t logs < <(find . -name '*-combined.log' -print)
Joey Armstrong479d2822023-08-29 15:56:01 -0400431 if [[ \${\#logs[@]} -gt 0 ]]; then
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400432 echo '** Bundle combined log'
433 gzip "${logs[@]}"
434 rm -f "${logs[@]}"
435 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400436
437 echo -e '** Gather robot Framework logs: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400438 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400439
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400440 // -----------------------------------------------------------------------
441 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400442 sh(
443 label : 'Monitor pod-mem-consumption',
444 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400445 echo -e '** Monitor pod-mem-consumption: ENTER'
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000446 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400447 cat <<EOM
448
449** -----------------------------------------------------------------------
450** Monitoring pod-memory-consumption using mem_consumption.py
451** -----------------------------------------------------------------------
452EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500453 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400454
455 echo '** Installing python virtualenv'
456 make venv-activate-patched
457
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400458 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000459 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400460 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000461 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400462 echo -e '** Monitor pod-mem-consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400463 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400464 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400465
466 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400467} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800468
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400469// -----------------------------------------------------------------------
470// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -0400471def collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400472 script {
473 String iam = getIam('collectArtifacts')
474 println("${iam}: ENTER (exitStatus=${exitStatus})")
475 }
Joey Armstrongcd419122023-08-07 14:56:39 -0400476
Joey Armstrong97a8b882023-08-02 16:08:52 -0400477 echo '''
478
479** -----------------------------------------------------------------------
480** collectArtifacts
481** -----------------------------------------------------------------------
482'''
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400483
Joey Armstrongcd419122023-08-07 14:56:39 -0400484 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400485
Joey Armstrongcd419122023-08-07 14:56:39 -0400486 sh """
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400487 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Hardik Windlassec9341b2021-06-07 11:58:29 +0000488 """
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400489
Joey Armstrongcd419122023-08-07 14:56:39 -0400490 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400491
Joey Armstrong0251e962023-08-25 20:51:31 -0400492 script {
493 println("${iam}: ENTER")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400494 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400495 pgrep_proc('kail-startup')
496 pkill_proc('kail')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400497 */
498 sh(label : 'pgrep_proc - kill-pre',
499 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400500pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400501""")
502 sh(label : 'pkill_proc - kail',
503 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400504if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then
505 pkill --uid "\$(id -u)" --echo --full 'kail'
506fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400507""")
508
509 sh(label : 'pgrep_proc - kill-post',
510 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400511pgrep --uid "\$(id -u)" --list-full --full 'kail || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400512""")
513
Joey Armstrong0251e962023-08-25 20:51:31 -0400514 println("${iam}: LEAVE")
515 }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400516
Joey Armstrongcd419122023-08-07 14:56:39 -0400517 println("${iam}: ENTER RobotPublisher")
518 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400519 disableArchiveOutput: false,
520 logFileName: '**/*/log*.html',
521 otherFiles: '',
522 outputFileName: '**/*/output*.xml',
523 outputPath: '.',
524 passThreshold: 100,
525 reportFileName: '**/*/report*.html',
526 unstableThreshold: 0,
527 onlyCritical: true])
Joey Armstrongcd419122023-08-07 14:56:39 -0400528 println("${iam}: LEAVE RobotPublisher")
529
530 println("${iam}: LEAVE (exitStatus=${exitStatus})")
531 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000532}
533
Joey Armstrongb65ada32023-08-03 12:50:20 -0400534// -----------------------------------------------------------------------
535// Intent: main
536// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800537pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400538 /* no label, executor is determined by JJB */
539 agent {
540 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800541 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400542
Joey Armstrong0251e962023-08-25 20:51:31 -0400543 options {
544 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800545 }
Joey Armstronged161f72023-04-11 13:16:59 -0400546
Joey Armstrong0251e962023-08-25 20:51:31 -0400547 environment {
548 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
549 VOLTCONFIG = "$HOME/.volt/config"
550 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
551 DIAGS_PROFILE = 'VOLTHA_PROFILE'
552 SSHPASS = 'karaf'
553 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400554
Joey Armstrong0251e962023-08-25 20:51:31 -0400555 stages {
556 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400557 steps {
Joey Armstrong0251e962023-08-25 20:51:31 -0400558 getVolthaCode([
559 branch: "${branch}",
560 gerritProject: "${gerritProject}",
561 gerritRefspec: "${gerritRefspec}",
562 volthaSystemTestsChange: "${volthaSystemTestsChange}",
563 volthaHelmChartsChange: "${volthaHelmChartsChange}",
564 ])
565 }
566 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400567
Joey Armstrong0251e962023-08-25 20:51:31 -0400568 stage('Build patch v1.1') {
569 // build the patch only if gerritProject is specified
570 when {
571 expression { return !gerritProject.isEmpty() }
572 }
573
574 steps {
575 // NOTE that the correct patch has already been checked out
576 // during the getVolthaCode step
577 buildVolthaComponent("${gerritProject}")
578 }
579 }
580
581 // -----------------------------------------------------------------------
582 // -----------------------------------------------------------------------
583 stage('Install Kail')
584 {
585 steps
586 {
587 script
588 {
589 String cmd = [
590 'make',
591 '--no-print-directory',
592 '-C', "$WORKSPACE/voltha-system-tests",
593 "KAIL_PATH=\"$WORKSPACE/bin\"",
594 'kail',
595 ].join(' ')
596
597 println(" ** Running: ${cmd}")
598 sh("${cmd}")
599 } // script
600 } // steps
601 } // stage
602
603 // -----------------------------------------------------------------------
604 // -----------------------------------------------------------------------
605 stage('Install Tools') {
606 steps {
607 script {
608 String branchName = branchName()
609 String iam = getIam('Install Tools')
610
611 println("${iam}: ENTER (branch=$branch)")
612 installKind(branch) // needed early by stage(Cleanup)
613 println("${iam}: LEAVE (branch=$branch)")
614 } // script
615 } // steps
616 } // stage
617
618 // -----------------------------------------------------------------------
619 // -----------------------------------------------------------------------
620 stage('Create K8s Cluster') {
621 steps {
622 script {
623 def clusterExists = sh(
624 returnStdout: true,
625 script: """kind get clusters | grep "${clusterName}" | wc -l""")
626
627 if (clusterExists.trim() == '0') {
628 createKubernetesCluster([nodes: 3, name: clusterName])
629 }
630 } // script
631 } // steps
632 } // stage('Create K8s Cluster')
633
634 // -----------------------------------------------------------------------
635 // -----------------------------------------------------------------------
636 stage('Replace voltctl') {
637 // if the project is voltctl, override the downloaded one with the built one
638 when {
639 expression { return gerritProject == 'voltctl' }
640 }
641
642 // Hmmmm(?) where did the voltctl download happen ?
643 // Likely Makefile but would be helpful to document here.
644 steps {
645 script {
646 String iam = getIam('Replace voltctl')
647
648 println("${iam} Running: installVoltctl($branch)")
649 println("${iam}: ENTER")
650 installVoltctl("$branch")
651 println("${iam}: LEAVE")
652 } // script
653 } // step
654 } // stage
655
656 // -----------------------------------------------------------------------
657 // -----------------------------------------------------------------------
658 stage('Load image in kind nodes')
659 {
660 when {
661 expression { return !gerritProject.isEmpty() }
662 }
663 steps {
664 loadToKind()
665 } // steps
666 } // stage
667
668 // -----------------------------------------------------------------------
669 // [TODO] verify testing output
670 // -----------------------------------------------------------------------
671 stage('Parse and execute tests')
672 {
673 steps {
674 script {
675 // Announce ourselves for log usability
676 enter('Parse and execute tests')
677
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400678 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400679 println("** [DEBUG]: tests=$tests")
680
681 // Display expected tests for times when output goes dark
682 tests.eachWithIndex { test, idx ->
683 String target = test['target']
684 println("** test[${idx}]: ${target}\n")
685 }
686
687 println('''
688** -----------------------------------------------------------------------
689** NOTE: For odd/silent job failures verify a few details
690** - All tests mentioned in the tests-to-run index were logged.
691** - Test suites display ENTER/LEAVE mesasge pairs.
692** - Processing terminated prematurely when LEAVE strings are missing.
693** -----------------------------------------------------------------------
694''')
695 tests.eachWithIndex { test, idx ->
696 println "** readYaml test suite[$idx]) test=[${test}]"
697
Joey Armstrongf404b642023-08-04 14:39:13 -0400698 String target = test['target']
699 String workflow = test['workflow']
700 String flags = test['flags']
701 Boolean teardown = test['teardown'].toBoolean()
702 Boolean logging = test['logging'].toBoolean()
703 String testLogging = (logging) ? 'True' : 'False'
704
Joey Armstrong0251e962023-08-25 20:51:31 -0400705 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400706** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400707** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400708** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400709""")
710
Joey Armstrong0251e962023-08-25 20:51:31 -0400711 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400712 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400713 execute_test(target, workflow, testLogging, teardown, flags)
714 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400715 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400716 catch (Exception err) {
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400717 String iamexc = getIam(name)
718 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400719 }
720 finally {
721 leave("execute_test (target=$target)")
722 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400723 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400724 // Premature exit if this message is not logged
725 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400726 } // script
727 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400728 } // stage
729 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400730
731 post
732 {
733 aborted { collectArtifacts('aborted') }
734 failure { collectArtifacts('failed') }
Joey Armstrongbf492922023-04-13 17:05:09 -0400735 always { collectArtifacts('always') }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800736 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400737} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400738
Joey Armstrong664c55a2023-08-28 14:22:33 -0400739// [EOF]