blob: e36c85bad02021065d07ed4b86cdfc9d3a22cbc5 [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Joey Armstronge5aae1c2023-07-24 14:11:20 -040018// [TODO] Update syntax below to the latest supported
Matteo Scandoloa156b572021-02-04 11:52:18 -080019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
21 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
23])
24
Joey Armstronge5aae1c2023-07-24 14:11:20 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040028String clusterName = 'kind-ci'
Joey Armstrongf076c312023-08-01 17:17:10 -040029
30// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040031// Intent:
Joey Armstrongf076c312023-08-01 17:17:10 -040032// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040033String branchName() {
Joey Armstrongf076c312023-08-01 17:17:10 -040034 String name = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040035
36 // [TODO] Sanity check the target branch
37 // if (name != jenkins.branch) { fatal }
Joey Armstrongf076c312023-08-01 17:17:10 -040038 return(name)
39}
Hardik Windlassec9341b2021-06-07 11:58:29 +000040
Joey Armstronge5aae1c2023-07-24 14:11:20 -040041// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040042// Intent: Difficult at times to determine when pipeline jobs have
43// regenerated. Hardcode a version string that can be assigned
44// per-script to be sure latest repository changes are being used.
45// -----------------------------------------------------------------------
46String pipelineVer() {
Joey Armstrong25637192023-08-30 09:53:05 -040047 String version = '45c11f80698d3be1416a86d2872d6e25aa24baa8'
Joey Armstrong0251e962023-08-25 20:51:31 -040048 return(version)
49}
50
51// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040052// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040053// Jenkins will re-write the call stack for serialization.S
54// -----------------------------------------------------------------------
55// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040056// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040057String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040058 String branchName = branchName()
Joey Armstrong0251e962023-08-25 20:51:31 -040059 String version = pipelineVer()
Joey Armstrong06a68372023-07-24 16:37:16 -040060 String src = [
61 'ci-management',
62 'jjb',
63 'pipeline',
64 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040065 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040066 'bbsim-tests.groovy'
67 ].join('/')
68
Joey Armstrong0251e962023-08-25 20:51:31 -040069 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040070 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040071}
72
73// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040074// Intent: Log progress message
75// -----------------------------------------------------------------------
76void enter(String name) {
77 // Announce ourselves for log usability
78 String iam = getIam(name)
79 println("${iam}: ENTER")
80 return
81}
82
83// -----------------------------------------------------------------------
84// Intent: Log progress message
85// -----------------------------------------------------------------------
86void leave(String name) {
87 // Announce ourselves for log usability
88 String iam = getIam(name)
89 println("${iam}: LEAVE")
90 return
91}
92
93// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -040094// Intent: Determine if working on a release branch.
95// Note: Conditional is legacy, should also check for *-dev or *-pre
96// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040097Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -040098 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -040099 // if branchName in modifiers
100 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400101}
102
103// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400104// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400105// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400106void execute_test\
107(
108 String testTarget, // functional-single-kind-dt
109 String workflow, // dt
110 String testLogging, // 'True'
111 Boolean teardown, // true
112 String testSpecificHelmFlags=''
113) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400114 String infraNamespace = 'default'
115 String volthaNamespace = 'voltha'
116 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500117
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400118 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400119 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400120 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400121 stage('Cleanup') {
122 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400123 timeout(15) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400124 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400125 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400126 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400127 } // timeout
128
129 timeout(5) {
130 script {
131 enter('Cleanup')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400132
Joey Armstrong0251e962023-08-25 20:51:31 -0400133 // remove orphaned port-forward from different namespaces
Joey Armstrong664c55a2023-08-28 14:22:33 -0400134 String proc = 'kubectl.*port-forward' // was 'port-forw'
135 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400136 pgrep_proc(proc)
137 pkill_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400138 pgrep_proc(proc) // todo: fatal unless (proc count==0)
139 */
Joey Armstrongebc18022023-08-26 13:20:49 -0400140
Joey Armstrong664c55a2023-08-28 14:22:33 -0400141 sh(label : 'pgrep_proc - kill-pre',
142 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400143pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400144""")
145
146 sh(label : 'pkill_proc - kubectl.*port-forward',
147 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400148if [[ \$(pgrep --count 'kubectl.*port-forward') -gt 0 ]]; then
149 pkill --uid "\$(id -u)" --echo --full 'kubectl.*port-forward'
150fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400151""")
152
153 sh(label : 'pgrep_proc - kill-post',
154 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400155pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400156""")
157
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400158 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400159 } // script
160 } // timeout
161 } // teardown
162 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500163
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400164 // -----------------------------------------------------------------------
165 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400166 stage('Deploy common infrastructure') {
167 script {
168 local dashargs = [
169 'kpi_exporter.enabled=false',
170 'dashboards.xos=false',
171 'dashboards.onos=false',
172 'dashboards.aaa=false',
173 'dashboards.voltha=false',
174 ].join(',')
175
176 local promargs = [
177 'prometheus.alertmanager.enabled=false',
178 'prometheus.pushgateway.enabled=false',
179 ].join(',')
180
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400181 sh(label : 'Deploy common infrastructure',
182 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000183 helm repo add onf https://charts.opencord.org
184 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400185
186 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000187 if [ ${withMonitoring} = true ] ; then
188 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400189 --set ${promargs} \
190 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000191 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400192 """)
193 } // script
194 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500195
Joey Armstrong0251e962023-08-25 20:51:31 -0400196 // -----------------------------------------------------------------------
197 // [TODO] Check onos_log output
198 // -----------------------------------------------------------------------
199 stage('Deploy Voltha') {
200 if (teardown) {
201 timeout(10) {
202 script {
203 String iam = getIam('Deploy Voltha')
204 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
205 sh("""
Joey Armstrongd1c45752023-08-29 16:56:05 -0400206 mkdir -p "$logsDir"
Joey Armstrong0251e962023-08-25 20:51:31 -0400207 touch "$onosLog"
208 echo "** kail-startup ENTER: \$(date)" > "$onosLog"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000209
Joey Armstrong0251e962023-08-25 20:51:31 -0400210 # Intermixed output (tee -a &) may get conflusing but let(s) see
211 # what messages are logged during startup.
212 # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
213 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
214 """)
Joey Armstrongf404b642023-08-04 14:39:13 -0400215
Joey Armstrong0251e962023-08-25 20:51:31 -0400216 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
217 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400218
Joey Armstrong0251e962023-08-25 20:51:31 -0400219 if (volthaHelmChartsChange != ''
220 || gerritProject == 'voltha-helm-charts'
221 || isReleaseBranch(branch) // branch != 'master'
222 ) {
223 localCharts = true
224 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000225
Joey Armstrong0251e962023-08-25 20:51:31 -0400226 String branchName = branchName()
227 Boolean isRelease = isReleaseBranch(branch)
228 println([
229 " ** localCharts=${localCharts}",
230 "branchName=${branchName}",
231 "branch=${branch}",
232 "branch=isReleaseBranch=${isRelease}",
233 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800234
Joey Armstrong0251e962023-08-25 20:51:31 -0400235 // -----------------------------------------------------------------------
236 // Rewrite localHelmFlags using array join, moving code around and
237 // refactoring into standalone functions
238 // -----------------------------------------------------------------------
239 // NOTE temporary workaround expose ONOS node ports
240 // -----------------------------------------------------------------------
241 String localHelmFlags = [
242 extraHelmFlags.trim(),
243 "--set global.log_level=${logLevel.toUpperCase()}",
244 '--set onos-classic.onosSshPort=30115',
245 '--set onos-classic.onosApiPort=30120',
246 '--set onos-classic.onosOfPort=31653',
247 '--set onos-classic.individualOpenFlowNodePorts=true',
248 testSpecificHelmFlags
249 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400250
Joey Armstrong0251e962023-08-25 20:51:31 -0400251 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000252
Joey Armstrong0251e962023-08-25 20:51:31 -0400253 if (gerritProject != '') {
254 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
255 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800256
Joey Armstrong0251e962023-08-25 20:51:31 -0400257 println("** ${iam}: ENTER")
258 volthaDeploy([
259 infraNamespace: infraNamespace,
260 volthaNamespace: volthaNamespace,
261 workflow: workflow.toLowerCase(),
262 withMacLearning: enableMacLearning.toBoolean(),
263 extraHelmFlags: localHelmFlags,
264 localCharts: localCharts,
265 bbsimReplica: olts.toInteger(),
266 dockerRegistry: registry,
267 ])
268 println("** ${iam}: LEAVE")
269 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400270
Joey Armstrong0251e962023-08-25 20:51:31 -0400271 // -----------------------------------------------------------------------
272 // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
273 // Why not simply use a pid file, capture _TAG=kail-startup above
274 // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
275 // -----------------------------------------------------------------------
276 script {
277 String proc = '_TAG=kail-startup'
278
279 println("${iam}: ENTER")
280 println("${iam}: Shutdown process $proc")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400281 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400282 pgrep_proc(proc)
283 pkill_proc(proc)
284 pgrep_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400285 */
286
287 sh(label : 'pgrep_proc - kill-pre',
288 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400289pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400290""")
291
292 sh(label : 'pkill_proc - _TAG=kail-startup',
293 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400294if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
295 pkill --uid "\$(id -u)" --echo --full '_TAG=kail-startup'
296fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400297""")
298
299 sh(label : 'pgrep_proc - kill-post',
300 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400301pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400302""")
303
Joey Armstrong0251e962023-08-25 20:51:31 -0400304 println("${iam}: LEAVE")
305 }
306
307 // -----------------------------------------------------------------------
308 // Bundle onos-voltha / kail logs
309 // -----------------------------------------------------------------------
310 sh("""
311cat <<EOM
312
313** -----------------------------------------------------------------------
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400314** Combine and compress voltha startup log(s)
Joey Armstrong0251e962023-08-25 20:51:31 -0400315** -----------------------------------------------------------------------
316EOM
317 pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000318 gzip -k onos-voltha-startup-combined.log
319 rm onos-voltha-startup-combined.log
Joey Armstrong0251e962023-08-25 20:51:31 -0400320 popd
321 """)
322 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400323
Joey Armstrong0251e962023-08-25 20:51:31 -0400324 // -----------------------------------------------------------------------
325 // -----------------------------------------------------------------------
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400326 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000327 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
328 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
329 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
330 bbsimDmiPortFwd=50075
331 for i in {0..${olts.toInteger() - 1}}; do
332 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
333 ((bbsimDmiPortFwd++))
334 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000335 if [ ${withMonitoring} = true ] ; then
336 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
337 fi
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400338# ps aux | grep port-forward
Hardik Windlassec9341b2021-06-07 11:58:29 +0000339 """
Joey Armstrong0251e962023-08-25 20:51:31 -0400340 // ---------------------------------
341 // Sanity check port-forward spawned
342 // ---------------------------------
343 script {
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400344 enter('Display port-forward procs')
Joey Armstrongebc18022023-08-26 13:20:49 -0400345 // String proc = 'kubectl.*port-forward' // was 'port-forward'
Joey Armstrong0251e962023-08-25 20:51:31 -0400346 String proc = 'port-forward'
347 println("Display spawned ${proc}")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400348 sh(label : 'pgrep_proc - check',
349 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400350pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400351""")
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400352 leave('Display port-forward procs')
Joey Armstrong0251e962023-08-25 20:51:31 -0400353 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500354
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400355 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400356 script {
357 enter('setOnosLogLevels')
358 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400359 onosNamespace: infraNamespace,
360 apps: [
361 'org.opencord.dhcpl2relay',
362 'org.opencord.olt',
363 'org.opencord.aaa',
364 'org.opencord.maclearner',
365 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
366 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
367 ],
368 logLevel: logLevel
369 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400370 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400371 } // script
372 } // if (teardown)
373 } // stage('Deploy Voltha')
374
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400375 // -----------------------------------------------------------------------
376 // -----------------------------------------------------------------------
Joey Armstrongfd896522023-08-04 13:30:48 -0400377 stage("Run test ${testTarget} on workflow ${workflow}")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400378 {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400379 sh(
380 label : 'Monitor using mem_consumption.py',
381 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400382 echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400383
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000384 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400385 cat <<EOM
386
387** -----------------------------------------------------------------------
388** Monitoring memory usage with mem_consumption.py
389** -----------------------------------------------------------------------
390EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500391 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
392 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400393
394 echo '** Installing python virtualenv'
395 make venv-activate-patched
396
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400397 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000398 # Collect initial memory consumption
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400399 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000400 fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400401
402 echo -e '** Monitor memory consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400403 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400404
405 sh """
Joey Armstrong54dec092023-08-03 18:21:38 -0400406 echo -e "\n** make testTarget=[${testTarget}]"
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700407 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000408 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700409 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000410 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800411
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400412 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Hardik Windlass72947302021-06-14 10:36:57 +0000413 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400414
415 getPodsInfo("${logsDir}")
416
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400417 sh(
418 label : 'Gather robot Framework logs',
419 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400420 echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400421
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400422 # set +e
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000423 # collect logs collected in the Robot Framework StartLogging keyword
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400424 cd "${logsDir}"
425
426 echo "** Available logs:"
Joey Armstrong2e274972023-08-29 17:39:18 -0400427 /bin/ls -l "$logsDir"
Joey Armstrong7ba23ac2023-08-29 15:21:53 -0400428 echo
429
Joey Armstrong2e274972023-08-29 17:39:18 -0400430 echo '** Bundle combined log'
431 gzip *-combined.log || true
432 rm -f *-combined.log || true
Joey Armstrong54dec092023-08-03 18:21:38 -0400433
434 echo -e '** Gather robot Framework logs: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400435 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400436
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400437 // -----------------------------------------------------------------------
438 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400439 sh(
440 label : 'Monitor pod-mem-consumption',
441 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400442 echo -e '** Monitor pod-mem-consumption: ENTER'
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000443 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400444 cat <<EOM
445
446** -----------------------------------------------------------------------
447** Monitoring pod-memory-consumption using mem_consumption.py
448** -----------------------------------------------------------------------
449EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500450 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400451
452 echo '** Installing python virtualenv'
453 make venv-activate-patched
454
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400455 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000456 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400457 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000458 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400459 echo -e '** Monitor pod-mem-consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400460 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400461 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400462
463 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400464} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800465
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400466// -----------------------------------------------------------------------
467// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -0400468def collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400469 script {
470 String iam = getIam('collectArtifacts')
471 println("${iam}: ENTER (exitStatus=${exitStatus})")
472 }
Joey Armstrongcd419122023-08-07 14:56:39 -0400473
Joey Armstrong97a8b882023-08-02 16:08:52 -0400474 echo '''
475
476** -----------------------------------------------------------------------
477** collectArtifacts
478** -----------------------------------------------------------------------
479'''
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400480
Joey Armstrongcd419122023-08-07 14:56:39 -0400481 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400482
Joey Armstrongcd419122023-08-07 14:56:39 -0400483 sh """
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400484 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Hardik Windlassec9341b2021-06-07 11:58:29 +0000485 """
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400486
Joey Armstrongcd419122023-08-07 14:56:39 -0400487 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400488
Joey Armstrong0251e962023-08-25 20:51:31 -0400489 script {
490 println("${iam}: ENTER")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400491 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400492 pgrep_proc('kail-startup')
493 pkill_proc('kail')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400494 */
495 sh(label : 'pgrep_proc - kill-pre',
496 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400497pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400498""")
499 sh(label : 'pkill_proc - kail',
500 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400501if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then
502 pkill --uid "\$(id -u)" --echo --full 'kail'
503fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400504""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400505 println("${iam}: LEAVE")
506 }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400507
Joey Armstrongcd419122023-08-07 14:56:39 -0400508 println("${iam}: ENTER RobotPublisher")
509 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400510 disableArchiveOutput: false,
511 logFileName: '**/*/log*.html',
512 otherFiles: '',
513 outputFileName: '**/*/output*.xml',
514 outputPath: '.',
515 passThreshold: 100,
516 reportFileName: '**/*/report*.html',
517 unstableThreshold: 0,
518 onlyCritical: true])
Joey Armstrongcd419122023-08-07 14:56:39 -0400519 println("${iam}: LEAVE RobotPublisher")
520
521 println("${iam}: LEAVE (exitStatus=${exitStatus})")
522 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000523}
524
Joey Armstrongb65ada32023-08-03 12:50:20 -0400525// -----------------------------------------------------------------------
526// Intent: main
527// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800528pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400529 /* no label, executor is determined by JJB */
530 agent {
531 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800532 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400533
Joey Armstrong0251e962023-08-25 20:51:31 -0400534 options {
535 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800536 }
Joey Armstronged161f72023-04-11 13:16:59 -0400537
Joey Armstrong0251e962023-08-25 20:51:31 -0400538 environment {
539 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
540 VOLTCONFIG = "$HOME/.volt/config"
541 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
542 DIAGS_PROFILE = 'VOLTHA_PROFILE'
543 SSHPASS = 'karaf'
544 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400545
Joey Armstrong0251e962023-08-25 20:51:31 -0400546 stages {
547 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400548 steps {
Joey Armstrong0251e962023-08-25 20:51:31 -0400549 getVolthaCode([
550 branch: "${branch}",
551 gerritProject: "${gerritProject}",
552 gerritRefspec: "${gerritRefspec}",
553 volthaSystemTestsChange: "${volthaSystemTestsChange}",
554 volthaHelmChartsChange: "${volthaHelmChartsChange}",
555 ])
556 }
557 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400558
Joey Armstrong0251e962023-08-25 20:51:31 -0400559 stage('Build patch v1.1') {
560 // build the patch only if gerritProject is specified
561 when {
562 expression { return !gerritProject.isEmpty() }
563 }
564
565 steps {
566 // NOTE that the correct patch has already been checked out
567 // during the getVolthaCode step
568 buildVolthaComponent("${gerritProject}")
569 }
570 }
571
572 // -----------------------------------------------------------------------
573 // -----------------------------------------------------------------------
574 stage('Install Kail')
575 {
576 steps
577 {
578 script
579 {
580 String cmd = [
581 'make',
582 '--no-print-directory',
583 '-C', "$WORKSPACE/voltha-system-tests",
584 "KAIL_PATH=\"$WORKSPACE/bin\"",
585 'kail',
586 ].join(' ')
587
588 println(" ** Running: ${cmd}")
589 sh("${cmd}")
590 } // script
591 } // steps
592 } // stage
593
594 // -----------------------------------------------------------------------
595 // -----------------------------------------------------------------------
596 stage('Install Tools') {
597 steps {
598 script {
599 String branchName = branchName()
600 String iam = getIam('Install Tools')
601
602 println("${iam}: ENTER (branch=$branch)")
603 installKind(branch) // needed early by stage(Cleanup)
604 println("${iam}: LEAVE (branch=$branch)")
605 } // script
606 } // steps
607 } // stage
608
609 // -----------------------------------------------------------------------
610 // -----------------------------------------------------------------------
611 stage('Create K8s Cluster') {
612 steps {
613 script {
614 def clusterExists = sh(
615 returnStdout: true,
616 script: """kind get clusters | grep "${clusterName}" | wc -l""")
617
618 if (clusterExists.trim() == '0') {
619 createKubernetesCluster([nodes: 3, name: clusterName])
620 }
621 } // script
622 } // steps
623 } // stage('Create K8s Cluster')
624
625 // -----------------------------------------------------------------------
626 // -----------------------------------------------------------------------
627 stage('Replace voltctl') {
628 // if the project is voltctl, override the downloaded one with the built one
629 when {
630 expression { return gerritProject == 'voltctl' }
631 }
632
633 // Hmmmm(?) where did the voltctl download happen ?
634 // Likely Makefile but would be helpful to document here.
635 steps {
636 script {
637 String iam = getIam('Replace voltctl')
638
639 println("${iam} Running: installVoltctl($branch)")
640 println("${iam}: ENTER")
641 installVoltctl("$branch")
642 println("${iam}: LEAVE")
643 } // script
644 } // step
645 } // stage
646
647 // -----------------------------------------------------------------------
648 // -----------------------------------------------------------------------
649 stage('Load image in kind nodes')
650 {
651 when {
652 expression { return !gerritProject.isEmpty() }
653 }
654 steps {
655 loadToKind()
656 } // steps
657 } // stage
658
659 // -----------------------------------------------------------------------
660 // [TODO] verify testing output
661 // -----------------------------------------------------------------------
662 stage('Parse and execute tests')
663 {
664 steps {
665 script {
666 // Announce ourselves for log usability
667 enter('Parse and execute tests')
668
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400669 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400670 println("** [DEBUG]: tests=$tests")
671
672 // Display expected tests for times when output goes dark
673 tests.eachWithIndex { test, idx ->
674 String target = test['target']
675 println("** test[${idx}]: ${target}\n")
676 }
677
678 println('''
679** -----------------------------------------------------------------------
680** NOTE: For odd/silent job failures verify a few details
681** - All tests mentioned in the tests-to-run index were logged.
682** - Test suites display ENTER/LEAVE mesasge pairs.
683** - Processing terminated prematurely when LEAVE strings are missing.
684** -----------------------------------------------------------------------
685''')
686 tests.eachWithIndex { test, idx ->
687 println "** readYaml test suite[$idx]) test=[${test}]"
688
Joey Armstrongf404b642023-08-04 14:39:13 -0400689 String target = test['target']
690 String workflow = test['workflow']
691 String flags = test['flags']
692 Boolean teardown = test['teardown'].toBoolean()
693 Boolean logging = test['logging'].toBoolean()
694 String testLogging = (logging) ? 'True' : 'False'
695
Joey Armstrong0251e962023-08-25 20:51:31 -0400696 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400697** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400698** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400699** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400700""")
701
Joey Armstrong0251e962023-08-25 20:51:31 -0400702 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400703 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400704 execute_test(target, workflow, testLogging, teardown, flags)
705 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400706 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400707 catch (Exception err) {
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400708 String iamexc = getIam(name)
709 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400710 }
711 finally {
712 leave("execute_test (target=$target)")
713 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400714 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400715 // Premature exit if this message is not logged
716 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400717 } // script
718 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400719 } // stage
720 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400721
722 post
723 {
724 aborted { collectArtifacts('aborted') }
725 failure { collectArtifacts('failed') }
Joey Armstrongbf492922023-04-13 17:05:09 -0400726 always { collectArtifacts('always') }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800727 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400728} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400729
Joey Armstrong664c55a2023-08-28 14:22:33 -0400730// [EOF]