blob: f71e75e8bd64a2c60a9c9941d752c5670a288186 [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Joey Armstronge5aae1c2023-07-24 14:11:20 -040018// [TODO] Update syntax below to the latest supported
Matteo Scandoloa156b572021-02-04 11:52:18 -080019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
21 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
23])
24
Joey Armstronge5aae1c2023-07-24 14:11:20 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstrongb65ada32023-08-03 12:50:20 -040028String clusterName = 'kind-ci' // was def
Joey Armstrongf076c312023-08-01 17:17:10 -040029
30// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040031// Intent:
Joey Armstrongf076c312023-08-01 17:17:10 -040032// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -040033String branchName() {
Joey Armstrongf076c312023-08-01 17:17:10 -040034 String name = 'master'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040035
36 // [TODO] Sanity check the target branch
37 // if (name != jenkins.branch) { fatal }
Joey Armstrongf076c312023-08-01 17:17:10 -040038 return(name)
39}
Hardik Windlassec9341b2021-06-07 11:58:29 +000040
Joey Armstronge5aae1c2023-07-24 14:11:20 -040041// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040042// Intent: Difficult at times to determine when pipeline jobs have
43// regenerated. Hardcode a version string that can be assigned
44// per-script to be sure latest repository changes are being used.
45// -----------------------------------------------------------------------
46String pipelineVer() {
Joey Armstrong6146f7e2023-08-28 09:05:38 -040047 String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
Joey Armstrong0251e962023-08-25 20:51:31 -040048 return(version)
49}
50
51// -----------------------------------------------------------------------
Joey Armstrong06a68372023-07-24 16:37:16 -040052// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrong0251e962023-08-25 20:51:31 -040053// Jenkins will re-write the call stack for serialization.S
54// -----------------------------------------------------------------------
55// Note: Hardcoded version string used to visualize changes in jenkins UI
Joey Armstrong06a68372023-07-24 16:37:16 -040056// -----------------------------------------------------------------------
Joey Armstrong97a8b882023-08-02 16:08:52 -040057String getIam(String func) {
Joey Armstrongb65ada32023-08-03 12:50:20 -040058 String branchName = branchName()
Joey Armstrong0251e962023-08-25 20:51:31 -040059 String version = pipelineVer()
Joey Armstrong06a68372023-07-24 16:37:16 -040060 String src = [
61 'ci-management',
62 'jjb',
63 'pipeline',
64 'voltha',
Joey Armstrongb65ada32023-08-03 12:50:20 -040065 branchName,
Joey Armstrong06a68372023-07-24 16:37:16 -040066 'bbsim-tests.groovy'
67 ].join('/')
68
Joey Armstrong0251e962023-08-25 20:51:31 -040069 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040070 return(name)
Joey Armstrong06a68372023-07-24 16:37:16 -040071}
72
73// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040074// Intent: Log progress message
75// -----------------------------------------------------------------------
76void enter(String name) {
77 // Announce ourselves for log usability
78 String iam = getIam(name)
79 println("${iam}: ENTER")
80 return
81}
82
83// -----------------------------------------------------------------------
84// Intent: Log progress message
85// -----------------------------------------------------------------------
86void leave(String name) {
87 // Announce ourselves for log usability
88 String iam = getIam(name)
89 println("${iam}: LEAVE")
90 return
91}
92
93// -----------------------------------------------------------------------
Joey Armstronge5aae1c2023-07-24 14:11:20 -040094// Intent: Determine if working on a release branch.
95// Note: Conditional is legacy, should also check for *-dev or *-pre
96// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -040097Boolean isReleaseBranch(String name) {
Joey Armstronge5aae1c2023-07-24 14:11:20 -040098 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrongb65ada32023-08-03 12:50:20 -040099 // if branchName in modifiers
100 return(name != 'master') // OR branchName.contains('-')
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400101}
102
103// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400104// Intent: Iterate over a list of test suites and invoke.
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400105// -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400106void execute_test\
107(
108 String testTarget, // functional-single-kind-dt
109 String workflow, // dt
110 String testLogging, // 'True'
111 Boolean teardown, // true
112 String testSpecificHelmFlags=''
113) {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400114 String infraNamespace = 'default'
115 String volthaNamespace = 'voltha'
116 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -0500117
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400118 // -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400119 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400120 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400121 stage('Cleanup') {
122 if (teardown) {
Joey Armstrong84adc542023-04-11 14:47:34 -0400123 timeout(15) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400124 script {
Joey Armstrongb65ada32023-08-03 12:50:20 -0400125 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong84adc542023-04-11 14:47:34 -0400126 }
Joey Armstrong0251e962023-08-25 20:51:31 -0400127 } // timeout
128
129 timeout(5) {
130 script {
131 enter('Cleanup')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400132
Joey Armstrong0251e962023-08-25 20:51:31 -0400133 // remove orphaned port-forward from different namespaces
Joey Armstrong664c55a2023-08-28 14:22:33 -0400134 String proc = 'kubectl.*port-forward' // was 'port-forw'
135 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400136 pgrep_proc(proc)
137 pkill_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400138 pgrep_proc(proc) // todo: fatal unless (proc count==0)
139 */
Joey Armstrongebc18022023-08-26 13:20:49 -0400140
Joey Armstrong664c55a2023-08-28 14:22:33 -0400141 sh(label : 'pgrep_proc - kill-pre',
142 script : """
143pgrep --uid "$(uid -u)" --list-full --full 'kubectl.*port-forward'
144""")
145
146 sh(label : 'pkill_proc - kubectl.*port-forward',
147 script : """
148pkill --uid "$(uid -u)" --echo --full 'kubectl.*port-forward'
149""")
150
151 sh(label : 'pgrep_proc - kill-post',
152 script : """
153pgrep --uid "$(uid -u)" --list-full --full 'kubectl.*port-forward'
154""")
155
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400156 leave('Cleanup')
Joey Armstrong0251e962023-08-25 20:51:31 -0400157 } // script
158 } // timeout
159 } // teardown
160 } // stage('Cleanup')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500161
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400162 // -----------------------------------------------------------------------
163 // -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400164 stage('Deploy common infrastructure') {
165 script {
166 local dashargs = [
167 'kpi_exporter.enabled=false',
168 'dashboards.xos=false',
169 'dashboards.onos=false',
170 'dashboards.aaa=false',
171 'dashboards.voltha=false',
172 ].join(',')
173
174 local promargs = [
175 'prometheus.alertmanager.enabled=false',
176 'prometheus.pushgateway.enabled=false',
177 ].join(',')
178
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400179 sh(label : 'Deploy common infrastructure',
180 script : """
Hardik Windlasse1660492022-03-14 15:12:46 +0000181 helm repo add onf https://charts.opencord.org
182 helm repo update
Joey Armstrong0251e962023-08-25 20:51:31 -0400183
184 echo -e "\nwithMonitoring=[$withMonitoring]"
Hardik Windlasse1660492022-03-14 15:12:46 +0000185 if [ ${withMonitoring} = true ] ; then
186 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong0251e962023-08-25 20:51:31 -0400187 --set ${promargs} \
188 --set ${dashargs}
Hardik Windlasse1660492022-03-14 15:12:46 +0000189 fi
Joey Armstrong0251e962023-08-25 20:51:31 -0400190 """)
191 } // script
192 } // stage('Deploy Common Infra')
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500193
Joey Armstrong0251e962023-08-25 20:51:31 -0400194 // -----------------------------------------------------------------------
195 // [TODO] Check onos_log output
196 // -----------------------------------------------------------------------
197 stage('Deploy Voltha') {
198 if (teardown) {
199 timeout(10) {
200 script {
201 String iam = getIam('Deploy Voltha')
202 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
203 sh("""
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700204 mkdir -p ${logsDir}
Joey Armstrong0251e962023-08-25 20:51:31 -0400205 touch "$onosLog"
206 echo "** kail-startup ENTER: \$(date)" > "$onosLog"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000207
Joey Armstrong0251e962023-08-25 20:51:31 -0400208 # Intermixed output (tee -a &) may get conflusing but let(s) see
209 # what messages are logged during startup.
210 # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
211 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
212 """)
Joey Armstrongf404b642023-08-04 14:39:13 -0400213
Joey Armstrong0251e962023-08-25 20:51:31 -0400214 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
215 Boolean localCharts = false
Joey Armstrongf404b642023-08-04 14:39:13 -0400216
Joey Armstrong0251e962023-08-25 20:51:31 -0400217 if (volthaHelmChartsChange != ''
218 || gerritProject == 'voltha-helm-charts'
219 || isReleaseBranch(branch) // branch != 'master'
220 ) {
221 localCharts = true
222 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000223
Joey Armstrong0251e962023-08-25 20:51:31 -0400224 String branchName = branchName()
225 Boolean isRelease = isReleaseBranch(branch)
226 println([
227 " ** localCharts=${localCharts}",
228 "branchName=${branchName}",
229 "branch=${branch}",
230 "branch=isReleaseBranch=${isRelease}",
231 ].join(', '))
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800232
Joey Armstrong0251e962023-08-25 20:51:31 -0400233 // -----------------------------------------------------------------------
234 // Rewrite localHelmFlags using array join, moving code around and
235 // refactoring into standalone functions
236 // -----------------------------------------------------------------------
237 // NOTE temporary workaround expose ONOS node ports
238 // -----------------------------------------------------------------------
239 String localHelmFlags = [
240 extraHelmFlags.trim(),
241 "--set global.log_level=${logLevel.toUpperCase()}",
242 '--set onos-classic.onosSshPort=30115',
243 '--set onos-classic.onosApiPort=30120',
244 '--set onos-classic.onosOfPort=31653',
245 '--set onos-classic.individualOpenFlowNodePorts=true',
246 testSpecificHelmFlags
247 ].join(' ')
Joey Armstrong28e86ee2023-08-03 15:22:29 -0400248
Joey Armstrong0251e962023-08-25 20:51:31 -0400249 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Hardik Windlassec9341b2021-06-07 11:58:29 +0000250
Joey Armstrong0251e962023-08-25 20:51:31 -0400251 if (gerritProject != '') {
252 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
253 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800254
Joey Armstrong0251e962023-08-25 20:51:31 -0400255 println("** ${iam}: ENTER")
256 volthaDeploy([
257 infraNamespace: infraNamespace,
258 volthaNamespace: volthaNamespace,
259 workflow: workflow.toLowerCase(),
260 withMacLearning: enableMacLearning.toBoolean(),
261 extraHelmFlags: localHelmFlags,
262 localCharts: localCharts,
263 bbsimReplica: olts.toInteger(),
264 dockerRegistry: registry,
265 ])
266 println("** ${iam}: LEAVE")
267 } // script
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400268
Joey Armstrong0251e962023-08-25 20:51:31 -0400269 // -----------------------------------------------------------------------
270 // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
271 // Why not simply use a pid file, capture _TAG=kail-startup above
272 // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
273 // -----------------------------------------------------------------------
274 script {
275 String proc = '_TAG=kail-startup'
276
277 println("${iam}: ENTER")
278 println("${iam}: Shutdown process $proc")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400279 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400280 pgrep_proc(proc)
281 pkill_proc(proc)
282 pgrep_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400283 */
284
285 sh(label : 'pgrep_proc - kill-pre',
286 script : """
287pgrep --uid "$(uid -u)" --list-full --full '_TAG=kail-startup'
288""")
289
290 sh(label : 'pkill_proc - _TAG=kail-startup',
291 script : """
292pkill --uid "$(uid -u)" --echo --full '_TAG=kail-startup'
293""")
294
295 sh(label : 'pgrep_proc - kill-post',
296 script : """
297pgrep --uid "$(uid -u)" --list-full --full '_TAG=kail-startup'
298""")
299
Joey Armstrong0251e962023-08-25 20:51:31 -0400300 println("${iam}: LEAVE")
301 }
302
303 // -----------------------------------------------------------------------
304 // Bundle onos-voltha / kail logs
305 // -----------------------------------------------------------------------
306 sh("""
307cat <<EOM
308
309** -----------------------------------------------------------------------
310** Combine an compress voltha startup log(s)
311** -----------------------------------------------------------------------
312EOM
313 pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000314 gzip -k onos-voltha-startup-combined.log
315 rm onos-voltha-startup-combined.log
Joey Armstrong0251e962023-08-25 20:51:31 -0400316 popd
317 """)
318 } // timeout(10)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400319
Joey Armstrong0251e962023-08-25 20:51:31 -0400320 // -----------------------------------------------------------------------
321 // -----------------------------------------------------------------------
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400322 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000323 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
324 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
325 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
326 bbsimDmiPortFwd=50075
327 for i in {0..${olts.toInteger() - 1}}; do
328 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
329 ((bbsimDmiPortFwd++))
330 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000331 if [ ${withMonitoring} = true ] ; then
332 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
333 fi
Hardik Windlassec9341b2021-06-07 11:58:29 +0000334 ps aux | grep port-forward
335 """
Joey Armstrong0251e962023-08-25 20:51:31 -0400336 // ---------------------------------
337 // Sanity check port-forward spawned
338 // ---------------------------------
339 script {
340 enter('port-forward check')
Joey Armstrongebc18022023-08-26 13:20:49 -0400341 // String proc = 'kubectl.*port-forward' // was 'port-forward'
Joey Armstrong0251e962023-08-25 20:51:31 -0400342 String proc = 'port-forward'
343 println("Display spawned ${proc}")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400344 sh(label : 'pgrep_proc - check',
345 script : """
346pgrep --uid "$(uid -u)" --list-full --full "port-forward"
347""")
Joey Armstrong0251e962023-08-25 20:51:31 -0400348 leave('port-forward check')
349 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500350
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400351 // setting ONOS log level
Joey Armstrong0251e962023-08-25 20:51:31 -0400352 script {
353 enter('setOnosLogLevels')
354 setOnosLogLevels([
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400355 onosNamespace: infraNamespace,
356 apps: [
357 'org.opencord.dhcpl2relay',
358 'org.opencord.olt',
359 'org.opencord.aaa',
360 'org.opencord.maclearner',
361 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
362 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
363 ],
364 logLevel: logLevel
365 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400366 leave('setOnosLogLevels')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400367 } // script
368 } // if (teardown)
369 } // stage('Deploy Voltha')
370
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400371 // -----------------------------------------------------------------------
372 // -----------------------------------------------------------------------
Joey Armstrongfd896522023-08-04 13:30:48 -0400373 stage("Run test ${testTarget} on workflow ${workflow}")
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400374 {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400375 sh(
376 label : 'Monitor using mem_consumption.py',
377 script : """
Joey Armstrong0251e962023-08-25 20:51:31 -0400378 echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400379
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000380 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400381 cat <<EOM
382
383** -----------------------------------------------------------------------
384** Monitoring memory usage with mem_consumption.py
385** -----------------------------------------------------------------------
386EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500387 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
388 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400389
390 echo '** Installing python virtualenv'
391 make venv-activate-patched
392
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400393 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000394 # Collect initial memory consumption
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400395 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000396 fi
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400397
398 echo -e '** Monitor memory consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400399 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400400
401 sh """
Joey Armstrong54dec092023-08-03 18:21:38 -0400402 echo -e "\n** make testTarget=[${testTarget}]"
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700403 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000404 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700405 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000406 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800407
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400408 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Hardik Windlass72947302021-06-14 10:36:57 +0000409 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400410
411 getPodsInfo("${logsDir}")
412
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400413 sh(
414 label : 'Gather robot Framework logs',
415 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400416 echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400417 # set +e
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000418 # collect logs collected in the Robot Framework StartLogging keyword
419 cd ${logsDir}
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400420 gzip *-combined.log
421 rm -f *-combined.log
Joey Armstrong54dec092023-08-03 18:21:38 -0400422
423 echo -e '** Gather robot Framework logs: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400424 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400425
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400426 // -----------------------------------------------------------------------
427 // -----------------------------------------------------------------------
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400428 sh(
429 label : 'Monitor pod-mem-consumption',
430 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400431 echo -e '** Monitor pod-mem-consumption: ENTER'
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000432 if [ ${withMonitoring} = true ] ; then
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400433 cat <<EOM
434
435** -----------------------------------------------------------------------
436** Monitoring pod-memory-consumption using mem_consumption.py
437** -----------------------------------------------------------------------
438EOM
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500439 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400440
441 echo '** Installing python virtualenv'
442 make venv-activate-patched
443
Joey Armstronge5aae1c2023-07-24 14:11:20 -0400444 set +u && source .venv/bin/activate && set -u
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000445 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400446 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000447 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400448 echo -e '** Monitor pod-mem-consumption: LEAVE\n'
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400449 """)
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400450 } // stage
Joey Armstrongb65ada32023-08-03 12:50:20 -0400451
452 return
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400453} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800454
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400455// -----------------------------------------------------------------------
456// -----------------------------------------------------------------------
Joey Armstrongb65ada32023-08-03 12:50:20 -0400457def collectArtifacts(exitStatus) {
Joey Armstrong0251e962023-08-25 20:51:31 -0400458 script {
459 String iam = getIam('collectArtifacts')
460 println("${iam}: ENTER (exitStatus=${exitStatus})")
461 }
Joey Armstrongcd419122023-08-07 14:56:39 -0400462
Joey Armstrong97a8b882023-08-02 16:08:52 -0400463 echo '''
464
465** -----------------------------------------------------------------------
466** collectArtifacts
467** -----------------------------------------------------------------------
468'''
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400469
Joey Armstrongcd419122023-08-07 14:56:39 -0400470 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400471
Joey Armstrongcd419122023-08-07 14:56:39 -0400472 sh """
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400473 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Hardik Windlassec9341b2021-06-07 11:58:29 +0000474 """
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400475
Joey Armstrongcd419122023-08-07 14:56:39 -0400476 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400477
Joey Armstrong0251e962023-08-25 20:51:31 -0400478 script {
479 println("${iam}: ENTER")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400480 /*
Joey Armstrong0251e962023-08-25 20:51:31 -0400481 pgrep_proc('kail-startup')
482 pkill_proc('kail')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400483 */
484 sh(label : 'pgrep_proc - kill-pre',
485 script : """
486pgrep --uid "$(uid -u)" --list-full --full "kail-startup',
487""")
488 sh(label : 'pkill_proc - kail',
489 script : """
490pkill --uid "$(uid -u)" --echo --full 'kail'
491""")
492
493 sh(label : 'pgrep_proc - kill-post',
494 script : """
495pgrep --uid "$(uid -u)" --list-full --full 'kail'
496""")
497
Joey Armstrong0251e962023-08-25 20:51:31 -0400498 println("${iam}: LEAVE")
499 }
Joey Armstrong97a8b882023-08-02 16:08:52 -0400500
Joey Armstrongcd419122023-08-07 14:56:39 -0400501 println("${iam}: ENTER RobotPublisher")
502 step([$class: 'RobotPublisher',
Joey Armstrong0251e962023-08-25 20:51:31 -0400503 disableArchiveOutput: false,
504 logFileName: '**/*/log*.html',
505 otherFiles: '',
506 outputFileName: '**/*/output*.xml',
507 outputPath: '.',
508 passThreshold: 100,
509 reportFileName: '**/*/report*.html',
510 unstableThreshold: 0,
511 onlyCritical: true])
Joey Armstrongcd419122023-08-07 14:56:39 -0400512 println("${iam}: LEAVE RobotPublisher")
513
514 println("${iam}: LEAVE (exitStatus=${exitStatus})")
515 return
Hardik Windlassec9341b2021-06-07 11:58:29 +0000516}
517
Joey Armstrongb65ada32023-08-03 12:50:20 -0400518// -----------------------------------------------------------------------
519// Intent: main
520// -----------------------------------------------------------------------
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800521pipeline {
Joey Armstrong0251e962023-08-25 20:51:31 -0400522 /* no label, executor is determined by JJB */
523 agent {
524 label "${params.buildNode}"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800525 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400526
Joey Armstrong0251e962023-08-25 20:51:31 -0400527 options {
528 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800529 }
Joey Armstronged161f72023-04-11 13:16:59 -0400530
Joey Armstrong0251e962023-08-25 20:51:31 -0400531 environment {
532 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
533 VOLTCONFIG = "$HOME/.volt/config"
534 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
535 DIAGS_PROFILE = 'VOLTHA_PROFILE'
536 SSHPASS = 'karaf'
537 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400538
Joey Armstrong0251e962023-08-25 20:51:31 -0400539 stages {
540 stage('Download Code') {
Joey Armstrongf404b642023-08-04 14:39:13 -0400541 steps {
Joey Armstrong0251e962023-08-25 20:51:31 -0400542 getVolthaCode([
543 branch: "${branch}",
544 gerritProject: "${gerritProject}",
545 gerritRefspec: "${gerritRefspec}",
546 volthaSystemTestsChange: "${volthaSystemTestsChange}",
547 volthaHelmChartsChange: "${volthaHelmChartsChange}",
548 ])
549 }
550 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400551
Joey Armstrong0251e962023-08-25 20:51:31 -0400552 stage('Build patch v1.1') {
553 // build the patch only if gerritProject is specified
554 when {
555 expression { return !gerritProject.isEmpty() }
556 }
557
558 steps {
559 // NOTE that the correct patch has already been checked out
560 // during the getVolthaCode step
561 buildVolthaComponent("${gerritProject}")
562 }
563 }
564
565 // -----------------------------------------------------------------------
566 // -----------------------------------------------------------------------
567 stage('Install Kail')
568 {
569 steps
570 {
571 script
572 {
573 String cmd = [
574 'make',
575 '--no-print-directory',
576 '-C', "$WORKSPACE/voltha-system-tests",
577 "KAIL_PATH=\"$WORKSPACE/bin\"",
578 'kail',
579 ].join(' ')
580
581 println(" ** Running: ${cmd}")
582 sh("${cmd}")
583 } // script
584 } // steps
585 } // stage
586
587 // -----------------------------------------------------------------------
588 // -----------------------------------------------------------------------
589 stage('Install Tools') {
590 steps {
591 script {
592 String branchName = branchName()
593 String iam = getIam('Install Tools')
594
595 println("${iam}: ENTER (branch=$branch)")
596 installKind(branch) // needed early by stage(Cleanup)
597 println("${iam}: LEAVE (branch=$branch)")
598 } // script
599 } // steps
600 } // stage
601
602 // -----------------------------------------------------------------------
603 // -----------------------------------------------------------------------
604 stage('Create K8s Cluster') {
605 steps {
606 script {
607 def clusterExists = sh(
608 returnStdout: true,
609 script: """kind get clusters | grep "${clusterName}" | wc -l""")
610
611 if (clusterExists.trim() == '0') {
612 createKubernetesCluster([nodes: 3, name: clusterName])
613 }
614 } // script
615 } // steps
616 } // stage('Create K8s Cluster')
617
618 // -----------------------------------------------------------------------
619 // -----------------------------------------------------------------------
620 stage('Replace voltctl') {
621 // if the project is voltctl, override the downloaded one with the built one
622 when {
623 expression { return gerritProject == 'voltctl' }
624 }
625
626 // Hmmmm(?) where did the voltctl download happen ?
627 // Likely Makefile but would be helpful to document here.
628 steps {
629 script {
630 String iam = getIam('Replace voltctl')
631
632 println("${iam} Running: installVoltctl($branch)")
633 println("${iam}: ENTER")
634 installVoltctl("$branch")
635 println("${iam}: LEAVE")
636 } // script
637 } // step
638 } // stage
639
640 // -----------------------------------------------------------------------
641 // -----------------------------------------------------------------------
642 stage('Load image in kind nodes')
643 {
644 when {
645 expression { return !gerritProject.isEmpty() }
646 }
647 steps {
648 loadToKind()
649 } // steps
650 } // stage
651
652 // -----------------------------------------------------------------------
653 // [TODO] verify testing output
654 // -----------------------------------------------------------------------
655 stage('Parse and execute tests')
656 {
657 steps {
658 script {
659 // Announce ourselves for log usability
660 enter('Parse and execute tests')
661
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400662 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong0251e962023-08-25 20:51:31 -0400663 println("** [DEBUG]: tests=$tests")
664
665 // Display expected tests for times when output goes dark
666 tests.eachWithIndex { test, idx ->
667 String target = test['target']
668 println("** test[${idx}]: ${target}\n")
669 }
670
671 println('''
672** -----------------------------------------------------------------------
673** NOTE: For odd/silent job failures verify a few details
674** - All tests mentioned in the tests-to-run index were logged.
675** - Test suites display ENTER/LEAVE mesasge pairs.
676** - Processing terminated prematurely when LEAVE strings are missing.
677** -----------------------------------------------------------------------
678''')
679 tests.eachWithIndex { test, idx ->
680 println "** readYaml test suite[$idx]) test=[${test}]"
681
Joey Armstrongf404b642023-08-04 14:39:13 -0400682 String target = test['target']
683 String workflow = test['workflow']
684 String flags = test['flags']
685 Boolean teardown = test['teardown'].toBoolean()
686 Boolean logging = test['logging'].toBoolean()
687 String testLogging = (logging) ? 'True' : 'False'
688
Joey Armstrong0251e962023-08-25 20:51:31 -0400689 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400690** -----------------------------------------------------------------------
Joey Armstrong0251e962023-08-25 20:51:31 -0400691** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400692** -----------------------------------------------------------------------
Joey Armstrong268442d2023-08-22 17:16:10 -0400693""")
694
Joey Armstrong0251e962023-08-25 20:51:31 -0400695 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400696 enter("execute_test (target=$target)")
Joey Armstrong0251e962023-08-25 20:51:31 -0400697 execute_test(target, workflow, testLogging, teardown, flags)
698 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400699 // groovylint-disable-next-line CatchException
Joey Armstrong0251e962023-08-25 20:51:31 -0400700 catch (Exception err) {
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400701 String iamexc = getIam(name)
702 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong0251e962023-08-25 20:51:31 -0400703 }
704 finally {
705 leave("execute_test (target=$target)")
706 }
Joey Armstrongb65ada32023-08-03 12:50:20 -0400707 } // for
Joey Armstrong0251e962023-08-25 20:51:31 -0400708 // Premature exit if this message is not logged
709 leave('Parse and execute tests')
Joey Armstrongb65ada32023-08-03 12:50:20 -0400710 } // script
711 } // steps
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400712 } // stage
713 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400714
715 post
716 {
717 aborted { collectArtifacts('aborted') }
718 failure { collectArtifacts('failed') }
Joey Armstrongbf492922023-04-13 17:05:09 -0400719 always { collectArtifacts('always') }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800720 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400721} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400722
Joey Armstrong664c55a2023-08-28 14:22:33 -0400723// [EOF]