blob: adf0808a401a899ff3422e9c5bc6a422e1df9fdb [file] [log] [blame]
Joey Armstrong7bcb5782023-06-07 12:25:57 -04001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// voltha-2.x e2e tests for openonu-go
16// uses bbsim to simulate OLT/ONUs
17
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040018// [TODO] Update syntax below to the latest supported
Joey Armstrong7bcb5782023-06-07 12:25:57 -040019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
21 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
23])
24
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstronge9725b12023-08-28 18:15:12 -040028String clusterName = 'kind-ci'
Joey Armstrong7bcb5782023-06-07 12:25:57 -040029
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040030// -----------------------------------------------------------------------
31// Intent:
32// -----------------------------------------------------------------------
Joey Armstrong54dec092023-08-03 18:21:38 -040033String branchName() {
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040034 String name = 'voltha-2.12'
35
36 // [TODO] Sanity check the target branch
37 // if (name != jenkins.branch) { fatal }
38 return(name)
39}
40
41// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040042// Intent: Difficult at times to determine when pipeline jobs have
43// regenerated. Hardcode a version string that can be assigned
44// per-script to be sure latest repository changes are being used.
45// -----------------------------------------------------------------------
46String pipelineVer() {
Joey Armstrong6146f7e2023-08-28 09:05:38 -040047 String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
Joey Armstrongaf4eef22023-08-25 16:14:45 -040048 return(version)
49}
50
51// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040052// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrongd0b5af02023-08-25 15:12:54 -040053// Jenkins will re-write the call stack for serialization.S
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040054// -----------------------------------------------------------------------
Joey Armstrongd0b5af02023-08-25 15:12:54 -040055// Note: Hardcoded version string used to visualize changes in jenkins UI
56// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040057String getIam(String func) {
Joey Armstrong54dec092023-08-03 18:21:38 -040058 String branchName = branchName()
Joey Armstrongaf4eef22023-08-25 16:14:45 -040059 String version = pipelineVer()
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040060 String src = [
61 'ci-management',
62 'jjb',
63 'pipeline',
64 'voltha',
Joey Armstrong54dec092023-08-03 18:21:38 -040065 branchName,
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040066 'bbsim-tests.groovy'
67 ].join('/')
68
Joey Armstrongd0b5af02023-08-25 15:12:54 -040069 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040070 return(name)
71}
72
73// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040074// Intent: Log progress message
75// -----------------------------------------------------------------------
Joey Armstrong7229d9b2023-08-25 18:03:13 -040076void enter(String name) {
Joey Armstrongaf4eef22023-08-25 16:14:45 -040077 // Announce ourselves for log usability
78 String iam = getIam(name)
79 println("${iam}: ENTER")
80 return
81}
82
83// -----------------------------------------------------------------------
84// Intent: Log progress message
85// -----------------------------------------------------------------------
Joey Armstrong7229d9b2023-08-25 18:03:13 -040086void leave(String name) {
Joey Armstrongaf4eef22023-08-25 16:14:45 -040087 // Announce ourselves for log usability
88 String iam = getIam(name)
89 println("${iam}: LEAVE")
90 return
91}
92
93// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040094// Intent: Determine if working on a release branch.
95// Note: Conditional is legacy, should also check for *-dev or *-pre
96// -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -040097Boolean isReleaseBranch(String name) {
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040098 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrong54dec092023-08-03 18:21:38 -040099 // if branchName in modifiers
100 return(name != 'master') // OR branchName.contains('-')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400101}
102
103// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -0400104// Intent: Iterate over a list of test suites and invoke.
Joey Armstrongf060aee2023-08-22 21:55:26 -0400105// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -0400106void execute_test\
107(
108 String testTarget, // functional-single-kind-dt
109 String workflow, // dt
110 String testLogging, // 'True'
111 Boolean teardown, // true
112 String testSpecificHelmFlags=''
113) {
Joey Armstrong54dec092023-08-03 18:21:38 -0400114 String infraNamespace = 'default'
115 String volthaNamespace = 'voltha'
116 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400117
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400118 // -----------------------------------------------------------------------
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400119 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400120 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400121 stage('Cleanup') {
Joey Armstrong38a87832023-08-23 17:02:50 -0400122 if (teardown) {
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400123 timeout(15) {
Joey Armstrong38a87832023-08-23 17:02:50 -0400124 script {
Joey Armstrong54dec092023-08-03 18:21:38 -0400125 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400126 }
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400127 } // timeout
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400128
Joey Armstrongf060aee2023-08-22 21:55:26 -0400129 timeout(5) {
130 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400131 enter('Cleanup')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400132
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400133 // remove orphaned port-forward from different namespaces
Joey Armstrong664c55a2023-08-28 14:22:33 -0400134 String proc = 'kubectl.*port-forward' // was 'port-forw'
135 /*
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400136 pgrep_proc(proc)
137 pkill_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400138 pgrep_proc(proc) // todo: fatal unless (proc count==0)
139 */
Joey Armstrongebc18022023-08-26 13:20:49 -0400140
Joey Armstrong664c55a2023-08-28 14:22:33 -0400141 sh(label : 'pgrep_proc - kill-pre',
142 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400143pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400144""")
145
146 sh(label : 'pkill_proc - kubectl.*port-forward',
147 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400148if [[ \$(pgrep --count 'kubectl.*port-forward') -gt 0 ]]; then
149 pkill --uid "\$(id -u)" --echo --full 'kubectl.*port-forward'
150fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400151""")
152
153 sh(label : 'pgrep_proc - kill-post',
154 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400155pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400156""")
157
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400158 leave('Cleanup')
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400159 } // script
160 } // timeout
Joey Armstrongf060aee2023-08-22 21:55:26 -0400161 } // teardown
Joey Armstrong38a87832023-08-23 17:02:50 -0400162 } // stage('Cleanup')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400163
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400164 // -----------------------------------------------------------------------
165 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400166 stage('Deploy common infrastructure') {
167 script {
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400168 local dashargs = [
169 'kpi_exporter.enabled=false',
170 'dashboards.xos=false',
171 'dashboards.onos=false',
172 'dashboards.aaa=false',
173 'dashboards.voltha=false',
174 ].join(',')
175
176 local promargs = [
177 'prometheus.alertmanager.enabled=false',
178 'prometheus.pushgateway.enabled=false',
179 ].join(',')
180
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400181 sh(label : 'Deploy common infrastructure',
182 script : """
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400183 helm repo add onf https://charts.opencord.org
184 helm repo update
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400185
186 echo -e "\nwithMonitoring=[$withMonitoring]"
187 if [ ${withMonitoring} = true ] ; then
188 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong38a87832023-08-23 17:02:50 -0400189 --set ${promargs} \
190 --set ${dashargs}
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400191 fi
Joey Armstrong38a87832023-08-23 17:02:50 -0400192 """)
193 } // script
194 } // stage('Deploy Common Infra')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400195
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400196 // -----------------------------------------------------------------------
Joey Armstrongf060aee2023-08-22 21:55:26 -0400197 // [TODO] Check onos_log output
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400198 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400199 stage('Deploy Voltha') {
200 if (teardown) {
201 timeout(10) {
202 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400203 String iam = getIam('Deploy Voltha')
204 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
205 sh("""
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400206 mkdir -p ${logsDir}
Joey Armstrong38a87832023-08-23 17:02:50 -0400207 touch "$onosLog"
208 echo "** kail-startup ENTER: \$(date)" > "$onosLog"
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400209
210 # Intermixed output (tee -a &) may get conflusing but let(s) see
211 # what messages are logged during startup.
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400212 # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
Joey Armstrong38a87832023-08-23 17:02:50 -0400213 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400214 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400215
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400216 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
217 Boolean localCharts = false
Joey Armstrong642540a2023-08-10 10:26:36 -0400218
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400219 if (volthaHelmChartsChange != ''
220 || gerritProject == 'voltha-helm-charts'
221 || isReleaseBranch(branch) // branch != 'master'
222 ) {
223 localCharts = true
224 }
Joey Armstrong642540a2023-08-10 10:26:36 -0400225
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400226 String branchName = branchName()
227 Boolean isRelease = isReleaseBranch(branch)
228 println([
229 " ** localCharts=${localCharts}",
230 "branchName=${branchName}",
231 "branch=${branch}",
232 "branch=isReleaseBranch=${isRelease}",
233 ].join(', '))
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400234
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400235 // -----------------------------------------------------------------------
236 // Rewrite localHelmFlags using array join, moving code around and
237 // refactoring into standalone functions
238 // -----------------------------------------------------------------------
239 // NOTE temporary workaround expose ONOS node ports
240 // -----------------------------------------------------------------------
241 String localHelmFlags = [
242 extraHelmFlags.trim(),
243 "--set global.log_level=${logLevel.toUpperCase()}",
244 '--set onos-classic.onosSshPort=30115',
245 '--set onos-classic.onosApiPort=30120',
246 '--set onos-classic.onosOfPort=31653',
247 '--set onos-classic.individualOpenFlowNodePorts=true',
248 testSpecificHelmFlags
249 ].join(' ')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400250
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400251 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400252
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400253 if (gerritProject != '') {
254 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
255 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400256
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400257 println("** ${iam}: ENTER")
258 volthaDeploy([
259 infraNamespace: infraNamespace,
260 volthaNamespace: volthaNamespace,
261 workflow: workflow.toLowerCase(),
262 withMacLearning: enableMacLearning.toBoolean(),
263 extraHelmFlags: localHelmFlags,
264 localCharts: localCharts,
265 bbsimReplica: olts.toInteger(),
266 dockerRegistry: registry,
267 ])
268 println("** ${iam}: LEAVE")
269 } // script
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400270
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400271 // -----------------------------------------------------------------------
272 // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
273 // Why not simply use a pid file, capture _TAG=kail-startup above
274 // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
275 // -----------------------------------------------------------------------
276 script {
277 String proc = '_TAG=kail-startup'
Joey Armstrongdddbbf92023-08-22 16:00:41 -0400278
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400279 println("${iam}: ENTER")
280 println("${iam}: Shutdown process $proc")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400281 /*
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400282 pgrep_proc(proc)
283 pkill_proc(proc)
284 pgrep_proc(proc)
Joey Armstrong664c55a2023-08-28 14:22:33 -0400285 */
286
287 sh(label : 'pgrep_proc - kill-pre',
288 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400289pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400290""")
291
292 sh(label : 'pkill_proc - _TAG=kail-startup',
293 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400294if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then
295 pkill --uid "\$(id -u)" --echo --full '_TAG=kail-startup'
296fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400297""")
298
299 sh(label : 'pgrep_proc - kill-post',
300 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400301pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400302""")
303
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400304 println("${iam}: LEAVE")
305 }
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400306
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400307 // -----------------------------------------------------------------------
308 // Bundle onos-voltha / kail logs
309 // -----------------------------------------------------------------------
310 sh("""
Joey Armstrongf060aee2023-08-22 21:55:26 -0400311cat <<EOM
312
313** -----------------------------------------------------------------------
314** Combine an compress voltha startup log(s)
315** -----------------------------------------------------------------------
316EOM
317 pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400318 gzip -k onos-voltha-startup-combined.log
319 rm onos-voltha-startup-combined.log
Joey Armstrongf060aee2023-08-22 21:55:26 -0400320 popd
321 """)
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400322 } // timeout(10)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400323
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400324 // -----------------------------------------------------------------------
325 // -----------------------------------------------------------------------
326 sh """
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400327 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
328 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
329 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
330 bbsimDmiPortFwd=50075
331 for i in {0..${olts.toInteger() - 1}}; do
332 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
333 ((bbsimDmiPortFwd++))
334 done
335 if [ ${withMonitoring} = true ] ; then
336 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
337 fi
338 ps aux | grep port-forward
339 """
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400340 // ---------------------------------
341 // Sanity check port-forward spawned
342 // ---------------------------------
343 script {
344 enter('port-forward check')
Joey Armstrongebc18022023-08-26 13:20:49 -0400345 // String proc = 'kubectl.*port-forward' // was 'port-forward'
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400346 String proc = 'port-forward'
347 println("Display spawned ${proc}")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400348 sh(label : 'pgrep_proc - check',
349 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400350pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400351""")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400352 leave('port-forward check')
353 }
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400354
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400355 // setting ONOS log level
356 script {
357 enter('setOnosLogLevels')
358 setOnosLogLevels([
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400359 onosNamespace: infraNamespace,
360 apps: [
361 'org.opencord.dhcpl2relay',
362 'org.opencord.olt',
363 'org.opencord.aaa',
364 'org.opencord.maclearner',
365 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
366 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
367 ],
368 logLevel: logLevel
369 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400370 leave('setOnosLogLevels')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400371 } // script
372 } // if (teardown)
373 } // stage('Deploy Voltha')
374
Joey Armstrongf060aee2023-08-22 21:55:26 -0400375 // -----------------------------------------------------------------------
376 // -----------------------------------------------------------------------
Joey Armstrong642540a2023-08-10 10:26:36 -0400377 stage("Run test ${testTarget} on workflow ${workflow}")
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400378 {
Joey Armstrongebc18022023-08-26 13:20:49 -0400379 sh(
380 label : 'Monitor using mem_consumption.py',
381 script : """
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400382 echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400383
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400384 if [ ${withMonitoring} = true ] ; then
Joey Armstrongf060aee2023-08-22 21:55:26 -0400385 cat <<EOM
386
387** -----------------------------------------------------------------------
388** Monitoring memory usage with mem_consumption.py
389** -----------------------------------------------------------------------
390EOM
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400391 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
392 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400393
394 echo '** Installing python virtualenv'
395 make venv-activate-patched
396
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400397 set +u && source .venv/bin/activate && set -u
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400398 # Collect initial memory consumption
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400399 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400400 fi
Joey Armstrongf060aee2023-08-22 21:55:26 -0400401
402 echo -e '** Monitor memory consumption: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400403 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400404
405 sh """
Joey Armstrong54dec092023-08-03 18:21:38 -0400406 echo -e "\n** make testTarget=[${testTarget}]"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400407 mkdir -p ${logsDir}
408 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
409 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
410 export KVSTOREPREFIX=voltha/voltha_voltha
411
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400412 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400413 """
414
415 getPodsInfo("${logsDir}")
416
Joey Armstrongebc18022023-08-26 13:20:49 -0400417 sh(
418 label : 'Gather robot Framework logs',
419 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400420 echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400421 # set +e
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400422 # collect logs collected in the Robot Framework StartLogging keyword
423 cd ${logsDir}
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400424 gzip *-combined.log
425 rm -f *-combined.log
Joey Armstrong54dec092023-08-03 18:21:38 -0400426
427 echo -e '** Gather robot Framework logs: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400428 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400429
Joey Armstrongf060aee2023-08-22 21:55:26 -0400430 // -----------------------------------------------------------------------
431 // -----------------------------------------------------------------------
Joey Armstrongebc18022023-08-26 13:20:49 -0400432 sh(
433 label : 'Monitor pod-mem-consumption',
434 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400435 echo -e '** Monitor pod-mem-consumption: ENTER'
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400436 if [ ${withMonitoring} = true ] ; then
Joey Armstrongf060aee2023-08-22 21:55:26 -0400437 cat <<EOM
438
439** -----------------------------------------------------------------------
440** Monitoring pod-memory-consumption using mem_consumption.py
441** -----------------------------------------------------------------------
442EOM
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400443 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400444
445 echo '** Installing python virtualenv'
446 make venv-activate-patched
Joey Armstrong38a87832023-08-23 17:02:50 -0400447
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400448 set +u && source .venv/bin/activate && set -u
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400449 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400450 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400451 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400452 echo -e '** Monitor pod-mem-consumption: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400453 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400454 } // stage
Joey Armstrong54dec092023-08-03 18:21:38 -0400455
456 return
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400457} // execute_test()
458
459// -----------------------------------------------------------------------
460// -----------------------------------------------------------------------
Joey Armstrong54dec092023-08-03 18:21:38 -0400461def collectArtifacts(exitStatus) {
Joey Armstrong6115fd62023-08-24 08:19:28 -0400462 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400463 String iam = getIam('collectArtifacts')
464 println("${iam}: ENTER (exitStatus=${exitStatus})")
Joey Armstrong6115fd62023-08-24 08:19:28 -0400465 }
Joey Armstrong642540a2023-08-10 10:26:36 -0400466
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400467 echo '''
468
469** -----------------------------------------------------------------------
470** collectArtifacts
471** -----------------------------------------------------------------------
472'''
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400473
Joey Armstrong642540a2023-08-10 10:26:36 -0400474 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400475
Joey Armstrong642540a2023-08-10 10:26:36 -0400476 sh """
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400477 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400478 """
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400479
Joey Armstrong642540a2023-08-10 10:26:36 -0400480 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400481
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400482 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400483 println("${iam}: ENTER")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400484 /*
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400485 pgrep_proc('kail-startup')
486 pkill_proc('kail')
Joey Armstrong664c55a2023-08-28 14:22:33 -0400487 */
488 sh(label : 'pgrep_proc - kill-pre',
489 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400490pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400491""")
492 sh(label : 'pkill_proc - kail',
493 script : """
Joey Armstrong12a8c832023-08-28 16:50:24 -0400494if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then
495 pkill --uid "\$(id -u)" --echo --full 'kail'
496fi
Joey Armstrong664c55a2023-08-28 14:22:33 -0400497""")
498
499 sh(label : 'pgrep_proc - kill-post',
500 script : """
Joey Armstronge9725b12023-08-28 18:15:12 -0400501pgrep --uid "\$(id -u)" --list-full --full 'kail || true
Joey Armstrong664c55a2023-08-28 14:22:33 -0400502""")
503
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400504 println("${iam}: LEAVE")
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400505 }
Joey Armstrongdd0cd6b2023-08-25 17:27:56 -0400506
Joey Armstrong642540a2023-08-10 10:26:36 -0400507 println("${iam}: ENTER RobotPublisher")
508 step([$class: 'RobotPublisher',
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400509 disableArchiveOutput: false,
510 logFileName: '**/*/log*.html',
511 otherFiles: '',
512 outputFileName: '**/*/output*.xml',
513 outputPath: '.',
514 passThreshold: 100,
515 reportFileName: '**/*/report*.html',
516 unstableThreshold: 0,
517 onlyCritical: true])
Joey Armstrong642540a2023-08-10 10:26:36 -0400518 println("${iam}: LEAVE RobotPublisher")
519
520 println("${iam}: LEAVE (exitStatus=${exitStatus})")
521 return
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400522}
523
Joey Armstrong54dec092023-08-03 18:21:38 -0400524// -----------------------------------------------------------------------
525// Intent: main
526// -----------------------------------------------------------------------
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400527pipeline {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400528 /* no label, executor is determined by JJB */
529 agent {
530 label "${params.buildNode}"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400531 }
532
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400533 options {
534 timeout(time: "${timeout}", unit: 'MINUTES')
535 }
536
537 environment {
538 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
539 VOLTCONFIG = "$HOME/.volt/config"
540 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
541 DIAGS_PROFILE = 'VOLTHA_PROFILE'
542 SSHPASS = 'karaf'
543 }
544
545 stages {
546 stage('Download Code') {
547 steps {
548 getVolthaCode([
549 branch: "${branch}",
550 gerritProject: "${gerritProject}",
551 gerritRefspec: "${gerritRefspec}",
552 volthaSystemTestsChange: "${volthaSystemTestsChange}",
553 volthaHelmChartsChange: "${volthaHelmChartsChange}",
554 ])
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400555 }
556 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400557
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400558 stage('Build patch v1.1') {
559 // build the patch only if gerritProject is specified
560 when {
561 expression { return !gerritProject.isEmpty() }
562 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400563
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400564 steps {
565 // NOTE that the correct patch has already been checked out
566 // during the getVolthaCode step
567 buildVolthaComponent("${gerritProject}")
568 }
569 }
570
571 // -----------------------------------------------------------------------
572 // -----------------------------------------------------------------------
573 stage('Install Kail')
Joey Armstrong2b2010d2023-08-02 21:47:20 -0400574 {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400575 steps
Joey Armstrong2b2010d2023-08-02 21:47:20 -0400576 {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400577 script
578 {
579 String cmd = [
580 'make',
581 '--no-print-directory',
582 '-C', "$WORKSPACE/voltha-system-tests",
583 "KAIL_PATH=\"$WORKSPACE/bin\"",
584 'kail',
585 ].join(' ')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400586
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400587 println(" ** Running: ${cmd}")
588 sh("${cmd}")
589 } // script
590 } // steps
591 } // stage
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400592
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400593 // -----------------------------------------------------------------------
594 // -----------------------------------------------------------------------
595 stage('Install Tools') {
596 steps {
597 script {
598 String branchName = branchName()
599 String iam = getIam('Install Tools')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400600
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400601 println("${iam}: ENTER (branch=$branch)")
602 installKind(branch) // needed early by stage(Cleanup)
603 println("${iam}: LEAVE (branch=$branch)")
604 } // script
605 } // steps
606 } // stage
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400607
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400608 // -----------------------------------------------------------------------
609 // -----------------------------------------------------------------------
610 stage('Create K8s Cluster') {
611 steps {
612 script {
613 def clusterExists = sh(
614 returnStdout: true,
615 script: """kind get clusters | grep "${clusterName}" | wc -l""")
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400616
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400617 if (clusterExists.trim() == '0') {
618 createKubernetesCluster([nodes: 3, name: clusterName])
619 }
620 } // script
621 } // steps
622 } // stage('Create K8s Cluster')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400623
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400624 // -----------------------------------------------------------------------
625 // -----------------------------------------------------------------------
626 stage('Replace voltctl') {
627 // if the project is voltctl, override the downloaded one with the built one
628 when {
629 expression { return gerritProject == 'voltctl' }
630 }
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400631
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400632 // Hmmmm(?) where did the voltctl download happen ?
633 // Likely Makefile but would be helpful to document here.
634 steps {
635 script {
636 String iam = getIam('Replace voltctl')
Joey Armstrong6115fd62023-08-24 08:19:28 -0400637
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400638 println("${iam} Running: installVoltctl($branch)")
639 println("${iam}: ENTER")
640 installVoltctl("$branch")
641 println("${iam}: LEAVE")
642 } // script
643 } // step
644 } // stage
Joey Armstrong268442d2023-08-22 17:16:10 -0400645
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400646 // -----------------------------------------------------------------------
647 // -----------------------------------------------------------------------
648 stage('Load image in kind nodes')
649 {
650 when {
651 expression { return !gerritProject.isEmpty() }
652 }
653 steps {
654 loadToKind()
655 } // steps
656 } // stage
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400657
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400658 // -----------------------------------------------------------------------
659 // [TODO] verify testing output
660 // -----------------------------------------------------------------------
661 stage('Parse and execute tests')
662 {
663 steps {
664 script {
665 // Announce ourselves for log usability
666 enter('Parse and execute tests')
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400667
Joey Armstrongebc18022023-08-26 13:20:49 -0400668 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400669 println("** [DEBUG]: tests=$tests")
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400670
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400671 // Display expected tests for times when output goes dark
672 tests.eachWithIndex { test, idx ->
673 String target = test['target']
674 println("** test[${idx}]: ${target}\n")
675 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400676
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400677 println('''
Joey Armstrongb29d5612023-08-24 12:53:46 -0400678** -----------------------------------------------------------------------
679** NOTE: For odd/silent job failures verify a few details
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400680** - All tests mentioned in the tests-to-run index were logged.
Joey Armstrongb29d5612023-08-24 12:53:46 -0400681** - Test suites display ENTER/LEAVE mesasge pairs.
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400682** - Processing terminated prematurely when LEAVE strings are missing.
Joey Armstrongb29d5612023-08-24 12:53:46 -0400683** -----------------------------------------------------------------------
Joey Armstrongbe7c9242023-08-24 16:20:31 -0400684''')
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400685 tests.eachWithIndex { test, idx ->
686 println "** readYaml test suite[$idx]) test=[${test}]"
Joey Armstrongf46b3ae2023-08-25 11:07:20 -0400687
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400688 String target = test['target']
689 String workflow = test['workflow']
690 String flags = test['flags']
691 Boolean teardown = test['teardown'].toBoolean()
692 Boolean logging = test['logging'].toBoolean()
693 String testLogging = (logging) ? 'True' : 'False'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400694
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400695 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400696** -----------------------------------------------------------------------
Joey Armstrong38a87832023-08-23 17:02:50 -0400697** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400698** -----------------------------------------------------------------------
Joey Armstrong38a87832023-08-23 17:02:50 -0400699""")
Joey Armstrong664c55a2023-08-28 14:22:33 -0400700
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400701 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400702 enter("execute_test (target=$target)")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400703 execute_test(target, workflow, testLogging, teardown, flags)
704 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400705 // groovylint-disable-next-line CatchException
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400706 catch (Exception err) {
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400707 String iamexc = getIam(name)
708 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400709 }
710 finally {
711 leave("execute_test (target=$target)")
712 }
713 } // for
714 // Premature exit if this message is not logged
715 leave('Parse and execute tests')
Joey Armstrong54dec092023-08-03 18:21:38 -0400716 } // script
717 } // steps
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400718 } // stage
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400719 } // stages
720
721 post
722 {
723 aborted { collectArtifacts('aborted') }
724 failure { collectArtifacts('failed') }
725 always { collectArtifacts('always') }
726 }
727} // pipeline
728
Joey Armstrong664c55a2023-08-28 14:22:33 -0400729// [EOF]