blob: 3909c4e1f4f4259db338890ed3e8efe741b8c672 [file] [log] [blame]
Joey Armstrong7bcb5782023-06-07 12:25:57 -04001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// voltha-2.x e2e tests for openonu-go
16// uses bbsim to simulate OLT/ONUs
17
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040018// [TODO] Update syntax below to the latest supported
Joey Armstrong7bcb5782023-06-07 12:25:57 -040019library identifier: 'cord-jenkins-libraries@master',
20 retriever: modernSCM([
21 $class: 'GitSCMSource',
22 remote: 'https://gerrit.opencord.org/ci-management.git'
23])
24
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040025//------------------//
26//---] GLOBAL [---//
27//------------------//
Joey Armstrong54dec092023-08-03 18:21:38 -040028String clusterName = 'kind-ci' // was def
Joey Armstrong7bcb5782023-06-07 12:25:57 -040029
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040030// -----------------------------------------------------------------------
31// Intent:
32// -----------------------------------------------------------------------
Joey Armstrong54dec092023-08-03 18:21:38 -040033String branchName() {
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040034 String name = 'voltha-2.12'
35
36 // [TODO] Sanity check the target branch
37 // if (name != jenkins.branch) { fatal }
38 return(name)
39}
40
41// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040042// Intent: Difficult at times to determine when pipeline jobs have
43// regenerated. Hardcode a version string that can be assigned
44// per-script to be sure latest repository changes are being used.
45// -----------------------------------------------------------------------
46String pipelineVer() {
Joey Armstrong6146f7e2023-08-28 09:05:38 -040047 String version = '5addce3fac89095d103ac5c6eedff2bb02e9ec63'
Joey Armstrongaf4eef22023-08-25 16:14:45 -040048 return(version)
49}
50
51// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040052// Intent: Due to lack of a reliable stack trace, construct a literal.
Joey Armstrongd0b5af02023-08-25 15:12:54 -040053// Jenkins will re-write the call stack for serialization.S
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040054// -----------------------------------------------------------------------
Joey Armstrongd0b5af02023-08-25 15:12:54 -040055// Note: Hardcoded version string used to visualize changes in jenkins UI
56// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040057String getIam(String func) {
Joey Armstrong54dec092023-08-03 18:21:38 -040058 String branchName = branchName()
Joey Armstrongaf4eef22023-08-25 16:14:45 -040059 String version = pipelineVer()
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040060 String src = [
61 'ci-management',
62 'jjb',
63 'pipeline',
64 'voltha',
Joey Armstrong54dec092023-08-03 18:21:38 -040065 branchName,
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040066 'bbsim-tests.groovy'
67 ].join('/')
68
Joey Armstrongd0b5af02023-08-25 15:12:54 -040069 String name = [src, version, func].join('::')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040070 return(name)
71}
72
73// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -040074// Intent: Log progress message
75// -----------------------------------------------------------------------
Joey Armstrong7229d9b2023-08-25 18:03:13 -040076void enter(String name) {
Joey Armstrongaf4eef22023-08-25 16:14:45 -040077 // Announce ourselves for log usability
78 String iam = getIam(name)
79 println("${iam}: ENTER")
80 return
81}
82
83// -----------------------------------------------------------------------
84// Intent: Log progress message
85// -----------------------------------------------------------------------
Joey Armstrong7229d9b2023-08-25 18:03:13 -040086void leave(String name) {
Joey Armstrongaf4eef22023-08-25 16:14:45 -040087 // Announce ourselves for log usability
88 String iam = getIam(name)
89 println("${iam}: LEAVE")
90 return
91}
92
93// -----------------------------------------------------------------------
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040094// Intent: Determine if working on a release branch.
95// Note: Conditional is legacy, should also check for *-dev or *-pre
96// -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -040097Boolean isReleaseBranch(String name) {
Joey Armstrong0e0a42b2023-08-02 21:04:21 -040098 // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
Joey Armstrong54dec092023-08-03 18:21:38 -040099 // if branchName in modifiers
100 return(name != 'master') // OR branchName.contains('-')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400101}
102
103// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -0400104// Intent: Iterate over a list of test suites and invoke.
Joey Armstrongf060aee2023-08-22 21:55:26 -0400105// -----------------------------------------------------------------------
Joey Armstrongaf4eef22023-08-25 16:14:45 -0400106void execute_test\
107(
108 String testTarget, // functional-single-kind-dt
109 String workflow, // dt
110 String testLogging, // 'True'
111 Boolean teardown, // true
112 String testSpecificHelmFlags=''
113) {
Joey Armstrong54dec092023-08-03 18:21:38 -0400114 String infraNamespace = 'default'
115 String volthaNamespace = 'voltha'
116 String logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400117
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400118 // -----------------------------------------------------------------------
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400119 // Intent: Cleanup stale port-forwarding
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400120 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400121 stage('Cleanup') {
Joey Armstrong38a87832023-08-23 17:02:50 -0400122 if (teardown) {
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400123 timeout(15) {
Joey Armstrong38a87832023-08-23 17:02:50 -0400124 script {
Joey Armstrong54dec092023-08-03 18:21:38 -0400125 helmTeardown(['default', infraNamespace, volthaNamespace])
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400126 }
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400127 } // timeout
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400128
Joey Armstrongf060aee2023-08-22 21:55:26 -0400129 timeout(5) {
130 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400131 enter('Cleanup')
132 // remove orphaned port-forward from different namespaces
Joey Armstrongebc18022023-08-26 13:20:49 -0400133 String proc = 'kubectl .*port-forward' // was 'port-forw'
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400134 pgrep_proc(proc)
135 pkill_proc(proc)
Joey Armstrongebc18022023-08-26 13:20:49 -0400136
137 // todo: fatal unless (proc count==0)
138 pgrep_proc(proc)
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400139 leave('Cleanup')
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400140 } // script
141 } // timeout
Joey Armstrongf060aee2023-08-22 21:55:26 -0400142 } // teardown
Joey Armstrong38a87832023-08-23 17:02:50 -0400143 } // stage('Cleanup')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400144
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400145 // -----------------------------------------------------------------------
146 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400147 stage('Deploy common infrastructure') {
148 script {
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400149 local dashargs = [
150 'kpi_exporter.enabled=false',
151 'dashboards.xos=false',
152 'dashboards.onos=false',
153 'dashboards.aaa=false',
154 'dashboards.voltha=false',
155 ].join(',')
156
157 local promargs = [
158 'prometheus.alertmanager.enabled=false',
159 'prometheus.pushgateway.enabled=false',
160 ].join(',')
161
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400162 sh(label : 'Deploy common infrastructure',
163 script : """
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400164 helm repo add onf https://charts.opencord.org
165 helm repo update
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400166
167 echo -e "\nwithMonitoring=[$withMonitoring]"
168 if [ ${withMonitoring} = true ] ; then
169 helm install nem-monitoring onf/nem-monitoring \
Joey Armstrong38a87832023-08-23 17:02:50 -0400170 --set ${promargs} \
171 --set ${dashargs}
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400172 fi
Joey Armstrong38a87832023-08-23 17:02:50 -0400173 """)
174 } // script
175 } // stage('Deploy Common Infra')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400176
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400177 // -----------------------------------------------------------------------
Joey Armstrongf060aee2023-08-22 21:55:26 -0400178 // [TODO] Check onos_log output
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400179 // -----------------------------------------------------------------------
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400180 stage('Deploy Voltha') {
181 if (teardown) {
182 timeout(10) {
183 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400184 String iam = getIam('Deploy Voltha')
185 String onosLog = "${logsDir}/onos-voltha-startup-combined.log"
186 sh("""
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400187 mkdir -p ${logsDir}
Joey Armstrong38a87832023-08-23 17:02:50 -0400188 touch "$onosLog"
189 echo "** kail-startup ENTER: \$(date)" > "$onosLog"
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400190
191 # Intermixed output (tee -a &) may get conflusing but let(s) see
192 # what messages are logged during startup.
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400193 # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" &
Joey Armstrong38a87832023-08-23 17:02:50 -0400194 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" &
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400195 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400196
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400197 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
198 Boolean localCharts = false
Joey Armstrong642540a2023-08-10 10:26:36 -0400199
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400200 if (volthaHelmChartsChange != ''
201 || gerritProject == 'voltha-helm-charts'
202 || isReleaseBranch(branch) // branch != 'master'
203 ) {
204 localCharts = true
205 }
Joey Armstrong642540a2023-08-10 10:26:36 -0400206
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400207 String branchName = branchName()
208 Boolean isRelease = isReleaseBranch(branch)
209 println([
210 " ** localCharts=${localCharts}",
211 "branchName=${branchName}",
212 "branch=${branch}",
213 "branch=isReleaseBranch=${isRelease}",
214 ].join(', '))
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400215
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400216 // -----------------------------------------------------------------------
217 // Rewrite localHelmFlags using array join, moving code around and
218 // refactoring into standalone functions
219 // -----------------------------------------------------------------------
220 // NOTE temporary workaround expose ONOS node ports
221 // -----------------------------------------------------------------------
222 String localHelmFlags = [
223 extraHelmFlags.trim(),
224 "--set global.log_level=${logLevel.toUpperCase()}",
225 '--set onos-classic.onosSshPort=30115',
226 '--set onos-classic.onosApiPort=30120',
227 '--set onos-classic.onosOfPort=31653',
228 '--set onos-classic.individualOpenFlowNodePorts=true',
229 testSpecificHelmFlags
230 ].join(' ')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400231
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400232 println("** ${iam} localHelmFlags = ${localHelmFlags}")
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400233
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400234 if (gerritProject != '') {
235 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
236 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400237
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400238 println("** ${iam}: ENTER")
239 volthaDeploy([
240 infraNamespace: infraNamespace,
241 volthaNamespace: volthaNamespace,
242 workflow: workflow.toLowerCase(),
243 withMacLearning: enableMacLearning.toBoolean(),
244 extraHelmFlags: localHelmFlags,
245 localCharts: localCharts,
246 bbsimReplica: olts.toInteger(),
247 dockerRegistry: registry,
248 ])
249 println("** ${iam}: LEAVE")
250 } // script
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400251
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400252 // -----------------------------------------------------------------------
253 // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
254 // Why not simply use a pid file, capture _TAG=kail-startup above
255 // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
256 // -----------------------------------------------------------------------
257 script {
258 String proc = '_TAG=kail-startup'
Joey Armstrongdddbbf92023-08-22 16:00:41 -0400259
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400260 println("${iam}: ENTER")
261 println("${iam}: Shutdown process $proc")
262 pgrep_proc(proc)
263 pkill_proc(proc)
264 pgrep_proc(proc)
265 println("${iam}: LEAVE")
266 }
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400267
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400268 // -----------------------------------------------------------------------
269 // Bundle onos-voltha / kail logs
270 // -----------------------------------------------------------------------
271 sh("""
Joey Armstrongf060aee2023-08-22 21:55:26 -0400272cat <<EOM
273
274** -----------------------------------------------------------------------
275** Combine an compress voltha startup log(s)
276** -----------------------------------------------------------------------
277EOM
278 pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400279 gzip -k onos-voltha-startup-combined.log
280 rm onos-voltha-startup-combined.log
Joey Armstrongf060aee2023-08-22 21:55:26 -0400281 popd
282 """)
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400283 } // timeout(10)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400284
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400285 // -----------------------------------------------------------------------
286 // -----------------------------------------------------------------------
287 sh """
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400288 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
289 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
290 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
291 bbsimDmiPortFwd=50075
292 for i in {0..${olts.toInteger() - 1}}; do
293 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
294 ((bbsimDmiPortFwd++))
295 done
296 if [ ${withMonitoring} = true ] ; then
297 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
298 fi
299 ps aux | grep port-forward
300 """
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400301 // ---------------------------------
302 // Sanity check port-forward spawned
303 // ---------------------------------
304 script {
305 enter('port-forward check')
Joey Armstrongebc18022023-08-26 13:20:49 -0400306 // String proc = 'kubectl.*port-forward' // was 'port-forward'
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400307 String proc = 'port-forward'
308 println("Display spawned ${proc}")
309 pgrep_proc(proc)
310 leave('port-forward check')
311 }
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400312
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400313 // setting ONOS log level
314 script {
315 enter('setOnosLogLevels')
316 setOnosLogLevels([
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400317 onosNamespace: infraNamespace,
318 apps: [
319 'org.opencord.dhcpl2relay',
320 'org.opencord.olt',
321 'org.opencord.aaa',
322 'org.opencord.maclearner',
323 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
324 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
325 ],
326 logLevel: logLevel
327 ])
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400328 leave('setOnosLogLevels')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400329 } // script
330 } // if (teardown)
331 } // stage('Deploy Voltha')
332
Joey Armstrongf060aee2023-08-22 21:55:26 -0400333 // -----------------------------------------------------------------------
334 // -----------------------------------------------------------------------
Joey Armstrong642540a2023-08-10 10:26:36 -0400335 stage("Run test ${testTarget} on workflow ${workflow}")
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400336 {
Joey Armstrongebc18022023-08-26 13:20:49 -0400337 sh(
338 label : 'Monitor using mem_consumption.py',
339 script : """
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400340 echo -e "\n** Monitor using mem_consumption.py ?"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400341
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400342 if [ ${withMonitoring} = true ] ; then
Joey Armstrongf060aee2023-08-22 21:55:26 -0400343 cat <<EOM
344
345** -----------------------------------------------------------------------
346** Monitoring memory usage with mem_consumption.py
347** -----------------------------------------------------------------------
348EOM
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400349 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
350 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400351
352 echo '** Installing python virtualenv'
353 make venv-activate-patched
354
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400355 set +u && source .venv/bin/activate && set -u
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400356 # Collect initial memory consumption
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400357 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400358 fi
Joey Armstrongf060aee2023-08-22 21:55:26 -0400359
360 echo -e '** Monitor memory consumption: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400361 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400362
363 sh """
Joey Armstrong54dec092023-08-03 18:21:38 -0400364 echo -e "\n** make testTarget=[${testTarget}]"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400365 mkdir -p ${logsDir}
366 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
367 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
368 export KVSTOREPREFIX=voltha/voltha_voltha
369
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400370 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400371 """
372
373 getPodsInfo("${logsDir}")
374
Joey Armstrongebc18022023-08-26 13:20:49 -0400375 sh(
376 label : 'Gather robot Framework logs',
377 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400378 echo -e '\n** Gather robot Framework logs: ENTER'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400379 # set +e
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400380 # collect logs collected in the Robot Framework StartLogging keyword
381 cd ${logsDir}
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400382 gzip *-combined.log
383 rm -f *-combined.log
Joey Armstrong54dec092023-08-03 18:21:38 -0400384
385 echo -e '** Gather robot Framework logs: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400386 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400387
Joey Armstrongf060aee2023-08-22 21:55:26 -0400388 // -----------------------------------------------------------------------
389 // -----------------------------------------------------------------------
Joey Armstrongebc18022023-08-26 13:20:49 -0400390 sh(
391 label : 'Monitor pod-mem-consumption',
392 script : """
Joey Armstrong54dec092023-08-03 18:21:38 -0400393 echo -e '** Monitor pod-mem-consumption: ENTER'
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400394 if [ ${withMonitoring} = true ] ; then
Joey Armstrongf060aee2023-08-22 21:55:26 -0400395 cat <<EOM
396
397** -----------------------------------------------------------------------
398** Monitoring pod-memory-consumption using mem_consumption.py
399** -----------------------------------------------------------------------
400EOM
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400401 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrongf060aee2023-08-22 21:55:26 -0400402
403 echo '** Installing python virtualenv'
404 make venv-activate-patched
Joey Armstrong38a87832023-08-23 17:02:50 -0400405
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400406 set +u && source .venv/bin/activate && set -u
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400407 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400408 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400409 fi
Joey Armstrong54dec092023-08-03 18:21:38 -0400410 echo -e '** Monitor pod-mem-consumption: LEAVE\n'
Joey Armstrongebc18022023-08-26 13:20:49 -0400411 """)
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400412 } // stage
Joey Armstrong54dec092023-08-03 18:21:38 -0400413
414 return
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400415} // execute_test()
416
417// -----------------------------------------------------------------------
418// -----------------------------------------------------------------------
Joey Armstrong54dec092023-08-03 18:21:38 -0400419def collectArtifacts(exitStatus) {
Joey Armstrong6115fd62023-08-24 08:19:28 -0400420 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400421 String iam = getIam('collectArtifacts')
422 println("${iam}: ENTER (exitStatus=${exitStatus})")
Joey Armstrong6115fd62023-08-24 08:19:28 -0400423 }
Joey Armstrong642540a2023-08-10 10:26:36 -0400424
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400425 echo '''
426
427** -----------------------------------------------------------------------
428** collectArtifacts
429** -----------------------------------------------------------------------
430'''
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400431
Joey Armstrong642540a2023-08-10 10:26:36 -0400432 getPodsInfo("$WORKSPACE/${exitStatus}")
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400433
Joey Armstrong642540a2023-08-10 10:26:36 -0400434 sh """
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400435 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400436 """
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400437
Joey Armstrong642540a2023-08-10 10:26:36 -0400438 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Joey Armstrong66fcb4e2023-08-11 12:09:24 -0400439
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400440 script {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400441 println("${iam}: ENTER")
442 pgrep_proc('kail-startup')
443 pkill_proc('kail')
444 println("${iam}: LEAVE")
Joey Armstrongec1ae0a2023-08-23 21:51:45 -0400445 }
Joey Armstrongdd0cd6b2023-08-25 17:27:56 -0400446
Joey Armstrong642540a2023-08-10 10:26:36 -0400447 println("${iam}: ENTER RobotPublisher")
448 step([$class: 'RobotPublisher',
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400449 disableArchiveOutput: false,
450 logFileName: '**/*/log*.html',
451 otherFiles: '',
452 outputFileName: '**/*/output*.xml',
453 outputPath: '.',
454 passThreshold: 100,
455 reportFileName: '**/*/report*.html',
456 unstableThreshold: 0,
457 onlyCritical: true])
Joey Armstrong642540a2023-08-10 10:26:36 -0400458 println("${iam}: LEAVE RobotPublisher")
459
460 println("${iam}: LEAVE (exitStatus=${exitStatus})")
461 return
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400462}
463
Joey Armstrong54dec092023-08-03 18:21:38 -0400464// -----------------------------------------------------------------------
465// Intent: main
466// -----------------------------------------------------------------------
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400467pipeline {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400468 /* no label, executor is determined by JJB */
469 agent {
470 label "${params.buildNode}"
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400471 }
472
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400473 options {
474 timeout(time: "${timeout}", unit: 'MINUTES')
475 }
476
477 environment {
478 KUBECONFIG = "$HOME/.kube/kind-${clusterName}"
479 VOLTCONFIG = "$HOME/.volt/config"
480 PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
481 DIAGS_PROFILE = 'VOLTHA_PROFILE'
482 SSHPASS = 'karaf'
483 }
484
485 stages {
486 stage('Download Code') {
487 steps {
488 getVolthaCode([
489 branch: "${branch}",
490 gerritProject: "${gerritProject}",
491 gerritRefspec: "${gerritRefspec}",
492 volthaSystemTestsChange: "${volthaSystemTestsChange}",
493 volthaHelmChartsChange: "${volthaHelmChartsChange}",
494 ])
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400495 }
496 }
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400497
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400498 stage('Build patch v1.1') {
499 // build the patch only if gerritProject is specified
500 when {
501 expression { return !gerritProject.isEmpty() }
502 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400503
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400504 steps {
505 // NOTE that the correct patch has already been checked out
506 // during the getVolthaCode step
507 buildVolthaComponent("${gerritProject}")
508 }
509 }
510
511 // -----------------------------------------------------------------------
512 // -----------------------------------------------------------------------
513 stage('Install Kail')
Joey Armstrong2b2010d2023-08-02 21:47:20 -0400514 {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400515 steps
Joey Armstrong2b2010d2023-08-02 21:47:20 -0400516 {
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400517 script
518 {
519 String cmd = [
520 'make',
521 '--no-print-directory',
522 '-C', "$WORKSPACE/voltha-system-tests",
523 "KAIL_PATH=\"$WORKSPACE/bin\"",
524 'kail',
525 ].join(' ')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400526
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400527 println(" ** Running: ${cmd}")
528 sh("${cmd}")
529 } // script
530 } // steps
531 } // stage
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400532
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400533 // -----------------------------------------------------------------------
534 // -----------------------------------------------------------------------
535 stage('Install Tools') {
536 steps {
537 script {
538 String branchName = branchName()
539 String iam = getIam('Install Tools')
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400540
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400541 println("${iam}: ENTER (branch=$branch)")
542 installKind(branch) // needed early by stage(Cleanup)
543 println("${iam}: LEAVE (branch=$branch)")
544 } // script
545 } // steps
546 } // stage
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400547
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400548 // -----------------------------------------------------------------------
549 // -----------------------------------------------------------------------
550 stage('Create K8s Cluster') {
551 steps {
552 script {
553 def clusterExists = sh(
554 returnStdout: true,
555 script: """kind get clusters | grep "${clusterName}" | wc -l""")
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400556
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400557 if (clusterExists.trim() == '0') {
558 createKubernetesCluster([nodes: 3, name: clusterName])
559 }
560 } // script
561 } // steps
562 } // stage('Create K8s Cluster')
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400563
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400564 // -----------------------------------------------------------------------
565 // -----------------------------------------------------------------------
566 stage('Replace voltctl') {
567 // if the project is voltctl, override the downloaded one with the built one
568 when {
569 expression { return gerritProject == 'voltctl' }
570 }
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400571
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400572 // Hmmmm(?) where did the voltctl download happen ?
573 // Likely Makefile but would be helpful to document here.
574 steps {
575 script {
576 String iam = getIam('Replace voltctl')
Joey Armstrong6115fd62023-08-24 08:19:28 -0400577
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400578 println("${iam} Running: installVoltctl($branch)")
579 println("${iam}: ENTER")
580 installVoltctl("$branch")
581 println("${iam}: LEAVE")
582 } // script
583 } // step
584 } // stage
Joey Armstrong268442d2023-08-22 17:16:10 -0400585
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400586 // -----------------------------------------------------------------------
587 // -----------------------------------------------------------------------
588 stage('Load image in kind nodes')
589 {
590 when {
591 expression { return !gerritProject.isEmpty() }
592 }
593 steps {
594 loadToKind()
595 } // steps
596 } // stage
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400597
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400598 // -----------------------------------------------------------------------
599 // [TODO] verify testing output
600 // -----------------------------------------------------------------------
601 stage('Parse and execute tests')
602 {
603 steps {
604 script {
605 // Announce ourselves for log usability
606 enter('Parse and execute tests')
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400607
Joey Armstrongebc18022023-08-26 13:20:49 -0400608 def tests = readYaml text: testTargets // typeof == Map (?)
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400609 println("** [DEBUG]: tests=$tests")
Joey Armstrong008cfaf2023-08-18 14:49:06 -0400610
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400611 // Display expected tests for times when output goes dark
612 tests.eachWithIndex { test, idx ->
613 String target = test['target']
614 println("** test[${idx}]: ${target}\n")
615 }
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400616
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400617 println('''
Joey Armstrongb29d5612023-08-24 12:53:46 -0400618** -----------------------------------------------------------------------
619** NOTE: For odd/silent job failures verify a few details
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400620** - All tests mentioned in the tests-to-run index were logged.
Joey Armstrongb29d5612023-08-24 12:53:46 -0400621** - Test suites display ENTER/LEAVE mesasge pairs.
Joey Armstrong5d65efe2023-08-25 09:43:18 -0400622** - Processing terminated prematurely when LEAVE strings are missing.
Joey Armstrongb29d5612023-08-24 12:53:46 -0400623** -----------------------------------------------------------------------
Joey Armstrongbe7c9242023-08-24 16:20:31 -0400624''')
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400625 tests.eachWithIndex { test, idx ->
626 println "** readYaml test suite[$idx]) test=[${test}]"
Joey Armstrongf46b3ae2023-08-25 11:07:20 -0400627
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400628 String target = test['target']
629 String workflow = test['workflow']
630 String flags = test['flags']
631 Boolean teardown = test['teardown'].toBoolean()
632 Boolean logging = test['logging'].toBoolean()
633 String testLogging = (logging) ? 'True' : 'False'
Joey Armstrong0e0a42b2023-08-02 21:04:21 -0400634
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400635 print("""
Joey Armstrong268442d2023-08-22 17:16:10 -0400636** -----------------------------------------------------------------------
Joey Armstrong38a87832023-08-23 17:02:50 -0400637** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}
Joey Armstrong268442d2023-08-22 17:16:10 -0400638** -----------------------------------------------------------------------
Joey Armstrong38a87832023-08-23 17:02:50 -0400639""")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400640 try {
Joey Armstrong6146f7e2023-08-28 09:05:38 -0400641 enter("execute_test (target=$target)")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400642 execute_test(target, workflow, testLogging, teardown, flags)
643 }
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400644 // groovylint-disable-next-line CatchException
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400645 catch (Exception err) {
Joey Armstrong9341c9a2023-08-28 12:09:19 -0400646 String iamexc = getIam(name)
647 println("** ${iamexc}: EXCEPTION ${err}")
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400648 }
649 finally {
650 leave("execute_test (target=$target)")
651 }
652 } // for
653 // Premature exit if this message is not logged
654 leave('Parse and execute tests')
Joey Armstrong54dec092023-08-03 18:21:38 -0400655 } // script
656 } // steps
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400657 } // stage
Joey Armstrong7bcb5782023-06-07 12:25:57 -0400658 } // stages
659
660 post
661 {
662 aborted { collectArtifacts('aborted') }
663 failure { collectArtifacts('failed') }
664 always { collectArtifacts('always') }
665 }
666} // pipeline
667
Joey Armstrong7229d9b2023-08-25 18:03:13 -0400668// [EOF] - 2