Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 1 | // Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | // voltha-2.x e2e tests for openonu-go |
| 16 | // uses bbsim to simulate OLT/ONUs |
| 17 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 18 | // [TODO] Update syntax below to the latest supported |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 19 | library identifier: 'cord-jenkins-libraries@master', |
| 20 | retriever: modernSCM([ |
| 21 | $class: 'GitSCMSource', |
| 22 | remote: 'https://gerrit.opencord.org/ci-management.git' |
| 23 | ]) |
| 24 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 25 | //------------------// |
| 26 | //---] GLOBAL [---// |
| 27 | //------------------// |
Joey Armstrong | e9725b1 | 2023-08-28 18:15:12 -0400 | [diff] [blame] | 28 | String clusterName = 'kind-ci' |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 29 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 30 | // ----------------------------------------------------------------------- |
| 31 | // Intent: |
| 32 | // ----------------------------------------------------------------------- |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 33 | String branchName() { |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 34 | String name = 'voltha-2.12' |
| 35 | |
| 36 | // [TODO] Sanity check the target branch |
| 37 | // if (name != jenkins.branch) { fatal } |
| 38 | return(name) |
| 39 | } |
| 40 | |
| 41 | // ----------------------------------------------------------------------- |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 42 | // Intent: Difficult at times to determine when pipeline jobs have |
| 43 | // regenerated. Hardcode a version string that can be assigned |
| 44 | // per-script to be sure latest repository changes are being used. |
| 45 | // ----------------------------------------------------------------------- |
Joey Armstrong | 2563719 | 2023-08-30 09:53:05 -0400 | [diff] [blame] | 46 | String pipelineVer() { |
| 47 | String version = '45c11f80698d3be1416a86d2872d6e25aa24baa8' |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 48 | return(version) |
| 49 | } |
| 50 | |
| 51 | // ----------------------------------------------------------------------- |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 52 | // Intent: Due to lack of a reliable stack trace, construct a literal. |
Joey Armstrong | d0b5af0 | 2023-08-25 15:12:54 -0400 | [diff] [blame] | 53 | // Jenkins will re-write the call stack for serialization.S |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 54 | // ----------------------------------------------------------------------- |
Joey Armstrong | d0b5af0 | 2023-08-25 15:12:54 -0400 | [diff] [blame] | 55 | // Note: Hardcoded version string used to visualize changes in jenkins UI |
| 56 | // ----------------------------------------------------------------------- |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 57 | String getIam(String func) { |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 58 | String branchName = branchName() |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 59 | String version = pipelineVer() |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 60 | String src = [ |
| 61 | 'ci-management', |
| 62 | 'jjb', |
| 63 | 'pipeline', |
| 64 | 'voltha', |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 65 | branchName, |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 66 | 'bbsim-tests.groovy' |
| 67 | ].join('/') |
| 68 | |
Joey Armstrong | d0b5af0 | 2023-08-25 15:12:54 -0400 | [diff] [blame] | 69 | String name = [src, version, func].join('::') |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 70 | return(name) |
| 71 | } |
| 72 | |
| 73 | // ----------------------------------------------------------------------- |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 74 | // Intent: Log progress message |
| 75 | // ----------------------------------------------------------------------- |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 76 | void enter(String name) { |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 77 | // Announce ourselves for log usability |
| 78 | String iam = getIam(name) |
| 79 | println("${iam}: ENTER") |
| 80 | return |
| 81 | } |
| 82 | |
| 83 | // ----------------------------------------------------------------------- |
| 84 | // Intent: Log progress message |
| 85 | // ----------------------------------------------------------------------- |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 86 | void leave(String name) { |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 87 | // Announce ourselves for log usability |
| 88 | String iam = getIam(name) |
| 89 | println("${iam}: LEAVE") |
| 90 | return |
| 91 | } |
| 92 | |
| 93 | // ----------------------------------------------------------------------- |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 94 | // Intent: Determine if working on a release branch. |
| 95 | // Note: Conditional is legacy, should also check for *-dev or *-pre |
| 96 | // ----------------------------------------------------------------------- |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 97 | Boolean isReleaseBranch(String name) { |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 98 | // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre'] |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 99 | // if branchName in modifiers |
| 100 | return(name != 'master') // OR branchName.contains('-') |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 101 | } |
| 102 | |
| 103 | // ----------------------------------------------------------------------- |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 104 | // Intent: Iterate over a list of test suites and invoke. |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 105 | // ----------------------------------------------------------------------- |
Joey Armstrong | af4eef2 | 2023-08-25 16:14:45 -0400 | [diff] [blame] | 106 | void execute_test\ |
| 107 | ( |
| 108 | String testTarget, // functional-single-kind-dt |
| 109 | String workflow, // dt |
| 110 | String testLogging, // 'True' |
| 111 | Boolean teardown, // true |
| 112 | String testSpecificHelmFlags='' |
| 113 | ) { |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 114 | String infraNamespace = 'default' |
| 115 | String volthaNamespace = 'voltha' |
| 116 | String logsDir = "$WORKSPACE/${testTarget}" |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 117 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 118 | // ----------------------------------------------------------------------- |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 119 | // Intent: Cleanup stale port-forwarding |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 120 | // ----------------------------------------------------------------------- |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 121 | stage('Cleanup') { |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 122 | if (teardown) { |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 123 | timeout(15) { |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 124 | script { |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 125 | helmTeardown(['default', infraNamespace, volthaNamespace]) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 126 | } |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 127 | } // timeout |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 128 | |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 129 | timeout(5) { |
| 130 | script { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 131 | enter('Cleanup') |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 132 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 133 | // remove orphaned port-forward from different namespaces |
Joey Armstrong | 2563719 | 2023-08-30 09:53:05 -0400 | [diff] [blame] | 134 | // String proc = 'kubectl.*port-forward' // was 'port-forw' |
| 135 | |
| 136 | /* Pass Gstring args to sh() having problems |
| 137 | implicit cast from GString to java.lang.String unsupported |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 138 | pgrep_proc(proc) |
| 139 | pkill_proc(proc) |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 140 | pgrep_proc(proc) // todo: fatal unless (proc count==0) |
| 141 | */ |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 142 | |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 143 | sh(label : 'pgrep_proc - kill-pre', |
| 144 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 145 | pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 146 | """) |
| 147 | |
| 148 | sh(label : 'pkill_proc - kubectl.*port-forward', |
| 149 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 150 | if [[ \$(pgrep --count 'kubectl.*port-forward') -gt 0 ]]; then |
| 151 | pkill --uid "\$(id -u)" --echo --full 'kubectl.*port-forward' |
| 152 | fi |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 153 | """) |
| 154 | |
| 155 | sh(label : 'pgrep_proc - kill-post', |
| 156 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 157 | pgrep --uid "\$(id -u)" --list-full --full 'kubectl.*port-forward' || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 158 | """) |
| 159 | |
Joey Armstrong | 6146f7e | 2023-08-28 09:05:38 -0400 | [diff] [blame] | 160 | leave('Cleanup') |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 161 | } // script |
| 162 | } // timeout |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 163 | } // teardown |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 164 | } // stage('Cleanup') |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 165 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 166 | // ----------------------------------------------------------------------- |
| 167 | // ----------------------------------------------------------------------- |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 168 | stage('Deploy common infrastructure') { |
| 169 | script { |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 170 | local dashargs = [ |
| 171 | 'kpi_exporter.enabled=false', |
| 172 | 'dashboards.xos=false', |
| 173 | 'dashboards.onos=false', |
| 174 | 'dashboards.aaa=false', |
| 175 | 'dashboards.voltha=false', |
| 176 | ].join(',') |
| 177 | |
| 178 | local promargs = [ |
| 179 | 'prometheus.alertmanager.enabled=false', |
| 180 | 'prometheus.pushgateway.enabled=false', |
| 181 | ].join(',') |
| 182 | |
Joey Armstrong | 6146f7e | 2023-08-28 09:05:38 -0400 | [diff] [blame] | 183 | sh(label : 'Deploy common infrastructure', |
| 184 | script : """ |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 185 | helm repo add onf https://charts.opencord.org |
| 186 | helm repo update |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 187 | |
| 188 | echo -e "\nwithMonitoring=[$withMonitoring]" |
| 189 | if [ ${withMonitoring} = true ] ; then |
| 190 | helm install nem-monitoring onf/nem-monitoring \ |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 191 | --set ${promargs} \ |
| 192 | --set ${dashargs} |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 193 | fi |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 194 | """) |
| 195 | } // script |
| 196 | } // stage('Deploy Common Infra') |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 197 | |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 198 | // ----------------------------------------------------------------------- |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 199 | // [TODO] Check onos_log output |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 200 | // ----------------------------------------------------------------------- |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 201 | stage('Deploy Voltha') { |
| 202 | if (teardown) { |
| 203 | timeout(10) { |
| 204 | script { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 205 | String iam = getIam('Deploy Voltha') |
| 206 | String onosLog = "${logsDir}/onos-voltha-startup-combined.log" |
| 207 | sh(""" |
Joey Armstrong | d1c4575 | 2023-08-29 16:56:05 -0400 | [diff] [blame] | 208 | mkdir -p "$logsDir" |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 209 | touch "$onosLog" |
| 210 | echo "** kail-startup ENTER: \$(date)" > "$onosLog" |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 211 | |
| 212 | # Intermixed output (tee -a &) may get conflusing but let(s) see |
| 213 | # what messages are logged during startup. |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 214 | # _TAG=ka7il-startup kail -n ${infraNamespace} -n ${volthaNamespace} >> "$onosLog" & |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 215 | _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} | tee -a "$onosLog" & |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 216 | """) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 217 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 218 | // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts |
| 219 | Boolean localCharts = false |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 220 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 221 | if (volthaHelmChartsChange != '' |
| 222 | || gerritProject == 'voltha-helm-charts' |
| 223 | || isReleaseBranch(branch) // branch != 'master' |
| 224 | ) { |
| 225 | localCharts = true |
| 226 | } |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 227 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 228 | String branchName = branchName() |
| 229 | Boolean isRelease = isReleaseBranch(branch) |
| 230 | println([ |
| 231 | " ** localCharts=${localCharts}", |
| 232 | "branchName=${branchName}", |
| 233 | "branch=${branch}", |
| 234 | "branch=isReleaseBranch=${isRelease}", |
| 235 | ].join(', ')) |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 236 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 237 | // ----------------------------------------------------------------------- |
| 238 | // Rewrite localHelmFlags using array join, moving code around and |
| 239 | // refactoring into standalone functions |
| 240 | // ----------------------------------------------------------------------- |
| 241 | // NOTE temporary workaround expose ONOS node ports |
| 242 | // ----------------------------------------------------------------------- |
| 243 | String localHelmFlags = [ |
| 244 | extraHelmFlags.trim(), |
| 245 | "--set global.log_level=${logLevel.toUpperCase()}", |
| 246 | '--set onos-classic.onosSshPort=30115', |
| 247 | '--set onos-classic.onosApiPort=30120', |
| 248 | '--set onos-classic.onosOfPort=31653', |
| 249 | '--set onos-classic.individualOpenFlowNodePorts=true', |
| 250 | testSpecificHelmFlags |
| 251 | ].join(' ') |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 252 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 253 | println("** ${iam} localHelmFlags = ${localHelmFlags}") |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 254 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 255 | if (gerritProject != '') { |
| 256 | localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}") |
| 257 | } |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 258 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 259 | println("** ${iam}: ENTER") |
| 260 | volthaDeploy([ |
| 261 | infraNamespace: infraNamespace, |
| 262 | volthaNamespace: volthaNamespace, |
| 263 | workflow: workflow.toLowerCase(), |
| 264 | withMacLearning: enableMacLearning.toBoolean(), |
| 265 | extraHelmFlags: localHelmFlags, |
| 266 | localCharts: localCharts, |
| 267 | bbsimReplica: olts.toInteger(), |
| 268 | dockerRegistry: registry, |
| 269 | ]) |
| 270 | println("** ${iam}: LEAVE") |
| 271 | } // script |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 272 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 273 | // ----------------------------------------------------------------------- |
| 274 | // Intent: Replacing P_IDS with pgrep/pkill is a step forward. |
| 275 | // Why not simply use a pid file, capture _TAG=kail-startup above |
| 276 | // Grep runs the risk of terminating stray commands (??-good <=> bad-??) |
| 277 | // ----------------------------------------------------------------------- |
| 278 | script { |
| 279 | String proc = '_TAG=kail-startup' |
Joey Armstrong | dddbbf9 | 2023-08-22 16:00:41 -0400 | [diff] [blame] | 280 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 281 | println("${iam}: ENTER") |
| 282 | println("${iam}: Shutdown process $proc") |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 283 | /* |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 284 | pgrep_proc(proc) |
| 285 | pkill_proc(proc) |
| 286 | pgrep_proc(proc) |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 287 | */ |
| 288 | |
| 289 | sh(label : 'pgrep_proc - kill-pre', |
| 290 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 291 | pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 292 | """) |
| 293 | |
| 294 | sh(label : 'pkill_proc - _TAG=kail-startup', |
| 295 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 296 | if [[ \$(pgrep --count '_TAG=kail-startup') -gt 0 ]]; then |
| 297 | pkill --uid "\$(id -u)" --echo --full '_TAG=kail-startup' |
| 298 | fi |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 299 | """) |
| 300 | |
| 301 | sh(label : 'pgrep_proc - kill-post', |
| 302 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 303 | pgrep --uid "\$(id -u)" --list-full --full '_TAG=kail-startup' || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 304 | """) |
| 305 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 306 | println("${iam}: LEAVE") |
| 307 | } |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 308 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 309 | // ----------------------------------------------------------------------- |
| 310 | // Bundle onos-voltha / kail logs |
| 311 | // ----------------------------------------------------------------------- |
| 312 | sh(""" |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 313 | cat <<EOM |
| 314 | |
| 315 | ** ----------------------------------------------------------------------- |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 316 | ** Combine and compress voltha startup log(s) |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 317 | ** ----------------------------------------------------------------------- |
| 318 | EOM |
| 319 | pushd "${logsDir}" || { echo "ERROR: pushd $logsDir failed"; exit 1; } |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 320 | gzip -k onos-voltha-startup-combined.log |
| 321 | rm onos-voltha-startup-combined.log |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 322 | popd |
| 323 | """) |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 324 | } // timeout(10) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 325 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 326 | // ----------------------------------------------------------------------- |
| 327 | // ----------------------------------------------------------------------- |
| 328 | sh """ |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 329 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"& |
| 330 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"& |
| 331 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"& |
| 332 | bbsimDmiPortFwd=50075 |
| 333 | for i in {0..${olts.toInteger() - 1}}; do |
| 334 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"& |
| 335 | ((bbsimDmiPortFwd++)) |
| 336 | done |
| 337 | if [ ${withMonitoring} = true ] ; then |
| 338 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"& |
| 339 | fi |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 340 | # ps aux | grep port-forward |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 341 | """ |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 342 | // --------------------------------- |
| 343 | // Sanity check port-forward spawned |
| 344 | // --------------------------------- |
| 345 | script { |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 346 | enter('Display port-forward procs') |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 347 | // String proc = 'kubectl.*port-forward' // was 'port-forward' |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 348 | String proc = 'port-forward' |
| 349 | println("Display spawned ${proc}") |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 350 | sh(label : 'pgrep_proc - check', |
| 351 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 352 | pgrep --uid "\$(id -u)" --list-full --full "port-forward" || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 353 | """) |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 354 | leave('Display port-forward procs') |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 355 | } |
Joey Armstrong | 5d65efe | 2023-08-25 09:43:18 -0400 | [diff] [blame] | 356 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 357 | // setting ONOS log level |
| 358 | script { |
| 359 | enter('setOnosLogLevels') |
| 360 | setOnosLogLevels([ |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 361 | onosNamespace: infraNamespace, |
| 362 | apps: [ |
| 363 | 'org.opencord.dhcpl2relay', |
| 364 | 'org.opencord.olt', |
| 365 | 'org.opencord.aaa', |
| 366 | 'org.opencord.maclearner', |
| 367 | 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager', |
| 368 | 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager' |
| 369 | ], |
| 370 | logLevel: logLevel |
| 371 | ]) |
Joey Armstrong | 6146f7e | 2023-08-28 09:05:38 -0400 | [diff] [blame] | 372 | leave('setOnosLogLevels') |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 373 | } // script |
| 374 | } // if (teardown) |
| 375 | } // stage('Deploy Voltha') |
| 376 | |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 377 | // ----------------------------------------------------------------------- |
| 378 | // ----------------------------------------------------------------------- |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 379 | stage("Run test ${testTarget} on workflow ${workflow}") |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 380 | { |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 381 | sh( |
| 382 | label : 'Monitor using mem_consumption.py', |
| 383 | script : """ |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 384 | echo -e "\n** Monitor using mem_consumption.py ?" |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 385 | |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 386 | if [ ${withMonitoring} = true ] ; then |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 387 | cat <<EOM |
| 388 | |
| 389 | ** ----------------------------------------------------------------------- |
| 390 | ** Monitoring memory usage with mem_consumption.py |
| 391 | ** ----------------------------------------------------------------------- |
| 392 | EOM |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 393 | mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}" |
| 394 | cd "$WORKSPACE/voltha-system-tests" |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 395 | |
| 396 | echo '** Installing python virtualenv' |
| 397 | make venv-activate-patched |
| 398 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 399 | set +u && source .venv/bin/activate && set -u |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 400 | # Collect initial memory consumption |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 401 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 402 | fi |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 403 | |
| 404 | echo -e '** Monitor memory consumption: LEAVE\n' |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 405 | """) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 406 | |
| 407 | sh """ |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 408 | echo -e "\n** make testTarget=[${testTarget}]" |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 409 | mkdir -p ${logsDir} |
| 410 | export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} " |
| 411 | ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}" |
| 412 | export KVSTOREPREFIX=voltha/voltha_voltha |
| 413 | |
Joey Armstrong | 0eb8bd8 | 2023-07-10 13:26:25 -0400 | [diff] [blame] | 414 | make -C "$WORKSPACE/voltha-system-tests" ${testTarget} |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 415 | """ |
| 416 | |
| 417 | getPodsInfo("${logsDir}") |
| 418 | |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 419 | sh( |
| 420 | label : 'Gather robot Framework logs', |
| 421 | script : """ |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 422 | echo -e '\n** Gather robot Framework logs: ENTER' |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 423 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 424 | # set +e |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 425 | # collect logs collected in the Robot Framework StartLogging keyword |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 426 | cd "${logsDir}" |
| 427 | |
| 428 | echo "** Available logs:" |
Joey Armstrong | d1c4575 | 2023-08-29 16:56:05 -0400 | [diff] [blame] | 429 | /bin/ls -l "$logsDir" |
Joey Armstrong | 7ba23ac | 2023-08-29 15:21:53 -0400 | [diff] [blame] | 430 | echo |
| 431 | |
Joey Armstrong | 2e27497 | 2023-08-29 17:39:18 -0400 | [diff] [blame] | 432 | echo '** Bundle combined log' |
| 433 | gzip *-combined.log || true |
| 434 | rm -f *-combined.log || true |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 435 | |
| 436 | echo -e '** Gather robot Framework logs: LEAVE\n' |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 437 | """) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 438 | |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 439 | // ----------------------------------------------------------------------- |
| 440 | // ----------------------------------------------------------------------- |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 441 | sh( |
| 442 | label : 'Monitor pod-mem-consumption', |
| 443 | script : """ |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 444 | echo -e '** Monitor pod-mem-consumption: ENTER' |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 445 | if [ ${withMonitoring} = true ] ; then |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 446 | cat <<EOM |
| 447 | |
| 448 | ** ----------------------------------------------------------------------- |
| 449 | ** Monitoring pod-memory-consumption using mem_consumption.py |
| 450 | ** ----------------------------------------------------------------------- |
| 451 | EOM |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 452 | cd "$WORKSPACE/voltha-system-tests" |
Joey Armstrong | f060aee | 2023-08-22 21:55:26 -0400 | [diff] [blame] | 453 | |
| 454 | echo '** Installing python virtualenv' |
| 455 | make venv-activate-patched |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 456 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 457 | set +u && source .venv/bin/activate && set -u |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 458 | # Collect memory consumption of voltha pods once all the tests are complete |
Joey Armstrong | 0eb8bd8 | 2023-07-10 13:26:25 -0400 | [diff] [blame] | 459 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 460 | fi |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 461 | echo -e '** Monitor pod-mem-consumption: LEAVE\n' |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 462 | """) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 463 | } // stage |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 464 | |
| 465 | return |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 466 | } // execute_test() |
| 467 | |
| 468 | // ----------------------------------------------------------------------- |
| 469 | // ----------------------------------------------------------------------- |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 470 | def collectArtifacts(exitStatus) { |
Joey Armstrong | 6115fd6 | 2023-08-24 08:19:28 -0400 | [diff] [blame] | 471 | script { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 472 | String iam = getIam('collectArtifacts') |
| 473 | println("${iam}: ENTER (exitStatus=${exitStatus})") |
Joey Armstrong | 6115fd6 | 2023-08-24 08:19:28 -0400 | [diff] [blame] | 474 | } |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 475 | |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 476 | echo ''' |
| 477 | |
| 478 | ** ----------------------------------------------------------------------- |
| 479 | ** collectArtifacts |
| 480 | ** ----------------------------------------------------------------------- |
| 481 | ''' |
Joey Armstrong | 66fcb4e | 2023-08-11 12:09:24 -0400 | [diff] [blame] | 482 | |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 483 | getPodsInfo("$WORKSPACE/${exitStatus}") |
Joey Armstrong | 66fcb4e | 2023-08-11 12:09:24 -0400 | [diff] [blame] | 484 | |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 485 | sh """ |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 486 | kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 487 | """ |
Joey Armstrong | 66fcb4e | 2023-08-11 12:09:24 -0400 | [diff] [blame] | 488 | |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 489 | archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*' |
Joey Armstrong | 66fcb4e | 2023-08-11 12:09:24 -0400 | [diff] [blame] | 490 | |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 491 | script { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 492 | println("${iam}: ENTER") |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 493 | /* |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 494 | pgrep_proc('kail-startup') |
| 495 | pkill_proc('kail') |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 496 | */ |
| 497 | sh(label : 'pgrep_proc - kill-pre', |
| 498 | script : """ |
Joey Armstrong | e9725b1 | 2023-08-28 18:15:12 -0400 | [diff] [blame] | 499 | pgrep --uid "\$(id -u)" --list-full --full 'kail-startup' || true |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 500 | """) |
| 501 | sh(label : 'pkill_proc - kail', |
| 502 | script : """ |
Joey Armstrong | 12a8c83 | 2023-08-28 16:50:24 -0400 | [diff] [blame] | 503 | if [[ \$(pgrep --count '_TAG=kail') -gt 0 ]]; then |
| 504 | pkill --uid "\$(id -u)" --echo --full 'kail' |
| 505 | fi |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 506 | """) |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 507 | println("${iam}: LEAVE") |
Joey Armstrong | ec1ae0a | 2023-08-23 21:51:45 -0400 | [diff] [blame] | 508 | } |
Joey Armstrong | dd0cd6b | 2023-08-25 17:27:56 -0400 | [diff] [blame] | 509 | |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 510 | println("${iam}: ENTER RobotPublisher") |
| 511 | step([$class: 'RobotPublisher', |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 512 | disableArchiveOutput: false, |
| 513 | logFileName: '**/*/log*.html', |
| 514 | otherFiles: '', |
| 515 | outputFileName: '**/*/output*.xml', |
| 516 | outputPath: '.', |
| 517 | passThreshold: 100, |
| 518 | reportFileName: '**/*/report*.html', |
| 519 | unstableThreshold: 0, |
| 520 | onlyCritical: true]) |
Joey Armstrong | 642540a | 2023-08-10 10:26:36 -0400 | [diff] [blame] | 521 | println("${iam}: LEAVE RobotPublisher") |
| 522 | |
| 523 | println("${iam}: LEAVE (exitStatus=${exitStatus})") |
| 524 | return |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 525 | } |
| 526 | |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 527 | // ----------------------------------------------------------------------- |
| 528 | // Intent: main |
| 529 | // ----------------------------------------------------------------------- |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 530 | pipeline { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 531 | /* no label, executor is determined by JJB */ |
| 532 | agent { |
| 533 | label "${params.buildNode}" |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 534 | } |
| 535 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 536 | options { |
| 537 | timeout(time: "${timeout}", unit: 'MINUTES') |
| 538 | } |
| 539 | |
| 540 | environment { |
| 541 | KUBECONFIG = "$HOME/.kube/kind-${clusterName}" |
| 542 | VOLTCONFIG = "$HOME/.volt/config" |
| 543 | PATH = "$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" |
| 544 | DIAGS_PROFILE = 'VOLTHA_PROFILE' |
| 545 | SSHPASS = 'karaf' |
| 546 | } |
| 547 | |
| 548 | stages { |
| 549 | stage('Download Code') { |
| 550 | steps { |
| 551 | getVolthaCode([ |
| 552 | branch: "${branch}", |
| 553 | gerritProject: "${gerritProject}", |
| 554 | gerritRefspec: "${gerritRefspec}", |
| 555 | volthaSystemTestsChange: "${volthaSystemTestsChange}", |
| 556 | volthaHelmChartsChange: "${volthaHelmChartsChange}", |
| 557 | ]) |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 558 | } |
| 559 | } |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 560 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 561 | stage('Build patch v1.1') { |
| 562 | // build the patch only if gerritProject is specified |
| 563 | when { |
| 564 | expression { return !gerritProject.isEmpty() } |
| 565 | } |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 566 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 567 | steps { |
| 568 | // NOTE that the correct patch has already been checked out |
| 569 | // during the getVolthaCode step |
| 570 | buildVolthaComponent("${gerritProject}") |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | // ----------------------------------------------------------------------- |
| 575 | // ----------------------------------------------------------------------- |
| 576 | stage('Install Kail') |
Joey Armstrong | 2b2010d | 2023-08-02 21:47:20 -0400 | [diff] [blame] | 577 | { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 578 | steps |
Joey Armstrong | 2b2010d | 2023-08-02 21:47:20 -0400 | [diff] [blame] | 579 | { |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 580 | script |
| 581 | { |
| 582 | String cmd = [ |
| 583 | 'make', |
| 584 | '--no-print-directory', |
| 585 | '-C', "$WORKSPACE/voltha-system-tests", |
| 586 | "KAIL_PATH=\"$WORKSPACE/bin\"", |
| 587 | 'kail', |
| 588 | ].join(' ') |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 589 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 590 | println(" ** Running: ${cmd}") |
| 591 | sh("${cmd}") |
| 592 | } // script |
| 593 | } // steps |
| 594 | } // stage |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 595 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 596 | // ----------------------------------------------------------------------- |
| 597 | // ----------------------------------------------------------------------- |
| 598 | stage('Install Tools') { |
| 599 | steps { |
| 600 | script { |
| 601 | String branchName = branchName() |
| 602 | String iam = getIam('Install Tools') |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 603 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 604 | println("${iam}: ENTER (branch=$branch)") |
| 605 | installKind(branch) // needed early by stage(Cleanup) |
| 606 | println("${iam}: LEAVE (branch=$branch)") |
| 607 | } // script |
| 608 | } // steps |
| 609 | } // stage |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 610 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 611 | // ----------------------------------------------------------------------- |
| 612 | // ----------------------------------------------------------------------- |
| 613 | stage('Create K8s Cluster') { |
| 614 | steps { |
| 615 | script { |
| 616 | def clusterExists = sh( |
| 617 | returnStdout: true, |
| 618 | script: """kind get clusters | grep "${clusterName}" | wc -l""") |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 619 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 620 | if (clusterExists.trim() == '0') { |
| 621 | createKubernetesCluster([nodes: 3, name: clusterName]) |
| 622 | } |
| 623 | } // script |
| 624 | } // steps |
| 625 | } // stage('Create K8s Cluster') |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 626 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 627 | // ----------------------------------------------------------------------- |
| 628 | // ----------------------------------------------------------------------- |
| 629 | stage('Replace voltctl') { |
| 630 | // if the project is voltctl, override the downloaded one with the built one |
| 631 | when { |
| 632 | expression { return gerritProject == 'voltctl' } |
| 633 | } |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 634 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 635 | // Hmmmm(?) where did the voltctl download happen ? |
| 636 | // Likely Makefile but would be helpful to document here. |
| 637 | steps { |
| 638 | script { |
| 639 | String iam = getIam('Replace voltctl') |
Joey Armstrong | 6115fd6 | 2023-08-24 08:19:28 -0400 | [diff] [blame] | 640 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 641 | println("${iam} Running: installVoltctl($branch)") |
| 642 | println("${iam}: ENTER") |
| 643 | installVoltctl("$branch") |
| 644 | println("${iam}: LEAVE") |
| 645 | } // script |
| 646 | } // step |
| 647 | } // stage |
Joey Armstrong | 268442d | 2023-08-22 17:16:10 -0400 | [diff] [blame] | 648 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 649 | // ----------------------------------------------------------------------- |
| 650 | // ----------------------------------------------------------------------- |
| 651 | stage('Load image in kind nodes') |
| 652 | { |
| 653 | when { |
| 654 | expression { return !gerritProject.isEmpty() } |
| 655 | } |
| 656 | steps { |
| 657 | loadToKind() |
| 658 | } // steps |
| 659 | } // stage |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 660 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 661 | // ----------------------------------------------------------------------- |
| 662 | // [TODO] verify testing output |
| 663 | // ----------------------------------------------------------------------- |
| 664 | stage('Parse and execute tests') |
| 665 | { |
| 666 | steps { |
| 667 | script { |
| 668 | // Announce ourselves for log usability |
| 669 | enter('Parse and execute tests') |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 670 | |
Joey Armstrong | ebc1802 | 2023-08-26 13:20:49 -0400 | [diff] [blame] | 671 | def tests = readYaml text: testTargets // typeof == Map (?) |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 672 | println("** [DEBUG]: tests=$tests") |
Joey Armstrong | 008cfaf | 2023-08-18 14:49:06 -0400 | [diff] [blame] | 673 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 674 | // Display expected tests for times when output goes dark |
| 675 | tests.eachWithIndex { test, idx -> |
| 676 | String target = test['target'] |
| 677 | println("** test[${idx}]: ${target}\n") |
| 678 | } |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 679 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 680 | println(''' |
Joey Armstrong | b29d561 | 2023-08-24 12:53:46 -0400 | [diff] [blame] | 681 | ** ----------------------------------------------------------------------- |
| 682 | ** NOTE: For odd/silent job failures verify a few details |
Joey Armstrong | 5d65efe | 2023-08-25 09:43:18 -0400 | [diff] [blame] | 683 | ** - All tests mentioned in the tests-to-run index were logged. |
Joey Armstrong | b29d561 | 2023-08-24 12:53:46 -0400 | [diff] [blame] | 684 | ** - Test suites display ENTER/LEAVE mesasge pairs. |
Joey Armstrong | 5d65efe | 2023-08-25 09:43:18 -0400 | [diff] [blame] | 685 | ** - Processing terminated prematurely when LEAVE strings are missing. |
Joey Armstrong | b29d561 | 2023-08-24 12:53:46 -0400 | [diff] [blame] | 686 | ** ----------------------------------------------------------------------- |
Joey Armstrong | be7c924 | 2023-08-24 16:20:31 -0400 | [diff] [blame] | 687 | ''') |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 688 | tests.eachWithIndex { test, idx -> |
| 689 | println "** readYaml test suite[$idx]) test=[${test}]" |
Joey Armstrong | f46b3ae | 2023-08-25 11:07:20 -0400 | [diff] [blame] | 690 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 691 | String target = test['target'] |
| 692 | String workflow = test['workflow'] |
| 693 | String flags = test['flags'] |
| 694 | Boolean teardown = test['teardown'].toBoolean() |
| 695 | Boolean logging = test['logging'].toBoolean() |
| 696 | String testLogging = (logging) ? 'True' : 'False' |
Joey Armstrong | 0e0a42b | 2023-08-02 21:04:21 -0400 | [diff] [blame] | 697 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 698 | print(""" |
Joey Armstrong | 268442d | 2023-08-22 17:16:10 -0400 | [diff] [blame] | 699 | ** ----------------------------------------------------------------------- |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 700 | ** Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags} |
Joey Armstrong | 268442d | 2023-08-22 17:16:10 -0400 | [diff] [blame] | 701 | ** ----------------------------------------------------------------------- |
Joey Armstrong | 38a8783 | 2023-08-23 17:02:50 -0400 | [diff] [blame] | 702 | """) |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 703 | |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 704 | try { |
Joey Armstrong | 6146f7e | 2023-08-28 09:05:38 -0400 | [diff] [blame] | 705 | enter("execute_test (target=$target)") |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 706 | execute_test(target, workflow, testLogging, teardown, flags) |
| 707 | } |
Joey Armstrong | 9341c9a | 2023-08-28 12:09:19 -0400 | [diff] [blame] | 708 | // groovylint-disable-next-line CatchException |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 709 | catch (Exception err) { |
Joey Armstrong | 9341c9a | 2023-08-28 12:09:19 -0400 | [diff] [blame] | 710 | String iamexc = getIam(name) |
| 711 | println("** ${iamexc}: EXCEPTION ${err}") |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 712 | } |
| 713 | finally { |
| 714 | leave("execute_test (target=$target)") |
| 715 | } |
| 716 | } // for |
| 717 | // Premature exit if this message is not logged |
| 718 | leave('Parse and execute tests') |
Joey Armstrong | 54dec09 | 2023-08-03 18:21:38 -0400 | [diff] [blame] | 719 | } // script |
| 720 | } // steps |
Joey Armstrong | 7229d9b | 2023-08-25 18:03:13 -0400 | [diff] [blame] | 721 | } // stage |
Joey Armstrong | 7bcb578 | 2023-06-07 12:25:57 -0400 | [diff] [blame] | 722 | } // stages |
| 723 | |
| 724 | post |
| 725 | { |
| 726 | aborted { collectArtifacts('aborted') } |
| 727 | failure { collectArtifacts('failed') } |
| 728 | always { collectArtifacts('always') } |
| 729 | } |
| 730 | } // pipeline |
| 731 | |
Joey Armstrong | 664c55a | 2023-08-28 14:22:33 -0400 | [diff] [blame] | 732 | // [EOF] |