Joey Armstrong | 980e37f | 2023-02-28 18:57:41 -0500 | [diff] [blame^] | 1 | // Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | // voltha-2.x e2e tests for openonu-go |
| 16 | // uses bbsim to simulate OLT/ONUs |
| 17 | |
| 18 | library identifier: 'cord-jenkins-libraries@master', |
| 19 | retriever: modernSCM([ |
| 20 | $class: 'GitSCMSource', |
| 21 | remote: 'https://gerrit.opencord.org/ci-management.git' |
| 22 | ]) |
| 23 | |
| 24 | def clusterName = "kind-ci" |
| 25 | |
| 26 | def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") { |
| 27 | def infraNamespace = "default" |
| 28 | def volthaNamespace = "voltha" |
| 29 | def logsDir = "$WORKSPACE/${testTarget}" |
| 30 | |
| 31 | stage('IAM') |
| 32 | { |
| 33 | script |
| 34 | { |
| 35 | String iam = [ |
| 36 | 'ci-management', |
| 37 | 'jjb', |
| 38 | 'pipeline', |
| 39 | 'voltha', |
| 40 | 'master', |
| 41 | 'bbsim-tests.groovy' |
| 42 | ].join('/') |
| 43 | println("** ${iam}: ENTER") |
| 44 | |
| 45 | String cmd = "which pkill" |
| 46 | def stream = sh( |
| 47 | returnStatus:false, |
| 48 | returnStdout: true, |
| 49 | script: cmd) |
| 50 | println(" ** ${cmd}:\n${stream}") |
| 51 | |
| 52 | println("** ${iam}: LEAVE") |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | stage('Cleanup') { |
| 57 | if (teardown) { |
| 58 | timeout(15) { |
| 59 | script { |
| 60 | helmTeardown(["default", infraNamespace, volthaNamespace]) |
| 61 | } |
| 62 | timeout(1) { |
| 63 | sh returnStdout: false, script: ''' |
| 64 | # remove orphaned port-forward from different namespaces |
| 65 | ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true |
| 66 | ''' |
| 67 | } |
| 68 | } |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | stage ('Initialize') |
| 73 | { |
| 74 | // VOL-4926 - Is voltha-system-tests available ? |
| 75 | String cmd = [ |
| 76 | 'make', |
| 77 | '-C', "$WORKSPACE/voltha-system-tests", |
| 78 | "KAIL_PATH=\"$WORKSPACE/bin\"", |
| 79 | 'kail', |
| 80 | ].join(' ') |
| 81 | println(" ** Running: ${cmd}:\n") |
| 82 | sh("${cmd}") |
| 83 | } |
| 84 | |
| 85 | stage('Deploy common infrastructure') { |
| 86 | sh ''' |
| 87 | helm repo add onf https://charts.opencord.org |
| 88 | helm repo update |
| 89 | if [ ${withMonitoring} = true ] ; then |
| 90 | helm install nem-monitoring onf/nem-monitoring \ |
| 91 | --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \ |
| 92 | --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false |
| 93 | fi |
| 94 | ''' |
| 95 | } |
| 96 | |
| 97 | stage('Deploy Voltha') { |
| 98 | if (teardown) { |
| 99 | timeout(10) { |
| 100 | script { |
| 101 | |
| 102 | sh """ |
| 103 | mkdir -p ${logsDir} |
| 104 | _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log & |
| 105 | """ |
| 106 | |
| 107 | // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts |
| 108 | def localCharts = false |
| 109 | if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") { |
| 110 | localCharts = true |
| 111 | } |
| 112 | |
| 113 | // NOTE temporary workaround expose ONOS node ports |
| 114 | def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " + |
| 115 | " --set onos-classic.onosSshPort=30115 " + |
| 116 | " --set onos-classic.onosApiPort=30120 " + |
| 117 | " --set onos-classic.onosOfPort=31653 " + |
| 118 | " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags |
| 119 | |
| 120 | if (gerritProject != "") { |
| 121 | localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}") |
| 122 | } |
| 123 | |
| 124 | volthaDeploy([ |
| 125 | infraNamespace: infraNamespace, |
| 126 | volthaNamespace: volthaNamespace, |
| 127 | workflow: workflow.toLowerCase(), |
| 128 | withMacLearning: enableMacLearning.toBoolean(), |
| 129 | extraHelmFlags: localHelmFlags, |
| 130 | localCharts: localCharts, |
| 131 | bbsimReplica: olts.toInteger(), |
| 132 | dockerRegistry: registry, |
| 133 | ]) |
| 134 | } |
| 135 | |
| 136 | // stop logging |
| 137 | sh """ |
| 138 | P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')" |
| 139 | if [ -n "\$P_IDS" ]; then |
| 140 | echo \$P_IDS |
| 141 | for P_ID in \$P_IDS; do |
| 142 | kill -9 \$P_ID |
| 143 | done |
| 144 | fi |
| 145 | cd ${logsDir} |
| 146 | gzip -k onos-voltha-startup-combined.log |
| 147 | rm onos-voltha-startup-combined.log |
| 148 | """ |
| 149 | } |
| 150 | sh """ |
| 151 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"& |
| 152 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"& |
| 153 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"& |
| 154 | bbsimDmiPortFwd=50075 |
| 155 | for i in {0..${olts.toInteger() - 1}}; do |
| 156 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"& |
| 157 | ((bbsimDmiPortFwd++)) |
| 158 | done |
| 159 | if [ ${withMonitoring} = true ] ; then |
| 160 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"& |
| 161 | fi |
| 162 | ps aux | grep port-forward |
| 163 | """ |
| 164 | // setting ONOS log level |
| 165 | script { |
| 166 | setOnosLogLevels([ |
| 167 | onosNamespace: infraNamespace, |
| 168 | apps: [ |
| 169 | 'org.opencord.dhcpl2relay', |
| 170 | 'org.opencord.olt', |
| 171 | 'org.opencord.aaa', |
| 172 | 'org.opencord.maclearner', |
| 173 | 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager', |
| 174 | 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager' |
| 175 | ], |
| 176 | logLevel: logLevel |
| 177 | ]) |
| 178 | } |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') { |
| 183 | sh """ |
| 184 | if [ ${withMonitoring} = true ] ; then |
| 185 | mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}" |
| 186 | cd "$WORKSPACE/voltha-system-tests" |
| 187 | make vst_venv |
| 188 | source ./vst_venv/bin/activate || true |
| 189 | # Collect initial memory consumption |
| 190 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true |
| 191 | fi |
| 192 | """ |
| 193 | sh """ |
| 194 | mkdir -p ${logsDir} |
| 195 | export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} " |
| 196 | ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}" |
| 197 | export KVSTOREPREFIX=voltha/voltha_voltha |
| 198 | |
| 199 | make -C "$WORKSPACE/voltha-system-tests" ${testTarget} || true |
| 200 | """ |
| 201 | getPodsInfo("${logsDir}") |
| 202 | sh """ |
| 203 | set +e |
| 204 | # collect logs collected in the Robot Framework StartLogging keyword |
| 205 | cd ${logsDir} |
| 206 | gzip *-combined.log || true |
| 207 | rm *-combined.log || true |
| 208 | """ |
| 209 | sh """ |
| 210 | if [ ${withMonitoring} = true ] ; then |
| 211 | cd "$WORKSPACE/voltha-system-tests" |
| 212 | source ./vst_venv/bin/activate || true |
| 213 | # Collect memory consumption of voltha pods once all the tests are complete |
| 214 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true |
| 215 | fi |
| 216 | """ |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | def collectArtifacts(exitStatus) { |
| 221 | getPodsInfo("$WORKSPACE/${exitStatus}") |
| 222 | sh """ |
| 223 | kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true |
| 224 | """ |
| 225 | archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*' |
| 226 | sh ''' |
| 227 | sync |
| 228 | pkill kail || true |
| 229 | which voltctl |
| 230 | md5sum $(which voltctl) |
| 231 | ''' |
| 232 | step([$class: 'RobotPublisher', |
| 233 | disableArchiveOutput: false, |
| 234 | logFileName: "**/*/log*.html", |
| 235 | otherFiles: '', |
| 236 | outputFileName: "**/*/output*.xml", |
| 237 | outputPath: '.', |
| 238 | passThreshold: 100, |
| 239 | reportFileName: "**/*/report*.html", |
| 240 | unstableThreshold: 0, |
| 241 | onlyCritical: true]); |
| 242 | } |
| 243 | |
| 244 | pipeline { |
| 245 | |
| 246 | /* no label, executor is determined by JJB */ |
| 247 | agent { |
| 248 | label "${params.buildNode}" |
| 249 | } |
| 250 | options { |
| 251 | timeout(time: "${timeout}", unit: 'MINUTES') |
| 252 | } |
| 253 | environment { |
| 254 | KUBECONFIG="$HOME/.kube/kind-${clusterName}" |
| 255 | VOLTCONFIG="$HOME/.volt/config" |
| 256 | PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" |
| 257 | DIAGS_PROFILE="VOLTHA_PROFILE" |
| 258 | SSHPASS="karaf" |
| 259 | } |
| 260 | stages { |
| 261 | stage('Download Code') { |
| 262 | steps { |
| 263 | getVolthaCode([ |
| 264 | branch: "${branch}", |
| 265 | gerritProject: "${gerritProject}", |
| 266 | gerritRefspec: "${gerritRefspec}", |
| 267 | volthaSystemTestsChange: "${volthaSystemTestsChange}", |
| 268 | volthaHelmChartsChange: "${volthaHelmChartsChange}", |
| 269 | ]) |
| 270 | } |
| 271 | } |
| 272 | stage('Build patch') { |
| 273 | // build the patch only if gerritProject is specified |
| 274 | when { |
| 275 | expression { |
| 276 | return !gerritProject.isEmpty() |
| 277 | } |
| 278 | } |
| 279 | steps { |
| 280 | // NOTE that the correct patch has already been checked out |
| 281 | // during the getVolthaCode step |
| 282 | buildVolthaComponent("${gerritProject}") |
| 283 | } |
| 284 | } |
| 285 | stage('Create K8s Cluster') { |
| 286 | steps { |
| 287 | script { |
| 288 | def clusterExists = sh returnStdout: true, script: """ |
| 289 | kind get clusters | grep ${clusterName} | wc -l |
| 290 | """ |
| 291 | if (clusterExists.trim() == "0") { |
| 292 | createKubernetesCluster([nodes: 3, name: clusterName]) |
| 293 | } |
| 294 | } |
| 295 | } |
| 296 | } |
| 297 | stage('Replace voltctl') { |
| 298 | // if the project is voltctl override the downloaded one with the built one |
| 299 | when { |
| 300 | expression { |
| 301 | return gerritProject == "voltctl" |
| 302 | } |
| 303 | } |
| 304 | steps{ |
| 305 | sh """ |
| 306 | # [TODO] - why is this platform specific (?) |
| 307 | # [TODO] - revisit, command alteration has masked an error (see: voltha-2.11). |
| 308 | # find will fail when no filsystem matches are found. |
| 309 | # mv(ls) succeded simply by accident/invoked at a different time. |
| 310 | mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl |
| 311 | chmod +x $WORKSPACE/bin/voltctl |
| 312 | """ |
| 313 | } |
| 314 | } |
| 315 | stage('Load image in kind nodes') { |
| 316 | when { |
| 317 | expression { |
| 318 | return !gerritProject.isEmpty() |
| 319 | } |
| 320 | } |
| 321 | steps { |
| 322 | loadToKind() |
| 323 | } |
| 324 | } |
| 325 | stage('Parse and execute tests') { |
| 326 | steps { |
| 327 | script { |
| 328 | def tests = readYaml text: testTargets |
| 329 | |
| 330 | for(int i = 0;i<tests.size();i++) { |
| 331 | def test = tests[i] |
| 332 | def target = test["target"] |
| 333 | def workflow = test["workflow"] |
| 334 | def flags = test["flags"] |
| 335 | def teardown = test["teardown"].toBoolean() |
| 336 | def logging = test["logging"].toBoolean() |
| 337 | def testLogging = 'False' |
| 338 | if (logging) { |
| 339 | testLogging = 'True' |
| 340 | } |
| 341 | println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}" |
| 342 | execute_test(target, workflow, testLogging, teardown, flags) |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | } |
| 347 | } |
| 348 | post { |
| 349 | aborted { |
| 350 | collectArtifacts("aborted") |
| 351 | } |
| 352 | failure { |
| 353 | collectArtifacts("failed") |
| 354 | } |
| 355 | always { |
| 356 | collectArtifacts("always") |
| 357 | } |
| 358 | } |
| 359 | } |