Joey Armstrong | 6540541 | 2022-11-22 10:43:06 -0500 | [diff] [blame^] | 1 | // -*- groovy -*- |
| 2 | // Copyright 2021-2022 Open Networking Foundation (ONF) and the ONF Contributors |
| 3 | // |
| 4 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | // you may not use this file except in compliance with the License. |
| 6 | // You may obtain a copy of the License at |
| 7 | // |
| 8 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | // |
| 10 | // Unless required by applicable law or agreed to in writing, software |
| 11 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | // See the License for the specific language governing permissions and |
| 14 | // limitations under the License. |
| 15 | |
| 16 | // voltha-2.x e2e tests for openonu-go |
| 17 | // uses bbsim to simulate OLT/ONUs |
| 18 | |
| 19 | library identifier: 'cord-jenkins-libraries@master', |
| 20 | retriever: modernSCM([ |
| 21 | $class: 'GitSCMSource', |
| 22 | remote: 'https://gerrit.opencord.org/ci-management.git' |
| 23 | ]) |
| 24 | |
| 25 | def clusterName = "kind-ci" |
| 26 | |
| 27 | def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") { |
| 28 | def infraNamespace = "default" |
| 29 | def volthaNamespace = "voltha" |
| 30 | def logsDir = "$WORKSPACE/${testTarget}" |
| 31 | |
| 32 | stage('IAM') |
| 33 | { |
| 34 | print(''' |
| 35 | jjb/pipeline/voltha/voltha-2.11 |
| 36 | ----------------------------------------------------------------------- |
| 37 | ''') |
| 38 | } |
| 39 | |
| 40 | stage('Cleanup') { |
| 41 | if (teardown) { |
| 42 | timeout(15) { |
| 43 | script { |
| 44 | helmTeardown(["default", infraNamespace, volthaNamespace]) |
| 45 | } |
| 46 | timeout(1) { |
| 47 | sh returnStdout: false, script: ''' |
| 48 | # remove orphaned port-forward from different namespaces |
| 49 | ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true |
| 50 | ''' |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | stage('Deploy common infrastructure') { |
| 56 | sh ''' |
| 57 | helm repo add onf https://charts.opencord.org |
| 58 | helm repo update |
| 59 | if [ ${withMonitoring} = true ] ; then |
| 60 | helm install nem-monitoring onf/nem-monitoring \ |
| 61 | --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \ |
| 62 | --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false |
| 63 | fi |
| 64 | ''' |
| 65 | } |
| 66 | stage('Deploy Voltha') { |
| 67 | if (teardown) { |
| 68 | timeout(10) { |
| 69 | script { |
| 70 | |
| 71 | sh """ |
| 72 | mkdir -p ${logsDir} |
| 73 | _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log & |
| 74 | """ |
| 75 | |
| 76 | // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts |
| 77 | def localCharts = false |
| 78 | if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") { |
| 79 | localCharts = true |
| 80 | } |
| 81 | |
| 82 | // NOTE temporary workaround expose ONOS node ports |
| 83 | def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " + |
| 84 | " --set onos-classic.onosSshPort=30115 " + |
| 85 | " --set onos-classic.onosApiPort=30120 " + |
| 86 | " --set onos-classic.onosOfPort=31653 " + |
| 87 | " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags |
| 88 | |
| 89 | if (gerritProject != "") { |
| 90 | localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}") |
| 91 | } |
| 92 | |
| 93 | volthaDeploy([ |
| 94 | infraNamespace: infraNamespace, |
| 95 | volthaNamespace: volthaNamespace, |
| 96 | workflow: workflow.toLowerCase(), |
| 97 | withMacLearning: enableMacLearning.toBoolean(), |
| 98 | extraHelmFlags: localHelmFlags, |
| 99 | localCharts: localCharts, |
| 100 | bbsimReplica: olts.toInteger(), |
| 101 | dockerRegistry: registry, |
| 102 | ]) |
| 103 | } |
| 104 | |
| 105 | // stop logging |
| 106 | sh """ |
| 107 | P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')" |
| 108 | if [ -n "\$P_IDS" ]; then |
| 109 | echo \$P_IDS |
| 110 | for P_ID in \$P_IDS; do |
| 111 | kill -9 \$P_ID |
| 112 | done |
| 113 | fi |
| 114 | cd ${logsDir} |
| 115 | gzip -k onos-voltha-startup-combined.log |
| 116 | rm onos-voltha-startup-combined.log |
| 117 | """ |
| 118 | } |
| 119 | sh """ |
| 120 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"& |
| 121 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"& |
| 122 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"& |
| 123 | bbsimDmiPortFwd=50075 |
| 124 | for i in {0..${olts.toInteger() - 1}}; do |
| 125 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"& |
| 126 | ((bbsimDmiPortFwd++)) |
| 127 | done |
| 128 | if [ ${withMonitoring} = true ] ; then |
| 129 | JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"& |
| 130 | fi |
| 131 | ps aux | grep port-forward |
| 132 | """ |
| 133 | // setting ONOS log level |
| 134 | script { |
| 135 | setOnosLogLevels([ |
| 136 | onosNamespace: infraNamespace, |
| 137 | apps: [ |
| 138 | 'org.opencord.dhcpl2relay', |
| 139 | 'org.opencord.olt', |
| 140 | 'org.opencord.aaa', |
| 141 | 'org.opencord.maclearner', |
| 142 | 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager', |
| 143 | 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager' |
| 144 | ], |
| 145 | logLevel: logLevel |
| 146 | ]) |
| 147 | } |
| 148 | } |
| 149 | } |
| 150 | stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') { |
| 151 | sh """ |
| 152 | if [ ${withMonitoring} = true ] ; then |
| 153 | mkdir -p $WORKSPACE/voltha-pods-mem-consumption-${workflow} |
| 154 | cd $WORKSPACE/voltha-system-tests |
| 155 | make vst_venv |
| 156 | source ./vst_venv/bin/activate || true |
| 157 | # Collect initial memory consumption |
| 158 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true |
| 159 | fi |
| 160 | """ |
| 161 | sh """ |
| 162 | mkdir -p ${logsDir} |
| 163 | export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} " |
| 164 | ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}" |
| 165 | export KVSTOREPREFIX=voltha/voltha_voltha |
| 166 | |
| 167 | make -C $WORKSPACE/voltha-system-tests ${testTarget} || true |
| 168 | """ |
| 169 | getPodsInfo("${logsDir}") |
| 170 | sh """ |
| 171 | set +e |
| 172 | # collect logs collected in the Robot Framework StartLogging keyword |
| 173 | cd ${logsDir} |
| 174 | gzip *-combined.log || true |
| 175 | rm *-combined.log || true |
| 176 | """ |
| 177 | sh """ |
| 178 | if [ ${withMonitoring} = true ] ; then |
| 179 | cd $WORKSPACE/voltha-system-tests |
| 180 | source ./vst_venv/bin/activate || true |
| 181 | # Collect memory consumption of voltha pods once all the tests are complete |
| 182 | python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true |
| 183 | fi |
| 184 | """ |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | def collectArtifacts(exitStatus) { |
| 189 | getPodsInfo("$WORKSPACE/${exitStatus}") |
| 190 | sh """ |
| 191 | kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true |
| 192 | """ |
| 193 | archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*' |
| 194 | sh ''' |
| 195 | sync |
| 196 | pkill kail || true |
| 197 | which voltctl |
| 198 | md5sum $(which voltctl) |
| 199 | ''' |
| 200 | step([$class: 'RobotPublisher', |
| 201 | disableArchiveOutput: false, |
| 202 | logFileName: "**/*/log*.html", |
| 203 | otherFiles: '', |
| 204 | outputFileName: "**/*/output*.xml", |
| 205 | outputPath: '.', |
| 206 | passThreshold: 100, |
| 207 | reportFileName: "**/*/report*.html", |
| 208 | unstableThreshold: 0, |
| 209 | onlyCritical: true]); |
| 210 | } |
| 211 | |
| 212 | pipeline { |
| 213 | |
| 214 | /* no label, executor is determined by JJB */ |
| 215 | agent { |
| 216 | label "${params.buildNode}" |
| 217 | } |
| 218 | options { |
| 219 | timeout(time: "${timeout}", unit: 'MINUTES') |
| 220 | } |
| 221 | environment { |
| 222 | KUBECONFIG="$HOME/.kube/kind-${clusterName}" |
| 223 | VOLTCONFIG="$HOME/.volt/config" |
| 224 | PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" |
| 225 | DIAGS_PROFILE="VOLTHA_PROFILE" |
| 226 | SSHPASS="karaf" |
| 227 | } |
| 228 | stages { |
| 229 | stage('Download Code') { |
| 230 | steps { |
| 231 | getVolthaCode([ |
| 232 | branch: "${branch}", |
| 233 | gerritProject: "${gerritProject}", |
| 234 | gerritRefspec: "${gerritRefspec}", |
| 235 | volthaSystemTestsChange: "${volthaSystemTestsChange}", |
| 236 | volthaHelmChartsChange: "${volthaHelmChartsChange}", |
| 237 | ]) |
| 238 | } |
| 239 | } |
| 240 | stage('Build patch') { |
| 241 | // build the patch only if gerritProject is specified |
| 242 | when { |
| 243 | expression { |
| 244 | return !gerritProject.isEmpty() |
| 245 | } |
| 246 | } |
| 247 | steps { |
| 248 | // NOTE that the correct patch has already been checked out |
| 249 | // during the getVolthaCode step |
| 250 | buildVolthaComponent("${gerritProject}") |
| 251 | } |
| 252 | } |
| 253 | stage('Create K8s Cluster') { |
| 254 | steps { |
| 255 | script { |
| 256 | def clusterExists = sh returnStdout: true, script: """ |
| 257 | kind get clusters | grep ${clusterName} | wc -l |
| 258 | """ |
| 259 | if (clusterExists.trim() == "0") { |
| 260 | createKubernetesCluster([nodes: 3, name: clusterName]) |
| 261 | } |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | stage('Replace voltctl') { |
| 266 | // if the project is voltctl override the downloaded one with the built one |
| 267 | when { |
| 268 | expression { |
| 269 | return gerritProject == "voltctl" |
| 270 | } |
| 271 | } |
| 272 | steps{ |
| 273 | sh """ |
| 274 | mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl |
| 275 | chmod +x $WORKSPACE/bin/voltctl |
| 276 | """ |
| 277 | } |
| 278 | } |
| 279 | stage('Load image in kind nodes') { |
| 280 | when { |
| 281 | expression { |
| 282 | return !gerritProject.isEmpty() |
| 283 | } |
| 284 | } |
| 285 | steps { |
| 286 | loadToKind() |
| 287 | } |
| 288 | } |
| 289 | stage('Parse and execute tests') { |
| 290 | steps { |
| 291 | script { |
| 292 | def tests = readYaml text: testTargets |
| 293 | |
| 294 | for(int i = 0;i<tests.size();i++) { |
| 295 | def test = tests[i] |
| 296 | def target = test["target"] |
| 297 | def workflow = test["workflow"] |
| 298 | def flags = test["flags"] |
| 299 | def teardown = test["teardown"].toBoolean() |
| 300 | def logging = test["logging"].toBoolean() |
| 301 | def testLogging = 'False' |
| 302 | if (logging) { |
| 303 | testLogging = 'True' |
| 304 | } |
| 305 | println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}" |
| 306 | execute_test(target, workflow, testLogging, teardown, flags) |
| 307 | } |
| 308 | } |
| 309 | } |
| 310 | } |
| 311 | } |
| 312 | post { |
| 313 | aborted { |
| 314 | collectArtifacts("aborted") |
| 315 | } |
| 316 | failure { |
| 317 | collectArtifacts("failed") |
| 318 | } |
| 319 | always { |
| 320 | collectArtifacts("always") |
| 321 | } |
| 322 | } |
| 323 | } |