blob: 6d0ba1ab0e354e53d9f75cd4354c49cbbf7a944d [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Matteo Scandoloa156b572021-02-04 11:52:18 -080018library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
Hardik Windlassec9341b2021-06-07 11:58:29 +000024def clusterName = "kind-ci"
25
Joey Armstrong2d5a7c12023-04-13 09:37:42 -040026def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
27{
Joey Armstrong8c6f6482023-01-12 12:31:44 -050028 def infraNamespace = "default"
29 def volthaNamespace = "voltha"
30 def logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -050031
32 stage('IAM')
33 {
Joey Armstrong84adc542023-04-11 14:47:34 -040034 script
35 {
36 String iam = [
37 'ci-management',
38 'jjb',
39 'pipeline',
40 'voltha',
41 'master',
42 'bbsim-tests.groovy'
43 ].join('/')
Joey Armstrong293e16b2022-11-26 20:16:33 -050044 println("** ${iam}: ENTER")
45
Joey Armstrong84adc542023-04-11 14:47:34 -040046 String cmd = "which pkill"
47 def stream = sh(
48 returnStatus:false,
49 returnStdout: true,
50 script: cmd)
51 println(" ** ${cmd}:\n${stream}")
52
Joey Armstrong293e16b2022-11-26 20:16:33 -050053 println("** ${iam}: LEAVE")
Joey Armstrong84adc542023-04-11 14:47:34 -040054 }
Joey Armstrong293e16b2022-11-26 20:16:33 -050055 }
56
57 stage('Cleanup') {
Joey Armstrong84adc542023-04-11 14:47:34 -040058 if (teardown) {
59 timeout(15) {
60 script {
61 helmTeardown(["default", infraNamespace, volthaNamespace])
62 }
63 timeout(1) {
64 sh returnStdout: false, script: '''
Hardik Windlassec9341b2021-06-07 11:58:29 +000065 # remove orphaned port-forward from different namespaces
66 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
67 '''
Joey Armstrong84adc542023-04-11 14:47:34 -040068 }
69 }
70 }
Hardik Windlassec9341b2021-06-07 11:58:29 +000071 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -050072
73 stage ('Initialize')
74 {
Joey Armstrong84adc542023-04-11 14:47:34 -040075 // VOL-4926 - Is voltha-system-tests available ?
76 String cmd = [
77 'make',
78 '-C', "$WORKSPACE/voltha-system-tests",
79 "KAIL_PATH=\"$WORKSPACE/bin\"",
80 'kail',
81 ].join(' ')
82 println(" ** Running: ${cmd}:\n")
Joey Armstrongbeef4cd2023-01-18 09:59:58 -050083 sh("${cmd}")
Joey Armstrong8c6f6482023-01-12 12:31:44 -050084 }
85
86 stage('Deploy common infrastructure') {
Joey Armstrong84adc542023-04-11 14:47:34 -040087 sh '''
Hardik Windlasse1660492022-03-14 15:12:46 +000088 helm repo add onf https://charts.opencord.org
89 helm repo update
90 if [ ${withMonitoring} = true ] ; then
91 helm install nem-monitoring onf/nem-monitoring \
92 --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
93 --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
94 fi
95 '''
Joey Armstrong8c6f6482023-01-12 12:31:44 -050096 }
97
98 stage('Deploy Voltha') {
Hardik Windlassec9341b2021-06-07 11:58:29 +000099 if (teardown) {
Matteo Scandoloea6348e2021-06-10 18:44:58 +0200100 timeout(10) {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000101 script {
102
103 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700104 mkdir -p ${logsDir}
105 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
Hardik Windlassec9341b2021-06-07 11:58:29 +0000106 """
107
108 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
109 def localCharts = false
Matteo Scandoloea6348e2021-06-10 18:44:58 +0200110 if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000111 localCharts = true
112 }
113
114 // NOTE temporary workaround expose ONOS node ports
Andrea Campanella9e41a602021-09-14 14:45:11 +0200115 def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
Matteo Scandolo9114c892021-05-20 11:41:55 -0700116 " --set onos-classic.onosSshPort=30115 " +
Hardik Windlassec9341b2021-06-07 11:58:29 +0000117 " --set onos-classic.onosApiPort=30120 " +
118 " --set onos-classic.onosOfPort=31653 " +
119 " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800120
Hardik Windlassec9341b2021-06-07 11:58:29 +0000121 if (gerritProject != "") {
122 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
123 }
124
125 volthaDeploy([
126 infraNamespace: infraNamespace,
127 volthaNamespace: volthaNamespace,
128 workflow: workflow.toLowerCase(),
Hardik Windlass2b164342022-02-28 03:50:48 +0000129 withMacLearning: enableMacLearning.toBoolean(),
Hardik Windlassec9341b2021-06-07 11:58:29 +0000130 extraHelmFlags: localHelmFlags,
131 localCharts: localCharts,
132 bbsimReplica: olts.toInteger(),
133 dockerRegistry: registry,
134 ])
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700135 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800136
Hardik Windlassec9341b2021-06-07 11:58:29 +0000137 // stop logging
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700138 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000139 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
140 if [ -n "\$P_IDS" ]; then
141 echo \$P_IDS
142 for P_ID in \$P_IDS; do
143 kill -9 \$P_ID
144 done
145 fi
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700146 cd ${logsDir}
Hardik Windlassec9341b2021-06-07 11:58:29 +0000147 gzip -k onos-voltha-startup-combined.log
148 rm onos-voltha-startup-combined.log
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700149 """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000150 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400151
152 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000153 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
154 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
155 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
156 bbsimDmiPortFwd=50075
157 for i in {0..${olts.toInteger() - 1}}; do
158 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
159 ((bbsimDmiPortFwd++))
160 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000161 if [ ${withMonitoring} = true ] ; then
162 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
163 fi
Hardik Windlassec9341b2021-06-07 11:58:29 +0000164 ps aux | grep port-forward
165 """
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500166
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400167 // setting ONOS log level
168 script
169 {
170 setOnosLogLevels([
171 onosNamespace: infraNamespace,
172 apps: [
173 'org.opencord.dhcpl2relay',
174 'org.opencord.olt',
175 'org.opencord.aaa',
176 'org.opencord.maclearner',
177 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
178 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
179 ],
180 logLevel: logLevel
181 ])
182 } // script
183 } // if (teardown)
184 } // stage('Deploy Voltha')
185
186 stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow')
187 {
188 sh """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000189 if [ ${withMonitoring} = true ] ; then
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500190 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
191 cd "$WORKSPACE/voltha-system-tests"
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000192 make vst_venv
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400193 source ./vst_venv/bin/activate
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000194 # Collect initial memory consumption
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400195 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000196 fi
Hardik Windlasse1660492022-03-14 15:12:46 +0000197 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400198
199 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700200 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000201 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700202 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000203 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800204
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400205 make -C "$WORKSPACE/voltha-system-tests" ${testTarget}
Hardik Windlass72947302021-06-14 10:36:57 +0000206 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400207
208 getPodsInfo("${logsDir}")
209
210 sh """
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000211 set +e
212 # collect logs collected in the Robot Framework StartLogging keyword
213 cd ${logsDir}
214 gzip *-combined.log || true
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400215 rm -f *-combined.log || true
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000216 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400217
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000218 sh """
219 if [ ${withMonitoring} = true ] ; then
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500220 cd "$WORKSPACE/voltha-system-tests"
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400221 source ./vst_venv/bin/activate
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000222 # Collect memory consumption of voltha pods once all the tests are complete
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400223 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000224 fi
225 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400226 } // stage
227} // execute_test()
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800228
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400229// -----------------------------------------------------------------------
230// -----------------------------------------------------------------------
231def collectArtifacts(exitStatus)
232{
Hardik Windlassec9341b2021-06-07 11:58:29 +0000233 getPodsInfo("$WORKSPACE/${exitStatus}")
234 sh """
Joey Armstrong0eb8bd82023-07-10 13:26:25 -0400235 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
Hardik Windlassec9341b2021-06-07 11:58:29 +0000236 """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000237 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Hardik Windlassec9341b2021-06-07 11:58:29 +0000238 sh '''
239 sync
240 pkill kail || true
241 which voltctl
242 md5sum $(which voltctl)
243 '''
244 step([$class: 'RobotPublisher',
245 disableArchiveOutput: false,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700246 logFileName: "**/*/log*.html",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000247 otherFiles: '',
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700248 outputFileName: "**/*/output*.xml",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000249 outputPath: '.',
250 passThreshold: 100,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700251 reportFileName: "**/*/report*.html",
Andrea Campanellaabc09772021-06-16 12:08:57 +0200252 unstableThreshold: 0,
253 onlyCritical: true]);
Hardik Windlassec9341b2021-06-07 11:58:29 +0000254}
255
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800256pipeline {
257
258 /* no label, executor is determined by JJB */
259 agent {
260 label "${params.buildNode}"
261 }
262 options {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000263 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800264 }
265 environment {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000266 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
267 VOLTCONFIG="$HOME/.volt/config"
268 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000269 DIAGS_PROFILE="VOLTHA_PROFILE"
Matteo Scandolo35ac7822021-10-28 18:08:54 -0700270 SSHPASS="karaf"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800271 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000272 stages {
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800273 stage('Download Code') {
274 steps {
275 getVolthaCode([
276 branch: "${branch}",
277 gerritProject: "${gerritProject}",
278 gerritRefspec: "${gerritRefspec}",
279 volthaSystemTestsChange: "${volthaSystemTestsChange}",
280 volthaHelmChartsChange: "${volthaHelmChartsChange}",
281 ])
282 }
283 }
Joey Armstrong84adc542023-04-11 14:47:34 -0400284
285 stage('Build patch v1.1')
286 {
287 // build the patch only if gerritProject is specified
288 when
289 {
290 expression
291 {
292 return !gerritProject.isEmpty()
293 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000294 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800295 steps {
296 // NOTE that the correct patch has already been checked out
297 // during the getVolthaCode step
298 buildVolthaComponent("${gerritProject}")
299 }
300 }
Joey Armstronged161f72023-04-11 13:16:59 -0400301
Joey Armstrongbf492922023-04-13 17:05:09 -0400302 stage('Create K8s Cluster')
Joey Armstrong84adc542023-04-11 14:47:34 -0400303 {
Joey Armstrongbf492922023-04-13 17:05:09 -0400304 steps
Joey Armstrong84adc542023-04-11 14:47:34 -0400305 {
Joey Armstrongbf492922023-04-13 17:05:09 -0400306 script
Joey Armstrong84adc542023-04-11 14:47:34 -0400307 {
Joey Armstrong6df65002023-04-14 12:24:13 -0400308 /*
Joey Armstrongbf492922023-04-13 17:05:09 -0400309 println(' ** Calling installKind.groovy: ENTER')
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400310 // chicken-n-egg problem, kind command needed
311 // to determine if kubernetes cluster is active
Joey Armstrong41feb7c2023-04-14 11:28:04 -0400312 installKind() { debug:true }
Joey Armstrongbf492922023-04-13 17:05:09 -0400313 println(' ** Calling installKind.groovy: LEAVE')
Joey Armstrong6df65002023-04-14 12:24:13 -0400314 */
Joey Armstrong84adc542023-04-11 14:47:34 -0400315
Joey Armstrongbf492922023-04-13 17:05:09 -0400316 def clusterExists = sh(
317 returnStdout: true,
318 script: """kind get clusters | grep ${clusterName} | wc -l""")
319
320 if (clusterExists.trim() == "0")
321 {
322 createKubernetesCluster([nodes: 3, name: clusterName])
323 }
324 } // script
325 } // steps
326 } // stage('Create K8s Cluster')
Joey Armstrong84adc542023-04-11 14:47:34 -0400327
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400328 stage('Replace voltctl')
329 {
330 // if the project is voltctl override the downloaded one with the built one
331 when {
332 expression {
333 return gerritProject == "voltctl"
334 }
335 }
336
337 steps
338 {
339 sh """
Joey Armstrong93c82732023-02-07 03:01:25 -0500340 # [TODO] - why is this platform specific (?)
341 # [TODO] - revisit, command alteration has masked an error (see: voltha-2.11).
342 # find will fail when no filsystem matches are found.
343 # mv(ls) succeded simply by accident/invoked at a different time.
Hardik Windlassc6ba23e2021-06-08 08:19:52 +0000344 mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
345 chmod +x $WORKSPACE/bin/voltctl
346 """
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400347 } // steps
348 } // stage
Hardik Windlassec9341b2021-06-07 11:58:29 +0000349
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400350 stage('Load image in kind nodes')
351 {
352 when {
353 expression {
354 return !gerritProject.isEmpty()
355 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000356 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400357 steps {
358 loadToKind()
359 }
Matteo Scandolo29597f02021-02-12 10:53:57 -0800360 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400361
362 stage('Parse and execute tests')
363 {
364 steps {
365 script {
366 def tests = readYaml text: testTargets
367
368 for(int i = 0;i<tests.size();i++) {
369 def test = tests[i]
370 def target = test["target"]
371 def workflow = test["workflow"]
372 def flags = test["flags"]
373 def teardown = test["teardown"].toBoolean()
374 def logging = test["logging"].toBoolean()
375 def testLogging = 'False'
376 if (logging) {
377 testLogging = 'True'
378 }
379 println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
380 execute_test(target, workflow, testLogging, teardown, flags)
381 }
382 }
383 }
384 } // stage
385 } // stages
Joey Armstrong84adc542023-04-11 14:47:34 -0400386
387 post
388 {
389 aborted { collectArtifacts('aborted') }
390 failure { collectArtifacts('failed') }
Joey Armstrongbf492922023-04-13 17:05:09 -0400391 always { collectArtifacts('always') }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800392 }
Joey Armstrong2d5a7c12023-04-13 09:37:42 -0400393} // pipeline
Joey Armstronged161f72023-04-11 13:16:59 -0400394
395// EOF