blob: 6a2330f02ae4b9d4d6beb5844963a46009547e63 [file] [log] [blame]
Joey Armstrong8c6f6482023-01-12 12:31:44 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Matteo Scandoloa156b572021-02-04 11:52:18 -080018library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
Hardik Windlassec9341b2021-06-07 11:58:29 +000024def clusterName = "kind-ci"
25
Hardik Windlass60fc0da2021-10-14 07:02:40 +000026def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
Joey Armstrong8c6f6482023-01-12 12:31:44 -050027 def infraNamespace = "default"
28 def volthaNamespace = "voltha"
29 def logsDir = "$WORKSPACE/${testTarget}"
Joey Armstrong293e16b2022-11-26 20:16:33 -050030
31 stage('IAM')
32 {
33 script
34 {
35 String iam = [
36 'ci-management',
37 'jjb',
38 'pipeline',
39 'voltha',
40 'master',
41 'bbsim-tests.groovy'
42 ].join('/')
43 println("** ${iam}: ENTER")
44
45 String cmd = "which pkill"
46 def stream = sh(
47 returnStatus:false,
48 returnStdout: true,
49 script: cmd)
50 println(" ** ${cmd}:\n${stream}")
51
52 println("** ${iam}: LEAVE")
53 }
54 }
55
56 stage('Cleanup') {
Joey Armstrong8c6f6482023-01-12 12:31:44 -050057 if (teardown) {
58 timeout(15) {
59 script {
60 helmTeardown(["default", infraNamespace, volthaNamespace])
61 }
62 timeout(1) {
63 sh returnStdout: false, script: '''
Hardik Windlassec9341b2021-06-07 11:58:29 +000064 # remove orphaned port-forward from different namespaces
65 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
66 '''
Joey Armstrong8c6f6482023-01-12 12:31:44 -050067 }
68 }
69 }
Hardik Windlassec9341b2021-06-07 11:58:29 +000070 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -050071
72 stage ('Initialize')
73 {
74 // VOL-4926 - Is voltha-system-tests available ?
75 String cmd = [
76 'make',
77 '-C', "$WORKSPACE/voltha-system-tests",
78 "KAIL_PATH=\"$WORKSPACE/bin\"",
79 'kail',
Joey Armstrongbeef4cd2023-01-18 09:59:58 -050080 ].join(' ')
Joey Armstrong8c6f6482023-01-12 12:31:44 -050081 println(" ** Running: ${cmd}:\n")
Joey Armstrongbeef4cd2023-01-18 09:59:58 -050082 sh("${cmd}")
Joey Armstrong8c6f6482023-01-12 12:31:44 -050083 }
84
85 stage('Deploy common infrastructure') {
86 sh '''
Hardik Windlasse1660492022-03-14 15:12:46 +000087 helm repo add onf https://charts.opencord.org
88 helm repo update
89 if [ ${withMonitoring} = true ] ; then
90 helm install nem-monitoring onf/nem-monitoring \
91 --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
92 --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
93 fi
94 '''
Joey Armstrong8c6f6482023-01-12 12:31:44 -050095 }
96
97 stage('Deploy Voltha') {
Hardik Windlassec9341b2021-06-07 11:58:29 +000098 if (teardown) {
Matteo Scandoloea6348e2021-06-10 18:44:58 +020099 timeout(10) {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000100 script {
101
102 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700103 mkdir -p ${logsDir}
104 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
Hardik Windlassec9341b2021-06-07 11:58:29 +0000105 """
106
107 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
108 def localCharts = false
Matteo Scandoloea6348e2021-06-10 18:44:58 +0200109 if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000110 localCharts = true
111 }
112
113 // NOTE temporary workaround expose ONOS node ports
Andrea Campanella9e41a602021-09-14 14:45:11 +0200114 def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
Matteo Scandolo9114c892021-05-20 11:41:55 -0700115 " --set onos-classic.onosSshPort=30115 " +
Hardik Windlassec9341b2021-06-07 11:58:29 +0000116 " --set onos-classic.onosApiPort=30120 " +
117 " --set onos-classic.onosOfPort=31653 " +
118 " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800119
Hardik Windlassec9341b2021-06-07 11:58:29 +0000120 if (gerritProject != "") {
121 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
122 }
123
124 volthaDeploy([
125 infraNamespace: infraNamespace,
126 volthaNamespace: volthaNamespace,
127 workflow: workflow.toLowerCase(),
Hardik Windlass2b164342022-02-28 03:50:48 +0000128 withMacLearning: enableMacLearning.toBoolean(),
Hardik Windlassec9341b2021-06-07 11:58:29 +0000129 extraHelmFlags: localHelmFlags,
130 localCharts: localCharts,
131 bbsimReplica: olts.toInteger(),
132 dockerRegistry: registry,
133 ])
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700134 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -0800135
Hardik Windlassec9341b2021-06-07 11:58:29 +0000136 // stop logging
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700137 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000138 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
139 if [ -n "\$P_IDS" ]; then
140 echo \$P_IDS
141 for P_ID in \$P_IDS; do
142 kill -9 \$P_ID
143 done
144 fi
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700145 cd ${logsDir}
Hardik Windlassec9341b2021-06-07 11:58:29 +0000146 gzip -k onos-voltha-startup-combined.log
147 rm onos-voltha-startup-combined.log
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700148 """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000149 }
150 sh """
151 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
152 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
153 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
154 bbsimDmiPortFwd=50075
155 for i in {0..${olts.toInteger() - 1}}; do
156 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
157 ((bbsimDmiPortFwd++))
158 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000159 if [ ${withMonitoring} = true ] ; then
160 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
161 fi
Hardik Windlassec9341b2021-06-07 11:58:29 +0000162 ps aux | grep port-forward
163 """
Matteo Scandolo35ac7822021-10-28 18:08:54 -0700164 // setting ONOS log level
Matteo Scandolo9e8ba502021-11-17 13:56:56 -0800165 script {
166 setOnosLogLevels([
167 onosNamespace: infraNamespace,
168 apps: [
169 'org.opencord.dhcpl2relay',
170 'org.opencord.olt',
171 'org.opencord.aaa',
Andrea Campanellac0762392022-03-11 15:01:13 +0100172 'org.opencord.maclearner',
Matteo Scandolo9e8ba502021-11-17 13:56:56 -0800173 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
174 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
175 ],
176 logLevel: logLevel
177 ])
178 }
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700179 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800180 }
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500181
Hardik Windlassec9341b2021-06-07 11:58:29 +0000182 stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
Hardik Windlass72947302021-06-14 10:36:57 +0000183 sh """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000184 if [ ${withMonitoring} = true ] ; then
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500185 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
186 cd "$WORKSPACE/voltha-system-tests"
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000187 make vst_venv
188 source ./vst_venv/bin/activate || true
189 # Collect initial memory consumption
190 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
191 fi
Hardik Windlasse1660492022-03-14 15:12:46 +0000192 """
193 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700194 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000195 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700196 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000197 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800198
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500199 make -C "$WORKSPACE/voltha-system-tests" ${testTarget} || true
Hardik Windlass72947302021-06-14 10:36:57 +0000200 """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700201 getPodsInfo("${logsDir}")
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000202 sh """
203 set +e
204 # collect logs collected in the Robot Framework StartLogging keyword
205 cd ${logsDir}
206 gzip *-combined.log || true
207 rm *-combined.log || true
208 """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000209 sh """
210 if [ ${withMonitoring} = true ] ; then
Joey Armstrong8c6f6482023-01-12 12:31:44 -0500211 cd "$WORKSPACE/voltha-system-tests"
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000212 source ./vst_venv/bin/activate || true
213 # Collect memory consumption of voltha pods once all the tests are complete
214 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
215 fi
216 """
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800217 }
218}
219
Hardik Windlassec9341b2021-06-07 11:58:29 +0000220def collectArtifacts(exitStatus) {
221 getPodsInfo("$WORKSPACE/${exitStatus}")
222 sh """
223 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
224 """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000225 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Hardik Windlassec9341b2021-06-07 11:58:29 +0000226 sh '''
227 sync
228 pkill kail || true
229 which voltctl
230 md5sum $(which voltctl)
231 '''
232 step([$class: 'RobotPublisher',
233 disableArchiveOutput: false,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700234 logFileName: "**/*/log*.html",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000235 otherFiles: '',
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700236 outputFileName: "**/*/output*.xml",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000237 outputPath: '.',
238 passThreshold: 100,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700239 reportFileName: "**/*/report*.html",
Andrea Campanellaabc09772021-06-16 12:08:57 +0200240 unstableThreshold: 0,
241 onlyCritical: true]);
Hardik Windlassec9341b2021-06-07 11:58:29 +0000242}
243
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800244pipeline {
245
246 /* no label, executor is determined by JJB */
247 agent {
248 label "${params.buildNode}"
249 }
250 options {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000251 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800252 }
253 environment {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000254 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
255 VOLTCONFIG="$HOME/.volt/config"
256 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000257 DIAGS_PROFILE="VOLTHA_PROFILE"
Matteo Scandolo35ac7822021-10-28 18:08:54 -0700258 SSHPASS="karaf"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800259 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000260 stages {
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800261 stage('Download Code') {
262 steps {
263 getVolthaCode([
264 branch: "${branch}",
265 gerritProject: "${gerritProject}",
266 gerritRefspec: "${gerritRefspec}",
267 volthaSystemTestsChange: "${volthaSystemTestsChange}",
268 volthaHelmChartsChange: "${volthaHelmChartsChange}",
269 ])
270 }
271 }
272 stage('Build patch') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000273 // build the patch only if gerritProject is specified
274 when {
275 expression {
276 return !gerritProject.isEmpty()
277 }
278 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800279 steps {
280 // NOTE that the correct patch has already been checked out
281 // during the getVolthaCode step
282 buildVolthaComponent("${gerritProject}")
283 }
284 }
285 stage('Create K8s Cluster') {
286 steps {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000287 script {
288 def clusterExists = sh returnStdout: true, script: """
289 kind get clusters | grep ${clusterName} | wc -l
290 """
291 if (clusterExists.trim() == "0") {
292 createKubernetesCluster([nodes: 3, name: clusterName])
293 }
294 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800295 }
296 }
Hardik Windlassc6ba23e2021-06-08 08:19:52 +0000297 stage('Replace voltctl') {
298 // if the project is voltctl override the downloaded one with the built one
299 when {
300 expression {
301 return gerritProject == "voltctl"
302 }
303 }
304 steps{
305 sh """
Joey Armstrong93c82732023-02-07 03:01:25 -0500306 # [TODO] - why is this platform specific (?)
307 # [TODO] - revisit, command alteration has masked an error (see: voltha-2.11).
308 # find will fail when no filsystem matches are found.
309 # mv(ls) succeded simply by accident/invoked at a different time.
Hardik Windlassc6ba23e2021-06-08 08:19:52 +0000310 mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
311 chmod +x $WORKSPACE/bin/voltctl
312 """
313 }
314 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800315 stage('Load image in kind nodes') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000316 when {
317 expression {
318 return !gerritProject.isEmpty()
319 }
320 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800321 steps {
322 loadToKind()
323 }
324 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000325 stage('Parse and execute tests') {
326 steps {
327 script {
328 def tests = readYaml text: testTargets
329
330 for(int i = 0;i<tests.size();i++) {
331 def test = tests[i]
332 def target = test["target"]
333 def workflow = test["workflow"]
334 def flags = test["flags"]
335 def teardown = test["teardown"].toBoolean()
Hardik Windlass24e275f2021-10-19 08:19:06 +0000336 def logging = test["logging"].toBoolean()
337 def testLogging = 'False'
338 if (logging) {
339 testLogging = 'True'
340 }
341 println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
342 execute_test(target, workflow, testLogging, teardown, flags)
Hardik Windlassec9341b2021-06-07 11:58:29 +0000343 }
344 }
Matteo Scandolo29597f02021-02-12 10:53:57 -0800345 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800346 }
347 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800348 post {
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800349 aborted {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000350 collectArtifacts("aborted")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800351 }
352 failure {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000353 collectArtifacts("failed")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800354 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800355 always {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000356 collectArtifacts("always")
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800357 }
358 }
359}