blob: daa2becbbcc5fcfe0c18d0a23f5e2764d948999d [file] [log] [blame]
Joey Armstrong7adbfb02023-01-24 17:02:37 -05001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// voltha-2.x e2e tests for openonu-go
16// uses bbsim to simulate OLT/ONUs
17
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
24def clusterName = "kind-ci"
25
26def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
27 def infraNamespace = "default"
28 def volthaNamespace = "voltha"
29 def logsDir = "$WORKSPACE/${testTarget}"
30
31 stage('IAM')
32 {
33 script
34 {
35 String iam = [
36 'ci-management',
37 'jjb',
38 'pipeline',
39 'voltha',
Joey Armstrong6baeaae2023-02-07 01:35:55 -050040 'voltha-2.11', // release-delta
Joey Armstrong7adbfb02023-01-24 17:02:37 -050041 'bbsim-tests.groovy'
42 ].join('/')
43 println("** ${iam}: ENTER")
44
45 String cmd = "which pkill"
46 def stream = sh(
47 returnStatus:false,
48 returnStdout: true,
49 script: cmd)
50 println(" ** ${cmd}:\n${stream}")
51
52 println("** ${iam}: LEAVE")
53 }
54 }
55
56 stage('Cleanup') {
57 if (teardown) {
58 timeout(15) {
59 script {
60 helmTeardown(["default", infraNamespace, volthaNamespace])
61 }
62 timeout(1) {
63 sh returnStdout: false, script: '''
64 # remove orphaned port-forward from different namespaces
65 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
66 '''
67 }
68 }
69 }
70 }
71
72 stage ('Initialize')
73 {
74 // VOL-4926 - Is voltha-system-tests available ?
75 String cmd = [
76 'make',
77 '-C', "$WORKSPACE/voltha-system-tests",
78 "KAIL_PATH=\"$WORKSPACE/bin\"",
79 'kail',
80 ].join(' ')
81 println(" ** Running: ${cmd}:\n")
82 sh("${cmd}")
83 }
84
85 stage('Deploy common infrastructure') {
86 sh '''
87 helm repo add onf https://charts.opencord.org
88 helm repo update
89 if [ ${withMonitoring} = true ] ; then
90 helm install nem-monitoring onf/nem-monitoring \
91 --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
92 --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
93 fi
94 '''
95 }
96
97 stage('Deploy Voltha') {
98 if (teardown) {
99 timeout(10) {
100 script {
101
102 sh """
103 mkdir -p ${logsDir}
104 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
105 """
106
107 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
108 def localCharts = false
109 if (volthaHelmChartsChange != ""
110 || gerritProject == "voltha-helm-charts"
111 || branch != 'master'
112 ) {
113 localCharts = true
114 }
115
116 // NOTE temporary workaround expose ONOS node ports
117 def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
118 " --set onos-classic.onosSshPort=30115 " +
119 " --set onos-classic.onosApiPort=30120 " +
120 " --set onos-classic.onosOfPort=31653 " +
121 " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
122
123 if (gerritProject != "") {
124 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
125 }
126
127 volthaDeploy([
128 infraNamespace: infraNamespace,
129 volthaNamespace: volthaNamespace,
130 workflow: workflow.toLowerCase(),
131 withMacLearning: enableMacLearning.toBoolean(),
132 extraHelmFlags: localHelmFlags,
133 localCharts: localCharts,
134 bbsimReplica: olts.toInteger(),
135 dockerRegistry: registry,
136 ])
137 }
138
139 // stop logging
140 sh """
141 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
142 if [ -n "\$P_IDS" ]; then
143 echo \$P_IDS
144 for P_ID in \$P_IDS; do
145 kill -9 \$P_ID
146 done
147 fi
148 cd ${logsDir}
149 gzip -k onos-voltha-startup-combined.log
150 rm onos-voltha-startup-combined.log
151 """
152 }
153 sh """
154 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
155 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
156 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
157 bbsimDmiPortFwd=50075
158 for i in {0..${olts.toInteger() - 1}}; do
159 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
160 ((bbsimDmiPortFwd++))
161 done
162 if [ ${withMonitoring} = true ] ; then
163 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
164 fi
165 ps aux | grep port-forward
166 """
167 // setting ONOS log level
168 script {
169 setOnosLogLevels([
170 onosNamespace: infraNamespace,
171 apps: [
172 'org.opencord.dhcpl2relay',
173 'org.opencord.olt',
174 'org.opencord.aaa',
175 'org.opencord.maclearner',
176 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
177 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
178 ],
179 logLevel: logLevel
180 ])
181 }
182 }
183 }
184
185 stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
186 sh """
187 if [ ${withMonitoring} = true ] ; then
188 mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
189 cd "$WORKSPACE/voltha-system-tests"
190 make vst_venv
191 source ./vst_venv/bin/activate || true
192 # Collect initial memory consumption
193 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
194 fi
195 """
196 sh """
197 mkdir -p ${logsDir}
198 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
199 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
200 export KVSTOREPREFIX=voltha/voltha_voltha
201
202 make -C "$WORKSPACE/voltha-system-tests" ${testTarget} || true
203 """
204 getPodsInfo("${logsDir}")
205 sh """
206 set +e
207 # collect logs collected in the Robot Framework StartLogging keyword
208 cd ${logsDir}
209 gzip *-combined.log || true
210 rm *-combined.log || true
211 """
212 sh """
213 if [ ${withMonitoring} = true ] ; then
214 cd "$WORKSPACE/voltha-system-tests"
215 source ./vst_venv/bin/activate || true
216 # Collect memory consumption of voltha pods once all the tests are complete
217 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
218 fi
219 """
220 }
221}
222
223def collectArtifacts(exitStatus) {
224 getPodsInfo("$WORKSPACE/${exitStatus}")
225 sh """
226 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
227 """
228 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
229 sh '''
230 sync
231 pkill kail || true
232 which voltctl
233 md5sum $(which voltctl)
234 '''
235 step([$class: 'RobotPublisher',
236 disableArchiveOutput: false,
237 logFileName: "**/*/log*.html",
238 otherFiles: '',
239 outputFileName: "**/*/output*.xml",
240 outputPath: '.',
241 passThreshold: 100,
242 reportFileName: "**/*/report*.html",
243 unstableThreshold: 0,
244 onlyCritical: true]);
245}
246
247pipeline {
248
249 /* no label, executor is determined by JJB */
250 agent {
251 label "${params.buildNode}"
252 }
253 options {
254 timeout(time: "${timeout}", unit: 'MINUTES')
255 }
256 environment {
257 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
258 VOLTCONFIG="$HOME/.volt/config"
259 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
260 DIAGS_PROFILE="VOLTHA_PROFILE"
261 SSHPASS="karaf"
262 }
263
264 stages {
265 stage('Download Code') {
266 steps {
267 getVolthaCode([
268 branch: "${branch}",
269 gerritProject: "${gerritProject}",
270 gerritRefspec: "${gerritRefspec}",
271 volthaSystemTestsChange: "${volthaSystemTestsChange}",
272 volthaHelmChartsChange: "${volthaHelmChartsChange}",
273 ])
274 }
275 }
276
277 stage('Build patch') {
278 // build the patch only if gerritProject is specified
279 when {
280 expression {
281 return !gerritProject.isEmpty()
282 }
283 }
284 steps {
285 // NOTE that the correct patch has already been checked out
286 // during the getVolthaCode step
287 buildVolthaComponent("${gerritProject}")
288 }
289 }
290 stage('Create K8s Cluster') {
291 steps {
292 script {
293 def clusterExists = sh returnStdout: true, script: """
294 kind get clusters | grep ${clusterName} | wc -l
295 """
296 if (clusterExists.trim() == "0") {
297 createKubernetesCluster([nodes: 3, name: clusterName])
298 }
299 }
300 }
301 }
302 stage('Replace voltctl') {
303 // if the project is voltctl override the downloaded one with the built one
304 when {
305 expression {
306 return gerritProject == "voltctl"
307 }
308 }
309 steps{
310 sh """
311 # [TODO] - why is this platform specific (?)
Joey Armstrong93c82732023-02-07 03:01:25 -0500312 # [TODO] - revisit, command alteration has masked an error (see: voltha-2.11).
313 # find will fail when no filsystem matches are found.
314 # mv(ls) succeded simply by accident/invoked at a different time.
Joey Armstrong7adbfb02023-01-24 17:02:37 -0500315 # find "$WORKSPACE/voltctl/release" -name 'voltctl-*-linux-amd*' \
Joey Armstrong93c82732023-02-07 03:01:25 -0500316 # -exec mv {} $WORKSPACE/bin/voltctl ;
Joey Armstrong7adbfb02023-01-24 17:02:37 -0500317 mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
318 chmod +x $WORKSPACE/bin/voltctl
319 """
320 }
321 }
322 stage('Load image in kind nodes') {
323 when {
324 expression {
325 return !gerritProject.isEmpty()
326 }
327 }
328 steps {
329 loadToKind()
330 }
331 }
332 stage('Parse and execute tests') {
333 steps {
334 script {
335 def tests = readYaml text: testTargets
336
337 for(int i = 0;i<tests.size();i++) {
338 def test = tests[i]
339 def target = test["target"]
340 def workflow = test["workflow"]
341 def flags = test["flags"]
342 def teardown = test["teardown"].toBoolean()
343 def logging = test["logging"].toBoolean()
344 def testLogging = 'False'
345 if (logging) {
346 testLogging = 'True'
347 }
348 println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
349 execute_test(target, workflow, testLogging, teardown, flags)
350 }
351 }
352 }
353 }
354 }
355 post {
356 aborted {
357 collectArtifacts("aborted")
358 }
359 failure {
360 collectArtifacts("failed")
361 }
362 always {
363 collectArtifacts("always")
364 }
365 }
366}