blob: edfcb6c098acc0ffdf203de1987deb1a4e89f369 [file] [log] [blame]
Hardik Windlassec9341b2021-06-07 11:58:29 +00001// Copyright 2021-present Open Networking Foundation
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Matteo Scandoloa156b572021-02-04 11:52:18 -080018library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
Hardik Windlassec9341b2021-06-07 11:58:29 +000024def clusterName = "kind-ci"
25
Hardik Windlass60fc0da2021-10-14 07:02:40 +000026def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
Hardik Windlassec9341b2021-06-07 11:58:29 +000027 def infraNamespace = "default"
28 def volthaNamespace = "voltha"
Matteo Scandolo9c230bb2021-10-22 12:48:39 -070029 def logsDir = "$WORKSPACE/${testTarget}"
Hardik Windlassec9341b2021-06-07 11:58:29 +000030 stage('Cleanup') {
31 if (teardown) {
32 timeout(15) {
33 script {
34 helmTeardown(["default", infraNamespace, volthaNamespace])
35 }
36 timeout(1) {
37 sh returnStdout: false, script: '''
38 # remove orphaned port-forward from different namespaces
39 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
40 '''
41 }
42 }
43 }
44 }
Hardik Windlasse1660492022-03-14 15:12:46 +000045 stage('Deploy common infrastructure') {
46 sh '''
47 helm repo add onf https://charts.opencord.org
48 helm repo update
49 if [ ${withMonitoring} = true ] ; then
50 helm install nem-monitoring onf/nem-monitoring \
51 --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
52 --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
53 fi
54 '''
55 }
Hardik Windlassec9341b2021-06-07 11:58:29 +000056 stage('Deploy Voltha') {
57 if (teardown) {
Matteo Scandoloea6348e2021-06-10 18:44:58 +020058 timeout(10) {
Hardik Windlassec9341b2021-06-07 11:58:29 +000059 script {
60
61 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -070062 mkdir -p ${logsDir}
63 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
Hardik Windlassec9341b2021-06-07 11:58:29 +000064 """
65
66 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
67 def localCharts = false
Matteo Scandoloea6348e2021-06-10 18:44:58 +020068 if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
Hardik Windlassec9341b2021-06-07 11:58:29 +000069 localCharts = true
70 }
71
72 // NOTE temporary workaround expose ONOS node ports
Andrea Campanella9e41a602021-09-14 14:45:11 +020073 def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
Matteo Scandolo9114c892021-05-20 11:41:55 -070074 " --set onos-classic.onosSshPort=30115 " +
Hardik Windlassec9341b2021-06-07 11:58:29 +000075 " --set onos-classic.onosApiPort=30120 " +
76 " --set onos-classic.onosOfPort=31653 " +
77 " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
Matteo Scandolo42f6e572021-01-25 15:11:34 -080078
Hardik Windlassec9341b2021-06-07 11:58:29 +000079 if (gerritProject != "") {
80 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
81 }
82
83 volthaDeploy([
84 infraNamespace: infraNamespace,
85 volthaNamespace: volthaNamespace,
86 workflow: workflow.toLowerCase(),
Hardik Windlass2b164342022-02-28 03:50:48 +000087 withMacLearning: enableMacLearning.toBoolean(),
Hardik Windlassec9341b2021-06-07 11:58:29 +000088 extraHelmFlags: localHelmFlags,
89 localCharts: localCharts,
90 bbsimReplica: olts.toInteger(),
91 dockerRegistry: registry,
92 ])
Matteo Scandolod17de3a2021-04-09 15:47:52 -070093 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -080094
Hardik Windlassec9341b2021-06-07 11:58:29 +000095 // stop logging
Matteo Scandolod17de3a2021-04-09 15:47:52 -070096 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +000097 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
98 if [ -n "\$P_IDS" ]; then
99 echo \$P_IDS
100 for P_ID in \$P_IDS; do
101 kill -9 \$P_ID
102 done
103 fi
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700104 cd ${logsDir}
Hardik Windlassec9341b2021-06-07 11:58:29 +0000105 gzip -k onos-voltha-startup-combined.log
106 rm onos-voltha-startup-combined.log
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700107 """
Hardik Windlassec9341b2021-06-07 11:58:29 +0000108 }
109 sh """
110 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
111 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
112 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
113 bbsimDmiPortFwd=50075
114 for i in {0..${olts.toInteger() - 1}}; do
115 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
116 ((bbsimDmiPortFwd++))
117 done
Hardik Windlass3bb089a2022-03-22 17:56:03 +0000118 if [ ${withMonitoring} = true ] ; then
119 JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
120 fi
Hardik Windlassec9341b2021-06-07 11:58:29 +0000121 ps aux | grep port-forward
122 """
Matteo Scandolo35ac7822021-10-28 18:08:54 -0700123 // setting ONOS log level
Matteo Scandolo9e8ba502021-11-17 13:56:56 -0800124 script {
125 setOnosLogLevels([
126 onosNamespace: infraNamespace,
127 apps: [
128 'org.opencord.dhcpl2relay',
129 'org.opencord.olt',
130 'org.opencord.aaa',
Andrea Campanellac0762392022-03-11 15:01:13 +0100131 'org.opencord.maclearner',
Matteo Scandolo9e8ba502021-11-17 13:56:56 -0800132 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
133 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
134 ],
135 logLevel: logLevel
136 ])
137 }
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700138 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800139 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000140 stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
Hardik Windlass72947302021-06-14 10:36:57 +0000141 sh """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000142 if [ ${withMonitoring} = true ] ; then
143 mkdir -p $WORKSPACE/voltha-pods-mem-consumption-${workflow}
144 cd $WORKSPACE/voltha-system-tests
145 make vst_venv
146 source ./vst_venv/bin/activate || true
147 # Collect initial memory consumption
148 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
149 fi
Hardik Windlasse1660492022-03-14 15:12:46 +0000150 """
151 sh """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700152 mkdir -p ${logsDir}
Hardik Windlassd62442d2021-11-30 10:51:20 +0000153 export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700154 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
Hardik Windlass72947302021-06-14 10:36:57 +0000155 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800156
Hardik Windlass72947302021-06-14 10:36:57 +0000157 make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
158 """
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700159 getPodsInfo("${logsDir}")
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000160 sh """
161 set +e
162 # collect logs collected in the Robot Framework StartLogging keyword
163 cd ${logsDir}
164 gzip *-combined.log || true
165 rm *-combined.log || true
166 """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000167 sh """
168 if [ ${withMonitoring} = true ] ; then
169 cd $WORKSPACE/voltha-system-tests
170 source ./vst_venv/bin/activate || true
171 # Collect memory consumption of voltha pods once all the tests are complete
172 python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
173 fi
174 """
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800175 }
176}
177
Hardik Windlassec9341b2021-06-07 11:58:29 +0000178def collectArtifacts(exitStatus) {
179 getPodsInfo("$WORKSPACE/${exitStatus}")
180 sh """
181 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
182 """
Hardik Windlass8cd517c2022-03-22 04:01:40 +0000183 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
Hardik Windlassec9341b2021-06-07 11:58:29 +0000184 sh '''
185 sync
186 pkill kail || true
187 which voltctl
188 md5sum $(which voltctl)
189 '''
190 step([$class: 'RobotPublisher',
191 disableArchiveOutput: false,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700192 logFileName: "**/*/log*.html",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000193 otherFiles: '',
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700194 outputFileName: "**/*/output*.xml",
Hardik Windlassec9341b2021-06-07 11:58:29 +0000195 outputPath: '.',
196 passThreshold: 100,
Matteo Scandolo9c230bb2021-10-22 12:48:39 -0700197 reportFileName: "**/*/report*.html",
Andrea Campanellaabc09772021-06-16 12:08:57 +0200198 unstableThreshold: 0,
199 onlyCritical: true]);
Hardik Windlassec9341b2021-06-07 11:58:29 +0000200}
201
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800202pipeline {
203
204 /* no label, executor is determined by JJB */
205 agent {
206 label "${params.buildNode}"
207 }
208 options {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000209 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800210 }
211 environment {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000212 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
213 VOLTCONFIG="$HOME/.volt/config"
214 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Hardik Windlassec9341b2021-06-07 11:58:29 +0000215 DIAGS_PROFILE="VOLTHA_PROFILE"
Matteo Scandolo35ac7822021-10-28 18:08:54 -0700216 SSHPASS="karaf"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800217 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000218 stages {
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800219 stage('Download Code') {
220 steps {
221 getVolthaCode([
222 branch: "${branch}",
223 gerritProject: "${gerritProject}",
224 gerritRefspec: "${gerritRefspec}",
225 volthaSystemTestsChange: "${volthaSystemTestsChange}",
226 volthaHelmChartsChange: "${volthaHelmChartsChange}",
227 ])
228 }
229 }
230 stage('Build patch') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000231 // build the patch only if gerritProject is specified
232 when {
233 expression {
234 return !gerritProject.isEmpty()
235 }
236 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800237 steps {
238 // NOTE that the correct patch has already been checked out
239 // during the getVolthaCode step
240 buildVolthaComponent("${gerritProject}")
241 }
242 }
243 stage('Create K8s Cluster') {
244 steps {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000245 script {
246 def clusterExists = sh returnStdout: true, script: """
247 kind get clusters | grep ${clusterName} | wc -l
248 """
249 if (clusterExists.trim() == "0") {
250 createKubernetesCluster([nodes: 3, name: clusterName])
251 }
252 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800253 }
254 }
Hardik Windlassc6ba23e2021-06-08 08:19:52 +0000255 stage('Replace voltctl') {
256 // if the project is voltctl override the downloaded one with the built one
257 when {
258 expression {
259 return gerritProject == "voltctl"
260 }
261 }
262 steps{
263 sh """
264 mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
265 chmod +x $WORKSPACE/bin/voltctl
266 """
267 }
268 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800269 stage('Load image in kind nodes') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000270 when {
271 expression {
272 return !gerritProject.isEmpty()
273 }
274 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800275 steps {
276 loadToKind()
277 }
278 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000279 stage('Parse and execute tests') {
280 steps {
281 script {
282 def tests = readYaml text: testTargets
283
284 for(int i = 0;i<tests.size();i++) {
285 def test = tests[i]
286 def target = test["target"]
287 def workflow = test["workflow"]
288 def flags = test["flags"]
289 def teardown = test["teardown"].toBoolean()
Hardik Windlass24e275f2021-10-19 08:19:06 +0000290 def logging = test["logging"].toBoolean()
291 def testLogging = 'False'
292 if (logging) {
293 testLogging = 'True'
294 }
295 println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
296 execute_test(target, workflow, testLogging, teardown, flags)
Hardik Windlassec9341b2021-06-07 11:58:29 +0000297 }
298 }
Matteo Scandolo29597f02021-02-12 10:53:57 -0800299 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800300 }
301 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800302 post {
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800303 aborted {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000304 collectArtifacts("aborted")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800305 }
306 failure {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000307 collectArtifacts("failed")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800308 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800309 always {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000310 collectArtifacts("always")
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800311 }
312 }
313}