blob: 6f053c7b5159a17f5e8b2e540a7e19b6e8c605f0 [file] [log] [blame]
Matteo Scandolo4b006ae2020-11-09 16:14:40 -08001// Copyright 2019-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// deploy VOLTHA using kind-voltha and performs a scale test
16
17pipeline {
18
19 /* no label, executor is determined by JJB */
20 agent {
21 label "${params.buildNode}"
22 }
23 options {
Matteo Scandolof7ca6312020-11-16 15:57:15 -080024 timeout(time: 120, unit: 'MINUTES')
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080025 }
26 environment {
27 JENKINS_NODE_COOKIE="dontKillMe" // do not kill processes after the build is done
28 KUBECONFIG="$HOME/.kube/config"
29 SSHPASS="karaf"
30 PATH="$PATH:$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
31 SCHEDULE_ON_CONTROL_NODES="yes"
32 FANCY=0
33 WAIT_ON_DOWN="yes"
34 WITH_SIM_ADAPTERS="no"
35 WITH_RADIUS="${withRadius}"
36 WITH_BBSIM="yes"
37 LEGACY_BBSIM_INDEX="no"
38 DEPLOY_K8S="no"
39 CONFIG_SADIS="external"
40 WITH_KAFKA="yes"
41 WITH_ETCD="yes"
42 VOLTHA_ETCD_PORT=9999
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080043 INFRA_NS="infra"
44
45 // configurable options
46 WITH_EAPOL="${withEapol}"
47 WITH_DHCP="${withDhcp}"
48 WITH_IGMP="${withIgmp}"
49 VOLTHA_LOG_LEVEL="${logLevel}"
50 NUM_OF_BBSIM="${olts}"
51 NUM_OF_OPENONU="${openonuAdapterReplicas}"
52 NUM_OF_ONOS="${onosReplicas}"
53 NUM_OF_ATOMIX="${atomixReplicas}"
54 NUM_OF_KAFKA="${kafkaReplicas}"
55 NUM_OF_ETCD="${etcdReplicas}"
56 WITH_PPROF="${withProfiling}"
57 EXTRA_HELM_FLAGS="${extraHelmFlags} " // note that the trailing space is required to separate the parameters from appends done later
58 VOLTHA_CHART="${volthaChart}"
59 VOLTHA_BBSIM_CHART="${bbsimChart}"
60 VOLTHA_ADAPTER_OPEN_OLT_CHART="${openoltAdapterChart}"
61 VOLTHA_ADAPTER_OPEN_ONU_CHART="${openonuAdapterChart}"
62 ONOS_CLASSIC_CHART="${onosChart}"
63 RADIUS_CHART="${radiusChart}"
64
65 APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius"
66 LOG_FOLDER="$WORKSPACE/logs"
67
68 GERRIT_PROJECT="${GERRIT_PROJECT}"
69 }
70
71 stages {
72 stage ('Cleanup') {
73 steps {
74 timeout(time: 11, unit: 'MINUTES') {
75 sh returnStdout: false, script: """
Matteo Scandolobe823242020-11-20 13:48:13 -080076 helm repo add stable https://charts.helm.sh/stable
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080077 helm repo add onf https://charts.opencord.org
78 helm repo add cord https://charts.opencord.org
79 helm repo add onos https://charts.onosproject.org
80 helm repo add atomix https://charts.atomix.io
81 helm repo add bbsim-sadis https://ciena.github.io/bbsim-sadis-server/charts
82 helm repo update
83
84 # removing ETCD port forward
85 P_ID="\$(ps e -ww -A | grep "_TAG=etcd-port-forward" | grep -v grep | awk '{print \$1}')"
86 if [ -n "\$P_ID" ]; then
87 kill -9 \$P_ID
88 fi
89
Matteo Scandolo9fe8cad2020-11-13 12:54:19 -080090 NAMESPACES="voltha1 voltha2 infra default"
91 for NS in \$NAMESPACES
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080092 do
Matteo Scandolo9fe8cad2020-11-13 12:54:19 -080093 for hchart in \$(helm list -n \$NS -q | grep -E -v 'docker-registry|kafkacat');
94 do
95 echo "Purging chart: \${hchart}"
96 helm delete -n \$NS "\${hchart}"
97 done
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080098 done
Matteo Scandolo4b006ae2020-11-09 16:14:40 -080099
100 test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
101
102 cd $WORKSPACE
103 rm -rf $WORKSPACE/*
104 """
105 }
106 }
107 }
108 stage('Clone kind-voltha') {
109 steps {
110 checkout([
111 $class: 'GitSCM',
112 userRemoteConfigs: [[
113 url: "https://gerrit.opencord.org/kind-voltha",
114 refspec: "${kindVolthaChange}"
115 ]],
116 branches: [[ name: "master", ]],
117 extensions: [
118 [$class: 'WipeWorkspace'],
119 [$class: 'RelativeTargetDirectory', relativeTargetDir: "kind-voltha"],
120 [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
121 ],
122 ])
123 script {
124 sh(script:"""
125 if [ '${kindVolthaChange}' != '' ] ; then
126 cd $WORKSPACE/kind-voltha;
127 git fetch https://gerrit.opencord.org/kind-voltha ${kindVolthaChange} && git checkout FETCH_HEAD
128 fi
129 """)
130 }
131 }
132 }
133 stage('Clone voltha-system-tests') {
134 steps {
135 checkout([
136 $class: 'GitSCM',
137 userRemoteConfigs: [[
138 url: "https://gerrit.opencord.org/voltha-system-tests",
139 refspec: "${volthaSystemTestsChange}"
140 ]],
141 branches: [[ name: "${release}", ]],
142 extensions: [
143 [$class: 'WipeWorkspace'],
144 [$class: 'RelativeTargetDirectory', relativeTargetDir: "voltha-system-tests"],
145 [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
146 ],
147 ])
148 script {
149 sh(script:"""
150 if [ '${volthaSystemTestsChange}' != '' ] ; then
151 cd $WORKSPACE/voltha-system-tests;
152 git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD
153 fi
154 """)
155 }
156 }
157 }
158 stage('Build patch') {
159 when {
160 expression {
161 return params.GERRIT_PROJECT
162 }
163 }
164 steps {
165 sh """
166 git clone https://\$GERRIT_HOST/\$GERRIT_PROJECT
167 cd \$GERRIT_PROJECT
168 git fetch https://\$GERRIT_HOST/\$GERRIT_PROJECT \$GERRIT_REFSPEC && git checkout FETCH_HEAD
169
170 DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-build
171 DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-push
172 """
173 }
174 }
175 stage('Deploy common infrastructure') {
176 // includes monitoring, kafka, etcd
177 steps {
178 sh '''
179 if [ ${withMonitoring} = true ] ; then
Matteo Scandolod4d55b72020-11-25 12:29:07 -0800180 helm install -n $INFRA_NS nem-monitoring cord/nem-monitoring \
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800181 -f $HOME/voltha-scale/grafana.yaml \
182 --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
183 --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
184 fi
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800185 '''
186 }
187 }
188 stage('Deploy VOLTHA infrastructure') {
189 steps {
190 sh returnStdout: false, script: '''
Matteo Scandolo128ee7a2020-12-03 14:16:54 -0800191
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800192 cd $WORKSPACE/kind-voltha/
193
Matteo Scandolof6656562020-11-16 14:45:44 -0800194 export ETCD_CHART=$HOME/teone/helm-charts/etcd
195 export KAFKA_CHART=$HOME/teone/helm-charts/kafka
Matteo Scandolob28a97b2020-11-13 10:58:32 -0800196
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800197 # KAFKA config
Matteo Scandolof6656562020-11-16 14:45:44 -0800198 export NUM_OF_KAFKA=${kafkaReplicas}
199 export EXTRA_HELM_FLAGS+=' --set prometheus.kafka.enabled=true,prometheus.operator.enabled=true,prometheus.jmx.enabled=true,prometheus.operator.serviceMonitor.namespace=default '
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800200
201 # ETCD config
Matteo Scandolof6656562020-11-16 14:45:44 -0800202 export EXTRA_HELM_FLAGS+=" --set memoryMode=${inMemoryEtcdStorage} "
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800203
204 NAME=infra JUST_INFRA=y ./voltha up
205
206 # Forward the ETCD port onto $VOLTHA_ETCD_PORT
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800207 _TAG=etcd-port-forward kubectl -n \$INFRA_NS port-forward --address 0.0.0.0 -n default service/etcd $VOLTHA_ETCD_PORT:2379&
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800208 '''
209 }
210 }
211 stage('Deploy Voltha') {
212 steps {
213 deploy_voltha_stacks(params.volthaStacks)
214 }
215 }
216 stage('Start logging') {
217 steps {
218 sh returnStdout: false, script: '''
219 # start logging with kail
220
221 mkdir -p $LOG_FOLDER
222
223 list=($APPS_TO_LOG)
224 for app in "${list[@]}"
225 do
226 echo "Starting logs for: ${app}"
227 _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log&
228 done
229 '''
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800230 }
231 }
232 stage('Configuration') {
233 steps {
234 script {
235 sh returnStdout: false, script: """
Matteo Scandolo68978752020-12-02 12:23:42 -0800236
Matteo Scandoloffe19c92020-11-24 15:25:25 -0800237 # TODO this needs to be repeated per stack
238 # kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false
239
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800240 #Setting link discovery
241 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${withLLDP}
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800242 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true
243 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800244 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 1000
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800245 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500
246
Matteo Scandoloffe19c92020-11-24 15:25:25 -0800247 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.onosproject
248 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.opencord
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800249
250 # Set Flows/Ports/Meters poll frequency
251 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${onosStatInterval}
252 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${onosStatInterval}
253
254 if [ ${withFlows} = false ]; then
255 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 app deactivate org.opencord.olt
256 fi
257
258 if [ ${withMibTemplate} = true ] ; then
259 rm -f BBSM-12345123451234512345-00000000000001-v1.json
260 wget https://raw.githubusercontent.com/opencord/voltha-openonu-adapter/master/templates/BBSM-12345123451234512345-00000000000001-v1.json
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800261 cat BBSM-12345123451234512345-00000000000001-v1.json | kubectl -n \$INFRA_NS exec -it \$(kubectl -n \$INFRA_NS get pods -l app=etcd | awk 'NR==2{print \$1}') -- etcdctl put service/voltha/omci_mibs/templates/BBSM/12345123451234512345/00000000000001
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800262 fi
263
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800264 if [ ${withPcap} = true ] && [ ${volthaStacks} -eq 1 ] ; then
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800265 # Start the tcp-dump in ofagent
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800266 export OF_AGENT=\$(kubectl -n \$INFRA_NS get pods -l app=ofagent -o name)
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800267 kubectl exec \$OF_AGENT -- apk update
268 kubectl exec \$OF_AGENT -- apk add tcpdump
269 kubectl exec \$OF_AGENT -- mv /usr/sbin/tcpdump /usr/bin/tcpdump
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800270 _TAG=ofagent-tcpdump kubectl -n \$INFRA_NS exec \$OF_AGENT -- tcpdump -nei eth0 -w out.pcap&
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800271
272 # Start the tcp-dump in radius
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800273 export RADIUS=\$(kubectl -n \$INFRA_NS get pods -l app=radius -o name)
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800274 kubectl exec \$RADIUS -- apt-get update
275 kubectl exec \$RADIUS -- apt-get install -y tcpdump
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800276 _TAG=radius-tcpdump kubectl -n \$INFRA_NS exec \$RADIUS -- tcpdump -w out.pcap&
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800277
278 # Start the tcp-dump in ONOS
279 for i in \$(seq 0 \$ONOSES); do
280 INSTANCE="onos-onos-classic-\$i"
281 kubectl exec \$INSTANCE -- apt-get update
282 kubectl exec \$INSTANCE -- apt-get install -y tcpdump
283 kubectl exec \$INSTANCE -- mv /usr/sbin/tcpdump /usr/bin/tcpdump
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800284 _TAG=\$INSTANCE kubectl -n \$INFRA_NS exec \$INSTANCE -- /usr/bin/tcpdump -nei eth0 port 1812 -w out.pcap&
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800285 done
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800286 else
287 echo "PCAP not supported for multiple VOLTHA stacks"
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800288 fi
289 """
290 }
291 }
292 }
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800293 stage('Setup Test') {
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800294 steps {
295 sh '''
296 mkdir -p $WORKSPACE/RobotLogs
297 cd $WORKSPACE/voltha-system-tests
298 make vst_venv
299 '''
300 sh '''
Matteo Scandolod1430a72020-12-04 15:14:44 -0800301 if [ ${withProfiling} = true ] && [ ${volthaStacks} -eq 1 ]; then
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800302 mkdir -p $LOG_FOLDER/pprof
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800303 cat << EOF > $WORKSPACE/pprof.sh
304timestamp() {
305 date +"%T"
306}
307
308i=0
309while [[ true ]]; do
310 ((i++))
311 ts=$(timestamp)
312 go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $LOG_FOLDER/pprof/rw-core-heap-\\$i-\\$ts.png
313 go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $LOG_FOLDER/pprof/rw-core-goroutine-\\$i-\\$ts.png
314 curl -o $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof http://127.0.0.1:6060/debug/pprof/profile?seconds=10
315 go tool pprof -png $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.png
316
317 go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $LOG_FOLDER/pprof/openolt-heap-\\$i-\\$ts.png
318 go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $LOG_FOLDER/pprof/openolt-goroutine-\\$i-\\$ts.png
319 curl -o $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof http://127.0.0.1:6061/debug/pprof/profile?seconds=10
320 go tool pprof -png $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.png
321
322 go tool pprof -png http://127.0.0.1:6062/debug/pprof/heap > $LOG_FOLDER/pprof/ofagent-heap-\\$i-\\$ts.png
323 go tool pprof -png http://127.0.0.1:6062/debug/pprof/goroutine > $LOG_FOLDER/pprof/ofagent-goroutine-\\$i-\\$ts.png
324 curl -o $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof http://127.0.0.1:6062/debug/pprof/profile?seconds=10
325 go tool pprof -png $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.png
326
327 sleep 10
328done
329EOF
330
331 _TAG="pprof"
332 _TAG=$_TAG bash $WORKSPACE/pprof.sh &
Matteo Scandolod1430a72020-12-04 15:14:44 -0800333 else
334 echo "Profiling not supported for multiple VOLTHA stacks"
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800335 fi
336 '''
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800337 }
338 }
339 stage('Run Test') {
340 steps {
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800341 test_voltha_stacks(params.volthaStacks)
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800342 }
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800343 }
344 }
345 post {
346 always {
347 // collect result, done in the "post" step so it's executed even in the
348 // event of a timeout in the tests
349 sh '''
350
351 # stop the kail processes
352 list=($APPS_TO_LOG)
353 for app in "${list[@]}"
354 do
355 echo "Stopping logs for: ${app}"
356 _TAG="kail-$app"
357 P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
358 if [ -n "$P_IDS" ]; then
359 echo $P_IDS
360 for P_ID in $P_IDS; do
361 kill -9 $P_ID
362 done
363 fi
364 done
365
Matteo Scandolod1430a72020-12-04 15:14:44 -0800366 if [ ${withPcap} = true ] && [ ${volthaStacks} -eq 1 ]; then
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800367 # stop ofAgent tcpdump
368 P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')"
369 if [ -n "\$P_ID" ]; then
370 kill -9 \$P_ID
371 fi
372
373 # stop radius tcpdump
374 P_ID="\$(ps e -ww -A | grep "_TAG=radius-tcpdump" | grep -v grep | awk '{print \$1}')"
375 if [ -n "\$P_ID" ]; then
376 kill -9 \$P_ID
377 fi
378
379 # stop onos tcpdump
380 LIMIT=$(($NUM_OF_ONOS - 1))
381 for i in $(seq 0 $LIMIT); do
382 INSTANCE="onos-onos-classic-$i"
383 P_ID="\$(ps e -ww -A | grep "_TAG=$INSTANCE" | grep -v grep | awk '{print \$1}')"
384 if [ -n "\$P_ID" ]; then
385 kill -9 \$P_ID
386 fi
387 done
388
389 # copy the file
390 export OF_AGENT=$(kubectl get pods -l app=ofagent | awk 'NR==2{print $1}') || true
391 kubectl cp $OF_AGENT:out.pcap $LOG_FOLDER/ofagent.pcap || true
392
393 export RADIUS=$(kubectl get pods -l app=radius | awk 'NR==2{print $1}') || true
394 kubectl cp $RADIUS:out.pcap $LOG_FOLDER/radius.pcap || true
395
396 LIMIT=$(($NUM_OF_ONOS - 1))
397 for i in $(seq 0 $LIMIT); do
398 INSTANCE="onos-onos-classic-$i"
399 kubectl cp $INSTANCE:out.pcap $LOG_FOLDER/$INSTANCE.pcap || true
400 done
401 fi
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800402 '''
403 sh '''
Matteo Scandolod1430a72020-12-04 15:14:44 -0800404 if [ ${withProfiling} = true ] && [ ${volthaStacks} -eq 1 ]; then
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800405 _TAG="pprof"
406 P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
407 if [ -n "$P_IDS" ]; then
408 echo $P_IDS
409 for P_ID in $P_IDS; do
410 kill -9 $P_ID
411 done
412 fi
413 fi
414 '''
415 plot([
416 csvFileName: 'scale-test.csv',
417 csvSeries: [
418 [file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
419 [file: 'plots/plot-onos-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
420 [file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
421 [file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
422 [file: 'plots/plot-onos-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
423 [file: 'plots/plot-onos-auth.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
424 [file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
425 [file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
426 [file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
427 [file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],
428 ],
Matteo Scandolod1430a72020-12-04 15:14:44 -0800429 group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (Stacks: ${params.volthaStacks}, OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800430 ])
431 step([$class: 'RobotPublisher',
432 disableArchiveOutput: false,
Matteo Scandolod4064492020-11-13 10:48:30 -0800433 logFileName: 'RobotLogs/**/log.html',
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800434 otherFiles: '',
Matteo Scandolod4064492020-11-13 10:48:30 -0800435 outputFileName: 'RobotLogs/**/output.xml',
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800436 outputPath: '.',
437 passThreshold: 100,
Matteo Scandolod4064492020-11-13 10:48:30 -0800438 reportFileName: 'RobotLogs/**/report.html',
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800439 unstableThreshold: 0]);
440 // get all the logs from kubernetes PODs
441 sh returnStdout: false, script: '''
442
443 # store information on running charts
Matteo Scandolod4064492020-11-13 10:48:30 -0800444 helm ls --all-namespaces > $LOG_FOLDER/helm-list.txt || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800445
446 # store information on the running pods
Matteo Scandolod4d55b72020-11-25 12:29:07 -0800447 kubectl get pods --all-namespaces -o wide > $LOG_FOLDER/pods.txt || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800448 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-images.txt || true
449 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-imagesId.txt || true
450
451 # copy the ONOS logs directly from the container to avoid the color codes
Matteo Scandolobe823242020-11-20 13:48:13 -0800452 printf '%s\n' $(kubectl -n \$INFRA_NS get pods -l app=onos-onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl -n \$INFRA_NS cp #:${karafHome}/data/log/karaf.log $LOG_FOLDER/#.log" || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800453
454 # get radius logs out of the container
Matteo Scandolod4064492020-11-13 10:48:30 -0800455 kubectl -n \$INFRA_NS cp $(kubectl -n \$INFRA_NS get pods -l app=radius --no-headers | awk '{print $1}'):/var/log/freeradius/radius.log $LOG_FOLDER//radius.log || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800456 '''
457 // dump all the BBSim(s) ONU information
Matteo Scandolod4064492020-11-13 10:48:30 -0800458 script {
459 for (int i = 1; i <= params.volthaStacks.toInteger(); i++) {
460 stack_ns="voltha"+i
461 sh """
462 BBSIM_IDS=\$(kubectl -n ${stack_ns} get pods | grep bbsim | grep -v server | awk '{print \$1}')
463 IDS=(\$BBSIM_IDS)
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800464
Matteo Scandolod4064492020-11-13 10:48:30 -0800465 for bbsim in "\${IDS[@]}"
466 do
Matteo Scandolod1430a72020-12-04 15:14:44 -0800467 kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl onu list > $LOG_FOLDER/${stack_ns}/\$bbsim-device-list.txt || true
468 kubectl -n ${stack_ns} exec -t \$bbsim -- bbsimctl service list > $LOG_FOLDER/${stack_ns}/\$bbsim-service-list.txt || true
Matteo Scandolod4064492020-11-13 10:48:30 -0800469 done
470 """
471 }
472 }
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800473 // get ONOS debug infos
474 sh '''
475
476 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 apps -a -s > $LOG_FOLDER/onos-apps.txt || true
477 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 nodes > $LOG_FOLDER/onos-nodes.txt || true
478 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt || true
479 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt || true
480
481 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt || true
482 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt || true
483
484 if [ ${withFlows} = true ] ; then
485 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-olts > $LOG_FOLDER/onos-olt-list.txt || true
486 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 flows -s > $LOG_FOLDER/onos-flows-list.txt || true
487 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 meters > $LOG_FOLDER/onos-meters-list.txt || true
488 fi
489
490 if [ ${provisionSubscribers} = true ]; then
491 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-subscribers > $LOG_FOLDER/onos-programmed-subscribers.txt || true
492 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-meters > $LOG_FOLDER/onos-programmed-meters.txt || true
493 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-bpmeter-mappings > $LOG_FOLDER/onos-bpmeter-mappings.txt || true
494 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-failed-subscribers > $LOG_FOLDER/onos-failed-subscribers.txt || true
495 fi
496
497 if [ ${withEapol} = true ] ; then
498 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-users > $LOG_FOLDER/onos-aaa-users.txt || true
499 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-statistics > $LOG_FOLDER/onos-aaa-statistics.txt || true
500 fi
501
502 if [ ${withDhcp} = true ] ; then
503 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 dhcpl2relay-allocations > $LOG_FOLDER/onos-dhcp-allocations.txt || true
504 fi
505 '''
506 // collect etcd metrics
507 sh '''
508 mkdir -p $WORKSPACE/etcd-metrics
509 curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_keys_total' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-key-count.json || true
510 curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=grpc_server_handled_total{grpc_service="etcdserverpb.KV"}' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-rpc-count.json || true
511 curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_db_total_size_in_bytes' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-db-size.json || true
512 curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time.json || true
513 '''
514 // get VOLTHA debug infos
515 script {
Matteo Scandolod4064492020-11-13 10:48:30 -0800516 for (int i = 1; i <= params.volthaStacks.toInteger(); i++) {
517 stack_ns="voltha"+i
518 voltcfg="~/.volt/config-voltha"+i
Matteo Scandolod4064492020-11-13 10:48:30 -0800519 try {
520 sh """
Matteo Scandolod1430a72020-12-04 15:14:44 -0800521 voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list -o json > $LOG_FOLDER/${stack_ns}/device-list.json || true
522 python -m json.tool $LOG_FOLDER/${stack_ns}/device-list.json > $LOG_FOLDER/${stack_ns}/voltha-devices-list.json || true
523 rm $LOG_FOLDER/${stack_ns}/device-list.json || true
524 voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list > $LOG_FOLDER/${stack_ns}/voltha-devices-list.txt || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800525
Matteo Scandolod4064492020-11-13 10:48:30 -0800526 DEVICE_LIST=
Matteo Scandolod1430a72020-12-04 15:14:44 -0800527 printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns}-m 8MB device flows # > $LOG_FOLDER/${stack_ns}/voltha-device-flows-#.txt" || true
528 printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device list | grep olt | awk '{print \$1}') | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/${stack_ns}/voltha-device-ports-#.txt" || true
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800529
Matteo Scandolod1430a72020-12-04 15:14:44 -0800530 printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice flows # > $LOG_FOLDER/${stack_ns}/voltha-logicaldevice-flows-#.txt" || true
531 printf '%s\n' \$(voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -c $HOME/.volt/config-${stack_ns} -m 8MB logicaldevice port list # > $LOG_FOLDER/${stack_ns}/voltha-logicaldevice-ports-#.txt" || true
Matteo Scandolod4064492020-11-13 10:48:30 -0800532 """
533 } catch(e) {
534 sh '''
535 echo "Can't get device list from voltclt"
536 '''
537 }
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800538 }
539 }
540 // get cpu usage by container
541 sh '''
542 if [ ${withMonitoring} = true ] ; then
543 cd $WORKSPACE/voltha-system-tests
544 source ./vst_venv/bin/activate
545 sleep 60 # we have to wait for prometheus to collect all the information
546 python tests/scale/sizing.py -o $WORKSPACE/plots || true
547 fi
548 '''
Matteo Scandolo89311a12020-12-05 14:43:58 -0800549 archiveArtifacts artifacts: 'kind-voltha/install-*.log,execution-time-*.txt,logs/**/*,RobotLogs/**/*,plots/*,etcd-metrics/*'
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800550 }
551 }
552}
553
554def deploy_voltha_stacks(numberOfStacks) {
555 for (int i = 1; i <= numberOfStacks.toInteger(); i++) {
556 stage("Deploy VOLTHA stack " + i) {
557 sh returnStdout: false, script: """
Matteo Scandolo77501cc2020-12-11 09:49:07 -0800558
559 # unset voltha-api port so that the port is forwarded on a new one
560 unset VOLTHA_API_PORT
561
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800562 cd $WORKSPACE/kind-voltha/
563
564 export NAME=voltha${i}
565 export VOLTHA_NS=voltha${i}
566 export ADAPTER_NS=voltha${i}
567 export BBSIM_NS=voltha${i}
568 export BBSIM_BASE_INDEX=${i}
569 export WITH_ETCD=etcd.\$INFRA_NS.svc:2379
570 export WITH_KAFKA=kafka.\$INFRA_NS.svc:9092
571 export WITH_ONOS=onos-onos-classic-hs.\$INFRA_NS.svc:6653
572
573 export EXTRA_HELM_FLAGS+=' '
574
575 # Load the release defaults
576 if [ '${release.trim()}' != 'master' ]; then
577 source $WORKSPACE/kind-voltha/releases/${release}
578 EXTRA_HELM_FLAGS+=" ${extraHelmFlags} "
579 fi
580
581 # BBSim custom image handling
582 if [ '${bbsimImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'bbsim' ]; then
583 IFS=: read -r bbsimRepo bbsimTag <<< '${bbsimImg.trim()}'
584 EXTRA_HELM_FLAGS+="--set images.bbsim.repository=\$bbsimRepo,images.bbsim.tag=\$bbsimTag "
585 fi
586
587 # VOLTHA and ofAgent custom image handling
588 # NOTE to override the rw-core image in a released version you must set the ofAgent image too
589 # TODO split ofAgent and voltha-go
590 if [ '${rwCoreImg.trim()}' != '' ] && [ '${ofAgentImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-go' ]; then
591 IFS=: read -r rwCoreRepo rwCoreTag <<< '${rwCoreImg.trim()}'
592 IFS=: read -r ofAgentRepo ofAgentTag <<< '${ofAgentImg.trim()}'
593 EXTRA_HELM_FLAGS+="--set images.rw_core.repository=\$rwCoreRepo,images.rw_core.tag=\$rwCoreTag,images.ofagent.repository=\$ofAgentRepo,images.ofagent.tag=\$ofAgentTag "
594 fi
595
596 # OpenOLT custom image handling
597 if [ '${openoltAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openolt-adapter' ]; then
598 IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< '${openoltAdapterImg.trim()}'
599 EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=\$openoltAdapterRepo,images.adapter_open_olt.tag=\$openoltAdapterTag "
600 fi
601
602 # OpenONU custom image handling
603 if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then
604 IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'
605 EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=\$openonuAdapterRepo,images.adapter_open_onu.tag=\$openonuAdapterTag "
606 fi
607
Andrea Campanella28763082020-11-18 10:55:31 +0100608 # OpenONU Go custom image handling
609 if [ '${openonuAdapterGoImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter-go' ]; then
610 IFS=: read -r openonuAdapterGoRepo openonuAdapterGoTag <<< '${openonuAdapterGoImg.trim()}'
Andrea Campanellaed026982020-11-20 10:06:42 +0100611 EXTRA_HELM_FLAGS+="--set images.adapter_open_onu_go.repository=\$openonuAdapterGoRepo,images.adapter_open_onu_go.tag=\$openonuAdapterGoTag "
Andrea Campanella28763082020-11-18 10:55:31 +0100612 fi
613
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800614 # ONOS custom image handling
615 if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then
616 IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'
617 EXTRA_HELM_FLAGS+="--set images.onos.repository=\$onosRepo,images.onos.tag=\$onosTag "
618 fi
619
620 # set BBSim parameters
621 EXTRA_HELM_FLAGS+='--set enablePerf=true,pon=${pons},onu=${onus} '
622
623 # disable the securityContext, this is a development cluster
624 EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '
625
626 # No persistent-volume-claims in Atomix
627 EXTRA_HELM_FLAGS+="--set atomix.persistence.enabled=false "
628
629 echo "Installing with the following extra arguments:"
630 echo $EXTRA_HELM_FLAGS
631
632 # if it's newer than voltha-2.4 set the correct BBSIM_CFG
633 if [ '${release.trim()}' != 'voltha-2.4' ]; then
634 export BBSIM_CFG="$WORKSPACE/kind-voltha/configs/bbsim-sadis-${workflow}.yaml"
635 fi
636
637 # Use custom built images
638
639 if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then
640 EXTRA_HELM_FLAGS+="--set images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,images.rw_core.tag=voltha-scale "
641 fi
642
643 if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then
644 EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,images.adapter_open_olt.tag=voltha-scale "
645 fi
646
647 if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then
648 EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,images.adapter_open_onu.tag=voltha-scale "
649 fi
650
Andrea Campanellaed026982020-11-20 10:06:42 +0100651 if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter-go' ]; then
652 EXTRA_HELM_FLAGS+="--set images.adapter_open_onu_go.repository=${dockerRegistry}/voltha/voltha-openonu-adapter-go,images.adapter_open_onu_go.tag=voltha-scale "
653 fi
654
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800655 if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then
656 EXTRA_HELM_FLAGS+="--set images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,images.ofagent.tag=voltha-scale "
657 fi
658
659 if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then
660 EXTRA_HELM_FLAGS+="--set images.onos.repository=${dockerRegistry}/voltha/voltha-onos,images.onos.tag=voltha-scale "
661 fi
662
663 if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then
664 EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "
665 fi
666
667 ./voltha up
668 """
669 }
670 }
671}
672
673def test_voltha_stacks(numberOfStacks) {
674 for (int i = 1; i <= numberOfStacks.toInteger(); i++) {
675 stage("Test VOLTHA stack " + i) {
676 timeout(time: 15, unit: 'MINUTES') {
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800677 sh """
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800678 export VOLTCONFIG="$HOME/.volt/config-voltha${i}"
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800679 ROBOT_PARAMS="-v stackId:${i} \
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800680 -v olt:${olts} \
681 -v pon:${pons} \
682 -v onu:${onus} \
683 -v workflow:${workflow} \
684 -v withEapol:${withEapol} \
685 -v withDhcp:${withDhcp} \
686 -v withIgmp:${withIgmp} \
687 --noncritical non-critical \
688 -e teardown "
689
690 if [ ${withEapol} = false ] ; then
691 ROBOT_PARAMS+="-e authentication "
692 fi
693
694 if [ ${withDhcp} = false ] ; then
695 ROBOT_PARAMS+="-e dhcp "
696 fi
697
698 if [ ${provisionSubscribers} = false ] ; then
699 # if we're not considering subscribers then we don't care about authentication and dhcp
700 ROBOT_PARAMS+="-e authentication -e provision -e flow-after -e dhcp "
701 fi
702
703 if [ ${withFlows} = false ] ; then
704 ROBOT_PARAMS+="-i setup -i activation "
705 fi
706
707 cd $WORKSPACE/voltha-system-tests
708 source ./vst_venv/bin/activate
Matteo Scandolo5f6c5492020-11-12 17:37:25 -0800709 robot -d $WORKSPACE/RobotLogs/voltha${i} \
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800710 \$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot
Matteo Scandolof7ca6312020-11-16 15:57:15 -0800711
712 # collect results
713 python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/voltha${i}/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time-voltha${i}.txt || true
714 cat $WORKSPACE/execution-time-voltha${i}.txt
Matteo Scandolodc4608c2020-11-12 16:48:16 -0800715 """
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800716 }
717 }
Matteo Scandolo4b006ae2020-11-09 16:14:40 -0800718 }
719}