blob: 7761ea4abf894b01b8b947b63a04c113b47d3d0c [file] [log] [blame]
Matteo Scandolo51b76302020-02-05 12:07:23 -08001/* voltha-scale-measurements pipeline */
2pipeline {
Matteo Scandolo51b76302020-02-05 12:07:23 -08003 /* no label, executor is determined by JJB */
4 agent {
5 label "${params.buildNode}"
6 }
Shrey Baid00ff5c42020-02-07 15:14:08 -08007 environment {
8 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
9 VOLTCONFIG="$HOME/.volt/config-minimal"
Shrey Baidb001c712020-02-11 11:12:46 -080010 SSHPASS="karaf"
Shrey Baide7614d02020-02-25 10:26:42 -080011 DEPLOY_K8S="n"
Shrey Baid00ff5c42020-02-07 15:14:08 -080012 }
Matteo Scandolo51b76302020-02-05 12:07:23 -080013 stages {
Shrey Baide7614d02020-02-25 10:26:42 -080014 stage('set-description') {
15 steps {
16 script {
Shrey Baid943e2072020-03-04 10:31:32 -080017 currentBuild.description = "$BUILD_TIMESTAMP"
Shrey Baide7614d02020-02-25 10:26:42 -080018 }
19 }
20 }
Shrey Baidb0447182020-02-27 16:39:35 -080021 stage('cleanup') {
Matteo Scandolo51b76302020-02-05 12:07:23 -080022 steps {
Shrey Baidb0447182020-02-27 16:39:35 -080023 sh '''
Shrey Baid49bf8f62020-03-09 17:52:01 -070024 rm -rf voltha-devices-count.txt voltha-devices-time.txt onos-ports-count.txt onos-ports-time.txt onos-ports-list.txt voltha-devices-list.json onos-ports-time-num.txt voltha-devices-time-num.txt
Matteo Scandolodd5bbe92020-03-27 11:43:08 -070025 for hchart in \$(helm list -q | grep -E -v 'docker-registry|kafkacat|etcd-operator');
Matteo Scandolo151e35c2020-02-28 15:45:15 -080026 do
27 echo "Purging chart: \${hchart}"
28 helm delete --purge "\${hchart}"
29 done
Shrey Baidb0447182020-02-27 16:39:35 -080030 bash /home/cord/voltha-scale/wait_for_pods.sh
31 bash /home/cord/voltha-scale/stop_port_forward.sh
32 '''
Matteo Scandolo51b76302020-02-05 12:07:23 -080033 }
34 }
35 stage('start') {
36 steps {
Shrey Baid00ff5c42020-02-07 15:14:08 -080037 sh '''
Matteo Scandolo51b76302020-02-05 12:07:23 -080038 #!/usr/bin/env bash
39 set -euo pipefail
Shrey Baid00ff5c42020-02-07 15:14:08 -080040 '''
41 }
42 }
Shrey Baid00ff5c42020-02-07 15:14:08 -080043 stage('deploy-voltha') {
Matteo Scandolo2fc1dac2020-03-09 08:39:53 -070044 options {
45 timeout(time:10)
46 }
Shrey Baid00ff5c42020-02-07 15:14:08 -080047 steps {
48 sh '''
Matteo Scandoloa3470b72020-03-17 12:26:47 -070049 helm repo update
Matteo Scandolodd5bbe92020-03-27 11:43:08 -070050 helm install -n cord-kafka incubator/kafka -f /home/cord/voltha-scale/voltha-values.yaml --version 0.13.3 --set replicas=3 --set persistence.enabled=false --set zookeeper.replicaCount=3 --set zookeeper.persistence.enabled=false
Matteo Scandoloe1c200f2020-03-24 10:33:58 -070051 helm install -n nem-monitoring cord/nem-monitoring --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
Shrey Baidb06b55a2020-03-12 15:04:58 -070052
Matteo Scandoloe69614f2020-03-10 11:35:35 -070053 IFS=: read -r onosRepo onosTag <<< ${onosImg}
54 helm install -n onos onf/onos --set images.onos.repository=${onosRepo} --set images.onos.tag=${onosTag} ${extraHelmFlags}
Shrey Baidb0447182020-02-27 16:39:35 -080055
Matteo Scandoloa2928e72020-03-02 16:38:22 -080056 IFS=: read -r volthaRepo volthaTag <<< ${volthaImg}
Matteo Scandoloe69614f2020-03-10 11:35:35 -070057 IFS=: read -r ofAgentRepo ofAgentTag <<< ${ofAgentImg}
Matteo Scandolofddcb002020-03-24 09:14:57 -070058 helm install -n voltha ${volthaChart} -f /home/cord/voltha-scale/voltha-values.yaml --set defaults.log_level=${logLevel},images.rw_core.repository=${volthaRepo},images.rw_core.tag=${volthaTag},images.ofagent_go.repository=${ofAgentRepo},images.ofagent_go.tag=${ofAgentTag} ${extraHelmFlags}
Matteo Scandoloa2928e72020-03-02 16:38:22 -080059
60 IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< ${openoltAdapterImg}
Matteo Scandolo330f7522020-03-17 17:15:53 -070061 helm install -n openolt ${openoltAdapterChart} -f /home/cord/voltha-scale/voltha-values.yaml --set defaults.log_level=${logLevel},images.adapter_open_olt.repository=${openoltAdapterRepo},images.adapter_open_olt.tag=${openoltAdapterTag} ${extraHelmFlags}
Matteo Scandoloa2928e72020-03-02 16:38:22 -080062
63 IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< ${openonuAdapterImg}
Matteo Scandolo330f7522020-03-17 17:15:53 -070064 helm install -n openonu ${openonuAdapterChart} -f /home/cord/voltha-scale/voltha-values.yaml --set defaults.log_level=${logLevel},images.adapter_open_onu.repository=${openonuAdapterRepo},images.adapter_open_onu.tag=${openonuAdapterTag} ${extraHelmFlags}
Matteo Scandoloa2928e72020-03-02 16:38:22 -080065
66 IFS=: read -r bbsimRepo bbsimTag <<< ${bbsimImg}
Matteo Scandolo463f31b2020-03-17 11:02:17 -070067 helm install -n bbsim ${bbsimChart} --set enablePerf=true,pon=${ponPorts},onu=${onuPerPon},auth=${bbsimAuth},dhcp=${bbsimDhcp},delay=${BBSIMdelay},images.bbsim.repository=${bbsimRepo},images.bbsim.tag=${bbsimTag} ${extraHelmFlags}
Matteo Scandolo7059b082020-03-17 09:05:03 -070068
Matteo Scandoloe69614f2020-03-10 11:35:35 -070069 helm install -n radius onf/freeradius ${extraHelmFlags}
Shrey Baidb0447182020-02-27 16:39:35 -080070
Shrey Baidb0447182020-02-27 16:39:35 -080071 bash /home/cord/voltha-scale/wait_for_pods.sh
72 bash /home/cord/voltha-scale/start_port_forward.sh
Shrey Baid00ff5c42020-02-07 15:14:08 -080073 '''
74 }
75 }
Matteo Scandoloa711ddf2020-03-06 16:41:34 -080076 stage('wait for adapters to be registered') {
Matteo Scandolo2fc1dac2020-03-09 08:39:53 -070077 options {
78 timeout(time:5)
79 }
Matteo Scandoloa711ddf2020-03-06 16:41:34 -080080 steps{
81 waitUntil {
82 script {
83 openolt_res = sh returnStdout: true, script: """
84 voltctl adapter list | grep openolt | wc -l
85 """
86
87 openonu_res = sh returnStdout: true, script: """
88 voltctl adapter list | grep brcm_openomci_onu | wc -l
89 """
90
Matteo Scandolo89f1cea2020-04-10 10:20:20 -070091 return openolt_res.toInteger() >= 1 && openonu_res.toInteger() >= 1
Matteo Scandoloa711ddf2020-03-06 16:41:34 -080092 }
93 }
94 }
95 }
Shrey Baide7614d02020-02-25 10:26:42 -080096 stage('MIB-template') {
97 steps {
98 sh '''
99 if [ ${withMibTemplate} = true ] ; then
Shrey Baidb0447182020-02-27 16:39:35 -0800100 rm -f BBSM-12345123451234512345-00000000000001-v1.json
Shrey Baide7614d02020-02-25 10:26:42 -0800101 wget https://raw.githubusercontent.com/opencord/voltha-openonu-adapter/master/templates/BBSM-12345123451234512345-00000000000001-v1.json
Shrey Baidb0447182020-02-27 16:39:35 -0800102 cat BBSM-12345123451234512345-00000000000001-v1.json | kubectl exec -it $(kubectl get pods | grep etcd-cluster | awk 'NR==1{print $1}') etcdctl put service/voltha/omci_mibs/templates/BBSM/12345123451234512345/00000000000001
Shrey Baide7614d02020-02-25 10:26:42 -0800103 fi
104 '''
Shrey Baid0d4e31d2020-02-11 13:28:47 -0800105 }
Shrey Baide7614d02020-02-25 10:26:42 -0800106 }
107 stage('disable-ONOS-apps') {
108 steps {
109 sh '''
110 #Check withOnosApps and disable apps accordingly
111 if [ ${withOnosApps} = false ] ; then
Shrey Baidb0447182020-02-27 16:39:35 -0800112 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost app deactivate org.opencord.olt
113 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost app deactivate org.opencord.aaa
114 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost app deactivate org.opencord.dhcpl2relay
Shrey Baide7614d02020-02-25 10:26:42 -0800115 fi
116 '''
Shrey Baid0d4e31d2020-02-11 13:28:47 -0800117 }
Shrey Baide7614d02020-02-25 10:26:42 -0800118 }
119 stage('configuration') {
120 steps {
121 sh '''
Matteo Scandolo330f7522020-03-17 17:15:53 -0700122 #Setting LOG level to ${logLevel}
123 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost log:set ${logLevel}
Shrey Baidb0447182020-02-27 16:39:35 -0800124 kubectl exec $(kubectl get pods | grep bbsim | awk 'NR==1{print $1}') bbsimctl log warn false
Shrey Baide7614d02020-02-25 10:26:42 -0800125 #Setting link discovery
Shrey Baidb0447182020-02-27 16:39:35 -0800126 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${setLinkDiscovery}
Shrey Baide7614d02020-02-25 10:26:42 -0800127 #Setting the flow stats collection interval
Shrey Baidb0447182020-02-27 16:39:35 -0800128 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${flowStatInterval}
Shrey Baide7614d02020-02-25 10:26:42 -0800129 #Setting the ports stats collection interval
Shrey Baidb0447182020-02-27 16:39:35 -0800130 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${portsStatInterval}
Matteo Scandolo7b3fde92020-02-26 16:17:16 -0800131 # extending voltctl timeout
Shrey Baidb0447182020-02-27 16:39:35 -0800132 sed -i 's/timeout: 10s/timeout: 5m/g' /home/cord/.volt/config
Shrey Baide7614d02020-02-25 10:26:42 -0800133 '''
134 }
135 }
136 stage('execute') {
137 options {
138 timeout(time:10)
139 }
140 stages {
Shrey Baida3b8aa62020-02-13 11:22:51 -0800141 stage('ONUs-enabled') {
142 steps {
143 sh '''
144 if [ -z ${expectedOnus} ]
145 then
146 echo -e "You need to set the target ONU number\n"
147 exit 1
148 fi
Shrey Baid00ff5c42020-02-07 15:14:08 -0800149
Shrey Baida3b8aa62020-02-13 11:22:51 -0800150 voltctl device create -t openolt -H bbsim:50060
151 voltctl device enable $(voltctl device list --filter Type~openolt -q)
152 # check ONUs reached Active State in VOLTHA
153 i=$(voltctl device list | grep -v OLT | grep ACTIVE | wc -l)
154 until [ $i -eq ${expectedOnus} ]
155 do
156 echo "$i ONUs ACTIVE of ${expectedOnus} expected (time: $SECONDS)"
157 sleep ${pollInterval}
158 i=$(voltctl device list | grep -v OLT | grep ACTIVE | wc -l)
159 done
160 echo "${expectedOnus} ONUs Activated in $SECONDS seconds (time: $SECONDS)"
Shrey Baid49bf8f62020-03-09 17:52:01 -0700161 echo $SECONDS > voltha-devices-time-num.txt
Shrey Baida3b8aa62020-02-13 11:22:51 -0800162 '''
163 }
164 }
165 stage('ONOS-ports') {
166 steps {
Shrey Baide7614d02020-02-25 10:26:42 -0800167 sh '''
Shrey Baida3b8aa62020-02-13 11:22:51 -0800168 # Check ports showed up in ONOS
Shrey Baidb0447182020-02-27 16:39:35 -0800169 z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports -e | grep BBSM | wc -l)
Shrey Baida3b8aa62020-02-13 11:22:51 -0800170 until [ $z -eq ${expectedOnus} ]
171 do
172 echo "${z} enabled ports of ${expectedOnus} expected (time: $SECONDS)"
173 sleep ${pollInterval}
Shrey Baidb0447182020-02-27 16:39:35 -0800174 z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports -e | grep BBSM | wc -l)
Shrey Baida3b8aa62020-02-13 11:22:51 -0800175 done
176 echo "${expectedOnus} ports enabled in $SECONDS seconds (time: $SECONDS)"
Shrey Baid49bf8f62020-03-09 17:52:01 -0700177 echo $SECONDS > temp.txt
178 paste voltha-devices-time-num.txt temp.txt | awk '{print ($1 + $2)}' > onos-ports-time-num.txt
179 echo "ONOS-Duration(s)" > onos-ports-time.txt
180 echo "VOLTHA-Duration(s)" > voltha-devices-time.txt
181 cat voltha-devices-time-num.txt >> voltha-devices-time.txt
182 cat onos-ports-time-num.txt >> onos-ports-time.txt
Shrey Baida3b8aa62020-02-13 11:22:51 -0800183 '''
184 }
185 }
Shrey Baidb001c712020-02-11 11:12:46 -0800186 }
187 }
Matteo Scandolo51b76302020-02-05 12:07:23 -0800188 }
189 post {
Shrey Baid4da57a62020-03-05 15:49:50 -0800190 success {
Shrey Baide7614d02020-02-25 10:26:42 -0800191 plot([
192 csvFileName: 'plot-onu-activation.csv',
Shrey Baid49bf8f62020-03-09 17:52:01 -0700193 csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-time.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-time.txt', inclusionFlag: 'OFF', url: '']],
Shrey Baid943e2072020-03-04 10:31:32 -0800194 group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Time (${BBSIMdelay}s Delay)", yaxis: 'Time (s)', useDescr: true
Shrey Baide7614d02020-02-25 10:26:42 -0800195 ])
Shrey Baid4da57a62020-03-05 15:49:50 -0800196 }
197 always {
198 sh '''
199 echo $(voltctl device list | grep -v OLT | grep ACTIVE | wc -l) > onus.txt
Shrey Baid49bf8f62020-03-09 17:52:01 -0700200 echo "#-of-ONUs" > voltha-devices-count.txt
201 cat onus.txt >> voltha-devices-count.txt
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700202 '''
203 sh '''
Shrey Baid4da57a62020-03-05 15:49:50 -0800204 echo $(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports -e | grep BBSM | wc -l) > ports.txt
Shrey Baid49bf8f62020-03-09 17:52:01 -0700205 echo "#-of-ports" > onos-ports-count.txt
206 cat ports.txt >> onos-ports-count.txt
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700207 '''
208 sh '''
Matteo Scandolo330f7522020-03-17 17:15:53 -0700209 kubectl get pods -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700210 '''
211 sh '''
Shrey Baid4da57a62020-03-05 15:49:50 -0800212 voltctl device list -o json > device-list.json
Shrey Baid49bf8f62020-03-09 17:52:01 -0700213 python -m json.tool device-list.json > voltha-devices-list.json
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700214 '''
215 sh '''
Shrey Baid49bf8f62020-03-09 17:52:01 -0700216 sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@localhost ports > onos-ports-list.txt
Shrey Baidce9ed552020-03-13 13:40:13 -0700217 curl -s -X GET -G http://127.0.0.1:31301/api/v1/query --data-urlencode 'query=avg(rate(container_cpu_usage_seconds_total[10m])*100) by (pod_name)' | jq . > cpu-usage.json
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700218 '''
219 sh '''
Matteo Scandolo89f1cea2020-04-10 10:20:20 -0700220 kubectl logs -l app=adapter-open-olt > open-olt-logs.txt
221 kubectl logs -l app=adapter-open-onu > open-onu-logs.txt
222 kubectl logs -l app=rw-core > voltha-rw-core-logs.txt
223 kubectl logs -l app=ofagent > voltha-ofagent-logs.txt
224 kubectl logs -l app=bbsim > bbsim-logs.txt
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700225 '''
226 sh '''
Shrey Baid49bf8f62020-03-09 17:52:01 -0700227 rm -rf BBSM-12345123451234512345-00000000000001-v1.json device-list.json onus.txt ports.txt temp.txt
Matteo Scandoloe1c200f2020-03-24 10:33:58 -0700228 '''
Shrey Baid4da57a62020-03-05 15:49:50 -0800229 plot([
230 csvFileName: 'plot-numbers.csv',
Shrey Baid49bf8f62020-03-09 17:52:01 -0700231 csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-count.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-count.txt', inclusionFlag: 'OFF', url: '']],
Shrey Baid4da57a62020-03-05 15:49:50 -0800232 group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Activated ONUs and Recognized Ports", yaxis: 'Number of Ports/ONUs', useDescr: true
233 ])
234
235 archiveArtifacts artifacts: '*.log,*.json,*txt'
236
Matteo Scandolo51b76302020-02-05 12:07:23 -0800237 }
238 }
Matteo Scandoloe69614f2020-03-10 11:35:35 -0700239}