[VOL-2839] Add support to check flows in scale-test
Change-Id: Idfee634203f96a0cd915e9a41ec2cbec225da802
diff --git a/jjb/pipeline/voltha-scale-measurements.groovy b/jjb/pipeline/voltha-scale-measurements.groovy
index f935bde..6c10215 100644
--- a/jjb/pipeline/voltha-scale-measurements.groovy
+++ b/jjb/pipeline/voltha-scale-measurements.groovy
@@ -8,17 +8,17 @@
SSHPASS="karaf"
}
stages {
- stage('set-description') {
+ stage('Set build description') {
steps {
script {
currentBuild.description = "$BUILD_TIMESTAMP"
}
}
}
- stage('cleanup') {
+ stage('Cleanup') {
steps {
sh '''
- rm -rf voltha-devices-count.txt voltha-devices-time.txt onos-ports-count.txt onos-ports-time.txt onos-ports-list.txt voltha-devices-list.json onos-ports-time-num.txt voltha-devices-time-num.txt
+ rm -rf *.txt
for hchart in \$(helm list -q | grep -E -v 'docker-registry|kafkacat|etcd-operator');
do
echo "Purging chart: \${hchart}"
@@ -29,7 +29,7 @@
'''
}
}
- stage('start') {
+ stage('Start') {
steps {
sh '''
#!/usr/bin/env bash
@@ -37,7 +37,7 @@
'''
}
}
- stage('deploy-voltha') {
+ stage('Deploy voltha') {
options {
timeout(time:10)
}
@@ -49,6 +49,9 @@
helm install -n radius onf/freeradius ${extraHelmFlags}
+ # Multi BBSim sadis server (to be migrated in ONF and included in the helm-chart)
+ # kubectl create -f https://raw.githubusercontent.com/ciena/bbsim-sadis-server/master/bbsim-sadis-server.yaml
+
# NOTE wait for the infrastructure to be running before installing VOLTHA
bash /home/cord/voltha-scale/wait_for_pods.sh
@@ -76,7 +79,7 @@
'''
}
}
- stage('wait for adapters to be registered') {
+ stage('Wait for Adapters to be registered in VOLTHA') {
options {
timeout(time:5)
}
@@ -96,7 +99,7 @@
}
}
}
- stage('MIB-template') {
+ stage('Push MIB template to ETCD') {
steps {
sh '''
if [ ${withMibTemplate} = true ] ; then
@@ -107,36 +110,56 @@
'''
}
}
- stage('disable-ONOS-apps') {
- steps {
- sh '''
- #Check withOnosApps and disable apps accordingly
- if [ ${withOnosApps} = false ] ; then
- sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.olt
- sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.aaa
- sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.dhcpl2relay
- fi
- '''
- }
- }
- stage('configuration') {
+ stage('Configure ONOS and VOLTHA') {
steps {
sh '''
#Setting LOG level to ${logLevel}
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 log:set ${logLevel}
kubectl exec $(kubectl get pods | grep bbsim | awk 'NR==1{print $1}') bbsimctl log warn false
+
#Setting link discovery
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${setLinkDiscovery}
- #Setting the flow stats collection interval
- sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${flowStatInterval}
- #Setting the ports stats collection interval
- sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${portsStatInterval}
+
# extending voltctl timeout
sed -i 's/timeout: 10s/timeout: 5m/g' /home/cord/.volt/config
+
+ #Setting the flow and ports stats collection interval
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${flowStatInterval}
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${portsStatInterval}
+
+ #Check withOnosApps and disable apps accordingly
+ if [ ${withOnosApps} = false ] ; then
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.olt
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.aaa
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.dhcpl2relay
+ else
+ curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json "http://127.0.0.1:30120/onos/v1/network/configuration/apps/org.opencord.sadis" --data '{
+ "sadis": {
+ "integration": {
+ "url": "http://bbsim-sadis-server.default.svc:58080/subscribers/%s",
+ "cache": {
+ "enabled": true,
+ "maxsize": 50,
+ "ttl": "PT1m"
+ }
+ }
+ },
+ "bandwidthprofile": {
+ "integration": {
+ "url": "http://bbsim-sadis-server.default.svc:58080/profiles/%s",
+ "cache": {
+ "enabled": true,
+ "maxsize": 50,
+ "ttl": "PT1m"
+ }
+ }
+ }
+ }'
+ fi
'''
}
}
- stage('execute') {
+ stage('Set timeout at 10 minutes') {
options {
timeout(time:10)
}
@@ -151,7 +174,7 @@
'''
}
}
- stage('ONUs-enabled') {
+ stage('Wait for ONUs to be enabled') {
steps {
sh '''
if [ -z ${expectedOnus} ]
@@ -169,11 +192,15 @@
i=$(voltctl device list | grep -v OLT | grep ACTIVE | wc -l)
done
echo "${expectedOnus} ONUs Activated in $SECONDS seconds (time: $SECONDS)"
+
echo $SECONDS > voltha-devices-time-num.txt
+
+ echo "VOLTHA Duration(s)" > voltha-devices-time.txt
+ cat voltha-devices-time-num.txt >> voltha-devices-time.txt
'''
}
}
- stage('ONOS-ports') {
+ stage('Wait for ports to be discovered in ONOS') {
steps {
sh '''
# Check ports showed up in ONOS
@@ -185,15 +212,40 @@
z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 ports -e | grep BBSM | wc -l)
done
echo "${expectedOnus} ports enabled in $SECONDS seconds (time: $SECONDS)"
+
echo $SECONDS > temp.txt
paste voltha-devices-time-num.txt temp.txt | awk '{print ($1 + $2)}' > onos-ports-time-num.txt
- echo "ONOS-Duration(s)" > onos-ports-time.txt
- echo "VOLTHA-Duration(s)" > voltha-devices-time.txt
- cat voltha-devices-time-num.txt >> voltha-devices-time.txt
+
+ echo "PORTs Duration(s)" > onos-ports-time.txt
cat onos-ports-time-num.txt >> onos-ports-time.txt
'''
}
}
+ stage('Wait for flows to be programmed') {
+ steps {
+ sh '''
+ if [ ${withOnosApps} = false ] ; then
+ echo "ONOS Apps are not enabled, nothing to check"
+ else
+ # wait until all flows are in added state
+ z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 flows -s | grep PENDING | wc -l)
+ until [ $z -eq 0 ]
+ do
+ echo "${z} flows in PENDING state (time: $SECONDS)"
+ sleep ${pollInterval}
+ z=$(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 flows -s | grep PENDING | wc -l)
+ done
+ echo "All flows correctly programmed (time: $SECONDS)"
+
+ echo $SECONDS > temp.txt
+ paste onos-ports-time-num.txt temp.txt | awk '{print ($1 + $2)}' > onos-flows-time-num.txt
+
+ echo "FLOWs Duration(s)" > onos-flows-time.txt
+ cat onos-flows-time-num.txt >> onos-flows-time.txt
+ fi
+ '''
+ }
+ }
}
}
}
@@ -201,33 +253,50 @@
success {
plot([
csvFileName: 'plot-onu-activation.csv',
- csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-time.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-time.txt', inclusionFlag: 'OFF', url: '']],
- group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Time (${BBSIMdelay}s Delay)", yaxis: 'Time (s)', useDescr: true
+ csvSeries: [
+ [displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-time.txt', inclusionFlag: 'OFF', url: ''],
+ [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-time.txt', inclusionFlag: 'OFF', url: ''],
+ [displayTableFlag: false, exclusionValues: '', file: 'onos-flows-time.txt', inclusionFlag: 'OFF', url: '']
+ ],
+ group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Time (${BBSIMdelay}s Delay)", yaxis: 'Time (s)', useDescr: true
])
}
always {
+ // count how many ONUs have been activated
sh '''
- echo $(voltctl device list | grep -v OLT | grep ACTIVE | wc -l) > onus.txt
echo "#-of-ONUs" > voltha-devices-count.txt
- cat onus.txt >> voltha-devices-count.txt
+ echo $(voltctl device list | grep -v OLT | grep ACTIVE | wc -l) >> voltha-devices-count.txt
'''
+ // count how many ports have been discovered
sh '''
- echo $(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 ports -e | grep BBSM | wc -l) > ports.txt
echo "#-of-ports" > onos-ports-count.txt
- cat ports.txt >> onos-ports-count.txt
+ echo $(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 ports -e | grep BBSM | wc -l) >> onos-ports-count.txt
'''
+ // count how many flows have been provisioned
+ sh '''
+ echo "#-of-flows" > onos-flows-count.txt
+ echo $(sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 flows -s | grep ADDED | wc -l) >> onos-flows-count.txt
+ '''
+ // check which containers were used in this build
sh '''
kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
'''
+ // dump all the VOLTHA devices informations
sh '''
voltctl device list -o json > device-list.json
python -m json.tool device-list.json > voltha-devices-list.json
'''
+ // get ports and flows from ONOS
sh '''
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 ports > onos-ports-list.txt
+ sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 flows -s > onos-flows-list.txt
+ '''
+ // collect the CPU usage
+ sh '''
curl -s -X GET -G http://127.0.0.1:31301/api/v1/query --data-urlencode 'query=avg(rate(container_cpu_usage_seconds_total[10m])*100) by (pod_name)' | jq . > cpu-usage.json
'''
+ // get all the logs from kubernetes PODs
sh '''
kubectl get pods -o wide
kubectl logs -l app=adapter-open-olt > open-olt-logs.txt
@@ -236,12 +305,17 @@
kubectl logs -l app=ofagent > voltha-ofagent-logs.txt
kubectl logs -l app=bbsim > bbsim-logs.txt
'''
+ // cleanup of things we don't want to archive
sh '''
- rm -rf BBSM-12345123451234512345-00000000000001-v1.json device-list.json onus.txt ports.txt temp.txt
+ rm -rf BBSM-12345123451234512345-00000000000001-v1.json device-list.json temp.txt
'''
+ // compile a plot of the activate informations
plot([
csvFileName: 'plot-numbers.csv',
- csvSeries: [[displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-count.txt', inclusionFlag: 'OFF', url: ''], [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-count.txt', inclusionFlag: 'OFF', url: '']],
+ csvSeries: [
+ [displayTableFlag: false, exclusionValues: '', file: 'voltha-devices-count.txt', inclusionFlag: 'OFF', url: ''],
+ [displayTableFlag: false, exclusionValues: '', file: 'onos-ports-count.txt', inclusionFlag: 'OFF', url: '']
+ ],
group: 'Voltha-Scale-Numbers', numBuilds: '100', style: 'line', title: "Activated ONUs and Recognized Ports", yaxis: 'Number of Ports/ONUs', useDescr: true
])
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 58ea1eb..971affe 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -100,6 +100,17 @@
numOfBbsim: 2
expectedOnus: 200
BBSIMdelay: 200
+ - 'voltha-scale-measurements-periodic':
+ name: 'voltha-scale-measurements-periodic-10-20-200ms-with-flows'
+ build-node: 'onf-pod1-head-node'
+ time-trigger: "H H/4 * * *"
+ onuPerPon: 20
+ ponPorts: 10
+ expectedOnus: 200
+ BBSIMdelay: 200
+ withOnosApps: true
+ flowStatInterval: 5
+ portsStatInterval: 5
- 'voltha-scale-measurements':
name: 'voltha-scale-measurements-dev'
build-node: 'voltha-scale'
@@ -144,6 +155,10 @@
withMibTemplate: true
numOfBbsim: 1
numOfKafka: 3
+ withOnosApps: false
+ setLinkDiscovery: false
+ flowStatInterval: 600
+ portsStatInterval: 600
parameters:
- string:
@@ -178,7 +193,7 @@
- bool:
name: withOnosApps
- default: false
+ default: '{withOnosApps}'
description: 'Option to deactivate certain ONOS apps'
- bool:
@@ -188,7 +203,7 @@
- bool:
name: setLinkDiscovery
- default: false
+ default: '{setLinkDiscovery}'
description: 'Option to toggle Link Discovery'
- string:
@@ -198,12 +213,12 @@
- string:
name: flowStatInterval
- default: 600
+ default: '{flowStatInterval}'
description: 'Flow Stats Collection Interval, milliseconds'
- string:
name: portsStatInterval
- default: 600
+ default: {portsStatInterval}
description: 'Ports Stats Collection Interval, milliseconds'
- bool:
@@ -326,7 +341,11 @@
extraHelmFlags: ''
withMibTemplate: true
numOfBbsim: 1
- numOfKafka: 1
+ numOfKafka: 3
+ withOnosApps: true
+ setLinkDiscovery: false
+ flowStatInterval: 600
+ portsStatInterval: 600
parameters:
- string:
@@ -361,7 +380,7 @@
- bool:
name: withOnosApps
- default: false
+ default: '{withOnosApps}'
description: 'Option to deactivate certain ONOS apps'
- bool:
@@ -371,7 +390,7 @@
- bool:
name: setLinkDiscovery
- default: false
+ default: '{setLinkDiscovery}'
description: 'Option to toggle Link Discovery'
- string:
@@ -381,12 +400,12 @@
- string:
name: flowStatInterval
- default: 600
+ default: '{flowStatInterval}'
description: 'Flow Stats Collection Interval, milliseconds'
- string:
name: portsStatInterval
- default: 600
+ default: {portsStatInterval}
description: 'Ports Stats Collection Interval, milliseconds'
- bool: