[VOL-2992] Adding profiler to scale pipeline
Change-Id: Ic79d30d09299e6b8101bd6bd03b9b7ac6d4452cf
diff --git a/jjb/pipeline/voltha-scale-test.groovy b/jjb/pipeline/voltha-scale-test.groovy
index b5785e3..aed5edc 100644
--- a/jjb/pipeline/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha-scale-test.groovy
@@ -27,7 +27,7 @@
KUBECONFIG="$HOME/.kube/config"
VOLTCONFIG="$HOME/.volt/config"
SSHPASS="karaf"
- PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ PATH="$PATH:$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
TYPE="minimal"
FANCY=0
WITH_SIM_ADAPTERS="no"
@@ -54,6 +54,8 @@
NUM_OF_OPENONU="${openonuAdapterReplicas}"
NUM_OF_ONOS="${onosReplicas}"
NUM_OF_ATOMIX="${atomixReplicas}"
+ WITH_PPROF="${withProfiling}"
+ EXTRA_HELM_FLAGS="${extraHelmFlags}"
VOLTHA_CHART="${volthaChart}"
VOLTHA_BBSIM_CHART="${bbsimChart}"
@@ -64,19 +66,21 @@
stages {
stage ('Cleanup') {
steps {
- sh returnStdout: false, script: """
- test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
+ timeout(time: 11, unit: 'MINUTES') {
+ sh returnStdout: false, script: """
+ test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
- for hchart in \$(helm list -q | grep -E -v 'docker-registry|kafkacat|etcd-operator');
- do
- echo "Purging chart: \${hchart}"
- helm delete --purge "\${hchart}"
- done
- bash /home/cord/voltha-scale/wait_for_pods.sh
+ for hchart in \$(helm list -q | grep -E -v 'docker-registry|kafkacat|etcd-operator');
+ do
+ echo "Purging chart: \${hchart}"
+ helm delete --purge "\${hchart}"
+ done
+ bash /home/cord/voltha-scale/wait_for_pods.sh
- cd $WORKSPACE
- rm -rf $WORKSPACE/*
- """
+ cd $WORKSPACE
+ rm -rf $WORKSPACE/*
+ """
+ }
}
}
stage('Clone kind-voltha') {
@@ -211,6 +215,38 @@
}
stage('Run Test') {
steps {
+ sh '''
+ mkdir -p $WORKSPACE/RobotLogs
+ cd $WORKSPACE/voltha-system-tests
+ make vst_venv
+ '''
+ sh '''
+ if [ ${withProfiling} = true ] ; then
+ mkdir -p $WORKSPACE/logs/pprof
+ echo $PATH
+ #Creating Python script for ONU Detection
+ cat << EOF > $WORKSPACE/pprof.sh
+i=0
+while [[ true ]]; do
+ ((i++))
+ go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $WORKSPACE/logs/pprof/rw-core-heap-\\$i.png
+ go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $WORKSPACE/logs/pprof/rw-core-goroutine-\\$i.png
+ go tool pprof -png http://127.0.0.1:6060/debug/pprof/block > $WORKSPACE/logs/pprof/rw-core-block-\\$i.png
+ go tool pprof -png http://127.0.0.1:6060/debug/pprof/mutex > $WORKSPACE/logs/pprof/rw-core-mutex-\\$i.png
+
+ go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $WORKSPACE/logs/pprof/openolt-heap-\\$i.png
+ go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $WORKSPACE/logs/pprof/openolt-goroutine-\\$i.png
+ go tool pprof -png http://127.0.0.1:6061/debug/pprof/block > $WORKSPACE/logs/pprof/openolt-block-\\$i.png
+ go tool pprof -png http://127.0.0.1:6061/debug/pprof/mutex > $WORKSPACE/logs/pprof/openolt-mutex-\\$i.png
+
+ sleep 10
+done
+EOF
+
+ _TAG="pprof"
+ _TAG=$_TAG bash $WORKSPACE/pprof.sh &
+ fi
+ '''
timeout(time: 11, unit: 'MINUTES') {
sh '''
ROBOT_PARAMS="-v olt:${olts} \
@@ -239,9 +275,7 @@
ROBOT_PARAMS+="-i setup -i activation "
fi
- mkdir -p $WORKSPACE/RobotLogs
- cd voltha-system-tests
- make vst_venv
+ cd $WORKSPACE/voltha-system-tests
source ./vst_venv/bin/activate
robot -d $WORKSPACE/RobotLogs \
$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot
@@ -262,6 +296,18 @@
}
post {
always {
+ sh '''
+ if [ ${withProfiling} = true ] ; then
+ _TAG="pprof"
+ P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
+ if [ -n "$P_IDS" ]; then
+ echo $P_IDS
+ for P_ID in $P_IDS; do
+ kill -9 $P_ID
+ done
+ fi
+ fi
+ '''
plot([
csvFileName: 'scale-test.csv',
csvSeries: [
@@ -354,7 +400,7 @@
kubectl logs -l app=bbsim > $WORKSPACE/logs/bbsim-logs.logs
kubectl logs -l app=onos > $WORKSPACE/logs/onos-logs.logs
'''
- archiveArtifacts artifacts: 'kind-voltha/install-minimal.log,execution-time.txt,logs/*'
+ archiveArtifacts artifacts: 'kind-voltha/install-minimal.log,execution-time.txt,logs/*,logs/pprof/*.png'
}
}
}
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 90424ee..6bd1ed7 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -7,7 +7,6 @@
project-name: '{name}'
jobs:
- # new Jobs
# name format is <job-template>-<olts>-<pon>-<onus>-<setup>
# needed for comparison with the openonu-go adapter
- 'voltha-scale-measurements':
@@ -61,6 +60,20 @@
withEapol: true
withDhcp: true
withIgmp: false
+ - 'voltha-scale-measurements':
+ name: 'voltha-scale-measurements-1-10-20-subscribers-profiled'
+ build-node: 'onf-pod1-head-node'
+ time-trigger: "H H/4 * * *"
+ olts: 1
+ pons: 10
+ onus: 20
+ withFlows: true
+ provisionSubscribers: true
+ withEapol: true
+ withDhcp: true
+ withIgmp: false
+ withProfiling: true
+ extraHelmFlags: '--set profiler.enabled=true '
# jobs for 512 ONUs with a 8 openonu-adapters and clustered ONOS
- 'voltha-scale-measurements':
@@ -207,9 +220,11 @@
withLLDP: false
withMibTemplate: true
withMonitoring: true
+ withProfiling: false
openonuAdapterReplicas: 1
onosReplicas: 1
atomixReplicas: 0
+ extraHelmFlags: ''
parameters:
- string:
@@ -282,6 +297,16 @@
default: '{withMonitoring}'
description: 'Option to install Prometheus'
+ - bool:
+ name: withProfiling
+ default: '{withProfiling}'
+ description: 'Option to collect profiling informations from rw-core and openolt (note that the appropriate -profile images needs to be used)'
+
+ - string:
+ name: extraHelmFlags
+ default: '{extraHelmFlags}'
+ description: 'Any extra helm parameters you want (passed to every helm install command)'
+
- string:
name: openonuAdapterReplicas
default: '{openonuAdapterReplicas}'
@@ -467,6 +492,16 @@
default: true
description: 'Option to install Prometheus'
+ - bool:
+ name: withProfiling
+ default: false
+ description: 'Option to collect profiling informations from rw-core and openolt (note that the appropriate -profile images needs to be used)'
+
+ - string:
+ name: extraHelmFlags
+ default: ' '
+ description: 'Any extra helm parameters you want (passed to every helm install command)'
+
- string:
name: openonuAdapterReplicas
default: 1