Merge "[VOL-2940] Exposing kafka metrics in scale test"
diff --git a/jjb/pipeline/voltha-scale-test.groovy b/jjb/pipeline/voltha-scale-test.groovy
index 2c3ff87..2e9c325 100644
--- a/jjb/pipeline/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha-scale-test.groovy
@@ -67,6 +67,8 @@
APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius"
LOG_FOLDER="$WORKSPACE/logs"
+
+ GERRIT_PROJECT="${GERRIT_PROJECT}"
}
stages {
@@ -154,6 +156,23 @@
}
}
}
+ stage('Build patch') {
+ when {
+ expression {
+ return env.hasProperty("GERRIT_PROJECT") && env.GERRIT_PROJECT != ""
+ }
+ }
+ steps {
+ sh """
+ git clone https://\$GERRIT_HOST/\$GERRIT_PROJECT
+ cd \$GERRIT_PROJECT
+ git fetch https://\$GERRIT_HOST/\$GERRIT_PROJECT \$GERRIT_REFSPEC && git checkout FETCH_HEAD
+
+ DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-build
+ DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-push
+ """
+ }
+ }
stage('Deploy common infrastructure') {
// includes monitoring, kafka, etcd
steps {
@@ -195,33 +214,34 @@
fi
# BBSim custom image handling
- if [ '${bbsimImg.trim()}' != '' ]; then
+ if [ '${bbsimImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'bbsim' ]; then
IFS=: read -r bbsimRepo bbsimTag <<< '${bbsimImg.trim()}'
EXTRA_HELM_FLAGS+="--set images.bbsim.repository=\$bbsimRepo,images.bbsim.tag=\$bbsimTag "
fi
# VOLTHA and ofAgent custom image handling
# NOTE to override the rw-core image in a released version you must set the ofAgent image too
- if [ '${rwCoreImg.trim()}' != '' ] && [ '${ofAgentImg.trim()}' != '' ]; then
+ # TODO split ofAgent and voltha-go
+ if [ '${rwCoreImg.trim()}' != '' ] && [ '${ofAgentImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-go' ]; then
IFS=: read -r rwCoreRepo rwCoreTag <<< '${rwCoreImg.trim()}'
IFS=: read -r ofAgentRepo ofAgentTag <<< '${ofAgentImg.trim()}'
EXTRA_HELM_FLAGS+="--set images.rw_core.repository=\$rwCoreRepo,images.rw_core.tag=\$rwCoreTag,images.ofagent.repository=\$ofAgentRepo,images.ofagent.tag=\$ofAgentTag "
fi
# OpenOLT custom image handling
- if [ '${openoltAdapterImg.trim()}' != '' ]; then
+ if [ '${openoltAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openolt-adapter' ]; then
IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< '${openoltAdapterImg.trim()}'
EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=\$openoltAdapterRepo,images.adapter_open_olt.tag=\$openoltAdapterTag "
fi
# OpenONU custom image handling
- if [ '${openonuAdapterImg.trim()}' != '' ]; then
+ if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then
IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'
EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=\$openonuAdapterRepo,images.adapter_open_onu.tag=\$openonuAdapterTag "
fi
# ONOS custom image handling
- if [ '${onosImg.trim()}' != '' ]; then
+ if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then
IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'
EXTRA_HELM_FLAGS+="--set images.onos.repository=\$onosRepo,images.onos.tag=\$onosTag "
fi
@@ -243,6 +263,32 @@
export BBSIM_CFG="$WORKSPACE/kind-voltha/configs/bbsim-sadis-${workflow}.yaml"
fi
+ # Use custom built images
+
+ if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then
+ EXTRA_HELM_FLAGS+="--set images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,images.rw_core.tag=voltha-scale "
+ fi
+
+ if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then
+ EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,images.adapter_open_olt.tag=voltha-scale "
+ fi
+
+ if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then
+ EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,images.adapter_open_onu.tag=voltha-scale "
+ fi
+
+ if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then
+ EXTRA_HELM_FLAGS+="--set images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,images.ofagent.tag=voltha-scale "
+ fi
+
+ if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then
+ EXTRA_HELM_FLAGS+="--set images.onos.repository=${dockerRegistry}/voltha/voltha-onos,images.onos.tag=voltha-scale "
+ fi
+
+ if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then
+ EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "
+ fi
+
./voltha up
# Forward the ETCD port onto $VOLTHA_ETCD_PORT
@@ -276,6 +322,7 @@
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true
+
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 1000
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 9dbef0f..8e1ea0b 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -24,8 +24,6 @@
withEapol: false
withDhcp: false
withIgmp: false
- onosReplicas: 3
- atomixReplicas: 3
openonuAdapterReplicas: 1
openonuAdapterImg: voltha/voltha-openonu-adapter-go:master
extraHelmFlags: '--set use_openonu_adapter_go=true'
@@ -43,8 +41,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- onosReplicas: 3
- atomixReplicas: 3
openonuAdapterReplicas: 1
openonuAdapterImg: voltha/voltha-openonu-adapter-go:master
extraHelmFlags: '--set use_openonu_adapter_go=true'
@@ -65,9 +61,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
withPcap: true
@@ -86,9 +79,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
- 'voltha-scale-measurements':
@@ -105,9 +95,6 @@
withEapol: false
withDhcp: true
withIgmp: true
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
- 'voltha-scale-measurements':
@@ -124,9 +111,6 @@
withEapol: false
withDhcp: false
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
# jobs for 1024 ONUs with a 8 openonu-adapters and clustered ONOS (2 OLTs)
@@ -143,9 +127,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
onosImg: matteoscandolo/voltha-onos:olt-log
@@ -163,9 +144,6 @@
withEapol: false
withDhcp: false
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
- 'voltha-scale-measurements':
@@ -201,9 +179,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.5
bbsimImg: ''
@@ -227,9 +202,6 @@
withEapol: false
withDhcp: false
withIgmp: false
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.5
bbsimImg: ''
@@ -253,9 +225,6 @@
withEapol: false
withDhcp: true
withIgmp: true
- openonuAdapterReplicas: 8
- onosReplicas: 3
- atomixReplicas: 3
extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.5
bbsimImg: ''
@@ -265,6 +234,51 @@
openonuAdapterImg: ''
onosImg: ''
+ # per patchset job
+ - 'voltha-scale-measurements':
+ name: 'voltha-scale-measurements-patchset-1-16-32-att-subscribers'
+ build-node: 'onf-pod1-head-node'
+ olts: 1
+ pons: 16
+ onus: 32
+ withFlows: true
+ provisionSubscribers: true
+ withEapol: true
+ withDhcp: true
+ withIgmp: false
+ withMonitoring: false
+ trigger-comment: "voltha scale test ATT"
+
+ - 'voltha-scale-measurements':
+ name: 'voltha-scale-measurements-patchset-1-16-32-tt-subscribers'
+ build-node: 'onf-pod1-head-node'
+ olts: 1
+ pons: 16
+ onus: 32
+ withFlows: true
+ provisionSubscribers: true
+ workflow: tt
+ withEapol: false
+ withDhcp: true
+ withIgmp: true
+ withMonitoring: false
+ trigger-comment: "voltha scale test TT"
+
+ - 'voltha-scale-measurements':
+ name: 'voltha-scale-measurements-patchset-1-16-32-dt-subscribers'
+ build-node: 'onf-pod1-head-node'
+ olts: 1
+ pons: 16
+ onus: 32
+ withFlows: true
+ provisionSubscribers: true
+ workflow: dt
+ withEapol: false
+ withDhcp: false
+ withIgmp: false
+ withMonitoring: false
+ trigger-comment: "voltha scale test DT"
+
# development pipeline
- 'voltha-scale-measurements-dev':
name: 'voltha-scale-measurements-master-dev'
@@ -293,6 +307,11 @@
description: 'Name of the Jenkins node to run the job on'
- string:
+ name: dockerRegistry
+ default: '{dockerRegistry}'
+ description: 'Address of the docker registry to use if we are building custom images'
+
+ - string:
name: logLevel
default: '{logLevel}'
description: 'Log level for all the components'
@@ -541,9 +560,9 @@
withProfiling: false
withPcap: false
withRadius: yes
- openonuAdapterReplicas: 1
- onosReplicas: 1
- atomixReplicas: 0
+ openonuAdapterReplicas: 8
+ onosReplicas: 3
+ atomixReplicas: 3
kafkaReplicas: 3
etcdReplicas: 3
extraHelmFlags: ''
@@ -553,6 +572,9 @@
inMemoryEtcdStorage: true
karafHome: 'apache-karaf-4.2.9'
logLevel: 'INFO'
+ dockerRegistry: 10.90.0.101:30500
+ trigger-comment: vv7CBoQQYYonvaN8xcru
+ time-trigger: 0 0 29 2 *
<<: *voltha-scale-job-parameters
@@ -562,9 +584,28 @@
dsl: !include-raw-escape: pipeline/{pipeline-script}
triggers:
+ # periodic jobs will set this every 4 hours
+ # patchset jobs will set this to "0 0 29 2 *" (feb 29th, it's once every 4 years)
- timed: |
TZ=America/Los_Angeles
{time-trigger}
+ # periodic jobs will set this to vv7CBoQQYYonvaN8xcru (hopefully no-one will comment with that)
+ # patchset jobs will set this to a meaningful comment
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ silent-start: false
+ successful-message: "PASSED scale test"
+ failure-message: "FAILED scale test"
+ unstable-message: "UNSTABLE scale test"
+ trigger-on:
+ - comment-added-contains-event:
+ comment-contains-value: '.*{trigger-comment}.*'
+ projects:
+ - project-compare-type: REG_EXP
+ project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos|bbsim)$'
+ branches:
+ - branch-compare-type: REG_EXP
+ branch-pattern: '{all-branches-regexp}'
- job-template:
id: 'voltha-scale-measurements-dev'
@@ -615,7 +656,7 @@
withIgmp: false
withLLDP: false
withMibTemplate: true
- withMonitoring: true
+ withMonitoring: false
withProfiling: false
withPcap: false
withRadius: yes
@@ -631,6 +672,7 @@
inMemoryEtcdStorage: true
karafHome: 'apache-karaf-4.2.9'
logLevel: 'INFO'
+ dockerRegistry: 10.90.0.69:30500
<<: *voltha-scale-job-parameters
@@ -648,10 +690,10 @@
unstable-message: "UNSTABLE scale test"
trigger-on:
- comment-added-contains-event:
- comment-contains-value: '.*scale test.*'
+ comment-contains-value: '.*test voltha scale.*'
projects:
- project-compare-type: REG_EXP
- project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos)$'
+ project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos|bbsim)$'
branches:
- branch-compare-type: REG_EXP
branch-pattern: '{all-branches-regexp}'