Merge "[VOL-2940] Exposing kafka metrics in scale test"
diff --git a/jjb/pipeline/voltha-scale-test.groovy b/jjb/pipeline/voltha-scale-test.groovy
index 2c3ff87..2e9c325 100644
--- a/jjb/pipeline/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha-scale-test.groovy
@@ -67,6 +67,8 @@
 
     APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius"
     LOG_FOLDER="$WORKSPACE/logs"
+
+    GERRIT_PROJECT="${GERRIT_PROJECT}"
   }
 
   stages {
@@ -154,6 +156,23 @@
         }
       }
     }
+    stage('Build patch') {
+      when {
+        expression {
+          return env.hasProperty("GERRIT_PROJECT") && env.GERRIT_PROJECT != ""
+        }
+      }
+      steps {
+        sh """
+        git clone https://\$GERRIT_HOST/\$GERRIT_PROJECT
+        cd \$GERRIT_PROJECT
+        git fetch https://\$GERRIT_HOST/\$GERRIT_PROJECT \$GERRIT_REFSPEC && git checkout FETCH_HEAD
+
+        DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-build
+        DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-push
+        """
+      }
+    }
     stage('Deploy common infrastructure') {
       // includes monitoring, kafka, etcd
       steps {
@@ -195,33 +214,34 @@
             fi
 
             # BBSim custom image handling
-            if [ '${bbsimImg.trim()}' != '' ]; then
+            if [ '${bbsimImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'bbsim' ]; then
               IFS=: read -r bbsimRepo bbsimTag <<< '${bbsimImg.trim()}'
               EXTRA_HELM_FLAGS+="--set images.bbsim.repository=\$bbsimRepo,images.bbsim.tag=\$bbsimTag "
             fi
 
             # VOLTHA and ofAgent custom image handling
             # NOTE to override the rw-core image in a released version you must set the ofAgent image too
-            if [ '${rwCoreImg.trim()}' != '' ] && [ '${ofAgentImg.trim()}' != '' ]; then
+            # TODO split ofAgent and voltha-go
+            if [ '${rwCoreImg.trim()}' != '' ] && [ '${ofAgentImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-go' ]; then
               IFS=: read -r rwCoreRepo rwCoreTag <<< '${rwCoreImg.trim()}'
               IFS=: read -r ofAgentRepo ofAgentTag <<< '${ofAgentImg.trim()}'
               EXTRA_HELM_FLAGS+="--set images.rw_core.repository=\$rwCoreRepo,images.rw_core.tag=\$rwCoreTag,images.ofagent.repository=\$ofAgentRepo,images.ofagent.tag=\$ofAgentTag "
             fi
 
             # OpenOLT custom image handling
-            if [ '${openoltAdapterImg.trim()}' != '' ]; then
+            if [ '${openoltAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openolt-adapter' ]; then
               IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< '${openoltAdapterImg.trim()}'
               EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=\$openoltAdapterRepo,images.adapter_open_olt.tag=\$openoltAdapterTag "
             fi
 
             # OpenONU custom image handling
-            if [ '${openonuAdapterImg.trim()}' != '' ]; then
+            if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then
               IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'
               EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=\$openonuAdapterRepo,images.adapter_open_onu.tag=\$openonuAdapterTag "
             fi
 
             # ONOS custom image handling
-            if [ '${onosImg.trim()}' != '' ]; then
+            if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then
               IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'
               EXTRA_HELM_FLAGS+="--set images.onos.repository=\$onosRepo,images.onos.tag=\$onosTag "
             fi
@@ -243,6 +263,32 @@
               export BBSIM_CFG="$WORKSPACE/kind-voltha/configs/bbsim-sadis-${workflow}.yaml"
             fi
 
+            # Use custom built images
+
+            if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then
+              EXTRA_HELM_FLAGS+="--set images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,images.rw_core.tag=voltha-scale "
+            fi
+
+            if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then
+              EXTRA_HELM_FLAGS+="--set images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,images.adapter_open_olt.tag=voltha-scale "
+            fi
+
+            if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then
+              EXTRA_HELM_FLAGS+="--set images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,images.adapter_open_onu.tag=voltha-scale "
+            fi
+
+            if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then
+              EXTRA_HELM_FLAGS+="--set images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,images.ofagent.tag=voltha-scale "
+            fi
+
+            if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then
+              EXTRA_HELM_FLAGS+="--set images.onos.repository=${dockerRegistry}/voltha/voltha-onos,images.onos.tag=voltha-scale "
+            fi
+
+            if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then
+              EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "
+            fi
+
             ./voltha up
 
             # Forward the ETCD port onto $VOLTHA_ETCD_PORT
@@ -276,6 +322,7 @@
           sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true
           sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true
 
+
           sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 1000
 
           sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index 9dbef0f..8e1ea0b 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -24,8 +24,6 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          onosReplicas: 3
-          atomixReplicas: 3
           openonuAdapterReplicas: 1
           openonuAdapterImg: voltha/voltha-openonu-adapter-go:master
           extraHelmFlags: '--set use_openonu_adapter_go=true'
@@ -43,8 +41,6 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          onosReplicas: 3
-          atomixReplicas: 3
           openonuAdapterReplicas: 1
           openonuAdapterImg: voltha/voltha-openonu-adapter-go:master
           extraHelmFlags: '--set use_openonu_adapter_go=true'
@@ -65,9 +61,6 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
           withPcap: true
 
@@ -86,9 +79,6 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
 
       - 'voltha-scale-measurements':
@@ -105,9 +95,6 @@
           withEapol: false
           withDhcp: true
           withIgmp: true
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
 
       - 'voltha-scale-measurements':
@@ -124,9 +111,6 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
 
       # jobs for 1024 ONUs with a 8 openonu-adapters and clustered ONOS (2 OLTs)
@@ -143,9 +127,6 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
           onosImg: matteoscandolo/voltha-onos:olt-log
 
@@ -163,9 +144,6 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
 
       - 'voltha-scale-measurements':
@@ -201,9 +179,6 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set defaults.rw_core.timeout=30s '
           release: voltha-2.5
           bbsimImg: ''
@@ -227,9 +202,6 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set defaults.rw_core.timeout=30s '
           release: voltha-2.5
           bbsimImg: ''
@@ -253,9 +225,6 @@
           withEapol: false
           withDhcp: true
           withIgmp: true
-          openonuAdapterReplicas: 8
-          onosReplicas: 3
-          atomixReplicas: 3
           extraHelmFlags: '--set defaults.rw_core.timeout=30s '
           release: voltha-2.5
           bbsimImg: ''
@@ -265,6 +234,51 @@
           openonuAdapterImg: ''
           onosImg: ''
 
+      # per patchset job
+      - 'voltha-scale-measurements':
+          name: 'voltha-scale-measurements-patchset-1-16-32-att-subscribers'
+          build-node: 'onf-pod1-head-node'
+          olts: 1
+          pons: 16
+          onus: 32
+          withFlows: true
+          provisionSubscribers: true
+          withEapol: true
+          withDhcp: true
+          withIgmp: false
+          withMonitoring: false
+          trigger-comment: "voltha scale test ATT"
+
+      - 'voltha-scale-measurements':
+          name: 'voltha-scale-measurements-patchset-1-16-32-tt-subscribers'
+          build-node: 'onf-pod1-head-node'
+          olts: 1
+          pons: 16
+          onus: 32
+          withFlows: true
+          provisionSubscribers: true
+          workflow: tt
+          withEapol: false
+          withDhcp: true
+          withIgmp: true
+          withMonitoring: false
+          trigger-comment: "voltha scale test TT"
+
+      - 'voltha-scale-measurements':
+          name: 'voltha-scale-measurements-patchset-1-16-32-dt-subscribers'
+          build-node: 'onf-pod1-head-node'
+          olts: 1
+          pons: 16
+          onus: 32
+          withFlows: true
+          provisionSubscribers: true
+          workflow: dt
+          withEapol: false
+          withDhcp: false
+          withIgmp: false
+          withMonitoring: false
+          trigger-comment: "voltha scale test DT"
+
       # development pipeline
       - 'voltha-scale-measurements-dev':
           name: 'voltha-scale-measurements-master-dev'
@@ -293,6 +307,11 @@
           description: 'Name of the Jenkins node to run the job on'
 
       - string:
+          name: dockerRegistry
+          default: '{dockerRegistry}'
+          description: 'Address of the docker registry to use if we are building custom images'
+
+      - string:
           name: logLevel
           default: '{logLevel}'
           description: 'Log level for all the components'
@@ -541,9 +560,9 @@
     withProfiling: false
     withPcap: false
     withRadius: yes
-    openonuAdapterReplicas: 1
-    onosReplicas: 1
-    atomixReplicas: 0
+    openonuAdapterReplicas: 8
+    onosReplicas: 3
+    atomixReplicas: 3
     kafkaReplicas: 3
     etcdReplicas: 3
     extraHelmFlags: ''
@@ -553,6 +572,9 @@
     inMemoryEtcdStorage: true
     karafHome: 'apache-karaf-4.2.9'
     logLevel: 'INFO'
+    dockerRegistry: 10.90.0.101:30500
+    trigger-comment: vv7CBoQQYYonvaN8xcru
+    time-trigger: 0 0 29 2 *
 
     <<: *voltha-scale-job-parameters
 
@@ -562,9 +584,28 @@
     dsl: !include-raw-escape: pipeline/{pipeline-script}
 
     triggers:
+      # periodic jobs will set this every 4 hours
+      # patchset jobs will set this to "0 0 29 2 *" (feb 29th, it's once every 4 years)
       - timed: |
                  TZ=America/Los_Angeles
                  {time-trigger}
+      # periodic jobs will set this to vv7CBoQQYYonvaN8xcru (hopefully no-one will comment with that)
+      # patchset jobs will set this to a meaningful comment
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          silent-start: false
+          successful-message: "PASSED scale test"
+          failure-message: "FAILED scale test"
+          unstable-message: "UNSTABLE scale test"
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: '.*{trigger-comment}.*'
+          projects:
+            - project-compare-type: REG_EXP
+              project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos|bbsim)$'
+              branches:
+                - branch-compare-type: REG_EXP
+                  branch-pattern: '{all-branches-regexp}'
 
 - job-template:
     id: 'voltha-scale-measurements-dev'
@@ -615,7 +656,7 @@
     withIgmp: false
     withLLDP: false
     withMibTemplate: true
-    withMonitoring: true
+    withMonitoring: false
     withProfiling: false
     withPcap: false
     withRadius: yes
@@ -631,6 +672,7 @@
     inMemoryEtcdStorage: true
     karafHome: 'apache-karaf-4.2.9'
     logLevel: 'INFO'
+    dockerRegistry: 10.90.0.69:30500
 
     <<: *voltha-scale-job-parameters
 
@@ -648,10 +690,10 @@
           unstable-message: "UNSTABLE scale test"
           trigger-on:
             - comment-added-contains-event:
-                comment-contains-value: '.*scale test.*'
+                comment-contains-value: '.*test voltha scale.*'
           projects:
             - project-compare-type: REG_EXP
-              project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos)$'
+              project-pattern: '^(voltha-go|voltha-openolt-adapter|voltha-openonu-adapter|ofagent-go|voltha-onos|bbsim)$'
               branches:
                 - branch-compare-type: REG_EXP
                   branch-pattern: '{all-branches-regexp}'