Merge "Call kali to collect the log before running the voltha"
diff --git a/jjb/cord-test/nightly-build-pipeline.yaml b/jjb/cord-test/nightly-build-pipeline.yaml
index a071906..6b9e854 100644
--- a/jjb/cord-test/nightly-build-pipeline.yaml
+++ b/jjb/cord-test/nightly-build-pipeline.yaml
@@ -862,6 +862,11 @@
           default: 'you@opennetworking.org, suchitra@opennetworking.org'
           description: ''
 
+      - string:
+          name: onosVersion
+          default: '{onos-version}'
+          description: 'ONOS version that needs to be installed'
+
       - bool:
           name: released
           default: '{released}'
diff --git a/jjb/cord-test/voltha.yaml b/jjb/cord-test/voltha.yaml
index 83b1095..7d62fd8 100644
--- a/jjb/cord-test/voltha.yaml
+++ b/jjb/cord-test/voltha.yaml
@@ -8,6 +8,8 @@
 
     build-timeout: '300'
 
+    onos-version: '1.13.9'
+
     jobs:
       # flex OCP pod with olt/onu - manual test job, voltha master build job
       - 'build_pod_manual':
@@ -55,6 +57,7 @@
           config-pod: 'flex-ocp-cord'
           release: 'master'
           branch: 'master'
+          onos-version: '2.2'
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
@@ -127,6 +130,7 @@
           config-pod: 'onf-demo-pod'
           release: 'master'
           branch: 'master'
+          onos-version: '1.13.10'
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
@@ -175,6 +179,7 @@
           config-pod: 'onf-demo-pod'
           release: 'master'
           branch: 'master'
+          onos-version: '1.13.10'
           released: false
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
@@ -199,6 +204,7 @@
           config-pod: 'onf-demo-pod'
           release: 'master'
           branch: 'master'
+          onos-version: '1.13.10'
           test-repo: 'voltha-system-tests'
           Jenkinsfile: 'Jenkinsfile-voltha-bal31-build'
           oltDebVersion: 'openolt_asfvolt16.deb'
diff --git a/jjb/pipeline/voltha-bbsim-tests.groovy b/jjb/pipeline/voltha-bbsim-tests.groovy
index a3033c1..c82501b 100644
--- a/jjb/pipeline/voltha-bbsim-tests.groovy
+++ b/jjb/pipeline/voltha-bbsim-tests.groovy
@@ -155,7 +155,20 @@
       steps {
         sh '''
            mkdir -p $WORKSPACE/RobotLogs
-           make -C $WORKSPACE/voltha/voltha-system-tests sanity-kind || true
+
+           # By default, all tests tagged 'sanity' are run.  This covers basic functionality
+           # like running through the ATT workflow for a single subscriber.
+           export TEST_TAGS=sanity
+
+           # If the Gerrit comment contains a line with "functional tests" then run the full
+           # functional test suite.  This covers tests tagged either 'sanity' or 'functional'.
+           # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
+           REGEX="functional tests"
+           if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
+             TEST_TAGS=sanityORfunctional
+           fi
+
+           make -C $WORKSPACE/voltha/voltha-system-tests single-kind || true
            '''
       }
     }
@@ -194,6 +207,8 @@
          extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
          extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
+         gzip $WORKSPACE/onos-voltha-combined.log
+
          ## shut down kind-voltha
          cd $WORKSPACE/kind-voltha
 	       WAIT_ON_DOWN=y ./voltha down
@@ -207,7 +222,7 @@
             passThreshold: 80,
             reportFileName: 'RobotLogs/report*.html',
             unstableThreshold: 0]);
-         archiveArtifacts artifacts: '*.log'
+         archiveArtifacts artifacts: '*.log,*.gz'
 
     }
   }
diff --git a/jjb/pipeline/voltha-go-multi-tests.groovy b/jjb/pipeline/voltha-go-multi-tests.groovy
index a167a29..c8886ff 100644
--- a/jjb/pipeline/voltha-go-multi-tests.groovy
+++ b/jjb/pipeline/voltha-go-multi-tests.groovy
@@ -130,6 +130,8 @@
          extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
          extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
+         gzip $WORKSPACE/onos-voltha-combined.log
+
          ## shut down voltha
          cd $WORKSPACE/kind-voltha/
          WAIT_ON_DOWN=y ./voltha down
@@ -143,7 +145,7 @@
             passThreshold: 100,
             reportFileName: 'RobotLogs/report*.html',
             unstableThreshold: 0]);
-         archiveArtifacts artifacts: '*.log'
+         archiveArtifacts artifacts: '*.log,*.gz'
 
     }
   }
diff --git a/jjb/pipeline/voltha-go-tests.groovy b/jjb/pipeline/voltha-go-tests.groovy
index d737988..e339fcc 100644
--- a/jjb/pipeline/voltha-go-tests.groovy
+++ b/jjb/pipeline/voltha-go-tests.groovy
@@ -125,6 +125,8 @@
          extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
          extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
+         gzip $WORKSPACE/onos-voltha-combined.log
+
          ## shut down voltha
          cd $WORKSPACE/kind-voltha/
          WAIT_ON_DOWN=y ./voltha down
@@ -138,7 +140,7 @@
             passThreshold: 100,
             reportFileName: 'RobotLogs/report*.html',
             unstableThreshold: 0]);
-         archiveArtifacts artifacts: '*.log'
+         archiveArtifacts artifacts: '*.log,*.gz'
 
     }
   }
diff --git a/jjb/pipeline/voltha-physical-build-and-tests.groovy b/jjb/pipeline/voltha-physical-build-and-tests.groovy
index ebc5174..3f9eda8 100644
--- a/jjb/pipeline/voltha-physical-build-and-tests.groovy
+++ b/jjb/pipeline/voltha-physical-build-and-tests.groovy
@@ -15,12 +15,18 @@
 // deploy VOLTHA built from patchset on a physical pod and run e2e test
 // uses kind-voltha to deploy voltha-2.X
 
-node {
-    // Need this so that deployment_config has global scope when it's read later
-    deployment_config = null
-    localDeploymentConfigFile = null
-    localKindVolthaValuesFile = null
-    localSadisConfigFile = null
+// Need this so that deployment_config has global scope when it's read later
+deployment_config = null
+localDeploymentConfigFile = null
+localKindVolthaValuesFile = null
+localSadisConfigFile = null
+
+// The pipeline assumes these variables are always defined
+if ( ! params.withPatchset ) {
+  GERRIT_EVENT_COMMENT_TEXT = ""
+  GERRIT_PROJECT = ""
+  GERRIT_CHANGE_NUMBER = ""
+  GERRIT_PATCHSET_NUMBER = ""
 }
 
 pipeline {
@@ -151,46 +157,36 @@
       }
       steps {
         script {
-          if ( params.withPatchset ) {
-            sh returnStdout: false, script: """
-            export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
+          sh returnStdout: false, script: """
+          export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
 
-            IMAGES=""
-            if [ "${gerritProject}" = "voltha-go" ]; then
-                IMAGES="rw_core ro_core "
-            elif [ "${gerritProject}" = "ofagent-py" ]; then
-                IMAGES="ofagent "
-            elif [ "${gerritProject}" = "voltha-onos" ]; then
-                IMAGES="onos "
-            elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
-                IMAGES="adapter_open_olt "
-            elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
-                IMAGES="adapter_open_onu "
-            elif [ "${gerritProject}" = "voltha-api-server" ]; then
-                IMAGES="afrouter afrouterd "
-            else
-                echo "No images to push"
-            fi
+          IMAGES=""
+          if [ "${gerritProject}" = "voltha-go" ]; then
+              IMAGES="rw_core ro_core "
+          elif [ "${gerritProject}" = "ofagent-py" ]; then
+              IMAGES="ofagent "
+          elif [ "${gerritProject}" = "voltha-onos" ]; then
+              IMAGES="onos "
+          elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
+              IMAGES="adapter_open_olt "
+          elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
+              IMAGES="adapter_open_onu "
+          elif [ "${gerritProject}" = "voltha-api-server" ]; then
+              IMAGES="afrouter afrouterd "
+          else
+              echo "No images to push"
+          fi
 
-            for I in \$IMAGES
-            do
-                EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
-            done
+          for I in \$IMAGES
+          do
+              EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
+          done
 
-            cd $WORKSPACE/kind-voltha/
-            echo \$EXTRA_HELM_FLAGS
-            kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
-            ./voltha up
-            """
-          } else {
-            sh returnStdout: false, script: """
-            export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
-            cd $WORKSPACE/kind-voltha/
-            echo \$EXTRA_HELM_FLAGS
-            kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
-            ./voltha up
-            """
-          }
+          cd $WORKSPACE/kind-voltha/
+          echo \$EXTRA_HELM_FLAGS
+          kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
+          ./voltha up
+          """
         }
       }
     }
@@ -282,7 +278,7 @@
     stage('Run E2E Tests') {
       environment {
         ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
-        ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs"
+        ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs "
         ROBOT_FILE="Voltha_PODTests.robot"
       }
       steps {
@@ -290,6 +286,15 @@
         cd voltha
         git clone -b ${branch} ${cordRepoUrl}/cord-tester
         mkdir -p $WORKSPACE/RobotLogs
+
+        # If the Gerrit comment contains a line with "functional tests" then run the full
+        # functional test suite.  This covers tests tagged either 'sanity' or 'functional'.
+        # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
+        REGEX="functional tests"
+        if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
+          ROBOT_MISC_ARGS+="-i functional"
+        fi
+
         make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
         """
       }
@@ -342,6 +347,8 @@
       extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
       extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
 
+      gzip $WORKSPACE/onos-voltha-combined.log
+
       ## collect events, the chart should be running by now
       kubectl get pods | grep -i voltha-kafka-dump | grep -i running
       if [[ $? == 0 ]]; then
@@ -367,7 +374,7 @@
         passThreshold: 80,
         reportFileName: 'RobotLogs/report*.html',
         unstableThreshold: 0]);
-      archiveArtifacts artifacts: '*.log'
+      archiveArtifacts artifacts: '*.log,*.gz'
     }
   }
 }
diff --git a/jjb/pipeline/voltha-physical-functional-tests.groovy b/jjb/pipeline/voltha-physical-functional-tests.groovy
index 05b6a90..5695c81 100644
--- a/jjb/pipeline/voltha-physical-functional-tests.groovy
+++ b/jjb/pipeline/voltha-physical-functional-tests.groovy
@@ -59,7 +59,6 @@
         git clone -b ${branch} ${cordRepoUrl}/cord-tester
         mkdir -p $WORKSPACE/bin
         bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/bin"
-        kail -n voltha -n default --since=20m > $WORKSPACE/onos-voltha-combined.log &
         """
       }
     }
@@ -71,6 +70,7 @@
       }
       steps {
         sh """
+        kail -n voltha -n default --since=20m > $WORKSPACE/Functional_onos-voltha-combined.log &
         mkdir -p $ROBOT_LOGS_DIR
         if  ( ${released} ); then
             export ROBOT_MISC_ARGS="--removekeywords wuks -i released -i sanity -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir}"
@@ -78,6 +78,8 @@
             export ROBOT_MISC_ARGS="--removekeywords wuks -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir}"
         fi
         make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
+        sync
+        pkill kail || true
         """
       }
     }
@@ -93,9 +95,12 @@
       }
       steps {
         sh """
+        kail -n voltha -n default > $WORKSPACE/FailureScenarios_onos-voltha-combined.log &
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir}"
         make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
+        sync
+        pkill kail || true
         """
       }
     }
@@ -111,9 +116,12 @@
       }
       steps {
         sh """
+        kail -n voltha -n default > $WORKSPACE/ErrorScenarios_onos-voltha-combined.log &
         mkdir -p $ROBOT_LOGS_DIR
         export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir}"
         make -C $WORKSPACE/voltha/voltha-system-tests voltha-test || true
+        sync
+        pkill kail || true
         """
       }
     }
@@ -133,14 +141,14 @@
       extract_errors_go() {
         echo
         echo "Error summary for $1:"
-        grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+        grep $1 $WORKSPACE/*onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
         echo
       }
 
       extract_errors_python() {
         echo
         echo "Error summary for $1:"
-        grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+        grep $1 $WORKSPACE/*onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
         echo
       }
 
@@ -148,6 +156,9 @@
       extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
       extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
       extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+
+      gzip $WORKSPACE/*onos-voltha-combined.log
+
       '''
       script {
         deployment_config.olts.each { olt ->
@@ -167,7 +178,7 @@
         reportFileName: '**/report*.html',
         unstableThreshold: 0
         ]);
-      archiveArtifacts artifacts: '*.log'
+      archiveArtifacts artifacts: '*.log,*.gz'
     }
     unstable {
       step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
diff --git a/jjb/pipeline/voltha-scale-measurements.groovy b/jjb/pipeline/voltha-scale-measurements.groovy
new file mode 100644
index 0000000..9059ad9
--- /dev/null
+++ b/jjb/pipeline/voltha-scale-measurements.groovy
@@ -0,0 +1,46 @@
+/* voltha-scale-measurements pipeline */
+pipeline {
+
+  /* no label, executor is determined by JJB */
+  agent {
+    label "${params.buildNode}"
+  }
+  stages {
+    stage('checkout') {
+      steps {
+        checkout([
+          $class: 'GitSCM',
+          userRemoteConfigs: [[ url: "https://github.com/ciena/kind-voltha.git", ]],
+          branches: [[ name: "master", ]],
+          extensions: [
+            [$class: 'WipeWorkspace'],
+            [$class: 'RelativeTargetDirectory', relativeTargetDir: "${params.projectName}"],
+            [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
+          ],
+        ])
+        script {
+          git_tags = sh(script:"cd $projectName; git tag -l --points-at HEAD", returnStdout: true).trim()
+        }
+      }
+    }
+    stage('start') {
+      steps {
+        sh """
+          #!/usr/bin/env bash
+          set -euo pipefail
+
+          echo "DO SOMETHING"
+          """
+      }
+    }
+  }
+  post {
+    cleanup {
+      sh '''
+        WAIT_ON_DOWN=y ./voltha down
+        cd $WORKSPACE/
+        rm -rf kind-voltha/ || true
+      '''
+    }
+  }
+}
diff --git a/jjb/verify/nem-ondemand-proxy.yaml b/jjb/verify/nem-ondemand-proxy.yaml
new file mode 100644
index 0000000..91d6286
--- /dev/null
+++ b/jjb/verify/nem-ondemand-proxy.yaml
@@ -0,0 +1,27 @@
+---
+# verification jobs for 'nem-ondemand-proxy' repo
+
+- project:
+    name: nem-ondemand-proxy
+    project: '{name}'
+
+    jobs:
+      - 'verify-nem-ondemand-proxy-jobs':
+          branch-regexp: '{all-branches-regexp}'
+      - 'publish-nem-ondemand-proxy-jobs'
+
+- job-group:
+    name: 'verify-nem-ondemand-proxy-jobs'
+    jobs:
+      - 'verify-licensed'
+      - 'tag-collision-reject':
+      - 'make-unit-test':
+          unit-test-targets: 'docker-build lint'
+          junit-allow-empty-results: true
+
+- job-group:
+    name: 'publish-nem-ondemand-proxy-jobs'
+    jobs:
+      - 'docker-publish':
+          docker-repo: 'opencord'
+          dependency-jobs: 'version-tag'
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index a69e79f..56510d3 100644
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -121,7 +121,7 @@
 
       - string:
           name: extraHelmFlags
-          default: '--set defaults.image_tag={default-image-tag},onu={onus},pon={pons}'
+          default: '--set defaults.image_tag={default-image-tag},onu={onus},pon={pons},use_ofagent_go=True,images.ofagent.repository=voltha/ofagent-go,images.ofagent.tag=master'
           description: 'Helm flags to pass to ./voltha up'
 
       - string:
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
new file mode 100644
index 0000000..af154e3
--- /dev/null
+++ b/jjb/voltha-scale.yaml
@@ -0,0 +1,54 @@
+---
+# VOLTHA 2.x scale measurements
+
+- project:
+    name: voltha-scale
+
+    project-name: '{name}'
+
+    jobs:
+      - 'voltha-scale-measurements':
+          name: 'voltha-scale-measurements-manual'
+          build-node: 'voltha-scale'
+
+- job-template:
+    id: 'voltha-scale-measurements'
+    name: '{name}'
+    pipeline-script: 'voltha-scale-measurements.groovy'
+
+    description: |
+      <!-- Managed by Jenkins Job Builder -->
+      Created by {id} job-template from ci-management/jjb/voltha-scale.yaml  <br /><br />
+      Using pipeline {pipeline-script} <br/><br/>
+      Scale measurements for VOLTHA 2.x
+
+    properties:
+      - cord-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+          artifact-num-to-keep: '{artifact-num-to-keep}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    parameters:
+      - string:
+          name: buildNode
+          default: '{build-node}'
+          description: 'Name of the Jenkins node to run the job on'
+
+      - string:
+          name: onuCount
+          default: 1
+          description: 'Number of ONUs to provision'
+
+      - string:
+          name: ponCount
+          default: 1
+          description: 'Number of PONs to provision'
+
+    project-type: pipeline
+    concurrent: true
+
+    dsl: !include-raw-escape: pipeline/{pipeline-script}