[VOL-5100] - periodic-voltha-test-bbsim

[VOL-5101] - voltha testing and kind command version

vars/installKind.groovy
=======================
  o Display command version for logging and to sanity check kind binary.
  o This script was created last year, using groovy + jenkins closures
    in call block to support native syntax like: installKail() { debug:true }.
    Encountered problems earlier, if they persist will revert to using
    normal script method with positional arguments.

jjb/pipeline/voltha/master/bbsim-tests.groovy
=============================================
  o Replaced hardcoded branch != master with isReleaseBranch(name).
  o Remove '|| true' command masking error status.
  o Try out pgrep/pkill commands to simplify usage.
  o Removed '|| true' from gzip/log concatenation commands.
  o Changed vst_venv/ to .venv/ -- newer viritual env directory name.
  o Invoke 'make venv-activate-script' near each python script call
    makefile target dependencies work properly.  Command always
    available and will only install once.
  o Replace "pkill kail || true" with "pgrep --count > 0 && pkill kail".
    The latter syntax will display command errors and will fail a job
    outright when the kail command refuses to terminate.

Change-Id: I6f6628347322486afe820a92f3a3bd3ce5bc6d4d
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 6d0ba1a..98c9beb 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -15,14 +15,33 @@
 // voltha-2.x e2e tests for openonu-go
 // uses bbsim to simulate OLT/ONUs
 
+// [TODO] Update syntax below to the latest supported
 library identifier: 'cord-jenkins-libraries@master',
     retriever: modernSCM([
       $class: 'GitSCMSource',
       remote: 'https://gerrit.opencord.org/ci-management.git'
 ])
 
+//------------------//
+//---]  GLOBAL  [---//
+//------------------//
 def clusterName = "kind-ci"
+String branch_name = 'master'
 
+// -----------------------------------------------------------------------
+// Intent: Determine if working on a release branch.
+//   Note: Conditional is legacy, should also check for *-dev or *-pre
+// -----------------------------------------------------------------------
+Boolean isReleaseBranch(String name)
+{
+    // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
+    // if branch_name in modifiers
+    return(name != 'master') // OR branch_name.contains('-')
+}
+
+// -----------------------------------------------------------------------
+// Intent:
+// -----------------------------------------------------------------------
 def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
 {
     def infraNamespace = "default"
@@ -38,19 +57,11 @@
                 'jjb',
                 'pipeline',
                 'voltha',
-                'master',
+		branch_name,
                 'bbsim-tests.groovy'
             ].join('/')
-            println("** ${iam}: ENTER")
-
-            String cmd = "which pkill"
-            def stream = sh(
-                returnStatus:false,
-                returnStdout: true,
-                script: cmd)
-            println(" ** ${cmd}:\n${stream}")
-
-            println("** ${iam}: LEAVE")
+	    println("${iam}: ENTER")
+	    println("${iam}: LEAVE")
         }
     }
 
@@ -63,7 +74,7 @@
             timeout(1) {
                     sh returnStdout: false, script: '''
           # remove orphaned port-forward from different namespaces
-          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
+          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
           '''
                 }
             }
@@ -95,11 +106,14 @@
     '''
     }
 
-    stage('Deploy Voltha') {
-    if (teardown) {
-      timeout(10) {
-        script {
-
+    stage('Deploy Voltha')
+    {
+	if (teardown)
+	{
+	    timeout(10)
+	    {
+		script
+		{
           sh """
           mkdir -p ${logsDir}
           _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
@@ -107,16 +121,24 @@
 
           // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
           def localCharts = false
-          if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
+		    
+          if (volthaHelmChartsChange != ""
+	      || gerritProject == "voltha-helm-charts"
+	      || isReleaseBranch(branch) // branch != 'master'
+          ) {
             localCharts = true
           }
+          Boolean is_release = isReleaseBranch(branch)
+	  println(" ** localCharts=${localCharts}, branch_name=${branch_name}, isReleaseBranch=${is_release}")
 
           // NOTE temporary workaround expose ONOS node ports
-          def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
-          " --set onos-classic.onosSshPort=30115 " +
-          " --set onos-classic.onosApiPort=30120 " +
-          " --set onos-classic.onosOfPort=31653 " +
-          " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
+	  def localHelmFlags = extraHelmFlags.trim()
+		  + " --set global.log_level=${logLevel.toUpperCase()} "
+		  + " --set onos-classic.onosSshPort=30115 "
+		  + " --set onos-classic.onosApiPort=30120 "
+		  + " --set onos-classic.onosOfPort=31653 "
+		  + " --set onos-classic.individualOpenFlowNodePorts=true "
+		  + testSpecificHelmFlags
 
           if (gerritProject != "") {
             localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
@@ -134,7 +156,22 @@
             ])
         }
 
-        // stop logging
+	// -----------------------------------------------------------------------
+	// Intent: Replacing P_IDS with pgrep/pkill is a step forward.
+	// Why not simply use a pid file, capture _TAG=kail-startup above
+	// Grep runs the risk of terminating stray commands (??-good <=> bad-??)
+	// -----------------------------------------------------------------------
+	println('Try out the pgrep/pkill commands')
+	def stream = sh(
+            returnStatus:false,
+            returnStdout:true,
+	    script: '''pgrep --list-full kail-startup || true'''
+	)
+	println("** pgrep output: ${stream}")
+
+	// -----------------------------------------------------------------------
+	// stop logging
+	// -----------------------------------------------------------------------
         sh """
           P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
           if [ -n "\$P_IDS" ]; then
@@ -189,8 +226,8 @@
     if [ ${withMonitoring} = true ] ; then
       mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
       cd "$WORKSPACE/voltha-system-tests"
-      make vst_venv
-      source ./vst_venv/bin/activate
+      make venv-activate-script
+      set +u && source .venv/bin/activate && set -u
       # Collect initial memory consumption
       python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
     fi
@@ -208,17 +245,18 @@
         getPodsInfo("${logsDir}")
 
         sh """
-      set +e
+      # set +e
       # collect logs collected in the Robot Framework StartLogging keyword
       cd ${logsDir}
-      gzip *-combined.log || true
-      rm -f *-combined.log || true
+      gzip *-combined.log
+      rm -f *-combined.log
     """
 
     sh """
     if [ ${withMonitoring} = true ] ; then
       cd "$WORKSPACE/voltha-system-tests"
-      source ./vst_venv/bin/activate
+      make venv-activate-script
+      set +u && source .venv/bin/activate && set -u
       # Collect memory consumption of voltha pods once all the tests are complete
       python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
     fi
@@ -237,7 +275,7 @@
   archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
   sh '''
     sync
-    pkill kail || true
+    [[ $(pgrep --count kail) -gt 0 ]] && pkill --echo kail
     which voltctl
     md5sum $(which voltctl)
   '''
@@ -299,23 +337,21 @@
       }
     }
 
-        stage('Create K8s Cluster')
-        {
+    stage('Create K8s Cluster')
+    {
             steps
             {
                 script
                 {
-		    /*
                     println(' ** Calling installKind.groovy: ENTER')
                     // chicken-n-egg problem, kind command needed
                     // to determine if kubernetes cluster is active
                     installKind() { debug:true }
                     println(' ** Calling installKind.groovy: LEAVE')
-		     */
 
-                    def clusterExists = sh(
+		    def clusterExists = sh(
                         returnStdout: true,
-                        script: """kind get clusters | grep ${clusterName} | wc -l""")
+                        script: """kind get clusters | grep "${clusterName}" | wc -l""")
 
                     if (clusterExists.trim() == "0")
                     {
diff --git a/vars/installKind.groovy b/vars/installKind.groovy
index e22281b..e7cc57a 100644
--- a/vars/installKind.groovy
+++ b/vars/installKind.groovy
@@ -47,7 +47,8 @@
 }
 
 dir="$WORKSPACE/bin"
-if [ ! -f "$dir/kind" ]; then
+cmd="$dir/kind"
+if [ ! -f "$cmd" ]; then
     mkdir -p "$dir"
     pushd "$dir" || error "pushd $dir failed"
     curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.0/kind-linux-amd64
@@ -55,6 +56,10 @@
     popd         || error "popd $dir failed"
 fi
 
+## Sanity check installed binary
+echo
+echo "Kind command verison: $("$cmd" --version)"
+
 return
 """)
     println("** ${iam}: LEAVE")