Merge "[VOL-4849] - Test suite: status=UNSTABLE periodic-software-upgrade-test-bbsim"
diff --git a/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy b/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
index 1238a53..cbbf43e 100755
--- a/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
@@ -50,14 +50,14 @@
 def test_software_upgrade(name) {
   def infraNamespace = "infra"
   def volthaNamespace = "voltha"
-  def openolt_adapter_deploy_tag = ""
-  def openolt_adapter_test_tag = ""
-  def openonu_adapter_deploy_tag = ""
-  def openonu_adapter_test_tag = ""
-  def rw_core_deploy_tag = ""
-  def rw_core_test_tag = ""
-  def ofagent_deploy_tag = ""
-  def ofagent_test_tag = ""
+  def openolt_adapter_deploy_tag = ''
+  def openolt_adapter_test_tag = ''
+  def openonu_adapter_deploy_tag = ''
+  def openonu_adapter_test_tag = ''
+  def rw_core_deploy_tag = ''
+  def rw_core_test_tag = ''
+  def ofagent_deploy_tag = ''
+  def ofagent_test_tag = ''
   def logsDir = "$WORKSPACE/${name}"
   stage('Deploy Voltha - '+ name) {
     timeout(10) {
@@ -67,7 +67,7 @@
       mkdir -p ${logsDir}
       _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
       """
-      def extraHelmFlags = extraHelmFlags.trim()
+      String extraHelmFlags = extraHelmFlags.trim()
       if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
           extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
       }
@@ -86,14 +86,14 @@
       if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
           extraHelmFlags = " --set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master " + extraHelmFlags
       }
-      extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
-      extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
+      extraHelmFlags += " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
+      extraHelmFlags += " --set voltha.onos_classic.replicas=3"
       //ONOS custom image handling
       if ( onosImg.trim() != '' ) {
          String[] split;
          onosImg = onosImg.trim()
          split = onosImg.split(':')
-        extraHelmFlags = extraHelmFlags + " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
+        extraHelmFlags += " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
       }
       Integer olts = 1
       if ("${name}" == "onu-image-dwl-simultaneously") {
@@ -102,18 +102,18 @@
       if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
         // fetch voltha components versions/tags
         (openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions("voltha-openolt-adapter", openoltAdapterDeployBaseTag.trim())
-        extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
+        extraHelmFlags += " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
         (openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions("voltha-openonu-adapter-go", openonuAdapterDeployBaseTag.trim())
-        extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
+        extraHelmFlags += " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
         (rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions("voltha-go", rwCoreDeployBaseTag.trim())
-        extraHelmFlags = extraHelmFlags + " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
+        extraHelmFlags += " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
         (ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions("ofagent-go", ofagentDeployBaseTag.trim())
-        extraHelmFlags = extraHelmFlags + " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
+        extraHelmFlags += " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
       }
       def localCharts = false
       // Currently only testing with ATT workflow
       // TODO: Support for other workflows
-      volthaDeploy([bbsimReplica: olts.toInteger(), workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
+      volthaDeploy([bbsimReplica: olts.toInteger(), workflow: 'att', extraHelmFlags: extraHelmFlags, localCharts: localCharts])
       // stop logging
       sh """
         P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
@@ -128,11 +128,11 @@
         rm onos-voltha-startup-combined.log
       """
       // forward ONOS and VOLTHA ports
-      sh """
+      sh('''
       JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8101:8101; done 2>&1 " &
       JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8181:8181; done 2>&1 " &
       JENKINS_NODE_COOKIE="dontKillMe" _TAG=port-forward-voltha-api /bin/bash -c "while true; do kubectl -n voltha port-forward --address 0.0.0.0 service/voltha-voltha-api 55555:55555; done 2>&1 " &
-      """
+      ''')
       sh """
       sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
       """
@@ -258,8 +258,44 @@
         ])
       }
     }
+
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
+    stage('Install Tools')
+    {
+        steps
+        {
+            script
+            {
+                String iam = getIam('Install Kind')
+                println("${iam}: ENTER")
+                installKind("$branch")   // needed early by stage(Cleanup)
+                println("${iam}: LEAVE")
+	    } // script
+	} // steps
+    } // stage
+
+    // -----------------------------------------------------------------------
+    // -----------------------------------------------------------------------
     stage('Cleanup') {
-      steps {
+        steps {
+
+	script {
+            println('''
+** -----------------------------------------------------------------------
+** Raw process listing
+** -----------------------------------------------------------------------
+''')
+        sh(''' ps faaux ''')
+
+            println('''
+** -----------------------------------------------------------------------
+** pgrep --list-full port
+** -----------------------------------------------------------------------
+''')
+        sh(''' set +euo pipefail && pgrep --list-full 'port' ''')
+		}
+
         // remove port-forwarding
         sh """
           # remove orphaned port-forward from different namespaces
diff --git a/vars/installKind.groovy b/vars/installKind.groovy
new file mode 100644
index 0000000..f5ec7c6
--- /dev/null
+++ b/vars/installKind.groovy
@@ -0,0 +1,79 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+    // Cannot rely on a stack trace due to jenkins manipulation
+    String src = 'vars/installKind.groovy'
+    String iam = [src, func].join('::')
+    return iam
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+Boolean process() {
+    String iam  = getIam('process')
+    Boolean ans = true
+
+    println("** ${iam}: ENTER")
+
+    String cmd = [
+        'make',
+        '--no-print-directory',
+        '-C', "$WORKSPACE/voltha-system-tests",
+        "KIND_PATH=\"$WORKSPACE/bin\"",
+        'install-command-kind',
+    ].join(' ')
+
+    println(" ** Running: ${cmd}")
+    sh(
+        label  : 'Install: kind', // jenkins usability: label log entry 'step'
+        script : "${cmd}",
+    )
+
+    println("** ${iam}: LEAVE")
+    return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Jenkins/groovy callback for installing the kind command.
+//    o Paramter branch is passed but not yet used.
+//    o Installer should be release friendly and checkout a frozen version
+// -----------------------------------------------------------------------
+def call(String branch) {
+    String iam = getIam('main')
+
+    println("** ${iam}: ENTER branch=${branch}")
+    println('** WARNING: branch= Not Yet Implemented')
+
+    try {
+        process()
+    }
+    catch (Exception err) {
+        println("** ${iam}: EXCEPTION ${err}")
+        throw err
+    }
+    finally {
+        println("** ${iam}: LEAVE")
+    }
+    return
+}
+
+// [EOF]