VOL-4925 - Build and release components.

jjb/shell/github-release.sh
---------------------------
  o Added a display banner to identify running script and version.
    Recent change to use cp -vs- rsync for release/* copy is not
    visible in the job log -- timing issue ?
  o Source common lib stacktrace.sh and traputils.sh to be more
    verbose when the script edits (courtesy of set -e).
  o Debugging statements added.
  o Use stacktrace.sh to display callstack when script exits with error.

jjb/shell/common/README.md
jjb/shell/common/common.sh
jjb/shell/common/example.sh
jjb/shell/common/preserve_argv.sh
jjb/shell/common/common/sh/tempdir.sh
jjb/shell/common/common/sh/traputils.sh
jjb/shell/common/common/sh/stacktrace.sh
-----------------------------------------
  o Create a common library of reusable utility shell scripts.
  o tempdir.sh    - automatic creation and removal of mktempdir()
  o stacktrace.sh - display script call stack.
  o traputils.sh  - register an interrupt handler calling stacktrace on exit.

Change-Id: I563948f078cf33fef4a58be2b7455f07a3bd9e3a
diff --git a/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy~ b/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy~
deleted file mode 100644
index 171648b..0000000
--- a/jjb/pipeline/voltha/voltha-2.11/bbsim-tests.groovy~
+++ /dev/null
@@ -1,355 +0,0 @@
-// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// voltha-2.x e2e tests for openonu-go
-// uses bbsim to simulate OLT/ONUs
-
-library identifier: 'cord-jenkins-libraries@master',
-    retriever: modernSCM([
-      $class: 'GitSCMSource',
-      remote: 'https://gerrit.opencord.org/ci-management.git'
-])
-
-def clusterName = "kind-ci"
-
-def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
-    def infraNamespace = "default"
-    def volthaNamespace = "voltha"
-    def logsDir = "$WORKSPACE/${testTarget}"
-
-    stage('IAM')
-    {
-	script
-	{
-	    String iam = [
-		'ci-management',
-		'jjb',
-		'pipeline',
-		'voltha',
-		'master',
-		'bbsim-tests.groovy'
-	    ].join('/')
-            println("** ${iam}: ENTER")
-
-	    String cmd = "which pkill"
-	    def stream = sh(
-		returnStatus:false,
-		returnStdout: true,
-		script: cmd)
-	    println(" ** ${cmd}:\n${stream}")
-	    
-            println("** ${iam}: LEAVE")
-	}
-    }
-
-    stage('Cleanup') {
-	if (teardown) {
-	    timeout(15) {
-		script {
-		    helmTeardown(["default", infraNamespace, volthaNamespace])
-		}
-	    timeout(1) {
-		    sh returnStdout: false, script: '''
-          # remove orphaned port-forward from different namespaces
-          ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
-          '''
-		}
-	    }
-	}
-    }
-
-    stage ('Initialize')
-    {
-	// VOL-4926 - Is voltha-system-tests available ?
-	String cmd = [
-	    'make',
-	    '-C', "$WORKSPACE/voltha-system-tests",
-	    "KAIL_PATH=\"$WORKSPACE/bin\"",
-	    'kail',
-	].join(' ')
-	println(" ** Running: ${cmd}:\n")
-        sh("${cmd}")
-    }
-
-    stage('Deploy common infrastructure') {
-	sh '''
-    helm repo add onf https://charts.opencord.org
-    helm repo update
-    if [ ${withMonitoring} = true ] ; then
-      helm install nem-monitoring onf/nem-monitoring \
-      --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
-      --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
-    fi
-    '''
-    }
-
-    stage('Deploy Voltha') {
-    if (teardown) {
-      timeout(10) {
-        script {
-
-          sh """
-          mkdir -p ${logsDir}
-          _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
-          """
-
-          // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
-          def localCharts = false
-          if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
-            localCharts = true
-          }
-
-          // NOTE temporary workaround expose ONOS node ports
-          def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
-          " --set onos-classic.onosSshPort=30115 " +
-          " --set onos-classic.onosApiPort=30120 " +
-          " --set onos-classic.onosOfPort=31653 " +
-          " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
-
-          if (gerritProject != "") {
-            localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
-          }
-
-          volthaDeploy([
-            infraNamespace: infraNamespace,
-            volthaNamespace: volthaNamespace,
-            workflow: workflow.toLowerCase(),
-            withMacLearning: enableMacLearning.toBoolean(),
-            extraHelmFlags: localHelmFlags,
-            localCharts: localCharts,
-            bbsimReplica: olts.toInteger(),
-            dockerRegistry: registry,
-            ])
-        }
-
-        // stop logging
-        sh """
-          P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
-          if [ -n "\$P_IDS" ]; then
-            echo \$P_IDS
-            for P_ID in \$P_IDS; do
-              kill -9 \$P_ID
-            done
-          fi
-          cd ${logsDir}
-          gzip -k onos-voltha-startup-combined.log
-          rm onos-voltha-startup-combined.log
-        """
-      }
-      sh """
-      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
-      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
-      JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
-      bbsimDmiPortFwd=50075
-      for i in {0..${olts.toInteger() - 1}}; do
-        JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
-        ((bbsimDmiPortFwd++))
-      done
-      if [ ${withMonitoring} = true ] ; then
-        JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
-      fi
-      ps aux | grep port-forward
-      """
-      // setting ONOS log level
-      script {
-        setOnosLogLevels([
-          onosNamespace: infraNamespace,
-          apps: [
-            'org.opencord.dhcpl2relay',
-            'org.opencord.olt',
-            'org.opencord.aaa',
-            'org.opencord.maclearner',
-            'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
-            'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
-          ],
-          logLevel: logLevel
-        ])
-      }
-    }
-  }
-
-  stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
-    sh """
-    if [ ${withMonitoring} = true ] ; then
-      mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
-      cd "$WORKSPACE/voltha-system-tests"
-      make vst_venv
-      source ./vst_venv/bin/activate || true
-      # Collect initial memory consumption
-      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
-    fi
-    """
-    sh """
-    mkdir -p ${logsDir}
-    export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
-    ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
-    export KVSTOREPREFIX=voltha/voltha_voltha
-
-    make -C "$WORKSPACE/voltha-system-tests" ${testTarget} || true
-    """
-    getPodsInfo("${logsDir}")
-    sh """
-      set +e
-      # collect logs collected in the Robot Framework StartLogging keyword
-      cd ${logsDir}
-      gzip *-combined.log || true
-      rm *-combined.log || true
-    """
-    sh """
-    if [ ${withMonitoring} = true ] ; then
-      cd "$WORKSPACE/voltha-system-tests"
-      source ./vst_venv/bin/activate || true
-      # Collect memory consumption of voltha pods once all the tests are complete
-      python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
-    fi
-    """
-  }
-}
-
-def collectArtifacts(exitStatus) {
-  getPodsInfo("$WORKSPACE/${exitStatus}")
-  sh """
-  kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
-  """
-  archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
-  sh '''
-    sync
-    pkill kail || true
-    which voltctl
-    md5sum $(which voltctl)
-  '''
-  step([$class: 'RobotPublisher',
-    disableArchiveOutput: false,
-    logFileName: "**/*/log*.html",
-    otherFiles: '',
-    outputFileName: "**/*/output*.xml",
-    outputPath: '.',
-    passThreshold: 100,
-    reportFileName: "**/*/report*.html",
-    unstableThreshold: 0,
-    onlyCritical: true]);
-}
-
-pipeline {
-
-  /* no label, executor is determined by JJB */
-  agent {
-    label "${params.buildNode}"
-  }
-  options {
-    timeout(time: "${timeout}", unit: 'MINUTES')
-  }
-  environment {
-    KUBECONFIG="$HOME/.kube/kind-${clusterName}"
-    VOLTCONFIG="$HOME/.volt/config"
-    PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    DIAGS_PROFILE="VOLTHA_PROFILE"
-    SSHPASS="karaf"
-  }
-  stages {
-    stage('Download Code') {
-      steps {
-        getVolthaCode([
-          branch: "${branch}",
-          gerritProject: "${gerritProject}",
-          gerritRefspec: "${gerritRefspec}",
-          volthaSystemTestsChange: "${volthaSystemTestsChange}",
-          volthaHelmChartsChange: "${volthaHelmChartsChange}",
-        ])
-      }
-    }
-    stage('Build patch') {
-      // build the patch only if gerritProject is specified
-      when {
-        expression {
-          return !gerritProject.isEmpty()
-        }
-      }
-      steps {
-        // NOTE that the correct patch has already been checked out
-        // during the getVolthaCode step
-        buildVolthaComponent("${gerritProject}")
-      }
-    }
-    stage('Create K8s Cluster') {
-      steps {
-        script {
-          def clusterExists = sh returnStdout: true, script: """
-          kind get clusters | grep ${clusterName} | wc -l
-          """
-          if (clusterExists.trim() == "0") {
-            createKubernetesCluster([nodes: 3, name: clusterName])
-          }
-        }
-      }
-    }
-    stage('Replace voltctl') {
-      // if the project is voltctl override the downloaded one with the built one
-      when {
-        expression {
-          return gerritProject == "voltctl"
-        }
-      }
-      steps{
-        sh """
-        mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
-        chmod +x $WORKSPACE/bin/voltctl
-        """
-      }
-    }
-    stage('Load image in kind nodes') {
-      when {
-        expression {
-          return !gerritProject.isEmpty()
-        }
-      }
-      steps {
-        loadToKind()
-      }
-    }
-    stage('Parse and execute tests') {
-        steps {
-          script {
-            def tests = readYaml text: testTargets
-
-            for(int i = 0;i<tests.size();i++) {
-              def test = tests[i]
-              def target = test["target"]
-              def workflow = test["workflow"]
-              def flags = test["flags"]
-              def teardown = test["teardown"].toBoolean()
-              def logging = test["logging"].toBoolean()
-              def testLogging = 'False'
-              if (logging) {
-                  testLogging = 'True'
-              }
-              println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
-              execute_test(target, workflow, testLogging, teardown, flags)
-            }
-          }
-        }
-    }
-  }
-  post {
-    aborted {
-      collectArtifacts("aborted")
-    }
-    failure {
-      collectArtifacts("failed")
-    }
-    always {
-      collectArtifacts("always")
-    }
-  }
-}
diff --git a/jjb/shell/common/README.md b/jjb/shell/common/README.md
new file mode 100644
index 0000000..4c171b6
--- /dev/null
+++ b/jjb/shell/common/README.md
@@ -0,0 +1,28 @@
+# Common shell utilities
+
+This subdirectory contains library shell scripts that support interrupt
+handling and displaying a stack trace.
+
+## Hierarchy may appear strange (common/common) but setup is intentional:
+
+common/
+├── common
+│   └── sh
+│       ├── stacktrace.sh
+│       ├── tempdir.sh
+│       └── traputils.sh
+├── common.sh
+└── preserve_argv.sh
+
+### Usage: Source individual libraries by path
+
+source common/common/sh/stacktrace.sh
+source common/common/sh/tempdir.sh
+
+### Usage: common.sh -- one-liner for sourcing sets of libraries.
+
+source common/common.sh
+source common/common.sh --tempdir
+source common/common.sh --traputils --stacktrace
+
+
diff --git a/jjb/shell/common/common.sh b/jjb/shell/common/common.sh
new file mode 100644
index 0000000..268c701
--- /dev/null
+++ b/jjb/shell/common/common.sh
@@ -0,0 +1,69 @@
+# -*- sh -*-
+## -----------------------------------------------------------------------
+## Intent:
+##   o This script can be used as a one-liner for sourcing common scripts.
+##   o Preserve command line arguments passed.
+##   o Accept common.sh arguments specifying a set of libraries to load.
+##   o Dependent common libraries will automatically be sourced.
+## -----------------------------------------------------------------------
+## Usage:
+##   o source common.sh --common-args-begin-- --tempdir
+##   o source common.sh --common-args-begin-- --stacktrace
+## -----------------------------------------------------------------------
+
+# __DEBUG_COMMON__=1
+[[ -v __DEBUG_COMMON__ ]] && echo " ** ${BASH_SOURCE[0]}: BEGIN"
+
+## -----------------------------------------------------------------------
+## Intent: Anonymous function used to source common shell libs
+## -----------------------------------------------------------------------
+## Usage: source common.sh '--stacktrace'
+## -----------------------------------------------------------------------
+function __anon_func__()
+{
+    local iam="${BASH_SOURCE[0]%/*}"
+    local cab='--common-args-begin--'
+
+    declare -a args=($*)
+
+    local raw
+    raw="$(readlink --canonicalize-existing --no-newline "${BASH_SOURCE[0]}")"
+    local top="${raw%/*}"
+    local common="${top}/common/sh"
+
+    local arg
+    for arg in "${args[@]}";
+    do
+	    case "$arg" in
+	        --tempdir)    source "${common}"/tempdir.sh    ;;
+	        --traputils)  source "${common}"/traputils.sh  ;;
+	        --stacktrace) source "${common}"/stacktrace.sh ;;
+            *) echo "ERROR ${BASH_SOURCE[0]}: [SKIP] unknown switch=$arg" ;;
+	    esac
+    done
+
+    return
+}
+
+##----------------##
+##---]  MAIN  [---##
+##----------------##
+source "${BASH_SOURCE[0]%/*}/preserve_argv.sh" # pushd @ARGV
+
+if [ $# -gt 0 ] && [ "$1" == '--common-args-begin--' ]; then
+    shift # remove arg marker
+fi
+
+if [ $# -eq 0 ]; then
+    # common.sh defaults
+    set -- '--tempdir' '--traputils' '--stacktrace'
+fi
+
+__anon_func__ "$@"
+unset __anon_func__
+source "${BASH_SOURCE[0]%/*}/preserve_argv.sh" # popd @ARGV
+
+[[ -v __DEBUG_COMMON__ ]] && echo " ** ${BASH_SOURCE[0]}: END"
+: # NOP
+
+# [EOF]
diff --git a/jjb/shell/common/common/sh/stacktrace.sh b/jjb/shell/common/common/sh/stacktrace.sh
new file mode 100644
index 0000000..4bbbe8e
--- /dev/null
+++ b/jjb/shell/common/common/sh/stacktrace.sh
@@ -0,0 +1,73 @@
+# -*- sh -*-
+## -----------------------------------------------------------------------
+## Intent  : Register an interrupt handler to generate a stack trace
+##           whenever shell commands fail or prematurely exist.
+## Usage   : source stacktrace.sh
+## See Also: traputils.sh
+##           https://gist.github.com/ahendrix/7030300
+## -----------------------------------------------------------------------
+## set -e silently exiting on error is less than helpful.
+## Use a call stack function to expose problem source.
+## -----------------------------------------------------------------------
+
+## -----------------------------------------------------------------------
+## Intent: Trap/exit on error displaying context on the way out.
+## -----------------------------------------------------------------------
+function errexit()
+{
+    local err=$?
+    set +o xtrace
+    local code="${1:-1}"
+
+    local prefix="${BASH_SOURCE[1]}:${BASH_LINENO[0]}"
+    echo -e "\nOFFENDER: ${prefix}"
+    if [ $# -gt 0 ] && [ "$1" == '--stacktrace-quiet' ]; then
+        code=1
+    else
+        echo "ERROR: '${BASH_COMMAND}' exited with status $err"
+    fi
+
+    # Print out the stack trace described by $function_stack
+    if [ ${#FUNCNAME[@]} -gt 2 ]
+    then
+	    echo "Call tree:"
+	    for ((i=1;i<${#FUNCNAME[@]}-1;i++))
+	    do
+	        echo " $i: ${BASH_SOURCE[$i+1]}:${BASH_LINENO[$i]} ${FUNCNAME[$i]}(...)"
+	    done
+    fi
+
+    echo "Exiting with status ${code}"
+    echo
+    exit "${code}"
+}
+
+##----------------##
+##---]  MAIN  [---##
+##----------------##
+
+# trap ERR to provide an error handler whenever a command exits nonzero
+#  this is a more verbose version of set -o errexit
+trap 'errexit' ERR
+# trap 'errexit' EXIT
+# trap 'errexit' DEBUG
+
+# setting errtrace allows our ERR trap handler to be propagated to functions,
+#  expansions and subshells
+set -o errtrace
+
+## Unit tests may need to disable interrupts to avoid control loss during segv
+# source $(dirname ${BASH_SOURCE})/traputils.sh
+source "${BASH_SOURCE[0]/stacktrace.sh/traputils.sh}"
+
+## -----------------------------------------------------------------------
+## Colon is a shell NOP operator that will set and return $?==0 to caller.
+## Sourced scripts adding ':' as the final operator helps prevent an ugly
+## manifestation.  Syntax errors normally will cause early script
+## termination with no context, NOP allows control to return back to the
+## caller where stacktrace can properly document the entry point.
+## -----------------------------------------------------------------------
+
+: # NOP w/side effects
+
+# EOF
diff --git a/jjb/shell/common/common/sh/tempdir.sh b/jjb/shell/common/common/sh/tempdir.sh
new file mode 100644
index 0000000..40d842e
--- /dev/null
+++ b/jjb/shell/common/common/sh/tempdir.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+## -----------------------------------------------------------------------
+## Intent: Automatic setup/teardown a scratch area for testing.
+## -----------------------------------------------------------------------
+## Usage:
+##   o source common/common.sh --common-args-begin-- --tempdir
+##   o source common/common/sh/tempdir.sh
+## -----------------------------------------------------------------------
+## Note:
+##   o Argument --preserve can be used to inhibit tempdir removal
+##     and test file cleanup at script exit.
+## -----------------------------------------------------------------------
+
+# __DEBUG_COMMON__=1
+[[ -v __DEBUG_COMMON__ ]] && echo "${BASH_SOURCE[0]}: BEGIN"
+
+##-------------------##
+##---]  GLOBALS  [---##
+##-------------------##
+if [[ ! -v __COMMON_TEMP_DIRS__ ]]; then
+    declare -g -a __COMMON_TEMP_DIRS__
+fi
+
+## -----------------------------------------------------------------------
+## Intent: Allocate a transient tmpdir
+## -----------------------------------------------------------------------
+## NOTE:
+##   o Caller should export TMPDIR="${path}"
+##   o TMPDIR= only visible when sourced form top level parent scope
+## -----------------------------------------------------------------------
+function common_tempdir_mkdir()
+{
+    local var="$1"; shift
+    
+    local pkgbase="${0##*/}" # basename
+    local pkgname="${pkgbase%.*}"
+
+    local __junk__
+    local __junk__="$(mktemp -d -t "${pkgname}.XXXXXXXXXX")"
+    
+    __COMMON_TEMP_DIRS__+=("$__junk__")
+
+    export TMPDIR="$__junk__"
+    eval "${var}=${__junk__}"# replace with typeset
+
+    # declare -p __COMMON_TEMP_DIRS__
+    return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Preserve allocated temp directories on exit
+## -----------------------------------------------------------------------
+## Usage:
+##   o kill -s 'SIGSYS' $$
+## -----------------------------------------------------------------------
+function sigtrap_preserve()
+{
+    local dir
+    for dir in "${__COMMON_TEMP_DIRS__[@]}";
+    do
+	    touch "$dir/.preserve"
+    done
+}
+trap sigtrap_preserve SIGSYS
+
+## -----------------------------------------------------------------------
+## Intent: Tempdir cleanup on exit
+## -----------------------------------------------------------------------
+function sigtrap()
+{
+    local dir
+    # declare -p __COMMON_TEMP_DIRS__
+    for dir in "${__COMMON_TEMP_DIRS__[@]}";
+    do
+	if [ -e "$dir/.preserve" ]; then
+	    echo "Preserving test output: $TMPDIR"
+	    find "$TMPDIR" -ls
+	else
+	    /bin/rm -fr "$TMPDIR"
+	fi
+    done
+    return
+}
+trap sigtrap EXIT
+
+[[ -v __DEBUG_COMMON__ ]] && echo "${BASH_SOURCE[0]}: END"
+unset __DEBUG_COMMON__
+
+: # NOP
+
+# [EOF]
diff --git a/jjb/shell/common/common/sh/traputils.sh b/jjb/shell/common/common/sh/traputils.sh
new file mode 100644
index 0000000..ded5cc7
--- /dev/null
+++ b/jjb/shell/common/common/sh/traputils.sh
@@ -0,0 +1,124 @@
+# -*- sh -*-
+## -----------------------------------------------------------------------
+## Intent: Helper script for disabling shell interrupt handlers
+## Usage:  source traputils.sh
+## -----------------------------------------------------------------------
+
+## -----------------------------------------------------------------------
+## Intent:
+## -----------------------------------------------------------------------
+function trap_stack_name()
+{
+    local sig=${1//[^a-zA-Z0-9]/_}
+    echo "__trap_stack_${sig}"
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## -----------------------------------------------------------------------
+function extract_trap()
+{
+    echo ${@:3:$(($#-3))}
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## -----------------------------------------------------------------------
+function get_trap()
+{
+    eval echo $(extract_trap `trap -p $1`)
+}
+
+## -----------------------------------------------------------------------
+## Intent: Push current signal handler so a new one can be installed.
+## -----------------------------------------------------------------------
+## Usage:
+##   trap_push 'func_one' 'SIGUSR1' 'SIGUSR2'
+##   trap_push 'func_two' 'SIGUSR1' 'SIGUSR2'
+##   kill -s 'SIGUSR1'   # func_two()
+##   trap_pop 'SIGUSR1'
+##   kill -s 'SIGUSR1'   # func_one()
+##   kill -s 'SIGUSR2'   # func_two()
+## -----------------------------------------------------------------------
+trap_push()
+{
+    local new_trap="$1"; shift
+    declare -a sigs=($*)
+    
+    # local sigs=$*
+    local sig
+    for sig in "${sigs[@]}";
+    do
+	    local stack_name="$(trap_stack_name "$sig")"
+	    local old_trap="$(get_trap "$sig")"
+        
+	    # eval '__trap_stack_SIGUSR1[${#__trap_stack_SIGUSR1[@]}]=$old_trap'
+	    # __trap_stack_SIGUSR1[${#__trap_stack_SIGUSR1[@]}]=one
+	    # trap two SIGUSR1
+        
+	    eval "${stack_name}"'[${#'"${stack_name}"'[@]}]=$old_trap'
+	    trap "${new_trap}" "$sig"
+    done
+    return
+}
+
+## -----------------------------------------------------------------------
+## Intent: Restore the last signal handler pushed
+## -----------------------------------------------------------------------
+function trap_pop()
+{
+    local sigs=$*
+    for sig in $sigs; do
+	    local stack_name=`trap_stack_name "$sig"`
+	    local count; eval 'count=${#'"${stack_name}"'[@]}'
+	    [[ $count -lt 1 ]] && return 127
+	    local new_trap
+	    local ref="${stack_name}"'[${#'"${stack_name}"'[@]}-1]'
+	    local cmd='new_trap=${'"$ref}"; eval $cmd
+	    trap "${new_trap}" "$sig"
+	    eval "unset $ref"
+    done
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## -----------------------------------------------------------------------
+function trap_prepend()
+{
+    local new_trap=$1
+    shift
+    local sigs=$*
+    for sig in $sigs; do
+	    if [[ -z $(get_trap $sig) ]]; then
+	        trap_push "$new_trap" "$sig"
+	    else
+	        trap_push "$new_trap ; $(get_trap $sig)" "$sig"
+	    fi
+    done
+}
+
+## -----------------------------------------------------------------------
+## Intent:
+## -----------------------------------------------------------------------
+function trap_append()
+{
+    local new_trap=$1
+    shift
+    local sigs=$*
+    for sig in $sigs; do
+	    if [[ -z $(get_trap $sig) ]]; then
+	        trap_push "$new_trap" "$sig"
+	    else
+	        trap_push "$(get_trap $sig) ; $new_trap" "$sig"
+	    fi
+    done
+}
+
+: # NOP
+
+# [SEE ALSO]
+# -----------------------------------------------------------------------
+# https://stackoverflow.com/questions/16115144/bash-save-and-restore-trap-state-easy-way-to-manage-multiple-handlers-for-trap
+# -----------------------------------------------------------------------
+
+# [EOF]
diff --git a/jjb/shell/common/example.sh b/jjb/shell/common/example.sh
new file mode 100755
index 0000000..0c0e46a
--- /dev/null
+++ b/jjb/shell/common/example.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+## -----------------------------------------------------------------------
+## Intent: An example script showing how to source common libraries
+##    and produce a stack trace on script error/exit
+## -----------------------------------------------------------------------
+
+declare -g fatal=1 # assign to view stack trace
+
+echo "$0: ENTER"
+
+echo "$0: Source library shell includes"
+
+declare -g pgmdir="${0%/*}" # dirname($script)
+declare -a common_args=()
+common_args+=('--common-args-begin--')
+# common_args+=('--traputils')
+# common_args+=('--stacktrace')
+# common_args+=('--tempdir')
+source "${pgmdir}/common.sh" "${common_args[@]}"
+
+echo "$0: define foo(), bar() & tans()"
+
+function foo()
+{
+    echo "${FUNCNAME}: hello - stack_frame[1]"
+    bar
+}
+
+function bar()
+{
+    echo "${FUNCNAME}: hello - stack_frame[2]"
+    tans
+}
+
+function tans()
+{
+    declare -g fatal
+    echo "${FUNCNAME}: early exit for stacktrace"
+    [[ $fatal -eq 1 ]] && exit 1
+    return
+}
+
+echo "$0: calling foo() for a stack trace"
+foo
+
+echo "$0: ENTER"
+
+## -----------------------------------------------------------------------
+# % ./example.sh
+# ./example.sh: ENTER
+# ./example.sh: Source library shell includes
+# ./example.sh: define foo(), bar() & tans()
+# ./example.sh: calling foo() for a stack trace
+# foo: hello - stack_frame[1]
+# bar: hello - stack_frame[2]
+# tans: early exit for stacktrace
+# 
+# OFFENDER: ./example.sh:1
+# ERROR: 'exit 1' exited with status 1
+# Call tree:
+# 1: ./example.sh:25 tans(...)
+# 2: ./example.sh:19 bar(...)
+# 3: ./example.sh:37 foo(...)
+# Exiting with status 1
+
+# [EOF]
diff --git a/jjb/shell/common/preserve_argv.sh b/jjb/shell/common/preserve_argv.sh
new file mode 100644
index 0000000..a0879b9
--- /dev/null
+++ b/jjb/shell/common/preserve_argv.sh
@@ -0,0 +1,56 @@
+# -*- sh -*-
+## -----------------------------------------------------------------------
+## Intent:
+##   o This is a helper module for common/common.sh to support passing
+##     switches into the library as part of the source command.
+##   o A delimiter must be inserted into @ARGV to prevent consumption
+##     of command line arguments passed into the parent script.
+## -----------------------------------------------------------------------
+## Usage: source preserve_argv.sh
+## -----------------------------------------------------------------------
+
+# __DEBUG_COMMON__=1
+[[ -v __DEBUG_COMMON__ ]] && echo " ** ${BASH_SOURCE[0]}: BEGIN"
+
+## -----------------------------------------------------------------------
+## Intent: Preserve command line args allowing common.sh sourcing.
+## -----------------------------------------------------------------------
+## Required: Caller is required to inline and shift the @ARGV {cab} marker
+## -----------------------------------------------------------------------
+## Usage: source preserve_argv.sh
+##   source "$(OPT_ROOT}"/common/common.sh --common-args-begin-- --stacktrace
+## -----------------------------------------------------------------------
+function __anon_func_preserve_argv__()
+{
+    local key='__preserve_argv_stack__'
+    local cab='--common-args-begin--'
+    local iam="${BASH_SOURCE[0]%/*}"
+
+    if [[ -v __preserve_argv_stack__ ]]; then
+        # echo " ** ${iam}: [POPD] environment: ${__preserve_argv_stack__[@]}"
+        set -- "${__preserve_argv_stack__[@]}"
+        unset __preserve_argv_stack__
+    elif [ $# -eq 0 ] || [ "$1" != "$cab" ]; then
+        echo " ** ${iam} ERROR: ARGV marker not found: ${cab}"
+        echo " ** command: $0"
+        exit 1
+    else
+        ## caller (common.sh) shift {cab} from argv
+        declare -g -a __preserve_argv_stack__=("$@")
+        # echo " ** ${iam}: [PUSHD] environment: ${__preserve_argv_stack__[@]}"
+    fi
+
+    return
+}
+
+##----------------##
+##---]  MAIN  [---##
+##----------------##
+__anon_func_preserve_argv__ "$@"
+unset __anon_func_preserve_argv__
+
+[[ -v __DEBUG_COMMON__ ]] && echo " ** ${BASH_SOURCE[0]}: CLOSE"
+
+: # NOP for set -e -vs- [[ -v __DEBUG_COMMON__ ]]
+
+# [EOF]
diff --git a/jjb/shell/github-release.sh b/jjb/shell/github-release.sh
index 8ff8754..32fa72b 100644
--- a/jjb/shell/github-release.sh
+++ b/jjb/shell/github-release.sh
@@ -20,6 +20,54 @@
 # message
 # -----------------------------------------------------------------------
 
+##-------------------##
+##---]  GLOBALS  [---##
+##-------------------##
+declare -g SCRIPT_VERSION='1.0' # git changeset needed
+declare -g TRACE=1              # uncomment to set -x
+declare -g ARGV="$*"            # archive for display
+
+##--------------------##
+##---]  INCLUDES  [---##
+##--------------------##
+declare -g pgmdir="${0%/*}" # dirname($script)
+declare -a common_args=()
+common_args+=('--common-args-begin--')
+common_args+=('--traputils')
+common_args+=('--stacktrace')
+# common_args+=('--tempdir')
+
+# shellcheck disable=SC1091
+source "${pgmdir}/common/common.sh" "${common_args[@]}"
+
+## -----------------------------------------------------------------------
+## Intent: Output a log banner to identify the running script/version.
+## -----------------------------------------------------------------------
+## TODO:
+##   o SCRIPT_VERSION => git changeset for repo:ci-managment
+##   o echo "library version: ${env."library.libName.version"}"
+# -----------------------------------------------------------------------
+# 14:18:38   > git fetch --no-tags --progress -- https://gerrit.opencord.org/ci-management.git +refs/heads/*:refs/remotes/origin/* # timeout=10
+# 14:18:39  Checking out Revision 50f6e0b97f449b32d32ec0e02d59642000351847 (master)
+# -----------------------------------------------------------------------
+function banner()
+{
+    local iam="${0##*/}"
+
+cat <<EOH
+
+** -----------------------------------------------------------------------
+** IAM: ${iam} :: ${FUNCNAME[0]}
+** ARGV: ${ARGV}
+** PWD: $(/bin/pwd)
+** NOW: $(date '+%Y/%m/%d %H:%M:%S')
+** VER: ${SCRIPT_VERSION:-'unknown'}
+** -----------------------------------------------------------------------
+EOH
+
+    return
+}
+
 ## -----------------------------------------------------------------------
 ## Intent:
 ##   Display available command versions
@@ -56,7 +104,9 @@
 
     # Copy artifacts into the release temp dir
     # shellcheck disable=SC2086
-    cp -v "$ARTIFACT_GLOB" "$RELEASE_TEMP"
+    # cp -v "$ARTIFACT_GLOB" "$RELEASE_TEMP"
+    echo "rsync -rv --checksum \"$ARTIFACT_GLOB\" \"$RELEASE_TEMP/.\""
+    rsync -rv --checksum "$ARTIFACT_GLOB" "$RELEASE_TEMP/."
     
     echo
     echo "** ${FUNCNAME[0]}: RELEASE_TEMP=${RELEASE_TEMP}"
@@ -72,6 +122,8 @@
 ##----------------##
 set -eu -o pipefail
 
+banner
+
 # when not running under Jenkins, use current dir as workspace and a blank
 # project name
 WORKSPACE=${WORKSPACE:-.}
@@ -82,7 +134,7 @@
 GITHUB_ORGANIZATION=${GITHUB_ORGANIZATION:-}
 
 # glob pattern relative to project dir matching release artifacts
-# ARTIFACT_GLOB=${ARTIFACT_GLOB:-"release/*"}
+# ARTIFACT_GLOB=${ARTIFACT_GLOB:-"release/*"} # stat -- release/* not found, literal string (?)
 ARTIFACT_GLOB=${ARTIFACT_GLOB:-"release/."}
 
 # Temporary staging directory to copy artifacts to
@@ -133,6 +185,11 @@
   exit 1
 else
 
+  declare -p release_path
+
+  # shellcheck disable=SC2015
+  [[ -v TRACE ]] && { set -x; } || { set +x; } # SC2015 (shellcheck -x)
+
   pushd "$release_path"
 
   # Release description is sanitized version of the log message
@@ -146,11 +203,11 @@
 
   # Are we failing on a literal string "release/*" ?
   # cp -v "$ARTIFACT_GLOB" "$RELEASE_TEMP"
-  echo "rsync -rv --checksum \"$ARTIFACT_GLOB\" \"$RELEASE_TEMP/.\""
-  rsync -rv --checksum "$ARTIFACT_GLOB" "$RELEASE_TEMP/."
+  # echo "rsync -rv --checksum \"$ARTIFACT_GLOB\" \"$RELEASE_TEMP/.\""
+  # rsync -rv --checksum "$ARTIFACT_GLOB" "$RELEASE_TEMP/."
 
   echo
-  echo "RELEASE_TEMP(${RELEASE_TMP}) contains:"
+  echo "RELEASE_TEMP(${RELEASE_TEMP}) contains:"
   find "$RELEASE_TEMP" -ls
 
   # create release