[VOL-5100] - periodic-voltha-test-bbsim
jjb/pipeline/voltha/master/bbsim-tests.groovy
jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
vars/installKind.groovy
vars/installKind.sh
--------------------------------------------------
o Remove groovy script and accompaning logic installing the kind command,
something is just not right in the land of OZ.
o Updated voltha-system-tests makefiles, added a target to install
the kind command (centrally, consistent version!) similar to kail.
Change-Id: I342a7e7bcf2d0cfe282b80c418feee0ee891302a
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 8ff8e6b..d38d004 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -28,9 +28,13 @@
def clusterName = 'kind-ci'
// -----------------------------------------------------------------------
+// Intent:
// -----------------------------------------------------------------------
String getBranchName() {
String name = 'master'
+
+ // [TODO] Sanity check the target branch
+ // if (name != jenkins.branch) { fatal }
return(name)
}
@@ -49,8 +53,8 @@
'bbsim-tests.groovy'
].join('/')
- String iam = [src, func].join('::')
- return iam
+ String name = [src, func].join('::')
+ return(name)
}
// -----------------------------------------------------------------------
@@ -65,46 +69,6 @@
}
// -----------------------------------------------------------------------
-// Intent: Phase helper method
-// -----------------------------------------------------------------------
-// NOTE: installKind temporarily disabled:
-// o error makes little sense, install.Kind.{groovy,sh} exist in vars/
-// o jenkins shared library -- checked out on server disk is stale
-// 2 changesets behind current ?!?!
-// o Disable call for now to revive the pipeline.
-// 10:26:05 org.codehaus.groovy.control.MultipleCompilationErrorsException: startup failed:
-// 10:26:05 WorkflowScript: 73: unexpected token: installKind @ line 73, column 2.
-// 10:26:05 installKind(name)
-// 10:26:05 ^
-// -----------------------------------------------------------------------
-Boolean install_kind(String name)
-{
- String iam = getIam('installKind')
- Boolean ans = False
-
- println("** ${iam}: ENTER")
- try
- {
- println("** ${iam} Running installKind(name): ENTER")
- installKind(name)
- println("** ${iam} Running installKind(name): LEAVE")
- ans = True // iff
- }
- catch (Exception err)
- {
- ans = False
- println("** ${iam}: EXCEPTION ${err}")
- throw err
- }
- finally
- {
- println("** ${iam}: ENTER")
- }
-
- return(ans)
-}
-
-// -----------------------------------------------------------------------
// Intent:
// -----------------------------------------------------------------------
def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
@@ -124,6 +88,8 @@
}
}
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
stage('Cleanup')
{
if (teardown) {
@@ -141,21 +107,8 @@
}
}
- stage ('Install Kail')
- {
- // VOL-4926 - Is voltha-system-tests available ?
- String cmd = [
- 'make',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KAIL_PATH=\"$WORKSPACE/bin\"",
- 'kail',
- ].join(' ')
- println(" ** Running: ${cmd}:\n")
- sh("${cmd}")
- // if (! my_install_kail())
- // throw new Exception('installKail() failed')
- }
-
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
stage('Deploy common infrastructure')
{
sh '''
@@ -420,20 +373,34 @@
}
// -----------------------------------------------------------------------
- // Intent: Install early so stage('Create K8s Cluster') will not
- // fail on bogus "script.sh: line 1: kind: command not found".
+ // -----------------------------------------------------------------------
+ stage ('Install Kail')
+ {
+ String cmd = [
+ 'make',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KAIL_PATH=\"$WORKSPACE/bin\"",
+ 'kail',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}:\n")
+ sh("${cmd}")
+ }
+
+ // -----------------------------------------------------------------------
// -----------------------------------------------------------------------
stage ('Install Kind')
{
- steps
- {
- script
- {
- String branch_name = getBranchName()
- install_kind(branch_name)
- } // script
- } // steps
- } // stage
+ String cmd = [
+ 'make',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KIND_PATH=\"$WORKSPACE/bin\"",
+ 'install-command-kind',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}:\n")
+ sh("${cmd}")
+ }
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index ce04a78..eeb2fff 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -15,14 +15,62 @@
// voltha-2.x e2e tests for openonu-go
// uses bbsim to simulate OLT/ONUs
+// [TODO] Update syntax below to the latest supported
library identifier: 'cord-jenkins-libraries@master',
retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://gerrit.opencord.org/ci-management.git'
])
-def clusterName = "kind-ci"
+//------------------//
+//---] GLOBAL [---//
+//------------------//
+def clusterName = 'kind-ci'
+// -----------------------------------------------------------------------
+// Intent:
+// -----------------------------------------------------------------------
+String getBranchName() {
+ String name = 'voltha-2.12'
+
+ // [TODO] Sanity check the target branch
+ // if (name != jenkins.branch) { fatal }
+ return(name)
+}
+
+// -----------------------------------------------------------------------
+// Intent: Due to lack of a reliable stack trace, construct a literal.
+// Jenkins will re-write the call stack for serialization.
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ String branch_name = getBranchName()
+ String src = [
+ 'ci-management',
+ 'jjb',
+ 'pipeline',
+ 'voltha',
+ branch_name,
+ 'bbsim-tests.groovy'
+ ].join('/')
+
+ String name = [src, func].join('::')
+ return(name)
+}
+
+// -----------------------------------------------------------------------
+// Intent: Determine if working on a release branch.
+// Note: Conditional is legacy, should also check for *-dev or *-pre
+// -----------------------------------------------------------------------
+Boolean isReleaseBranch(String name)
+{
+ // List modifiers = ['-dev', '-pre', 'voltha-x.y.z-pre']
+ // if branch_name in modifiers
+ return(name != 'master') // OR branch_name.contains('-')
+}
+
+// -----------------------------------------------------------------------
+// Intent:
+// -----------------------------------------------------------------------
def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "")
{
def infraNamespace = "default"
@@ -33,28 +81,17 @@
{
script
{
- String iam = [
- 'ci-management',
- 'jjb',
- 'pipeline',
- 'voltha',
- 'voltha-2.12',
- 'bbsim-tests.groovy'
- ].join('/')
- println("** ${iam}: ENTER")
-
- String cmd = "which pkill"
- def stream = sh(
- returnStatus:false,
- returnStdout:true,
- script: cmd)
- println(" ** ${cmd}:\n${stream}")
-
- println("** ${iam}: LEAVE")
+ // Announce ourselves for log usability
+ String iam = getIam('execute_test')
+ println("${iam}: ENTER")
+ println("${iam}: LEAVE")
}
}
-
- stage('Cleanup') {
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Cleanup')
+ {
if (teardown) {
timeout(15) {
script {
@@ -63,27 +100,17 @@
timeout(1) {
sh returnStdout: false, script: '''
# remove orphaned port-forward from different namespaces
- ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
+ ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9
'''
}
}
}
}
- stage ('Initialize')
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Deploy common infrastructure')
{
- // VOL-4926 - Is voltha-system-tests available ?
- String cmd = [
- 'make',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KAIL_PATH=\"$WORKSPACE/bin\"",
- 'kail',
- ].join(' ')
- println(" ** Running: ${cmd}:\n")
- sh("${cmd}")
- }
-
- stage('Deploy common infrastructure') {
sh '''
helm repo add onf https://charts.opencord.org
helm repo update
@@ -95,11 +122,14 @@
'''
}
- stage('Deploy Voltha') {
- if (teardown) {
- timeout(10) {
- script {
-
+ stage('Deploy Voltha')
+ {
+ if (teardown)
+ {
+ timeout(10)
+ {
+ script
+ {
sh """
mkdir -p ${logsDir}
_TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
@@ -107,19 +137,30 @@
// if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
def localCharts = false
- if (volthaHelmChartsChange != ""
- || gerritProject == "voltha-helm-charts"
- || branch != 'master'
- ) {
+
+ if (volthaHelmChartsChange != ""
+ || gerritProject == "voltha-helm-charts"
+ || isReleaseBranch(branch) // branch != 'master'
+ ) {
localCharts = true
}
+ String branch_name = getBranchName()
+ Boolean is_release = isReleaseBranch(branch)
+ println([
+ " ** localCharts=${localCharts}",
+ "branch_name=${branch_name}",
+ "branch=${branch}",
+ "branch=isReleaseBranch=${is_release}",
+ ].join(', '))
// NOTE temporary workaround expose ONOS node ports
- def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
- " --set onos-classic.onosSshPort=30115 " +
- " --set onos-classic.onosApiPort=30120 " +
- " --set onos-classic.onosOfPort=31653 " +
- " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
+ def localHelmFlags = extraHelmFlags.trim()
+ + " --set global.log_level=${logLevel.toUpperCase()} "
+ + " --set onos-classic.onosSshPort=30115 "
+ + " --set onos-classic.onosApiPort=30120 "
+ + " --set onos-classic.onosOfPort=31653 "
+ + " --set onos-classic.individualOpenFlowNodePorts=true "
+ + testSpecificHelmFlags
if (gerritProject != "") {
localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
@@ -137,7 +178,22 @@
])
}
+ // -----------------------------------------------------------------------
+ // Intent: Replacing P_IDS with pgrep/pkill is a step forward.
+ // Why not simply use a pid file, capture _TAG=kail-startup above
+ // Grep runs the risk of terminating stray commands (??-good <=> bad-??)
+ // -----------------------------------------------------------------------
+ echo 'Try out pgrep/pkill commands'
+ def stream = sh(
+ returnStatus:false,
+ returnStdout:true,
+ script: '''pgrep --list-full kail-startup || true'''
+ )
+ println("** pgrep output: ${stream}")
+
+ // -----------------------------------------------------------------------
// stop logging
+ // -----------------------------------------------------------------------
sh """
P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
if [ -n "\$P_IDS" ]; then
@@ -192,10 +248,10 @@
if [ ${withMonitoring} = true ] ; then
mkdir -p "$WORKSPACE/voltha-pods-mem-consumption-${workflow}"
cd "$WORKSPACE/voltha-system-tests"
- make vst_venv
- source ./vst_venv/bin/activate || true
+ make venv-activate-script
+ set +u && source .venv/bin/activate && set -u
# Collect initial memory consumption
- python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+ python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
"""
@@ -211,17 +267,18 @@
getPodsInfo("${logsDir}")
sh """
- set +e
+ # set +e
# collect logs collected in the Robot Framework StartLogging keyword
cd ${logsDir}
- gzip *-combined.log || true
- rm *-combined.log || true
+ gzip *-combined.log
+ rm -f *-combined.log
"""
sh """
if [ ${withMonitoring} = true ] ; then
cd "$WORKSPACE/voltha-system-tests"
- source ./vst_venv/bin/activate
+ make venv-activate-script
+ set +u && source .venv/bin/activate && set -u
# Collect memory consumption of voltha pods once all the tests are complete
python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace}
fi
@@ -233,17 +290,28 @@
// -----------------------------------------------------------------------
def collectArtifacts(exitStatus)
{
+ echo '''
+
+** -----------------------------------------------------------------------
+** collectArtifacts
+** -----------------------------------------------------------------------
+'''
+
getPodsInfo("$WORKSPACE/${exitStatus}")
+
sh """
- kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
+ kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log
"""
+
archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
+
sh '''
sync
- pkill kail || true
+ [[ $(pgrep --count kail) -gt 0 ]] && pkill --echo kail
which voltctl
md5sum $(which voltctl)
'''
+
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
logFileName: "**/*/log*.html",
@@ -295,75 +363,103 @@
return !gerritProject.isEmpty()
}
}
- steps {
- // NOTE that the correct patch has already been checked out
- // during the getVolthaCode step
- buildVolthaComponent("${gerritProject}")
- }
+
+ steps
+ {
+ // NOTE that the correct patch has already been checked out
+ // during the getVolthaCode step
+ buildVolthaComponent("${gerritProject}")
+ }
}
- stage('Create K8s Cluster')
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage ('Install Kail')
+ {
+ String cmd = [
+ 'make',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KAIL_PATH=\"$WORKSPACE/bin\"",
+ 'kail',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}:\n")
+ sh("${cmd}")
+ }
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage ('Install Kind')
+ {
+ String cmd = [
+ 'make',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KIND_PATH=\"$WORKSPACE/bin\"",
+ 'install-command-kind',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}:\n")
+ sh("${cmd}")
+ }
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Create K8s Cluster')
+ {
+ steps
{
- steps
+ script
{
- script
+ def clusterExists = sh(
+ returnStdout: true,
+ script: """kind get clusters | grep "${clusterName}" | wc -l""")
+
+ if (clusterExists.trim() == "0")
{
- /*
- println(' ** Calling installKind.groovy: ENTER')
- // chicken-n-egg problem, kind command needed
- // to determine if kubernetes cluster is active
- installKind() { debug:true }
- println(' ** Calling installKind.groovy: LEAVE')
- */
-
- def clusterExists = sh(
- returnStdout: true,
- script: """kind get clusters | grep ${clusterName} | wc -l""")
-
- if (clusterExists.trim() == "0")
- {
- createKubernetesCluster([nodes: 3, name: clusterName])
- }
- } // script
- } // steps
- } // stage('Create K8s Cluster')
-
- stage('Replace voltctl')
- {
- // if the project is voltctl override the downloaded one with the built one
- when {
- expression {
- return gerritProject == "voltctl"
+ createKubernetesCluster([nodes: 3, name: clusterName])
}
- }
+ } // script
+ } // steps
+ } // stage('Create K8s Cluster')
- steps
- {
- sh """
- # [TODO] - why is this platform specific (?)
- # [TODO] - revisit, command alteration has masked an error (see: voltha-2.11).
- # find will fail when no filsystem matches are found.
- # mv(ls) succeded simply by accident/invoked at a different time.
- mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
- chmod +x $WORKSPACE/bin/voltctl
- """
- } // steps
- } // stage
-
- stage('Load image in kind nodes')
- {
- when {
- expression {
- return !gerritProject.isEmpty()
- }
- }
- steps {
- loadToKind()
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Replace voltctl')
+ {
+ // if the project is voltctl, override the downloaded one with the built one
+ when {
+ expression {
+ return gerritProject == "voltctl"
}
}
- stage('Parse and execute tests')
+ // Hmmmm(?) where did the voltctl download happen ?
+ // Likely Makefile but would be helpful to document here.
+ steps
{
+ println("${iam} Running: installVoltctl($branch)")
+ installVoltctl("$branch")
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Load image in kind nodes')
+ {
+ when {
+ expression {
+ return !gerritProject.isEmpty()
+ }
+ }
+ steps {
+ loadToKind()
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Parse and execute tests')
+ {
steps {
script {
def tests = readYaml text: testTargets
diff --git a/vars/installKind.groovy b/vars/installKind.groovy
deleted file mode 100644
index be791d0..0000000
--- a/vars/installKind.groovy
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env groovy
-// -----------------------------------------------------------------------
-// Copyright 2023 Open Networking Foundation (ONF) and the ONF Contributors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-// -----------------------------------------------------------------------
-
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-String getIam(String func) {
- // Cannot rely on a stack trace due to jenkins manipulation
- String src = 'vars/installKind.groovy'
- String iam = [src, func].join('::')
- return iam
-}
-
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-void process(Map args) {
- String iam = getIam('process')
-
- println("** ${iam}: ENTER branch=${args.branch}")
- println('args = ' + args)
-
- // go install sigs.k8s.io/kind@v0.18.0
- sh(
- script: './installKind.sh',
- returnStdout: true
- )
-
- println("** ${iam}: LEAVE")
- return
-}
-
-// -----------------------------------------------------------------------
-// TODO: Support native syntax: installKind() { debug:true }
-// -----------------------------------------------------------------------
-/*
-Boolean call\
- (
- // def self, // jenkins env object for access to primitives like echo()
- Closure body // jenkins closure attached to the call iam() {closure}
- )
-{
- Map config = [:] // propogate block parameters
- body.resolveStrategy = Closure.DELEGATE_FIRST
- body.delegate = config // make parameters visible down below
- body()
- */
-
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-Boolean call(String branch) {
- String iam = getIam('main')
- Boolean ans = true
-
- println("** ${iam}: ENTER")
- println("** ${iam}: Debug= is " + config.contains(debug))
-
- try {
- // Will be passed in eventually
- Map config = [debug :false, branch:branch]
- process(config)
- }
- /*
- catch (Exception err)
- {
- println("** ${iam}: EXCEPTION ${err}")
- throw err
- }
-*/
- finally {
- println("** ${iam}: LEAVE")
- }
-
- return(ans)
-}
-
-// [EOF]
diff --git a/vars/installKind.sh b/vars/installKind.sh
deleted file mode 100755
index 24eb6cc..0000000
--- a/vars/installKind.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-## -----------------------------------------------------------------------
-## Intent:
-## -----------------------------------------------------------------------
-
-##-------------------##
-##---] GLOBALS [---##
-##-------------------##
-set -eu -o pipefail
-umask 0
-
-## -----------------------------------------------------------------------
-## Intent: Display a message then exit with non-zero shell exit status.
-## -----------------------------------------------------------------------
-function error()
-{
- echo "** ERROR: $*"
- exit 1
-}
-
-## -----------------------------------------------------------------------
-## Intent: Display program usage
-## -----------------------------------------------------------------------
-function usage
-{
- cat <<EOH
-Usage: $0
- --cmd Kind command name to download (w/arch type)
- --dir Target bin directory to download into
- --ver Kind command version to download (default=v0.11.0)
-
-[MODEs]
- --clean Clean download, remove kind binary if it exists.
-EOH
- exit 1
-}
-
-##----------------##
-##---] MAIN [---##
-##----------------##
-
-WORKSPACE="${WORKSPACE:-$(/bin/pwd)}"
-dir="$WORKSPACE/bin"
-
-bin='kind-linux-amd64'
-ver='v0.11.0'
-
-while [ $# -gt 0 ]; do
- arg="$1"; shift
- case "$arg" in
- -*clean) declare -i clean_mode=1 ;;
- -*cmd) bin="$1";s shift ;; # cmd-by-arch
- -*dir) dir="$1"; shift ;; # target dir
- -*help) usage; exit 0 ;;
- -*ver) ver="$1"; shift ;; # version
- *) echo "[SKIP] Unknown argument [$arg]" ;;
- esac
-done
-
-
-cmd="$dir/kind"
-[[ -v clean_mode ]] && /bin/rm -f "$cmd"
-
-if [ ! -f "$cmd" ]; then
- mkdir -p "$dir"
- pushd "$dir" || error "pushd $dir failed"
- echo "** ${0##*/}: Download ${bin} ${ver}"
-
- curl --silent -Lo ./kind "https://kind.sigs.k8s.io/dl/${ver}/${bin}"
- readarray -t file_info < <(file "$cmd")
-
- ## Validate binary
- case "${file_info[@]}" in
- *ASCII*)
- echo "ERROR: kind command is invalid"
- set -x; cat "$cmd"; set +x
- echo
- /bin/rm -f "$cmd"
- ;;
- *) chmod +x ./kind ;;
- esac
-
- popd || error "popd $dir failed"
-fi
-
-echo "Kind command version: $($cmd --version)"
-
-# [EOF]