Merge "Disable jobs dependent on offline menlo-3 node"
diff --git a/edit.sh b/edit.sh
deleted file mode 100755
index fbea2d7..0000000
--- a/edit.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-# args+=('--quick')
-emacs ./jjb/pipeline/voltha/{master,voltha-2.12}/bbsim-tests.groovy
diff --git a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
index 25aaee9..5d97b82 100644
--- a/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/bbsim-tests.groovy
@@ -70,52 +70,6 @@
// -----------------------------------------------------------------------
// Intent:
// -----------------------------------------------------------------------
-void pgrep_proc(String proc) {
-
- println("** RAW PROCESS OUTPUT:")
- def stream = sh(returnStdout: true, scirpt: 'ps faaux')
- println(stream)
-
- String cmd = [
- 'pgrep',
- // '--older', 5, // switch not supported, nodes using older version
- '--list-full',
- "\"${proc}\"",
- ';',
- 'echo', 'DONE',
- ';',
- 'true',
- ].join(' ')
-
- println("** Running: ${cmd}")
- sh(
- returnStdout: true,
- "set +euo pipefail && ${cmd}"
- )
- return
-}
-
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-void pkill_proc(String proc) {
- String cmd = [
- 'pkill',
- // switch not supported, nodes using older version
- // NOTE: pkill should not kill it-self
- // good old kill (ps | grep -v -e grep -e '$$-me') }
- // '--older', 5,
- '--echo',
- "\"${proc}\"",
- ].join(' ')
-
- println("** Running: ${cmd}")
- sh(""" if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then "$cmd"; fi" """)
- return
-}
-
-// -----------------------------------------------------------------------
-// Intent:
-// -----------------------------------------------------------------------
void execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags='') {
String infraNamespace = 'default'
String volthaNamespace = 'voltha'
@@ -280,6 +234,7 @@
println("${iam}: Shutdown process $proc")
pgrep_proc(proc)
pkill_proc(proc)
+ pgrep_proc(proc)
println("${iam}: LEAVE")
}
@@ -317,6 +272,7 @@
fi
ps aux | grep port-forward
"""
+
script {
String proc = 'port-forward'
@@ -547,23 +503,15 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
- stage('Install Kind')
- {
- steps
- {
- script
- {
+ stage('Install Tools') {
+ steps {
+ script {
+ String branchName = branchName()
+ String iam = getIam('Install Tools')
- String cmd = [
- 'make',
- '--no-print-directory',
- '-C', "$WORKSPACE/voltha-system-tests",
- "KIND_PATH=\"$WORKSPACE/bin\"",
- 'install-command-kind',
- ].join(' ')
-
- println(" ** Running: ${cmd}")
- sh("${cmd}")
+ println("${iam}: ENTER (branch=$branch)")
+ installKind(branch) // needed early by stage(Cleanup)
+ println("${iam}: LEAVE (branch=$branch)")
} // script
} // steps
} // stage
@@ -630,23 +578,30 @@
println("${iam}: ENTER")
def tests = readYaml text: testTargets
+ println("** [DEBUG]: tests=$tests")
- String buffer = []
+ // Display expected tests for times when output goes dark
+ ArrayList buffer = [] as String
tests.eachWithIndex { test, idx ->
String target = test['target']
- buffer += sprintf(" test[%02d]: %s\n", idx, target)
+ println(" ** Build test index [$idx]: target=$target")
+ buffer.add(" test[${idx}]: ${target}\n")
+ println("** buffer contains: $buffer")
}
println("** Testing index: tests-to-run")
- println(buffer)
- println("""
+ println(buffer.join(''))
+
+ println('''
** -----------------------------------------------------------------------
** NOTE: For odd/silent job failures verify a few details
-** - All tests mentioned in the index have been processed.
+** - All tests mentioned in the tests-to-run index were logged.
** - Test suites display ENTER/LEAVE mesasge pairs.
+** - Processing terminated prematurely when LEAVE strings are missing.
** -----------------------------------------------------------------------
-""")
- tests.eachWithIndex { test, idx ->
+''')
+
+ tests.eachWithIndex { test, idx ->
println "** readYaml test suite[$idx]) test=[${test}]"
String target = test['target']
diff --git a/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy b/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
index 1238a53..d7427bb 100755
--- a/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/voltha-2.12/software-upgrades.groovy
@@ -21,6 +21,36 @@
remote: 'https://gerrit.opencord.org/ci-management.git'
])
+// -----------------------------------------------------------------------
+// Intent:
+// -----------------------------------------------------------------------
+String branchName() {
+ String name = 'voltha-2.12'
+
+ // [TODO] Sanity check the target branch
+ // if (name != jenkins.branch) { fatal }
+ return(name)
+}
+
+// -----------------------------------------------------------------------
+// Intent: Due to lack of a reliable stack trace, construct a literal.
+// Jenkins will re-write the call stack for serialization.
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ String branchName = branchName()
+ String src = [
+ 'ci-management',
+ 'jjb',
+ 'pipeline',
+ 'voltha',
+ branchName,
+ 'software-upgrades.groovy'
+ ].join('/')
+
+ String name = [src, func].join('::')
+ return(name)
+}
+
// fetches the versions/tags of the voltha component
// returns the deployment version which is one less than the latest available tag of the repo, first voltha stack gets deployed using this;
// returns the test version which is the latest tag of the repo, the component upgrade gets tested on this.
@@ -50,14 +80,14 @@
def test_software_upgrade(name) {
def infraNamespace = "infra"
def volthaNamespace = "voltha"
- def openolt_adapter_deploy_tag = ""
- def openolt_adapter_test_tag = ""
- def openonu_adapter_deploy_tag = ""
- def openonu_adapter_test_tag = ""
- def rw_core_deploy_tag = ""
- def rw_core_test_tag = ""
- def ofagent_deploy_tag = ""
- def ofagent_test_tag = ""
+ def openolt_adapter_deploy_tag = ''
+ def openolt_adapter_test_tag = ''
+ def openonu_adapter_deploy_tag = ''
+ def openonu_adapter_test_tag = ''
+ def rw_core_deploy_tag = ''
+ def rw_core_test_tag = ''
+ def ofagent_deploy_tag = ''
+ def ofagent_test_tag = ''
def logsDir = "$WORKSPACE/${name}"
stage('Deploy Voltha - '+ name) {
timeout(10) {
@@ -67,7 +97,7 @@
mkdir -p ${logsDir}
_TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
"""
- def extraHelmFlags = extraHelmFlags.trim()
+ String extraHelmFlags = extraHelmFlags.trim()
if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
}
@@ -86,14 +116,14 @@
if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
extraHelmFlags = " --set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master " + extraHelmFlags
}
- extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
- extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
+ extraHelmFlags += " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
+ extraHelmFlags += " --set voltha.onos_classic.replicas=3"
//ONOS custom image handling
if ( onosImg.trim() != '' ) {
String[] split;
onosImg = onosImg.trim()
split = onosImg.split(':')
- extraHelmFlags = extraHelmFlags + " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
+ extraHelmFlags += " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
}
Integer olts = 1
if ("${name}" == "onu-image-dwl-simultaneously") {
@@ -102,19 +132,21 @@
if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
// fetch voltha components versions/tags
(openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions("voltha-openolt-adapter", openoltAdapterDeployBaseTag.trim())
- extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
+ extraHelmFlags += " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
(openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions("voltha-openonu-adapter-go", openonuAdapterDeployBaseTag.trim())
- extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
+ extraHelmFlags += " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
(rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions("voltha-go", rwCoreDeployBaseTag.trim())
- extraHelmFlags = extraHelmFlags + " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
+ extraHelmFlags += " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
(ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions("ofagent-go", ofagentDeployBaseTag.trim())
- extraHelmFlags = extraHelmFlags + " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
+ extraHelmFlags += " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
}
def localCharts = false
// Currently only testing with ATT workflow
// TODO: Support for other workflows
- volthaDeploy([bbsimReplica: olts.toInteger(), workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
- // stop logging
+ volthaDeploy([bbsimReplica: olts.toInteger(), workflow: 'att', extraHelmFlags: extraHelmFlags, localCharts: localCharts])
+
+ // [TODO] pkill_proc("_TAG=kail-${name}")
+ // stop logging
sh """
P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
if [ -n "\$P_IDS" ]; then
@@ -128,11 +160,11 @@
rm onos-voltha-startup-combined.log
"""
// forward ONOS and VOLTHA ports
- sh """
+ sh('''
JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8101:8101; done 2>&1 " &
JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8181:8181; done 2>&1 " &
JENKINS_NODE_COOKIE="dontKillMe" _TAG=port-forward-voltha-api /bin/bash -c "while true; do kubectl -n voltha port-forward --address 0.0.0.0 service/voltha-voltha-api 55555:55555; done 2>&1 " &
- """
+ ''')
sh """
sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
"""
@@ -201,11 +233,16 @@
# Run the specified tests
make -C $WORKSPACE/voltha-system-tests \$TARGET || true
"""
- // remove port-forwarding
+
+ pkill_proc('port-forw')
+ // remove port-forwarding
+ /*
sh """
# remove orphaned port-forward from different namespaces
ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
"""
+ */
+
// collect pod details
get_pods_info("$WORKSPACE/${name}")
sh """
@@ -258,21 +295,40 @@
])
}
}
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
+ stage('Install Tools') {
+ steps {
+ script {
+ String branchName = branchName()
+ String iam = getIam('Install Kind')
+
+ println("${iam}: ENTER (branch=$branch)")
+ installKind(branch) // needed early by stage(Cleanup)
+ println("${iam}: LEAVE (branch=$branch)")
+ } // script
+ } // steps
+ } // stage
+
+ // -----------------------------------------------------------------------
+ // -----------------------------------------------------------------------
stage('Cleanup') {
- steps {
- // remove port-forwarding
- sh """
- # remove orphaned port-forward from different namespaces
- ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
- """
- helmTeardown(['infra', 'voltha'])
- }
- }
+ steps {
+ script {
+ pgrep_proc('port-forw')
+ pkill_proc('port-forw')
+ } // script
+ helmTeardown(['infra', 'voltha'])
+ } // steps
+ } // stage
+
stage('Create K8s Cluster') {
- steps {
- createKubernetesCluster([nodes: 3])
- }
+ steps {
+ createKubernetesCluster([nodes: 3])
+ }
}
+
stage('Run Test') {
steps {
test_software_upgrade("onos-app-upgrade")
@@ -306,3 +362,5 @@
}
}
}
+
+// [EOF]
diff --git a/vars/installKind.groovy b/vars/installKind.groovy
new file mode 100644
index 0000000..f5ec7c6
--- /dev/null
+++ b/vars/installKind.groovy
@@ -0,0 +1,79 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ // Cannot rely on a stack trace due to jenkins manipulation
+ String src = 'vars/installKind.groovy'
+ String iam = [src, func].join('::')
+ return iam
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+Boolean process() {
+ String iam = getIam('process')
+ Boolean ans = true
+
+ println("** ${iam}: ENTER")
+
+ String cmd = [
+ 'make',
+ '--no-print-directory',
+ '-C', "$WORKSPACE/voltha-system-tests",
+ "KIND_PATH=\"$WORKSPACE/bin\"",
+ 'install-command-kind',
+ ].join(' ')
+
+ println(" ** Running: ${cmd}")
+ sh(
+ label : 'Install: kind', // jenkins usability: label log entry 'step'
+ script : "${cmd}",
+ )
+
+ println("** ${iam}: LEAVE")
+ return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Jenkins/groovy callback for installing the kind command.
+// o Paramter branch is passed but not yet used.
+// o Installer should be release friendly and checkout a frozen version
+// -----------------------------------------------------------------------
+def call(String branch) {
+ String iam = getIam('main')
+
+ println("** ${iam}: ENTER branch=${branch}")
+ println('** WARNING: branch= Not Yet Implemented')
+
+ try {
+ process()
+ }
+ catch (Exception err) {
+ println("** ${iam}: EXCEPTION ${err}")
+ throw err
+ }
+ finally {
+ println("** ${iam}: LEAVE")
+ }
+ return
+}
+
+// [EOF]
diff --git a/vars/pgrep_proc.groovy b/vars/pgrep_proc.groovy
new file mode 100644
index 0000000..1ad5dfc
--- /dev/null
+++ b/vars/pgrep_proc.groovy
@@ -0,0 +1,77 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ // Cannot rely on a stack trace due to jenkins manipulation
+ String src = 'vars/pgrep_proc.groovy'
+ String iam = [src, func].join('::')
+ return iam
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+Boolean process(String proc) {
+ String iam = getIam('process')
+ Boolean ans = true
+
+ println("** ${iam}: ENTER")
+
+ String cmdFull = "pgrep --list-full '${proc}'"
+ String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdFull}; fi"""
+
+ print("""
+** -----------------------------------------------------------------------
+** Running: $cmd
+** -----------------------------------------------------------------------
+""")
+ sh(
+ label : 'pgrep_proc', // jenkins usability: label log entry 'step'
+ script : "${cmd}",
+ )
+
+ println("** ${iam}: LEAVE")
+ return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Jenkins/groovy callback for installing the kind command.
+// o Paramter branch is passed but not yet used.
+// o Installer should be release friendly and checkout a frozen version
+// -----------------------------------------------------------------------
+def call(String proc) {
+ String iam = getIam('main')
+
+ println("** ${iam}: ENTER")
+
+ try {
+ process(proc)
+ }
+ catch (Exception err) {
+ println("** ${iam}: EXCEPTION ${err}")
+ throw err
+ }
+ finally {
+ println("** ${iam}: LEAVE")
+ }
+ return
+}
+
+// [EOF]
diff --git a/vars/pkill_proc.groovy b/vars/pkill_proc.groovy
new file mode 100644
index 0000000..753a1f1
--- /dev/null
+++ b/vars/pkill_proc.groovy
@@ -0,0 +1,78 @@
+#!/usr/bin/env groovy
+// -----------------------------------------------------------------------
+// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------
+// Install the voltctl command by branch name "voltha-xx"
+// -----------------------------------------------------------------------
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+String getIam(String func) {
+ // Cannot rely on a stack trace due to jenkins manipulation
+ String src = 'vars/pkill_proc.groovy'
+ String iam = [src, func].join('::')
+ return iam
+}
+
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+Boolean process(String proc) {
+ String iam = getIam('process')
+ Boolean ans = true
+
+ println("** ${iam}: ENTER")
+
+ String cmdKill = "pkill --echo '${proc}'"
+ String cmd = """if [[ \$(pgrep --count "${proc}") -gt 0 ]]; then ${cmdKill}; fi"""
+
+ print("""
+** -----------------------------------------------------------------------
+** Running: $cmd
+** -----------------------------------------------------------------------
+""")
+
+ sh(
+ label : 'pkill_proc', // jenkins usability: label log entry 'step'
+ script : "${cmd}",
+ )
+
+ println("** ${iam}: LEAVE")
+ return(ans)
+}
+
+// -----------------------------------------------------------------------
+// Install: Jenkins/groovy callback for installing the kind command.
+// o Paramter branch is passed but not yet used.
+// o Installer should be release friendly and checkout a frozen version
+// -----------------------------------------------------------------------
+def call(String proc) {
+ String iam = getIam('main')
+
+ println("** ${iam}: ENTER")
+
+ try {
+ process(proc)
+ }
+ catch (Exception err) {
+ println("** ${iam}: EXCEPTION ${err}")
+ throw err
+ }
+ finally {
+ println("** ${iam}: LEAVE")
+ }
+ return
+}
+
+// [EOF]