blob: f0b67873a92f4df27a853dea5f453930a2f1866e [file] [log] [blame]
// Copyright 2019-present Open Networking Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// deploy VOLTHA built from patchset on a physical pod and run e2e test
// uses kind-voltha to deploy voltha-2.X
// NOTE we are importing the library even if it's global so that it's
// easier to change the keywords during a replay
library identifier: 'cord-jenkins-libraries@master',
retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://gerrit.opencord.org/ci-management.git'
])
// Need this so that deployment_config has global scope when it's read later
deployment_config = null
localDeploymentConfigFile = null
localKindVolthaValuesFile = null
localSadisConfigFile = null
// The pipeline assumes these variables are always defined
if ( params.manualBranch != "" ) {
GERRIT_EVENT_COMMENT_TEXT = ""
GERRIT_PROJECT = ""
GERRIT_BRANCH = "${params.manualBranch}"
GERRIT_CHANGE_NUMBER = ""
GERRIT_PATCHSET_NUMBER = ""
}
pipeline {
/* no label, executor is determined by JJB */
agent {
label "${params.buildNode}"
}
options {
timeout(time: 120, unit: 'MINUTES')
}
environment {
KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
VOLTCONFIG="$HOME/.volt/config-minimal"
PATH="$WORKSPACE/bin:$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
NAME="minimal"
FANCY=0
//VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
ONOS_SSH_PORT=30115
ONOS_API_PORT=30120
}
stages {
stage ('Initialize') {
steps {
sh returnStdout: false, script: """
if [ "${branch}" != "master" ]; then
echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
source "$WORKSPACE/kind-voltha/releases/${branch}"
else
echo "on master, using default settings for kind-voltha"
fi
test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
cd $WORKSPACE
rm -rf $WORKSPACE/*
"""
script {
if (env.configRepo && ! env.localConfigDir) {
env.localConfigDir = "$WORKSPACE"
sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
}
localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
}
}
}
stage('Download Code') {
steps {
checkout([
$class: 'GitSCM',
userRemoteConfigs: [[
url: "https://gerrit.opencord.org/kind-voltha",
]],
branches: [[ name: "master", ]],
extensions: [
[$class: 'WipeWorkspace'],
[$class: 'RelativeTargetDirectory', relativeTargetDir: "kind-voltha"],
[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
],
])
getVolthaCode([
branch: "${branch}",
gerritProject: "${gerritProject}",
gerritRefspec: "${gerritRefspec}",
volthaSystemTestsChange: "${volthaSystemTestsChange}",
volthaHelmChartsChange: "${volthaHelmChartsChange}",
])
}
}
stage('Check config files') {
steps {
script {
try {
deployment_config = readYaml file: "${localDeploymentConfigFile}"
} catch (err) {
echo "Error reading ${localDeploymentConfigFile}"
throw err
}
sh returnStdout: false, script: """
if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
"""
}
}
}
stage('Build patch') {
steps {
// NOTE that the correct patch has already been checked out
// during the getVolthaCode step
buildVolthaComponent("${gerritProject}")
}
}
stage('Create KinD Cluster') {
steps {
sh returnStdout: false, script: """
if [ "${branch}" != "master" ]; then
echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
source "$WORKSPACE/kind-voltha/releases/${branch}"
else
echo "on master, using default settings for kind-voltha"
fi
cd $WORKSPACE/kind-voltha/
JUST_K8S=y ./voltha up
"""
}
}
stage('Load image in kind nodes') {
when {
expression { params.manualBranch == "" }
}
steps {
sh returnStdout: false, script: """
if [ "${branch}" != "master" ]; then
echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
source "$WORKSPACE/kind-voltha/releases/${branch}"
else
echo "on master, using default settings for kind-voltha"
fi
if ! [[ "${gerritProject}" =~ ^(voltha-system-tests|kind-voltha|voltha-helm-charts)\$ ]]; then
docker images | grep citest
for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}")
do
echo "Pushing \$image to nodes"
kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2
docker rmi \$image:citest \$image:latest || true
done
fi
"""
}
}
stage('Deploy Voltha') {
environment {
WITH_SIM_ADAPTERS="no"
WITH_RADIUS="yes"
DEPLOY_K8S="no"
VOLTHA_LOG_LEVEL="DEBUG"
}
steps {
script {
sh returnStdout: false, script: """
if [ "${branch}" != "master" ]; then
echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}"
source "$WORKSPACE/kind-voltha/releases/${branch}"
else
echo "on master, using default settings for kind-voltha"
fi
export EXTRA_HELM_FLAGS+='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
IMAGES=""
if [ "${gerritProject}" = "voltha-go" ]; then
IMAGES="rw_core ro_core "
elif [ "${gerritProject}" = "ofagent-py" ]; then
IMAGES="ofagent "
elif [ "${gerritProject}" = "voltha-onos" ]; then
IMAGES="onos "
elif [ "${gerritProject}" = "voltha-openolt-adapter" ]; then
IMAGES="adapter_open_olt "
elif [ "${gerritProject}" = "voltha-openonu-adapter" ]; then
IMAGES="adapter_open_onu "
elif [ "${gerritProject}" = "voltha-openonu-adapter-go" ]; then
IMAGES="adapter_open_onu_go "
elif [ "${gerritProject}" = "voltha-api-server" ]; then
IMAGES="afrouter afrouterd "
else
echo "No images to push"
fi
for I in \$IMAGES
do
EXTRA_HELM_FLAGS+="--set images.\$I.tag=citest,images.\$I.pullPolicy=Never "
done
if [ "${gerritProject}" = "voltha-helm-charts" ]; then
export CHART_PATH=$WORKSPACE/voltha-helm-charts
export VOLTHA_CHART=\$CHART_PATH/voltha
export VOLTHA_ADAPTER_OPEN_OLT_CHART=\$CHART_PATH/voltha-adapter-openolt
export VOLTHA_ADAPTER_OPEN_ONU_CHART=\$CHART_PATH/voltha-adapter-openonu
helm dep update \$VOLTHA_CHART
helm dep update \$VOLTHA_ADAPTER_OPEN_OLT_CHART
helm dep update \$VOLTHA_ADAPTER_OPEN_ONU_CHART
fi
cd $WORKSPACE/kind-voltha/
echo \$EXTRA_HELM_FLAGS
kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
./voltha up
set +e
# Remove noise from voltha-core logs
voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
# Remove noise from openolt logs
voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
"""
}
}
}
stage('Deploy Kafka Dump Chart') {
steps {
script {
sh returnStdout: false, script: """
helm repo add cord https://charts.opencord.org
helm repo update
if helm version -c --short|grep v2 -q; then
helm install -n voltha-kafka-dump cord/voltha-kafka-dump
else
helm install voltha-kafka-dump cord/voltha-kafka-dump
fi
"""
}
}
}
stage('Push Tech-Profile') {
when {
expression { params.profile != "Default" }
}
steps {
sh returnStdout: false, script: """
etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
"""
}
}
stage('Push Sadis-config') {
steps {
sh returnStdout: false, script: """
ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115
ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts
sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay"
sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa"
sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt"
sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.FlowObjectiveManager"
sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager"
curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
"""
}
}
stage('Reinstall OLT software') {
when {
expression { params.reinstallOlt }
}
steps {
script {
deployment_config.olts.each { olt ->
sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
waitUntil {
olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
return olt_sw_present.toInteger() == 0
}
if ( params.branch == 'voltha-2.3' ) {
oltDebVersion = oltDebVersionVoltha23
} else {
oltDebVersion = oltDebVersionMaster
}
sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
waitUntil {
olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
return olt_sw_present.toInteger() == 1
}
if ( olt.fortygig ) {
// If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
}
}
}
}
}
stage('Restart OLT processes') {
steps {
script {
deployment_config.olts.each { olt ->
sh returnStdout: false, script: """
ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; reboot'
sleep 120
"""
waitUntil {
onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
return onu_discovered.toInteger() > 0
}
}
}
}
}
stage('Run E2E Tests') {
environment {
ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
ROBOT_FILE="Voltha_PODTests.robot"
}
steps {
sh returnStdout: false, script: """
mkdir -p $WORKSPACE/RobotLogs
# If the Gerrit comment contains a line with "functional tests" then run the full
# functional test suite. This covers tests tagged either 'sanity' or 'functional'.
# Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
REGEX="functional tests"
if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
ROBOT_MISC_ARGS+="-i functional"
fi
# Likewise for dataplane tests
REGEX="dataplane tests"
if [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]]; then
ROBOT_MISC_ARGS+="-i dataplane"
fi
make -C $WORKSPACE/voltha-system-tests voltha-test || true
"""
}
}
stage('After-Test Delay') {
when {
expression { params.manualBranch == "" }
}
steps {
sh returnStdout: false, script: """
# Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
REGEX="hardware test with delay\$"
[[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
"""
}
}
}
post {
always {
sh returnStdout: false, script: '''
set +e
cp $WORKSPACE/kind-voltha/install-minimal.log $WORKSPACE/
kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
kubectl get nodes -o wide
kubectl get pods -o wide
kubectl get pods -n voltha -o wide
sync
pkill kail || true
## Pull out errors from log files
extract_errors_go() {
echo
echo "Error summary for $1:"
grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
echo
}
extract_errors_python() {
echo
echo "Error summary for $1:"
grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
echo
}
extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
gzip $WORKSPACE/onos-voltha-combined.log
## collect events, the chart should be running by now
kubectl get pods | grep -i voltha-kafka-dump | grep -i running
if [[ $? == 0 ]]; then
kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
fi
'''
script {
deployment_config.olts.each { olt ->
sh returnStdout: false, script: """
until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log
do
echo "Fetching openolt.log log failed, retrying..."
sleep 10
done
sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
until sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log
do
echo "Fetching dev_mgmt_daemon.log failed, retrying..."
sleep 10
done
sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
"""
}
}
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
logFileName: 'RobotLogs/log*.html',
otherFiles: '',
outputFileName: 'RobotLogs/output*.xml',
outputPath: '.',
passThreshold: 100,
reportFileName: 'RobotLogs/report*.html',
unstableThreshold: 0]);
archiveArtifacts artifacts: '*.log,*.gz'
}
}
}