blob: 3877af3ce7952e96f506371e84651c6623ed52c2 [file] [log] [blame]
Hardik Windlass0f9621d2021-02-24 21:23:19 +05301// Copyright 2021-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
Hardik Windlass0f9621d2021-02-24 21:23:19 +053014// voltha-2.x e2e tests
15// uses bbsim to simulate OLT/ONUs
Hardik Windlass0f9621d2021-02-24 21:23:19 +053016// NOTE we are importing the library even if it's global so that it's
17// easier to change the keywords during a replay
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
Hardik Windlass0f9621d2021-02-24 21:23:19 +053023def test_software_upgrade(name) {
24 stage('Deploy Voltha - '+ name) {
Andrea Campanella5764e182021-03-18 14:17:59 +010025 def extraHelmFlags = "${extraHelmFlags} --set global.log_level=DEBUG,onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
Hardik Windlassd87caad2021-05-06 06:55:09 +000026 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade") {
27 extraHelmFlags = extraHelmFlags + "--set global.image_tag=master --set onos-classic.image.tag=master "
28 }
29 if ("${name}" == "voltha-component-upgrade") {
30 extraHelmFlags = extraHelmFlags + "--set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master "
Hardik Windlassff97f652021-03-18 15:11:13 +053031 }
Andrea Campanella5764e182021-03-18 14:17:59 +010032 extraHelmFlags = extraHelmFlags + """ --set voltha.services.controller[0].service=voltha-infra-onos-classic-0.voltha-infra-onos-classic-hs.infra.svc \
33 --set voltha.services.controller[0].port=6653 \
34 --set voltha.services.controller[0].address=voltha-infra-onos-classic-0.voltha-infra-onos-classic-hs.infra.svc:6653 \
35 --set voltha.services.controller[1].service=voltha-infra-onos-classic-1.voltha-infra-onos-classic-hs.infra.svc \
36 --set voltha.services.controller[1].port=6653 \
37 --set voltha.services.controller[1].address=voltha-infra-onos-classic-1.voltha-infra-onos-classic-hs.infra.svc:6653 \
38 --set voltha.services.controller[2].service=voltha-infra-onos-classic-2.voltha-infra-onos-classic-hs.infra.svc \
39 --set voltha.services.controller[2].port=6653 \
40 --set voltha.services.controller[2].address=voltha-infra-onos-classic-2.voltha-infra-onos-classic-hs.infra.svc:6653 """
41 //ONOS custom image handling
42 if ( onosImg.trim() != '' ) {
43 String[] split;
44 onosImg = onosImg.trim()
45 split = onosImg.split(':')
Andrea Campanellaea2379a2021-03-22 13:56:26 +010046 extraHelmFlags = extraHelmFlags + "--set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
Andrea Campanella5764e182021-03-18 14:17:59 +010047 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +053048 def localCharts = false
Hardik Windlass0f9621d2021-02-24 21:23:19 +053049 // Currently only testing with ATT workflow
50 // TODO: Support for other workflows
51 volthaDeploy([workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
52 // start logging
53 sh """
Hardik Windlass3db9adf2021-03-25 15:49:45 +053054 rm -rf $WORKSPACE/${name} || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +053055 mkdir -p $WORKSPACE/${name}
56 _TAG=kail-${name} kail -n infra -n voltha > $WORKSPACE/${name}/onos-voltha-combined.log &
57 """
58 // forward ONOS and VOLTHA ports
59 sh """
Matteo Scandolof593a0f2021-04-14 10:16:06 -070060 _TAG=onos-port-forward bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8101:8101; done &"
61 _TAG=onos-port-forward bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8181:8181; done &"
62 _TAG=port-forward-voltha-api bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-api 55555:55555; done &"
Hardik Windlass0f9621d2021-02-24 21:23:19 +053063 """
Andrea Campanella9eeb4e72021-03-19 10:00:43 +010064 sh """
65 sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
66 """
Hardik Windlass0f9621d2021-02-24 21:23:19 +053067 }
68 stage('Test - '+ name) {
69 sh """
Hardik Windlass25c8ddb2021-03-05 20:26:16 +053070 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
71 mkdir -p \$ROBOT_LOGS_DIR
72 if [[ ${name} == 'onos-app-upgrade' ]]; then
73 export ONOS_APPS_UNDER_TEST+=''
Hardik Windlassb360f6e2021-03-09 17:46:21 +053074 if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053075 ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
76 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053077 if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053078 ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
79 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053080 if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053081 ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
82 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053083 if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053084 ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
85 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053086 if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053087 ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
88 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053089 if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053090 ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
91 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053092 if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053093 ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +053094 fi
95 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
96 export TARGET=onos-app-upgrade-test
97 fi
98 if [[ ${name} == 'voltha-component-upgrade' ]]; then
99 export VOLTHA_COMPS_UNDER_TEST+=''
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530100 if [ ${adapterOpenOltImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530101 VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}*"
102 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530103 if [ ${adapterOpenOnuImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530104 VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,${adapterOpenOnuImage.trim()}*"
105 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530106 if [ ${rwCoreImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530107 VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,${rwCoreImage.trim()}*"
108 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530109 if [ ${ofAgentImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530110 VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,${ofAgentImage.trim()}*"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530111 fi
112 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
113 export TARGET=voltha-comp-upgrade-test
114 fi
Hardik Windlass834afbb2021-03-23 11:48:15 +0530115 if [[ ${name} == 'onu-software-upgrade' ]]; then
116 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onu_image_name:${onuImageName.trim()} -v onu_image_url:${onuImageUrl.trim()} -v onu_image_version:${onuImageVersion.trim()} -v onu_image_crc:${onuImageCrc.trim()} -v onu_image_local_dir:${onuImageLocalDir.trim()} -e PowerSwitch"
117 export TARGET=onu-upgrade-test
118 fi
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530119 export VOLTCONFIG=$HOME/.volt/config-minimal
120 export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
121 # Run the specified tests
122 make -C $WORKSPACE/voltha-system-tests \$TARGET || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530123 """
124 // stop logging
125 sh """
126 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
127 if [ -n "\$P_IDS" ]; then
128 echo \$P_IDS
129 for P_ID in \$P_IDS; do
130 kill -9 \$P_ID
131 done
132 fi
133 """
134 // remove port-forwarding
135 sh """
136 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200137 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530138 """
139 // collect pod details
Hardik Windlass428f5132021-03-15 12:17:47 +0530140 get_pods_info("$WORKSPACE/${name}")
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530141 helmTeardown(['infra', 'voltha'])
142 }
143}
Hardik Windlass428f5132021-03-15 12:17:47 +0530144def get_pods_info(dest) {
145 // collect pod details, this is here in case of failure
146 sh """
Hardik Windlass3db9adf2021-03-25 15:49:45 +0530147 mkdir -p ${dest} || true
Hardik Windlass428f5132021-03-15 12:17:47 +0530148 kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
149 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
150 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee ${dest}/pod-imagesId.txt || true
Andrea Campanellaea2379a2021-03-22 13:56:26 +0100151 kubectl describe pods --all-namespaces -l app.kubernetes.io/part-of=voltha > ${dest}/voltha-pods-describe.txt
152 kubectl describe pods -n infra -l app=onos-classic > ${dest}/onos-pods-describe.txt
Hardik Windlass428f5132021-03-15 12:17:47 +0530153 helm ls --all-namespaces > ${dest}/helm-charts.txt
154 """
Andrea Campanella5764e182021-03-18 14:17:59 +0100155 sh '''
156 # copy the ONOS logs directly from the container to avoid the color codes
Andrea Campanella9eeb4e72021-03-19 10:00:43 +0100157 printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.9/data/log/karaf.log ''' + dest + '''/#.log' || true
Andrea Campanella5764e182021-03-18 14:17:59 +0100158 '''
Hardik Windlass428f5132021-03-15 12:17:47 +0530159}
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530160pipeline {
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530161 /* no label, executor is determined by JJB */
162 agent {
163 label "${params.buildNode}"
164 }
165 options {
Hardik Windlass973230d2021-05-17 11:08:59 +0530166 timeout(time: 60, unit: 'MINUTES')
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530167 }
168 environment {
169 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530170 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
Andrea Campanella9eeb4e72021-03-19 10:00:43 +0100171 SSHPASS="karaf"
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530172 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530173 stages{
174 stage('Download Code') {
175 steps {
176 getVolthaCode([
177 branch: "${branch}",
178 volthaSystemTestsChange: "${volthaSystemTestsChange}",
179 volthaHelmChartsChange: "${volthaHelmChartsChange}",
180 ])
181 }
182 }
Hardik Windlass834afbb2021-03-23 11:48:15 +0530183 stage('Cleanup') {
184 steps {
185 // remove port-forwarding
186 sh """
187 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200188 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Hardik Windlass834afbb2021-03-23 11:48:15 +0530189 """
190 helmTeardown(['infra', 'voltha'])
191 }
192 }
Hardik Windlassabd342c2021-03-25 12:20:43 +0530193 stage('Install latest voltctl') {
194 steps {
195 sh """
Hardik Windlass60230a22021-04-01 12:20:52 +0530196 mkdir -p $WORKSPACE/bin || true
Hardik Windlassabd342c2021-03-25 12:20:43 +0530197 # install voltctl
198 HOSTOS="\$(uname -s | tr "[:upper:]" "[:lower:"])"
199 HOSTARCH="\$(uname -m | tr "[:upper:]" "[:lower:"])"
200 if [ "\$HOSTARCH" == "x86_64" ]; then
201 HOSTARCH="amd64"
202 fi
203 VC_VERSION="\$(curl --fail -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')"
204 curl -Lo $WORKSPACE/bin/voltctl https://github.com/opencord/voltctl/releases/download/v\$VC_VERSION/voltctl-\$VC_VERSION-\$HOSTOS-\$HOSTARCH
205 chmod +x $WORKSPACE/bin/voltctl
206 """
207 }
208 }
Hardik Windlass9f5bee12021-05-07 06:47:21 +0000209 stage('Create K8s Cluster') {
210 steps {
211 createKubernetesCluster([nodes: 3])
212 }
213 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530214 stage('Run Test') {
215 steps {
216 test_software_upgrade("onos-app-upgrade")
217 test_software_upgrade("voltha-component-upgrade")
Hardik Windlass834afbb2021-03-23 11:48:15 +0530218 test_software_upgrade("onu-software-upgrade")
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530219 }
220 }
221 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530222 post {
Hardik Windlass428f5132021-03-15 12:17:47 +0530223 aborted {
224 get_pods_info("$WORKSPACE/failed")
225 }
226 failure {
227 get_pods_info("$WORKSPACE/failed")
228 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530229 always {
230 sh '''
231 gzip $WORKSPACE/onos-app-upgrade/onos-voltha-combined.log || true
232 gzip $WORKSPACE/voltha-component-upgrade/onos-voltha-combined.log || true
Hardik Windlass834afbb2021-03-23 11:48:15 +0530233 gzip $WORKSPACE/onu-software-upgrade/onos-voltha-combined.log || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530234 '''
235 step([$class: 'RobotPublisher',
236 disableArchiveOutput: false,
237 logFileName: 'RobotLogs/*/log*.html',
238 otherFiles: '',
239 outputFileName: 'RobotLogs/*/output*.xml',
240 outputPath: '.',
241 passThreshold: 100,
242 reportFileName: 'RobotLogs/*/report*.html',
243 unstableThreshold: 0]);
Hardik Windlass428f5132021-03-15 12:17:47 +0530244 archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz,*.txt,**/*.txt'
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530245 }
246 }
247}