blob: a6a56b458f7d5305ac548e2433cd95d33b46ddf4 [file] [log] [blame]
Hardik Windlass0f9621d2021-02-24 21:23:19 +05301// Copyright 2021-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
Hardik Windlass0f9621d2021-02-24 21:23:19 +053014// voltha-2.x e2e tests
15// uses bbsim to simulate OLT/ONUs
Hardik Windlass0f9621d2021-02-24 21:23:19 +053016// NOTE we are importing the library even if it's global so that it's
17// easier to change the keywords during a replay
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
Hardik Windlass0f9621d2021-02-24 21:23:19 +053023def test_software_upgrade(name) {
Hardik Windlass7e4e3152021-09-29 06:42:05 +000024 def infraNamespace = "infra"
25 def volthaNamespace = "voltha"
Hardik Windlassdad8e5c2021-11-11 05:19:47 +000026 def logsDir = "$WORKSPACE/${name}"
Hardik Windlass0f9621d2021-02-24 21:23:19 +053027 stage('Deploy Voltha - '+ name) {
Matteo Scandolo4b040342021-10-08 14:26:06 -070028 timeout(10) {
Hardik Windlasscb955822021-10-21 14:59:11 +000029 // start logging
30 sh """
Hardik Windlassdad8e5c2021-11-11 05:19:47 +000031 rm -rf ${logsDir} || true
32 mkdir -p ${logsDir}
33 _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
Hardik Windlasscb955822021-10-21 14:59:11 +000034 """
Andrea Campanella9e41a602021-09-14 14:45:11 +020035 def extraHelmFlags = extraHelmFlags.trim()
Hardik Windlass1bb96a32022-01-19 10:10:08 +000036 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
Andrea Campanella0442dd02021-12-30 15:44:34 +010037 extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
TorstenThiemecf20f4b2021-11-12 13:38:26 +000038 }
39 if ("${name}" == "onu-image-dwl-simultaneously") {
Andrea Campanella0442dd02021-12-30 15:44:34 +010040 extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=2,pon=2 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
TorstenThiemecf20f4b2021-11-12 13:38:26 +000041 }
42 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-image-dwl-simultaneously") {
Andrea Campanella0442dd02021-12-30 15:44:34 +010043 extraHelmFlags = " --set global.image_tag=master --set onos-classic.image.tag=master " + extraHelmFlags
Hardik Windlassd87caad2021-05-06 06:55:09 +000044 }
Hardik Windlass1bb96a32022-01-19 10:10:08 +000045 if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
Andrea Campanella0442dd02021-12-30 15:44:34 +010046 extraHelmFlags = " --set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master " + extraHelmFlags
Hardik Windlassff97f652021-03-18 15:11:13 +053047 }
Andrea Campanella834f27b2021-06-08 17:08:41 +020048 extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
49 extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
Andrea Campanella5764e182021-03-18 14:17:59 +010050 //ONOS custom image handling
51 if ( onosImg.trim() != '' ) {
52 String[] split;
53 onosImg = onosImg.trim()
54 split = onosImg.split(':')
Andrea Campanella0442dd02021-12-30 15:44:34 +010055 extraHelmFlags = extraHelmFlags + " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
Andrea Campanella5764e182021-03-18 14:17:59 +010056 }
TorstenThiemecf20f4b2021-11-12 13:38:26 +000057 def olts = 1
58 if ("${name}" == "onu-image-dwl-simultaneously") {
59 olts = 2
60 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +053061 def localCharts = false
Hardik Windlass0f9621d2021-02-24 21:23:19 +053062 // Currently only testing with ATT workflow
63 // TODO: Support for other workflows
TorstenThiemecf20f4b2021-11-12 13:38:26 +000064 volthaDeploy([bbsimReplica: olts.toInteger(), workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
Hardik Windlasscb955822021-10-21 14:59:11 +000065 // stop logging
Hardik Windlass0f9621d2021-02-24 21:23:19 +053066 sh """
Hardik Windlasscb955822021-10-21 14:59:11 +000067 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
68 if [ -n "\$P_IDS" ]; then
69 echo \$P_IDS
70 for P_ID in \$P_IDS; do
71 kill -9 \$P_ID
72 done
73 fi
Hardik Windlassdad8e5c2021-11-11 05:19:47 +000074 cd ${logsDir}
Hardik Windlasscb955822021-10-21 14:59:11 +000075 gzip -k onos-voltha-startup-combined.log
76 rm onos-voltha-startup-combined.log
Hardik Windlass0f9621d2021-02-24 21:23:19 +053077 """
78 // forward ONOS and VOLTHA ports
79 sh """
Andrea Campanella1b7c0942021-09-14 11:02:58 +020080 JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8101:8101; done 2>&1 " &
81 JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8181:8181; done 2>&1 " &
82 JENKINS_NODE_COOKIE="dontKillMe" _TAG=port-forward-voltha-api /bin/bash -c "while true; do kubectl -n voltha port-forward --address 0.0.0.0 service/voltha-voltha-api 55555:55555; done 2>&1 " &
Hardik Windlass0f9621d2021-02-24 21:23:19 +053083 """
Andrea Campanella9eeb4e72021-03-19 10:00:43 +010084 sh """
85 sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
86 """
Matteo Scandolo4b040342021-10-08 14:26:06 -070087 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +053088 }
89 stage('Test - '+ name) {
TorstenThiemed697a402021-11-10 08:43:20 +000090 timeout(60) {
Hardik Windlass0f9621d2021-02-24 21:23:19 +053091 sh """
Hardik Windlass25c8ddb2021-03-05 20:26:16 +053092 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
93 mkdir -p \$ROBOT_LOGS_DIR
94 if [[ ${name} == 'onos-app-upgrade' ]]; then
95 export ONOS_APPS_UNDER_TEST+=''
Hardik Windlassb360f6e2021-03-09 17:46:21 +053096 if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +053097 ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
98 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +053099 if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530100 ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
101 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530102 if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530103 ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
104 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530105 if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530106 ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
107 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530108 if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530109 ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
110 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530111 if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530112 ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
113 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530114 if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530115 ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530116 fi
117 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
118 export TARGET=onos-app-upgrade-test
119 fi
Hardik Windlass1bb96a32022-01-19 10:10:08 +0000120 if [ ${name} == 'voltha-component-upgrade' ] || [ ${name} == 'voltha-component-rolling-upgrade' ]; then
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530121 export VOLTHA_COMPS_UNDER_TEST+=''
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530122 if [ ${adapterOpenOltImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530123 VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}*"
124 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530125 if [ ${adapterOpenOnuImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530126 VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,${adapterOpenOnuImage.trim()}*"
127 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530128 if [ ${rwCoreImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530129 VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,${rwCoreImage.trim()}*"
130 fi
Hardik Windlassb360f6e2021-03-09 17:46:21 +0530131 if [ ${ofAgentImage.trim()} != '' ]; then
Hardik Windlass98506e72021-03-09 14:29:17 +0530132 VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,${ofAgentImage.trim()}*"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530133 fi
134 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
Hardik Windlass1bb96a32022-01-19 10:10:08 +0000135 fi
136 if [[ ${name} == 'voltha-component-upgrade' ]]; then
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530137 export TARGET=voltha-comp-upgrade-test
138 fi
Hardik Windlass1bb96a32022-01-19 10:10:08 +0000139 if [[ ${name} == 'voltha-component-rolling-upgrade' ]]; then
140 export TARGET=voltha-comp-rolling-upgrade-test
141 fi
Hardik Windlass834afbb2021-03-23 11:48:15 +0530142 if [[ ${name} == 'onu-software-upgrade' ]]; then
Hardik Windlass88814542021-06-22 09:43:46 +0000143 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
Hardik Windlass834afbb2021-03-23 11:48:15 +0530144 export TARGET=onu-upgrade-test
145 fi
TorstenThiemecf20f4b2021-11-12 13:38:26 +0000146 if [[ ${name} == 'onu-image-dwl-simultaneously' ]]; then
147 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
148 export TARGET=onu-upgrade-test-multiolt-kind-att
149 fi
Hardik Windlass85491a02021-10-21 17:14:37 +0000150 testLogging='False'
Hardik Windlasscb955822021-10-21 14:59:11 +0000151 if [ ${logging} = true ]; then
Hardik Windlass85491a02021-10-21 17:14:37 +0000152 testLogging='True'
Hardik Windlasscb955822021-10-21 14:59:11 +0000153 fi
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530154 export VOLTCONFIG=$HOME/.volt/config-minimal
155 export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000156 ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:\$testLogging"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530157 # Run the specified tests
158 make -C $WORKSPACE/voltha-system-tests \$TARGET || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530159 """
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530160 // remove port-forwarding
161 sh """
162 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200163 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530164 """
165 // collect pod details
Hardik Windlass428f5132021-03-15 12:17:47 +0530166 get_pods_info("$WORKSPACE/${name}")
Hardik Windlassdad8e5c2021-11-11 05:19:47 +0000167 sh """
168 set +e
169 # collect logs collected in the Robot Framework StartLogging keyword
170 cd ${logsDir}
171 gzip *-combined.log || true
172 rm *-combined.log || true
173 """
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530174 helmTeardown(['infra', 'voltha'])
Matteo Scandolo4b040342021-10-08 14:26:06 -0700175 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530176 }
177}
Hardik Windlass428f5132021-03-15 12:17:47 +0530178def get_pods_info(dest) {
179 // collect pod details, this is here in case of failure
180 sh """
Hardik Windlass3db9adf2021-03-25 15:49:45 +0530181 mkdir -p ${dest} || true
Hardik Windlass428f5132021-03-15 12:17:47 +0530182 kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
183 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
184 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee ${dest}/pod-imagesId.txt || true
Andrea Campanellaea2379a2021-03-22 13:56:26 +0100185 kubectl describe pods --all-namespaces -l app.kubernetes.io/part-of=voltha > ${dest}/voltha-pods-describe.txt
186 kubectl describe pods -n infra -l app=onos-classic > ${dest}/onos-pods-describe.txt
Hardik Windlass428f5132021-03-15 12:17:47 +0530187 helm ls --all-namespaces > ${dest}/helm-charts.txt
188 """
Andrea Campanella5764e182021-03-18 14:17:59 +0100189 sh '''
190 # copy the ONOS logs directly from the container to avoid the color codes
Andrea Campanella188e50d2022-01-14 11:30:05 +0100191 printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.14/data/log/karaf.log ''' + dest + '''/#.log' || true
Andrea Campanella5764e182021-03-18 14:17:59 +0100192 '''
Hardik Windlass428f5132021-03-15 12:17:47 +0530193}
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530194pipeline {
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530195 /* no label, executor is determined by JJB */
196 agent {
197 label "${params.buildNode}"
198 }
199 options {
TorstenThiemecf20f4b2021-11-12 13:38:26 +0000200 timeout(time: 120, unit: 'MINUTES')
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530201 }
202 environment {
203 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
Hardik Windlass25c8ddb2021-03-05 20:26:16 +0530204 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
Andrea Campanella9eeb4e72021-03-19 10:00:43 +0100205 SSHPASS="karaf"
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530206 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530207 stages{
208 stage('Download Code') {
209 steps {
210 getVolthaCode([
211 branch: "${branch}",
212 volthaSystemTestsChange: "${volthaSystemTestsChange}",
213 volthaHelmChartsChange: "${volthaHelmChartsChange}",
214 ])
215 }
216 }
Hardik Windlass834afbb2021-03-23 11:48:15 +0530217 stage('Cleanup') {
218 steps {
219 // remove port-forwarding
220 sh """
221 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200222 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Hardik Windlass834afbb2021-03-23 11:48:15 +0530223 """
224 helmTeardown(['infra', 'voltha'])
225 }
226 }
Hardik Windlass9f5bee12021-05-07 06:47:21 +0000227 stage('Create K8s Cluster') {
228 steps {
229 createKubernetesCluster([nodes: 3])
230 }
231 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530232 stage('Run Test') {
233 steps {
234 test_software_upgrade("onos-app-upgrade")
235 test_software_upgrade("voltha-component-upgrade")
Hardik Windlass1bb96a32022-01-19 10:10:08 +0000236 test_software_upgrade("voltha-component-rolling-upgrade")
Hardik Windlass834afbb2021-03-23 11:48:15 +0530237 test_software_upgrade("onu-software-upgrade")
TorstenThiemecf20f4b2021-11-12 13:38:26 +0000238 test_software_upgrade("onu-image-dwl-simultaneously")
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530239 }
240 }
241 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530242 post {
Hardik Windlass428f5132021-03-15 12:17:47 +0530243 aborted {
244 get_pods_info("$WORKSPACE/failed")
245 }
246 failure {
247 get_pods_info("$WORKSPACE/failed")
248 }
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530249 always {
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530250 step([$class: 'RobotPublisher',
251 disableArchiveOutput: false,
252 logFileName: 'RobotLogs/*/log*.html',
253 otherFiles: '',
254 outputFileName: 'RobotLogs/*/output*.xml',
255 outputPath: '.',
256 passThreshold: 100,
257 reportFileName: 'RobotLogs/*/report*.html',
Andrea Campanellaabc09772021-06-16 12:08:57 +0200258 unstableThreshold: 0,
259 onlyCritical: true]);
Hardik Windlass428f5132021-03-15 12:17:47 +0530260 archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz,*.txt,**/*.txt'
Hardik Windlass0f9621d2021-02-24 21:23:19 +0530261 }
262 }
263}