blob: 1238a536ff3214bc48d43eef31955679eb487258 [file] [log] [blame]
Joey Armstrong7bcb5782023-06-07 12:25:57 -04001// Copyright 2021-2023 Open Networking Foundation (ONF) and the ONF Contributors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14// voltha-2.x e2e tests
15// uses bbsim to simulate OLT/ONUs
16// NOTE we are importing the library even if it's global so that it's
17// easier to change the keywords during a replay
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
24// fetches the versions/tags of the voltha component
25// returns the deployment version which is one less than the latest available tag of the repo, first voltha stack gets deployed using this;
26// returns the test version which is the latest tag of the repo, the component upgrade gets tested on this.
27// Note: if there is a major version change between deployment and test tags, then deployment tag will be same as test tag, i.e. both as latest.
28def get_voltha_comp_versions(component, base_deploy_tag) {
29 def comp_test_tag = sh (
30 script: "git ls-remote --refs --tags https://github.com/opencord/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=1 | sed 's/v//'",
31 returnStdout: true
32 ).trim()
33 def comp_deploy_tag = sh (
34 script: "git ls-remote --refs --tags https://github.com/opencord/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=2 | head -n 1 | sed 's/v//'",
35 returnStdout: true
36 ).trim()
37 def comp_deploy_major = comp_deploy_tag.substring(0, comp_deploy_tag.indexOf('.'))
38 def comp_test_major = comp_test_tag.substring(0, comp_test_tag.indexOf('.'))
39 if ( "${comp_deploy_major.trim()}" != "${comp_test_major.trim()}") {
40 comp_deploy_tag = comp_test_tag
41 }
42 if ( "${comp_test_tag.trim()}" == "${base_deploy_tag.trim()}") {
43 comp_deploy_tag = comp_test_tag
44 comp_test_tag = "master"
45 }
46 println "${component}: deploy_tag: ${comp_deploy_tag}, test_tag: ${comp_test_tag}"
47 return [comp_deploy_tag, comp_test_tag]
48}
49
50def test_software_upgrade(name) {
51 def infraNamespace = "infra"
52 def volthaNamespace = "voltha"
53 def openolt_adapter_deploy_tag = ""
54 def openolt_adapter_test_tag = ""
55 def openonu_adapter_deploy_tag = ""
56 def openonu_adapter_test_tag = ""
57 def rw_core_deploy_tag = ""
58 def rw_core_test_tag = ""
59 def ofagent_deploy_tag = ""
60 def ofagent_test_tag = ""
61 def logsDir = "$WORKSPACE/${name}"
62 stage('Deploy Voltha - '+ name) {
63 timeout(10) {
64 // start logging
65 sh """
66 rm -rf ${logsDir} || true
67 mkdir -p ${logsDir}
68 _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
69 """
70 def extraHelmFlags = extraHelmFlags.trim()
71 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
72 extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
73 }
74 if ("${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg") {
75 extraHelmFlags = " --set global.extended_omci_support.enabled=true " + extraHelmFlags
76 }
77 if ("${name}" == "onu-software-upgrade-omci-extended-msg") {
78 extraHelmFlags = " --set omccVersion=180 " + extraHelmFlags
79 }
80 if ("${name}" == "onu-image-dwl-simultaneously") {
81 extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=2,pon=2 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
82 }
83 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "onu-image-dwl-simultaneously") {
84 extraHelmFlags = " --set global.image_tag=master --set onos-classic.image.tag=master " + extraHelmFlags
85 }
86 if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
87 extraHelmFlags = " --set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master " + extraHelmFlags
88 }
89 extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
90 extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
91 //ONOS custom image handling
92 if ( onosImg.trim() != '' ) {
93 String[] split;
94 onosImg = onosImg.trim()
95 split = onosImg.split(':')
96 extraHelmFlags = extraHelmFlags + " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
97 }
98 Integer olts = 1
99 if ("${name}" == "onu-image-dwl-simultaneously") {
100 olts = 2
101 }
102 if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
103 // fetch voltha components versions/tags
104 (openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions("voltha-openolt-adapter", openoltAdapterDeployBaseTag.trim())
105 extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
106 (openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions("voltha-openonu-adapter-go", openonuAdapterDeployBaseTag.trim())
107 extraHelmFlags = extraHelmFlags + " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
108 (rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions("voltha-go", rwCoreDeployBaseTag.trim())
109 extraHelmFlags = extraHelmFlags + " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
110 (ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions("ofagent-go", ofagentDeployBaseTag.trim())
111 extraHelmFlags = extraHelmFlags + " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
112 }
113 def localCharts = false
114 // Currently only testing with ATT workflow
115 // TODO: Support for other workflows
116 volthaDeploy([bbsimReplica: olts.toInteger(), workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
117 // stop logging
118 sh """
119 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
120 if [ -n "\$P_IDS" ]; then
121 echo \$P_IDS
122 for P_ID in \$P_IDS; do
123 kill -9 \$P_ID
124 done
125 fi
126 cd ${logsDir}
127 gzip -k onos-voltha-startup-combined.log
128 rm onos-voltha-startup-combined.log
129 """
130 // forward ONOS and VOLTHA ports
131 sh """
132 JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8101:8101; done 2>&1 " &
133 JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8181:8181; done 2>&1 " &
134 JENKINS_NODE_COOKIE="dontKillMe" _TAG=port-forward-voltha-api /bin/bash -c "while true; do kubectl -n voltha port-forward --address 0.0.0.0 service/voltha-voltha-api 55555:55555; done 2>&1 " &
135 """
136 sh """
137 sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
138 """
139 }
140 }
141 stage('Test - '+ name) {
142 timeout(75) {
143 sh """
144 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
145 mkdir -p \$ROBOT_LOGS_DIR
146 if [[ ${name} == 'onos-app-upgrade' ]]; then
147 export ONOS_APPS_UNDER_TEST+=''
148 if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
149 ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
150 fi
151 if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
152 ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
153 fi
154 if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
155 ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
156 fi
157 if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
158 ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
159 fi
160 if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
161 ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
162 fi
163 if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
164 ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
165 fi
166 if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
167 ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
168 fi
169 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
170 export TARGET=onos-app-upgrade-test
171 fi
172 if [ ${name} == 'voltha-component-upgrade' ] || [ ${name} == 'voltha-component-rolling-upgrade' ]; then
173 export VOLTHA_COMPS_UNDER_TEST+=''
174 VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,voltha/voltha-openolt-adapter:${openolt_adapter_test_tag}*"
175 VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,voltha/voltha-openonu-adapter-go:${openonu_adapter_test_tag}*"
176 VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,voltha/voltha-rw-core:${rw_core_test_tag}*"
177 VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,voltha/voltha-ofagent-go:${ofagent_test_tag}*"
178 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
179 fi
180 if [[ ${name} == 'voltha-component-upgrade' ]]; then
181 export TARGET=voltha-comp-upgrade-test
182 fi
183 if [[ ${name} == 'voltha-component-rolling-upgrade' ]]; then
184 export TARGET=voltha-comp-rolling-upgrade-test
185 fi
186 if [ ${name} == 'onu-software-upgrade' ] || [ ${name} == 'onu-software-upgrade-omci-extended-msg' ]; then
187 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
188 export TARGET=onu-upgrade-test
189 fi
190 if [[ ${name} == 'onu-image-dwl-simultaneously' ]]; then
191 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
192 export TARGET=onu-upgrade-test-multiolt-kind-att
193 fi
194 testLogging='False'
195 if [ ${logging} = true ]; then
196 testLogging='True'
197 fi
198 export VOLTCONFIG=$HOME/.volt/config-minimal
199 export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
200 ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:\$testLogging"
201 # Run the specified tests
202 make -C $WORKSPACE/voltha-system-tests \$TARGET || true
203 """
204 // remove port-forwarding
205 sh """
206 # remove orphaned port-forward from different namespaces
207 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
208 """
209 // collect pod details
210 get_pods_info("$WORKSPACE/${name}")
211 sh """
212 set +e
213 # collect logs collected in the Robot Framework StartLogging keyword
214 cd ${logsDir}
215 gzip *-combined.log || true
216 rm *-combined.log || true
217 """
218 helmTeardown(['infra', 'voltha'])
219 }
220 }
221}
222def get_pods_info(dest) {
223 // collect pod details, this is here in case of failure
224 sh """
225 mkdir -p ${dest} || true
226 kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
227 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
228 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee ${dest}/pod-imagesId.txt || true
229 kubectl describe pods --all-namespaces -l app.kubernetes.io/part-of=voltha > ${dest}/voltha-pods-describe.txt
230 kubectl describe pods -n infra -l app=onos-classic > ${dest}/onos-pods-describe.txt
231 helm ls --all-namespaces > ${dest}/helm-charts.txt
232 """
233 sh '''
234 # copy the ONOS logs directly from the container to avoid the color codes
235 printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.14/data/log/karaf.log ''' + dest + '''/#.log' || true
236 '''
237}
238pipeline {
239 /* no label, executor is determined by JJB */
240 agent {
241 label "${params.buildNode}"
242 }
243 options {
244 timeout(time: 220, unit: 'MINUTES')
245 }
246 environment {
247 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
248 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
249 SSHPASS="karaf"
250 }
251 stages{
252 stage('Download Code') {
253 steps {
254 getVolthaCode([
255 branch: "${branch}",
256 volthaSystemTestsChange: "${volthaSystemTestsChange}",
257 volthaHelmChartsChange: "${volthaHelmChartsChange}",
258 ])
259 }
260 }
261 stage('Cleanup') {
262 steps {
263 // remove port-forwarding
264 sh """
265 # remove orphaned port-forward from different namespaces
266 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
267 """
268 helmTeardown(['infra', 'voltha'])
269 }
270 }
271 stage('Create K8s Cluster') {
272 steps {
273 createKubernetesCluster([nodes: 3])
274 }
275 }
276 stage('Run Test') {
277 steps {
278 test_software_upgrade("onos-app-upgrade")
279 test_software_upgrade("voltha-component-upgrade")
280 test_software_upgrade("voltha-component-rolling-upgrade")
281 test_software_upgrade("onu-software-upgrade")
282 test_software_upgrade("onu-software-upgrade-omci-extended-msg")
283 test_software_upgrade("onu-image-dwl-simultaneously")
284 }
285 }
286 }
287 post {
288 aborted {
289 get_pods_info("$WORKSPACE/failed")
290 }
291 failure {
292 get_pods_info("$WORKSPACE/failed")
293 }
294 always {
295 step([$class: 'RobotPublisher',
296 disableArchiveOutput: false,
297 logFileName: 'RobotLogs/*/log*.html',
298 otherFiles: '',
299 outputFileName: 'RobotLogs/*/output*.xml',
300 outputPath: '.',
301 passThreshold: 100,
302 reportFileName: 'RobotLogs/*/report*.html',
303 unstableThreshold: 0,
304 onlyCritical: true]);
305 archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz,*.txt,**/*.txt'
306 }
307 }
308}