blob: 665ef57f4e3602918c5328a017253f21e0bcf8cd [file] [log] [blame]
Matteo Scandolof593a0f2021-04-14 10:16:06 -07001// Copyright 2021-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14// voltha-2.x e2e tests
15// uses bbsim to simulate OLT/ONUs
16// NOTE we are importing the library even if it's global so that it's
17// easier to change the keywords during a replay
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23def test_software_upgrade(name) {
24 stage('Deploy Voltha - '+ name) {
25 def extraHelmFlags = "${extraHelmFlags} --set global.log_level=DEBUG,onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 "
Hardik Windlass6d9a82e2021-07-08 16:23:21 +000026 if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade") {
27 extraHelmFlags = extraHelmFlags + "--set global.image_tag=master --set onos-classic.image.tag=master "
28 }
29 if ("${name}" == "voltha-component-upgrade") {
30 extraHelmFlags = extraHelmFlags + "--set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master "
31 }
Andrea Campanella834f27b2021-06-08 17:08:41 +020032 extraHelmFlags = extraHelmFlags + " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
Hardik Windlass6d9a82e2021-07-08 16:23:21 +000033 extraHelmFlags = extraHelmFlags + " --set voltha.onos_classic.replicas=3"
Matteo Scandolof593a0f2021-04-14 10:16:06 -070034 //ONOS custom image handling
35 if ( onosImg.trim() != '' ) {
36 String[] split;
37 onosImg = onosImg.trim()
38 split = onosImg.split(':')
39 extraHelmFlags = extraHelmFlags + "--set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
40 }
Hardik Windlass6d9a82e2021-07-08 16:23:21 +000041 def localCharts = false
42 if (branch != "master") {
43 localCharts = true
44 }
Matteo Scandolof593a0f2021-04-14 10:16:06 -070045 // Currently only testing with ATT workflow
46 // TODO: Support for other workflows
Hardik Windlass6d9a82e2021-07-08 16:23:21 +000047 volthaDeploy([workflow: "att", extraHelmFlags: extraHelmFlags, localCharts: localCharts])
Matteo Scandolof593a0f2021-04-14 10:16:06 -070048 // start logging
49 sh """
50 rm -rf $WORKSPACE/${name} || true
51 mkdir -p $WORKSPACE/${name}
52 _TAG=kail-${name} kail -n infra -n voltha > $WORKSPACE/${name}/onos-voltha-combined.log &
53 """
54 // forward ONOS and VOLTHA ports
55 sh """
56 _TAG=onos-port-forward bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8101:8101; done &"
57 _TAG=onos-port-forward bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8181:8181; done &"
58 _TAG=port-forward-voltha-api bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-api 55555:55555; done &"
59 """
60 sh """
61 sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
62 """
63 }
64 stage('Test - '+ name) {
65 sh """
66 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
67 mkdir -p \$ROBOT_LOGS_DIR
68 if [[ ${name} == 'onos-app-upgrade' ]]; then
69 export ONOS_APPS_UNDER_TEST+=''
70 if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
71 ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
72 fi
73 if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
74 ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
75 fi
76 if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
77 ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
78 fi
79 if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
80 ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
81 fi
82 if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
83 ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
84 fi
85 if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
86 ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
87 fi
88 if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
89 ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
90 fi
91 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
92 export TARGET=onos-app-upgrade-test
93 fi
94 if [[ ${name} == 'voltha-component-upgrade' ]]; then
95 export VOLTHA_COMPS_UNDER_TEST+=''
96 if [ ${adapterOpenOltImage.trim()} != '' ]; then
97 VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,${adapterOpenOltImage.trim()}*"
98 fi
99 if [ ${adapterOpenOnuImage.trim()} != '' ]; then
100 VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,${adapterOpenOnuImage.trim()}*"
101 fi
102 if [ ${rwCoreImage.trim()} != '' ]; then
103 VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,${rwCoreImage.trim()}*"
104 fi
105 if [ ${ofAgentImage.trim()} != '' ]; then
106 VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,${ofAgentImage.trim()}*"
107 fi
108 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
109 export TARGET=voltha-comp-upgrade-test
110 fi
111 if [[ ${name} == 'onu-software-upgrade' ]]; then
Hardik Windlass6d9a82e2021-07-08 16:23:21 +0000112 export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700113 export TARGET=onu-upgrade-test
114 fi
115 export VOLTCONFIG=$HOME/.volt/config-minimal
116 export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
Andrea Campanella834f27b2021-06-08 17:08:41 +0200117 ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120"
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700118 # Run the specified tests
119 make -C $WORKSPACE/voltha-system-tests \$TARGET || true
120 """
121 // stop logging
122 sh """
123 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
124 if [ -n "\$P_IDS" ]; then
125 echo \$P_IDS
126 for P_ID in \$P_IDS; do
127 kill -9 \$P_ID
128 done
129 fi
130 """
131 // remove port-forwarding
132 sh """
133 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200134 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700135 """
136 // collect pod details
137 get_pods_info("$WORKSPACE/${name}")
138 helmTeardown(['infra', 'voltha'])
139 }
140}
141def get_pods_info(dest) {
142 // collect pod details, this is here in case of failure
143 sh """
144 mkdir -p ${dest} || true
145 kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
146 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
147 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee ${dest}/pod-imagesId.txt || true
148 kubectl describe pods --all-namespaces -l app.kubernetes.io/part-of=voltha > ${dest}/voltha-pods-describe.txt
149 kubectl describe pods -n infra -l app=onos-classic > ${dest}/onos-pods-describe.txt
150 helm ls --all-namespaces > ${dest}/helm-charts.txt
151 """
152 sh '''
153 # copy the ONOS logs directly from the container to avoid the color codes
154 printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.9/data/log/karaf.log ''' + dest + '''/#.log' || true
155 '''
156}
157pipeline {
158 /* no label, executor is determined by JJB */
159 agent {
160 label "${params.buildNode}"
161 }
162 options {
Hardik Windlass973230d2021-05-17 11:08:59 +0530163 timeout(time: 60, unit: 'MINUTES')
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700164 }
165 environment {
166 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
167 KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
168 SSHPASS="karaf"
169 }
170 stages{
171 stage('Download Code') {
172 steps {
173 getVolthaCode([
174 branch: "${branch}",
175 volthaSystemTestsChange: "${volthaSystemTestsChange}",
176 volthaHelmChartsChange: "${volthaHelmChartsChange}",
177 ])
178 }
179 }
180 stage('Cleanup') {
181 steps {
182 // remove port-forwarding
183 sh """
184 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +0200185 ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700186 """
187 helmTeardown(['infra', 'voltha'])
188 }
189 }
Hardik Windlass9f5bee12021-05-07 06:47:21 +0000190 stage('Create K8s Cluster') {
191 steps {
Hardik Windlass6f854a12021-07-12 13:20:21 +0000192 createKubernetesCluster([branch: "${branch}", nodes: 3])
Hardik Windlass9f5bee12021-05-07 06:47:21 +0000193 }
194 }
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700195 stage('Run Test') {
196 steps {
197 test_software_upgrade("onos-app-upgrade")
198 test_software_upgrade("voltha-component-upgrade")
199 test_software_upgrade("onu-software-upgrade")
200 }
201 }
202 }
203 post {
204 aborted {
205 get_pods_info("$WORKSPACE/failed")
206 }
207 failure {
208 get_pods_info("$WORKSPACE/failed")
209 }
210 always {
211 sh '''
212 gzip $WORKSPACE/onos-app-upgrade/onos-voltha-combined.log || true
213 gzip $WORKSPACE/voltha-component-upgrade/onos-voltha-combined.log || true
214 gzip $WORKSPACE/onu-software-upgrade/onos-voltha-combined.log || true
215 '''
216 step([$class: 'RobotPublisher',
217 disableArchiveOutput: false,
218 logFileName: 'RobotLogs/*/log*.html',
219 otherFiles: '',
220 outputFileName: 'RobotLogs/*/output*.xml',
221 outputPath: '.',
222 passThreshold: 100,
223 reportFileName: 'RobotLogs/*/report*.html',
Hardik Windlass6d9a82e2021-07-08 16:23:21 +0000224 unstableThreshold: 0,
225 onlyCritical: true]);
Matteo Scandolof593a0f2021-04-14 10:16:06 -0700226 archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz,*.txt,**/*.txt'
227 }
228 }
229}