blob: e715bb4128a53daac6998b8940a225ad9ea7f152 [file] [log] [blame]
Kailash6f5acb62019-08-28 14:38:45 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15node ("${TestNodeName}") {
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070016 def withKind = false
17 if (params.withKind != null){
18 withKind = params.withKind
19 }
Kailash6f5acb62019-08-28 14:38:45 -070020 timeout (100) {
21 try {
22 stage ("Parse deployment configuration file") {
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070023 sh returnStdout: true, script: "rm -rf ${configBaseDir} voltha-system-tests kind-voltha"
Suchitra Vemurie514b8c2019-09-23 19:37:18 -070024 sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/${configBaseDir}"
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070025 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/voltha-system-tests"
Suchitra Vemuri6f787782020-04-10 20:56:34 -070026 sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/kind-voltha"
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070027 if ( params.workFlow == "DT" ) {
28 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
29 }
30 else
31 {
32 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
33 }
Kailash6f5acb62019-08-28 14:38:45 -070034 }
35 stage('Clean up') {
You Wang25e58612019-10-01 16:30:28 -070036 timeout(10) {
Kailash6f5acb62019-08-28 14:38:45 -070037 sh returnStdout: true, script: """
38 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
Matteo Scandolo1294aeb2019-09-24 16:20:32 -070039 for hchart in \$(helm list -q | grep -E -v 'docker-registry|mavenrepo|ponnet');
40 do
41 echo "Purging chart: \${hchart}"
42 helm delete --purge "\${hchart}"
43 done
Kailash6f5acb62019-08-28 14:38:45 -070044 """
45 timeout(5) {
46 waitUntil {
47 helm_deleted = sh returnStdout: true, script: """
48 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
49 helm ls -q | grep -E -v 'docker-registry|mavenrepo|ponnet' | wc -l
50 """
51 return helm_deleted.toInteger() == 0
52 }
53 }
54 timeout(5) {
55 waitUntil {
56 kubectl_deleted = sh returnStdout: true, script: """
57 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
58 kubectl get pods --all-namespaces --no-headers | grep -E -v 'kube-system|docker-registry|mavenrepo|ponnet' | wc -l
59 """
60 return kubectl_deleted.toInteger() == 0
61 }
62 }
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070063 timeout(1) {
Hung-Wei Chiu7ea92dd2020-04-07 16:40:35 -070064 sh returnStdout: false, script: """
65 #Remove all while true; do kubectl port-forward ...done; rules
66 for port_fwd_id in `ps -ax | grep "port-forward" | grep -E "onos|voltha" | grep "while true"| awk '{print \$1}'`;
67 do
68 cat /proc/\$port_fwd_id/cmdline
69 kill -9 \$port_fwd_id
70 done
71
72 #Remove all kubectl port-forward rules
73 for port_fwd_id in `ps -ax | grep "port-forward" | grep -E "onos|voltha" | awk '{print \$1}'`;
74 do
75 cat /proc/\$port_fwd_id/cmdline
76 kill -9 \$port_fwd_id
77 done
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070078 """
79 }
Kailash6f5acb62019-08-28 14:38:45 -070080 }
81 }
Suchitra Vemuri1934abc2019-11-05 12:41:11 -080082 stage('Install Voltha') {
Kailash6f5acb62019-08-28 14:38:45 -070083 timeout(10) {
84 sh returnStdout: true, script: """
85 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
86 cd kind-voltha/
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070087 if [ "${branch}" != "master" ]; then
88 cd $WORKSPACE/kind-voltha
89 source releases/${branch}
Suchitra Vemurib9483862019-11-05 21:37:29 -080090 else
Andy Bavier7cee6f42019-11-05 16:49:47 -070091 export EXTRA_HELM_FLAGS='-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml'
Suchitra Vemuri1934abc2019-11-05 12:41:11 -080092 fi
Andy Bavier7cee6f42019-11-05 16:49:47 -070093
94 # VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
hwchiuf2a24142020-04-04 05:00:29 +000095 # JENKINS_NODE_COOKIE=dontKillMe ask Jenkins doesn't kill the proces after this job. we want the `while true; do kubectl port-forward` keep alive
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070096 if [[ "${workFlow}" == "DT" ]]; then
hwchiuf2a24142020-04-04 05:00:29 +000097 JENKINS_NODE_COOKIE=dontKillMe ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no WITH_EAPOL=no WITH_DHCP=no WITH_IGMP=no WITH_RADIUS=no FANCY=0 ./voltha up
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -070098 else
hwchiuf2a24142020-04-04 05:00:29 +000099 JENKINS_NODE_COOKIE=dontKillMe ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_RADIUS=y WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no FANCY=0 ./voltha up
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700100 fi
Andy Bavier7cee6f42019-11-05 16:49:47 -0700101
Suchitra Vemuri1934abc2019-11-05 12:41:11 -0800102 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
103 kubectl get nodes -o wide
104 kubectl get pods -n voltha -o wide
Kailash6f5acb62019-08-28 14:38:45 -0700105 """
Suchitra Vemuri1934abc2019-11-05 12:41:11 -0800106 }
Kailash6f5acb62019-08-28 14:38:45 -0700107 }
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700108 if ( withKind && deployment_config.fabric_switches.size() > 0 ) {
109 stage('OpenFlow port forward for aggregation switch') {
110 timeout(1) {
111 sh returnStdout: true, script: """
112 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
113 #This uses the default route out, porperly configured is the management
114 mgmt_address=`ip route get 1 | awk '{print \$NF;exit}'`
115 JENKINS_NODE_COOKIE=dontkill kubectl port-forward -n default service/onos-openflow --address=\$mgmt_address 6653:6653 >> /dev/null 2>&1 &
116 """
117 }
118 }
119 }
120 if ( params.configurePod && params.profile != "Default" ) {
Suchitra Vemuri72680092019-10-03 12:39:20 -0700121 stage('Push Tech-Profile') {
122 timeout(1) {
123 out_push_tp = sh returnStatus: true, script: """
124 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
Suchitra Vemuri2dc7e1a2020-06-08 17:18:06 -0700125 etcd_container=\$(kubectl get pods | grep etcd | awk 'NR==1{print \$1}')
126 kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-${profile}.json \$etcd_container:/tmp/flexpod.json
127 put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64')
Suchitra Vemuri72680092019-10-03 12:39:20 -0700128 """
129 return out_push_tp == 0
130 }
131 timeout(1) {
132 out_get_tp = sh returnStatus: true, script: """
Suchitra Vemuri2dc7e1a2020-06-08 17:18:06 -0700133 etcd_container=\$(kubectl get pods | grep etcd | awk 'NR==1{print \$1}')
134 get_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/voltha/technology_profiles/XGS-PON/64')
Suchitra Vemuri72680092019-10-03 12:39:20 -0700135 """
136 return out_get_tp == 0
137 }
hwchiuc77c7d22019-10-18 13:58:35 -0700138 }
139 }
140 stage('Push Sadis-config') {
141 timeout(1) {
142 sadis_out = sh returnStatus: true, script: """
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700143 if [[ "${workFlow}" == "DT" ]]; then
144 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis-DT.json
145 else
Suchitra Vemuri72680092019-10-03 12:39:20 -0700146 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis.json
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700147 fi
148 """
149 return sadis_out == 0
You Wangb9526e62019-10-07 15:25:48 -0700150 }
hwchiuc77c7d22019-10-18 13:58:35 -0700151 }
Kailash6f5acb62019-08-28 14:38:45 -0700152 if ( params.reinstallOlt ) {
153 stage('Reinstall OLT software') {
154 for(int i=0; i < deployment_config.olts.size(); i++) {
155 sh returnStdout: true, script: """
Suchitra Vemuri9c972262020-05-14 16:44:28 -0700156 if [ "${branch}" != "master" ]; then
157 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${deployment_config.olts[i].oltDebVersion23}"
158 else
159 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${deployment_config.olts[i].oltDebVersion}"
160 fi
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700161 sleep 10
Kailash6f5acb62019-08-28 14:38:45 -0700162 """
163 timeout(5) {
164 waitUntil {
165 olt_sw_present = sh returnStdout: true, script: """
Suchitra Vemuri95165762020-05-18 15:22:45 -0700166 if [[ "${deployment_config.olts[i].oltDebVersion}" == *"asfvolt16-2.4"* ]] || [[ "${deployment_config.olts[i].oltDebVersion23}" == *"asfvolt16"* ]]; then
Suchitra Vemuri9c972262020-05-14 16:44:28 -0700167 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
168 else
169 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asgvolt64 | wc -l'
170 fi
171 if (${deployment_config.olts[i].fortygig}); then
172 if [[ "${deployment_config.olts[i].oltDebVersion}" == *"asfvolt16-2.4"* ]] || [[ "${deployment_config.olts[i].oltDebVersion23}" == *"asfvolt16-2.4"* ]]; then
173 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000 if=XLAUI" >> /broadcom/qax.soc'
174 else
175 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc'
176 fi
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700177 fi
Kailash6f5acb62019-08-28 14:38:45 -0700178 """
179 return olt_sw_present.toInteger() == 1
180 }
181 }
Kailash6f5acb62019-08-28 14:38:45 -0700182 }
183 }
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700184 stage('Restart OLT processes') {
Kailash6f5acb62019-08-28 14:38:45 -0700185 for(int i=0; i < deployment_config.olts.size(); i++) {
186 timeout(5) {
187 sh returnStdout: true, script: """
188 ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700189 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'reboot' || true
190 sleep 120
Kailash6f5acb62019-08-28 14:38:45 -0700191 """
192 }
193 timeout(15) {
194 waitUntil {
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700195 devprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep dev_mgmt_daemon | wc -l'"
196 return devprocess.toInteger() > 0
197 }
198 }
199 timeout(15) {
200 waitUntil {
201 openoltprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep openolt | wc -l'"
202 return openoltprocess.toInteger() > 0
Kailash6f5acb62019-08-28 14:38:45 -0700203 }
204 }
205 }
206 }
207 }
Matteo Scandolo1294aeb2019-09-24 16:20:32 -0700208 if ( deployment_config.fabric_switches.size() > 0 ) {
You Wang25e58612019-10-01 16:30:28 -0700209 stage('Switch Configurations in ONOS') {
210 timeout(1) {
211 netcfg_out = sh returnStatus: true, script: """
212 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/${configBaseDir}/${configToscaDir}/voltha/${configFileName}-onos-netcfg-switch.json
213 curl -sSL --user karaf:karaf -X POST http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting/active
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700214
You Wang25e58612019-10-01 16:30:28 -0700215 """
216 return netcfg_out == 0
217 }
218 timeout(1) {
219 waitUntil {
220 sr_active_out = sh returnStatus: true, script: """
You Wangeb76aeb2019-12-09 22:08:23 -0800221 ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115
Suchitra Vemuric3257262019-12-05 19:06:42 -0800222 ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts
223 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay"
224 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa"
225 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt"
You Wang25e58612019-10-01 16:30:28 -0700226 curl -sSL --user karaf:karaf -X GET http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting | jq '.state' | grep ACTIVE
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700227 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "cfg set org.onosproject.net.flow.impl.FlowRuleManager purgeOnDisconnection false"
Andrea Campanellaa1b06512020-04-01 09:56:23 +0200228 sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "cfg set org.onosproject.net.meter.impl.MeterManager purgeOnDisconnection false"
You Wang25e58612019-10-01 16:30:28 -0700229 """
230 return sr_active_out == 0
231 }
232 }
Suchitra Vemuricf5c5c52020-05-05 17:54:07 -0700233 timeout(5) {
You Wang25e58612019-10-01 16:30:28 -0700234 for(int i=0; i < deployment_config.hosts.src.size(); i++) {
Suchitra Vemuricf5c5c52020-05-05 17:54:07 -0700235 def aggPort = -1
236 for(int j=0; j < deployment_config.olts.size(); j++) {
237 if(deployment_config.olts[j].serial == deployment_config.hosts.src[j].olt){
238 aggPort = deployment_config.olts[j].aggPort
239 }
240 }
241 if(aggPort == -1){
242 throw new Exception("Upstream port for the olt is not configured, field aggPort is empty")
243 }
You Wang25e58612019-10-01 16:30:28 -0700244 xconnect_out = sh returnStatus: true, script: """
Suchitra Vemurif82086d2020-01-16 19:55:16 -0800245 version=\$(sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "summary" | grep version)
Suchitra Vemuri0c40ebe2020-03-27 15:09:36 -0700246 sleep 10
Suchitra Vemurif82086d2020-01-16 19:55:16 -0800247 if [[ \$version == *"version=2.2"* ]]; then
Suchitra Vemuricf5c5c52020-05-05 17:54:07 -0700248 curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "endpoints": [${deployment_config.fabric_switches[0].bngPort},${aggPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect'
Suchitra Vemurif82086d2020-01-16 19:55:16 -0800249 else
Suchitra Vemuricf5c5c52020-05-05 17:54:07 -0700250 curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "ports": [${deployment_config.fabric_switches[0].bngPort},${aggPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect'
Suchitra Vemurif82086d2020-01-16 19:55:16 -0800251 fi
You Wang25e58612019-10-01 16:30:28 -0700252 """
253 }
254 }
Kailash6f5acb62019-08-28 14:38:45 -0700255 }
Matteo Scandolo1294aeb2019-09-24 16:20:32 -0700256 }
Kailash6f5acb62019-08-28 14:38:45 -0700257 currentBuild.result = 'SUCCESS'
258 } catch (err) {
259 currentBuild.result = 'FAILURE'
260 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
261 throw err
262 }
263 echo "RESULT: ${currentBuild.result}"
264 }
265}