blob: 97f3de9229f523f62356237ffd40478f002fa1c6 [file] [log] [blame]
Suchitra Vemuriddb27792018-08-23 15:29:42 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Suchitra Vemuriddb27792018-08-23 15:29:42 -070015node ("${TestNodeName}") {
Luca Preteb3029bd2018-08-28 16:31:39 -070016 timeout (100) {
17 try {
18 stage ("Parse deployment configuration file") {
19 sh returnStdout: true, script: "rm -rf ${configBaseDir}"
You Wangd6c5e8c2018-08-29 12:49:51 -070020 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
Luca Prete43f52cd2018-08-28 16:36:36 -070021 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yml"
Suchitra Vemuriddb27792018-08-23 15:29:42 -070022 }
Luca Preteb3029bd2018-08-28 16:31:39 -070023 stage('Clean up') {
24 timeout(10) {
25 sh returnStdout: true, script: """
26 rm -rf helm-charts cord-tester
You Wangd6c5e8c2018-08-29 12:49:51 -070027 git clone -b ${branch} ${cordRepoUrl}/helm-charts
Luca Preteb3029bd2018-08-28 16:31:39 -070028 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
Luca Prete5a970152018-08-29 15:20:47 -070029 for hchart in \$(helm list -q | grep -E -v 'docker-registry|mavenrepo|ponnet');
Luca Preteb3029bd2018-08-28 16:31:39 -070030 do
31 echo "Purging chart: \${hchart}"
32 helm delete --purge "\${hchart}"
33 done
Suchitra Vemuriddb27792018-08-23 15:29:42 -070034 """
Luca Preteb3029bd2018-08-28 16:31:39 -070035 timeout(5) {
36 waitUntil {
37 helm_deleted = sh returnStdout: true, script: """
38 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
Luca Preteb4966672018-08-29 16:49:48 -070039 helm ls -q | grep -E -v 'docker-registry|mavenrepo|ponnet' | wc -l
Suchitra Vemuriddb27792018-08-23 15:29:42 -070040 """
Luca Preteb3029bd2018-08-28 16:31:39 -070041 return helm_deleted.toInteger() == 0
42 }
43 }
44 timeout(5) {
45 waitUntil {
46 kubectl_deleted = sh returnStdout: true, script: """
47 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
Luca Preteb4966672018-08-29 16:49:48 -070048 kubectl get pods --all-namespaces --no-headers | grep -E -v 'kube-system|docker-registry|mavenrepo|ponnet' | wc -l
Luca Preteb3029bd2018-08-28 16:31:39 -070049 """
50 return kubectl_deleted.toInteger() == 0
Suchitra Vemuriddb27792018-08-23 15:29:42 -070051 }
52 }
Luca Prete79fbc602018-09-24 15:33:31 -070053 // timeout(5) {
54 // waitUntil {
55 // kubectl_deleted = sh returnStdout: true, script: """
56 // export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
57 // kubectl get statefulset --all-namespaces --no-headers | grep -E -v 'kube-system|docker-registry|mavenrepo|ponnet' | wc -l
58 // """
59 // return kubectl_deleted.toInteger() == 0
60 // }
61 // }
Suchitra Vemuriddb27792018-08-23 15:29:42 -070062 }
63 }
Luca Preteb3029bd2018-08-28 16:31:39 -070064 dir ("helm-charts") {
Kailash Khalasi100cf462018-09-20 10:24:57 -070065 stage('Install CORD Kafka') {
66 timeout(10) {
67 sh returnStdout: true, script: """
68 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
69 helm install -f examples/kafka-single.yaml --version 0.8.8 -n cord-kafka incubator/kafka
70 """
71 }
72 timeout(10) {
73 waitUntil {
74 kafka_instances_running = sh returnStdout: true, script: """
75 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
76 kubectl get pods | grep cord-kafka | grep -i running | grep 1/1 | wc -l
77 """
78 return kafka_instances_running.toInteger() == 2
79 }
80 }
81 }
Luca Pretecfbf71b2018-09-24 15:39:23 -070082 // stage('Install Logging Infrastructure') {
83 // timeout(10) {
84 // sh returnStdout: true, script: """
85 // export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
86 // helm dep update logging
87 // helm install -f examples/logging-single.yaml -n logging logging
88 // scripts/wait_for_pods.sh
89 // """
90 // }
91 // }
Kailash Khalasi100cf462018-09-20 10:24:57 -070092 stage('Install Monitoring Infrastructure') {
93 timeout(10) {
94 sh returnStdout: true, script: """
95 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
96 helm dep update nem-monitoring
97 helm install -n nem-monitoring nem-monitoring
98 scripts/wait_for_pods.sh
99 """
100 }
101 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700102 stage('Install voltha') {
103 timeout(10) {
104 sh returnStdout: true, script: """
105 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
106 helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
107 helm dep build voltha
108 helm install -n voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcd-operator.customResources.createEtcdClusterCRD=false voltha
Matteo Scandoloa6e1f9d2018-08-29 17:23:19 -0700109 helm upgrade -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha
Luca Preteb3029bd2018-08-28 16:31:39 -0700110 """
111 }
112 timeout(10) {
113 waitUntil {
114 voltha_completed = sh returnStdout: true, script: """
115 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
116 kubectl get pods -n voltha | grep -i running | grep 1/1 | wc -l
117 """
118 return voltha_completed.toInteger() == 8
119 }
120 }
121 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700122 stage('Install ONOS') {
123 timeout(10) {
124 sh returnStdout: true, script: """
125 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
Luca Prete5a970152018-08-29 15:20:47 -0700126 helm install -n onos -f configs/onos.yaml -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml onos
Luca Preteb3029bd2018-08-28 16:31:39 -0700127 """
128 }
129 timeout(10) {
130 waitUntil {
131 onos_completed = sh returnStdout: true, script: """
132 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
Luca Preteb1ae41b2018-08-28 16:47:54 -0700133 kubectl get pods | grep -i onos | grep -i running | grep 1/1 | wc -l
Luca Preteb3029bd2018-08-28 16:31:39 -0700134 """
135 return onos_completed.toInteger() == 1
136 }
137 }
138 }
139 stage('Install xos-core') {
140 timeout(10) {
141 sh returnStdout: true, script: """
142 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
143 helm dep update xos-core
144 helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n xos-core xos-core
145 """
146 }
147 timeout(10) {
148 waitUntil {
149 xos_core_completed = sh returnStdout: true, script: """
150 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
151 kubectl get pods | grep -i xos | grep -i running | grep 1/1 | wc -l
152 """
Scott Bakerde6374f2018-09-24 11:34:55 -0700153 return xos_core_completed.toInteger() == 6
Luca Preteb3029bd2018-08-28 16:31:39 -0700154 }
155 }
156 }
157 stage('Install att-workflow') {
158 timeout(10) {
159 sh returnStdout: true, script: """
160 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
161 helm dep update xos-profiles/att-workflow
162 helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n att-workflow xos-profiles/att-workflow
163 """
164 }
165 timeout(10) {
166 waitUntil {
167 att_workflow_tosca_completed = sh returnStdout: true, script: """
168 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
169 kubectl get pods | grep -i att-workflow-tosca-loader | grep -i completed | wc -l
170 """
171 return att_workflow_tosca_completed.toInteger() == 1
172 }
173 }
174 }
175 stage('Install base-kubernetes') {
176 timeout(10) {
177 sh returnStdout: true, script: """
178 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
179 helm dep update xos-profiles/base-kubernetes
180 helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n base-kubernetes xos-profiles/base-kubernetes
181 """
182 }
183 timeout(10) {
184 waitUntil {
185 base_kubernetes_tosca_running = sh returnStdout: true, script: """
186 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
187 kubectl get pods | grep -i base-kubernetes-tosca-loader | grep -i completed | wc -l
188 """
189 return base_kubernetes_tosca_running.toInteger() == 1
190 }
Suchitra Vemuriddb27792018-08-23 15:29:42 -0700191 }
192 }
193 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700194 stage('Reinstall OLT software') {
195 for(int i=0; i < deployment_config.olts.size(); i++) {
196 sh returnStdout: true, script: """
197 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'
Suchitra Vemuriddb27792018-08-23 15:29:42 -0700198 """
Luca Preteb3029bd2018-08-28 16:31:39 -0700199 timeout(5) {
200 waitUntil {
201 olt_sw_present = sh returnStdout: true, script: """
202 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
203 """
204 return olt_sw_present.toInteger() == 0
205 }
206 }
207 sh returnStdout: true, script: """
208 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}"
209 """
210 timeout(5) {
211 waitUntil {
212 olt_sw_present = sh returnStdout: true, script: """
213 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
214 """
215 return olt_sw_present.toInteger() == 1
216 }
217 }
218 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
219 if ("${deployment_config.olts[i].fortygig}" != null && "${deployment_config.olts[i].fortygig}" == 'true') {
220 sh returnStdout: true, script: """
221 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc'
222 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '/opt/bcm68620/svk_init.sh'
223 """
224 }
225 }
Suchitra Vemuriddb27792018-08-23 15:29:42 -0700226 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700227 stage('Restart OLT processes') {
228 for(int i=0; i < deployment_config.olts.size(); i++) {
229 timeout(5) {
230 sh returnStdout: true, script: """
231 ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts
232 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'service bal_core_dist stop' || true
233 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'service openolt stop' || true
234 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /var/log/bal_core_dist.log'
235 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /var/log/openolt.log'
236 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'service bal_core_dist start &'
237 sleep 5
238 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'service openolt start &'
239 """
240 }
241 timeout(15) {
242 waitUntil {
Kailash Khalasi23b693c2018-09-14 14:46:50 -0700243 onu_discovered = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cat /var/log/openolt.log | grep \"oper_state: up\" | wc -l'"
Luca Preteb3029bd2018-08-28 16:31:39 -0700244 return onu_discovered.toInteger() > 0
245 }
246 }
247 }
248 }
Kailash Khalasi4d18e202018-09-14 13:32:16 -0700249 if ( params.configurePod ) {
Kailash Khalasidef2f4e2018-09-17 10:39:14 -0700250 dir ("${configBaseDir}/${configToscaDir}/att-workflow") {
Kailash Khalasi4d18e202018-09-14 13:32:16 -0700251 stage('Configure R-CORD - Fabric and whitelist') {
252 timeout(1) {
253 waitUntil {
254 out_fabric = sh returnStdout: true, script: """
255 curl -s -H "xos-username:admin@opencord.org" -H "xos-password:letmein" -X POST --data-binary @${configFileName}-fabric.yaml http://${deployment_config.nodes[0].ip}:30007/run | grep -i "created models" | wc -l
256 """
257 return out_fabric.toInteger() == 1
258 }
Kailash Khalasic1a90f12018-09-13 13:13:22 -0700259 }
260 }
Kailash Khalasi4d18e202018-09-14 13:32:16 -0700261 stage('Configure R-CORD - Subscriber') {
262 timeout(1) {
263 waitUntil {
264 out_subscriber = sh returnStdout: true, script: """
265 curl -s -H 'xos-username:admin@opencord.org' -H 'xos-password:letmein' -X POST --data-binary @${configFileName}-subscriber.yaml http://${deployment_config.nodes[0].ip}:30007/run | grep -i "created models" | wc -l
266 """
267 return out_subscriber.toInteger() == 1
268 }
Kailash Khalasic1a90f12018-09-13 13:13:22 -0700269 }
270 }
Kailash Khalasi4d18e202018-09-14 13:32:16 -0700271 stage('Configure R-CORD - OLT') {
272 timeout(1) {
273 waitUntil {
274 out_olt = sh returnStdout: true, script: """
275 curl -H 'xos-username:admin@opencord.org' -H 'xos-password:letmein' -X POST --data-binary @${configFileName}-olt.yaml http://${deployment_config.nodes[0].ip}:30007/run | grep -i "created models" | wc -l
276 """
277 return out_olt.toInteger() == 1
278 }
Kailash Khalasic1a90f12018-09-13 13:13:22 -0700279 }
280 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700281 }
Suchitra Vemuriddb27792018-08-23 15:29:42 -0700282 }
Luca Preteb3029bd2018-08-28 16:31:39 -0700283 currentBuild.result = 'SUCCESS'
284 } catch (err) {
285 currentBuild.result = 'FAILURE'
286 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
287 }
288 echo "RESULT: ${currentBuild.result}"
Suchitra Vemuriddb27792018-08-23 15:29:42 -0700289 }
290}