blob: 81c3421b06b3fda7e64b7b0d9a369f7b7eab730a [file] [log] [blame]
Kailash Khalasideeead02018-04-11 13:27:27 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Kailash Khalasi85d82142018-04-18 11:02:04 -070015import groovy.json.JsonSlurper
Kailash Khalasideeead02018-04-11 13:27:27 -070016
Kailash Khalasib8e174e2018-04-17 09:03:59 -070017def filename = 'manifest-${branch}.xml'
Kailash Khalasideeead02018-04-11 13:27:27 -070018def manifestUrl = 'https://gerrit.opencord.org/manifest'
19deployment_config = null;
20pod_config = null;
21
22node ("${devNodeName}") {
23 timeout (time: 240) {
Kailash Khalasideeead02018-04-11 13:27:27 -070024 stage ("Parse deployment configuration file") {
25 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
Kailash Khalasib8e174e2018-04-17 09:03:59 -070026 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
Kailash Khalasideeead02018-04-11 13:27:27 -070027 deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
28 pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}"
29 }
30 stage ('Remove old head node from known hosts') {
31 sh "ssh-keygen -R ${deployment_config.head.ip}"
32 }
33 stage ('Checkout cord repo') {
34 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
35 }
36
37 try {
38 dir('build') {
39 stage ("Re-deploy head node") {
40 maasOps: {
41 sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}"
42 sh "maas maas machine release ${deployment_config.maas.head_system_id}"
43
44 timeout(time: 15) {
45 waitUntil {
46 try {
47 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready"
48 return true
49 } catch (exception) {
50 return false
51 }
52 }
53 }
54
55 sh 'maas maas machines allocate'
56 sh "maas maas machine deploy ${deployment_config.maas.head_system_id}"
57
58 timeout(time: 30) {
59 waitUntil {
60 try {
61 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed"
62 return true
63 } catch (exception) {
64 return false
65 }
66 }
67 }
68 }
69 }
70
71 stage ("Download CORD POD configuration") {
Kailash Khalasib8e174e2018-04-17 09:03:59 -070072 sh "cd ../orchestration/profiles; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation"
Kailash Khalasideeead02018-04-11 13:27:27 -070073 }
74
75 stage ("Generate CORD configuration") {
76 sh "make PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config"
77 }
78
79 if (deployment_config.fabric_switches != null) {
80 stage("Reserve IPs for fabric switches") {
81 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
82 def str = createMACIPbindingStr("fabric", i+1,
83 "${deployment_config.fabric_switches[i].mac}",
84 "${deployment_config.fabric_switches[i].ip}")
85 sh "echo $str >> maas/roles/maas/files/dhcpd.reservations"
86 }
87 }
88 }
89
90 stage ("Deploy") {
91 sh "make build"
92 }
93
94 if (deployment_config.compute_nodes != null) {
95 stage ("Power cycle compute nodes") {
96 for(int i=0; i < deployment_config.compute_nodes.size(); i++) {
97 sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle"
98 }
99 }
100
101 stage ("Wait for compute nodes to get deployed") {
102 sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}"
103 def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}")
104 runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
105 timeout(time: 90) {
106 waitUntil {
107 try {
108 num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim()
109 return num.toInteger() == deployment_config.compute_nodes.size()
110 } catch (exception) {
111 return false
112 }
113 }
114 }
115 }
116
117 stage ("Wait for compute nodes to be provisioned") {
118 timeout(time:45) {
119 waitUntil {
120 try {
121 num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim()
122 return num.toInteger() == deployment_config.compute_nodes.size()
123 } catch (exception) {
124 return false
125 }
126 }
127 }
128 }
129 }
130
131 if (deployment_config.fabric_switches != null) {
132 stage ("Wait for fabric switches to get deployed") {
133 runFabricSwitchCmdAll("sudo onl-onie-boot-mode install")
134 runFabricSwitchCmdAll("sudo reboot")
135 // Ensure that switches get provisioned after ONIE reinstall.
136 // Delete them if they were provisioned earlier. If the switches are not
137 // present in 'cord prov list', this command has no effect.
138 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
139 runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}")
140 }
141 timeout(time: 45) {
142 waitUntil {
143 try {
144 def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim()
145 return harvestCompleted.toInteger() == deployment_config.fabric_switches.size()
146 } catch (exception) {
147 return false
148 }
149 }
150 }
151 }
152
153 stage ("Wait for fabric switches to be provisioned") {
154 timeout(time:45) {
155 waitUntil {
156 try {
157 def provCompleted = 0
158 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
159 def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim()
160 provCompleted = provCompleted + count.toInteger()
161 }
162 return provCompleted == deployment_config.fabric_switches.size()
163 } catch (exception) {
164 return false
165 }
166 }
167 }
168 }
169
170 // Post installation configuration starts here
171 fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0]
172 xosUser = "xosadmin@opencord.org"
173 xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim()
174 stage ("Connect fabric switches and compute nodes to ONOS") {
175 // Configure breakout ports
176 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
177 if (deployment_config.fabric_switches[i].breakout_ports != null) {
178 for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) {
179 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
180 "${deployment_config.fabric_switches[i].user}",
181 "${deployment_config.fabric_switches[i].pass}",
182 "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf")
183 }
184 }
185 }
186 connectFabricAndComputeToOnos()
187 }
188 stage ("Configure the compute nodes") {
189 leafSwitchNum = 0
190 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
191 if(deployment_config.fabric_switches[i].containsKey("role")) {
192 if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) {
193 leafSwitchNum += 1
194 }
195 }
196 }
197 for(int i=1; i<=leafSwitchNum; i++) {
198 // Figure out which compute node connects to which switch
199 leafName = "leaf-" + i.toString()
200 computeNames = getComputeNames(leafName)
201 echo "Compute nodes connnected to " + leafName + " switch:"
202 for(name in computeNames) { echo "${name}" }
203 index = 1
204 for(name in computeNames) {
205 if(i>1) {
206 //Update fabric IP of compute nodes
207 index += 1
208 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim()
209 if (fabricIp != "") {
210 nodeId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/nodes | jq '.[\"items\"][] | select(.dataPlaneIp==\"${fabricIp}\") | .id'").trim()
211 newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24"
212 out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"dataPlaneIp\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/nodes/${nodeId}").trim()
213 // Wait until the new fabric IP gets configured
214 timeout(time: 5) {
215 waitUntil {
216 try {
217 num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim()
218 return num.toInteger() == 1
219 } catch (exception) {
220 return false
221 }
222 }
223 }
224 }
225 else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*"
226 }
227 //Add routes to fabric subnets
228 for(int j=1; j<=leafSwitchNum; j++) {
229 if(j!=i) {
230 runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists")
231 }
232 }
233 }
234 }
235 }
236 stage ("Generate and load network configuration") {
237 // Reconnect compute nodes to update the fabric IP in ONOS
238 connectFabricAndComputeToOnos()
239 // Refresh fabric configurations
240 sh "make fabric-refresh"
241 }
242 }
243 }
244
245 if (deployment_config.make_release == true) {
246 stage ("Trigger Build") {
247 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
248 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
249 }
250 }
251
252 currentBuild.result = 'SUCCESS'
253 } catch (err) {
254 currentBuild.result = 'FAILURE'
255 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
Kailash Khalasi5ed4e032018-04-26 15:15:10 -0700256 } finally {
257 sh "make -C build clean-all || true"
258 sh "rm -rf *"
Kailash Khalasideeead02018-04-11 13:27:27 -0700259 }
260 echo "RESULT: ${currentBuild.result}"
261 }
262}
263
264/**
265 * Returns a string used to bind IPs and MAC addresses, substituting the values
266 * given.
267 *
268 * @param word the word used to generate the host name
269 * @param counter the counter used to generate the host name
270 * @param mac the MAC address to substitute
271 * @param ip the IP address to substitute
272 */
273def createMACIPbindingStr(word, counter, mac, ip) {
274 return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}"""
275}
276
277/**
278 * Runs a command on a remote host using sshpass.
279 *
280 * @param ip the node IP address
281 * @param user the node user name
282 * @param pass the node password
283 * @param command the command to run
284 * @param sshArgs arguments for the ssh command
285 * @return the output of the command
286 */
287def runCmd(ip, user, pass, command, sshArgs="") {
288 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"")
289}
290
291/**
292 * Runs a command on the head node.
293 *
294 * @param command the command to run
295 * @param sshArgs arguments for the ssh command
296 * @return the output of the command
297 */
298def runHeadNodeCmd(command, sshArgs="") {
299 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"")
300}
301
302/**
303 * Runs a command on a fabric switch.
304 *
305 * @param ip the mgmt IP of the fabric switch, reachable from the head node
306 * @param user the mgmt user name of the fabric switch
307 * @param pass the mgmt password of the fabric switch
308 * @param command the command to run on the fabric switch
309 * @param ssgArgs arguments for the ssh command
310 * @return the output of the command
311 */
312def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") {
313 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
314}
315
316/**
317 * Runs a command on all fabric switches
318 *
319 * @param command the command to run on the fabric switches
320 * @param ssgArgs arguments for the ssh command
321 */
322def runFabricSwitchCmdAll(command, sshArgs="") {
323 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
324 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
325 "${deployment_config.fabric_switches[i].user}",
326 "${deployment_config.fabric_switches[i].pass}",
327 "${command}",
328 "${sshArgs}")
329 }
330}
331
332/**
333 * Runs a command on a compute node.
334 *
335 * @param name the name of the compute node
336 * @param command the command to run on the compute node
337 * @param ssgArgs arguments for the ssh command
338 * @return the output of the command
339 */
340def runComputeNodeCmd(name, command, sshArgs="") {
341 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"")
342}
343
344/**
345 * Runs a command on all compute nodes
346 *
347 * @param command the command to run on the compute nodes
348 * @param ssgArgs arguments for the ssh command
349 */
350def runComputeNodeCmdAll(command, sshArgs="") {
351 computeNamesAll = getComputeNames()
352 for (name in computeNamesAll) {
353 runComputeNodeCmd("${name}", "${command}", "${sshArgs}")
354 }
355}
356
357/**
358 * Runs an ONOS CLI command
359 *
360 * @param name the onos node name, reachable from the head node
361 * @param port the port used to login to ONOS CLI
362 * @param user the user name to login to ONOS CLI
363 * @param pass the password to login to ONOS CLI
364 * @param command the command to run in ONOS CLI
365 * @return the output of the command
366 */
367def runOnosCliCmd(name, port, user, pass, command) {
368 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"")
369}
370
371/**
372 * Returns a list of compute node names. When "role" is specified, returns only
373 * names of compute nodes connected to the switch
374 *
375 * @param role the switch role, i.e. "leaf-1"
376 */
377def getComputeNames(role="") {
378 computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim()
379 computeNamesAll = "${computeNamesAll}".split()
380 computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"")
Kailash Khalasi85d82142018-04-18 11:02:04 -0700381 computeNamesAll = new JsonSlurper().parseText("${computeNamesAll}")
Kailash Khalasideeead02018-04-11 13:27:27 -0700382 if ("${role}" == "") return computeNamesAll
383 computeNames = []
384 switchMac = ""
385 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
386 if(deployment_config.fabric_switches[i].containsKey("role")) {
387 if ("${deployment_config.fabric_switches[i].role}" == "${role}")
388 switchMac = "${deployment_config.fabric_switches[i].mac}"
389 }
390 }
391 if ("${switchMac}" != "") {
392 switchMac = switchMac.toLowerCase().replaceAll(':','')
393 // Get fabric IPs of compute nodes connected to the switch
394 try {
395 computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.locations[].elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'",
396 "-q").trim()
397 }catch (exception) {
398 return computeNames
399 }
400 computeFabricIps = "${computeFabricIps}".split()
Kailash Khalasi85d82142018-04-18 11:02:04 -0700401 computeFabricIps = new JsonSlurper().parseText("${computeFabricIps}")
Kailash Khalasideeead02018-04-11 13:27:27 -0700402 // Figure out which compute node connects to the switch
403 for (name in computeNamesAll) {
404 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim()
405 if (fabricIp in computeFabricIps) {
406 computeNames.add("${name}")
407 }
408 }
409 }
410 return computeNames
411}
412
413/**
414 * Connects all fabric switches and compute nodes to ONOS
415 */
416def connectFabricAndComputeToOnos() {
417 // Kill existing switch connections
418 runFabricSwitchCmdAll("./killit || echo no ofagentapp running")
419 // Clean stale ONOS data
420 runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please")
421 // Connect switches to ONOS
422 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
423 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
424 "${deployment_config.fabric_switches[i].user}",
425 "${deployment_config.fabric_switches[i].pass}",
426 "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log",
427 "-qftn")
428 }
429 // Verify ONOS has recognized the switches
430 timeout(time: 5) {
431 waitUntil {
432 try {
433 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim()
434 return num.toInteger() == deployment_config.fabric_switches.size()
435 } catch (exception) {
436 return false
437 }
438 }
439 }
440 // Connect compute nodes to ONOS
441 runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn")
442 // Verify ONOS has recognized the hosts
443 timeout(time: 5) {
444 waitUntil {
445 try {
446 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim()
447 return num.toInteger() >= deployment_config.compute_nodes.size()
448 } catch (exception) {
449 return false
450 }
451 }
452 }
453}