blob: 63ad81da3f7a1c2041a4b1efbdf79d906f108743 [file] [log] [blame]
Kailash Khalasideeead02018-04-11 13:27:27 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Kailash Khalasi85d82142018-04-18 11:02:04 -070015import groovy.json.JsonSlurper
Kailash Khalasideeead02018-04-11 13:27:27 -070016
Kailash Khalasib8e174e2018-04-17 09:03:59 -070017def filename = 'manifest-${branch}.xml'
Kailash Khalasideeead02018-04-11 13:27:27 -070018def manifestUrl = 'https://gerrit.opencord.org/manifest'
19deployment_config = null;
20pod_config = null;
21
22node ("${devNodeName}") {
23 timeout (time: 240) {
Kailash Khalasideeead02018-04-11 13:27:27 -070024 stage ("Parse deployment configuration file") {
25 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
Kailash Khalasib8e174e2018-04-17 09:03:59 -070026 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
Kailash Khalasideeead02018-04-11 13:27:27 -070027 deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
28 pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}"
29 }
Kailash Khalasi9f2ba4e2018-04-19 14:17:17 -070030 stage ("Clean-up previous build") {
31 sh "make -C build PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config"
32 sh "make -C build clean-all || true"
33 sh "rm -rf *"
34 }
Kailash Khalasideeead02018-04-11 13:27:27 -070035 stage ('Remove old head node from known hosts') {
36 sh "ssh-keygen -R ${deployment_config.head.ip}"
37 }
38 stage ('Checkout cord repo') {
39 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
40 }
41
42 try {
43 dir('build') {
44 stage ("Re-deploy head node") {
45 maasOps: {
46 sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}"
47 sh "maas maas machine release ${deployment_config.maas.head_system_id}"
48
49 timeout(time: 15) {
50 waitUntil {
51 try {
52 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready"
53 return true
54 } catch (exception) {
55 return false
56 }
57 }
58 }
59
60 sh 'maas maas machines allocate'
61 sh "maas maas machine deploy ${deployment_config.maas.head_system_id}"
62
63 timeout(time: 30) {
64 waitUntil {
65 try {
66 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed"
67 return true
68 } catch (exception) {
69 return false
70 }
71 }
72 }
73 }
74 }
75
76 stage ("Download CORD POD configuration") {
Kailash Khalasib8e174e2018-04-17 09:03:59 -070077 sh "cd ../orchestration/profiles; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation"
Kailash Khalasideeead02018-04-11 13:27:27 -070078 }
79
80 stage ("Generate CORD configuration") {
81 sh "make PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config"
82 }
83
84 if (deployment_config.fabric_switches != null) {
85 stage("Reserve IPs for fabric switches") {
86 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
87 def str = createMACIPbindingStr("fabric", i+1,
88 "${deployment_config.fabric_switches[i].mac}",
89 "${deployment_config.fabric_switches[i].ip}")
90 sh "echo $str >> maas/roles/maas/files/dhcpd.reservations"
91 }
92 }
93 }
94
95 stage ("Deploy") {
96 sh "make build"
97 }
98
99 if (deployment_config.compute_nodes != null) {
100 stage ("Power cycle compute nodes") {
101 for(int i=0; i < deployment_config.compute_nodes.size(); i++) {
102 sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle"
103 }
104 }
105
106 stage ("Wait for compute nodes to get deployed") {
107 sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}"
108 def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}")
109 runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
110 timeout(time: 90) {
111 waitUntil {
112 try {
113 num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim()
114 return num.toInteger() == deployment_config.compute_nodes.size()
115 } catch (exception) {
116 return false
117 }
118 }
119 }
120 }
121
122 stage ("Wait for compute nodes to be provisioned") {
123 timeout(time:45) {
124 waitUntil {
125 try {
126 num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim()
127 return num.toInteger() == deployment_config.compute_nodes.size()
128 } catch (exception) {
129 return false
130 }
131 }
132 }
133 }
134 }
135
136 if (deployment_config.fabric_switches != null) {
137 stage ("Wait for fabric switches to get deployed") {
138 runFabricSwitchCmdAll("sudo onl-onie-boot-mode install")
139 runFabricSwitchCmdAll("sudo reboot")
140 // Ensure that switches get provisioned after ONIE reinstall.
141 // Delete them if they were provisioned earlier. If the switches are not
142 // present in 'cord prov list', this command has no effect.
143 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
144 runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}")
145 }
146 timeout(time: 45) {
147 waitUntil {
148 try {
149 def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim()
150 return harvestCompleted.toInteger() == deployment_config.fabric_switches.size()
151 } catch (exception) {
152 return false
153 }
154 }
155 }
156 }
157
158 stage ("Wait for fabric switches to be provisioned") {
159 timeout(time:45) {
160 waitUntil {
161 try {
162 def provCompleted = 0
163 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
164 def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim()
165 provCompleted = provCompleted + count.toInteger()
166 }
167 return provCompleted == deployment_config.fabric_switches.size()
168 } catch (exception) {
169 return false
170 }
171 }
172 }
173 }
174
175 // Post installation configuration starts here
176 fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0]
177 xosUser = "xosadmin@opencord.org"
178 xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim()
179 stage ("Connect fabric switches and compute nodes to ONOS") {
180 // Configure breakout ports
181 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
182 if (deployment_config.fabric_switches[i].breakout_ports != null) {
183 for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) {
184 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
185 "${deployment_config.fabric_switches[i].user}",
186 "${deployment_config.fabric_switches[i].pass}",
187 "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf")
188 }
189 }
190 }
191 connectFabricAndComputeToOnos()
192 }
193 stage ("Configure the compute nodes") {
194 leafSwitchNum = 0
195 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
196 if(deployment_config.fabric_switches[i].containsKey("role")) {
197 if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) {
198 leafSwitchNum += 1
199 }
200 }
201 }
202 for(int i=1; i<=leafSwitchNum; i++) {
203 // Figure out which compute node connects to which switch
204 leafName = "leaf-" + i.toString()
205 computeNames = getComputeNames(leafName)
206 echo "Compute nodes connnected to " + leafName + " switch:"
207 for(name in computeNames) { echo "${name}" }
208 index = 1
209 for(name in computeNames) {
210 if(i>1) {
211 //Update fabric IP of compute nodes
212 index += 1
213 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim()
214 if (fabricIp != "") {
215 nodeId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/nodes | jq '.[\"items\"][] | select(.dataPlaneIp==\"${fabricIp}\") | .id'").trim()
216 newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24"
217 out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"dataPlaneIp\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/nodes/${nodeId}").trim()
218 // Wait until the new fabric IP gets configured
219 timeout(time: 5) {
220 waitUntil {
221 try {
222 num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim()
223 return num.toInteger() == 1
224 } catch (exception) {
225 return false
226 }
227 }
228 }
229 }
230 else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*"
231 }
232 //Add routes to fabric subnets
233 for(int j=1; j<=leafSwitchNum; j++) {
234 if(j!=i) {
235 runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists")
236 }
237 }
238 }
239 }
240 }
241 stage ("Generate and load network configuration") {
242 // Reconnect compute nodes to update the fabric IP in ONOS
243 connectFabricAndComputeToOnos()
244 // Refresh fabric configurations
245 sh "make fabric-refresh"
246 }
247 }
248 }
249
250 if (deployment_config.make_release == true) {
251 stage ("Trigger Build") {
252 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
253 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
254 }
255 }
256
257 currentBuild.result = 'SUCCESS'
258 } catch (err) {
259 currentBuild.result = 'FAILURE'
260 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
261 }
262 echo "RESULT: ${currentBuild.result}"
263 }
264}
265
266/**
267 * Returns a string used to bind IPs and MAC addresses, substituting the values
268 * given.
269 *
270 * @param word the word used to generate the host name
271 * @param counter the counter used to generate the host name
272 * @param mac the MAC address to substitute
273 * @param ip the IP address to substitute
274 */
275def createMACIPbindingStr(word, counter, mac, ip) {
276 return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}"""
277}
278
279/**
280 * Runs a command on a remote host using sshpass.
281 *
282 * @param ip the node IP address
283 * @param user the node user name
284 * @param pass the node password
285 * @param command the command to run
286 * @param sshArgs arguments for the ssh command
287 * @return the output of the command
288 */
289def runCmd(ip, user, pass, command, sshArgs="") {
290 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"")
291}
292
293/**
294 * Runs a command on the head node.
295 *
296 * @param command the command to run
297 * @param sshArgs arguments for the ssh command
298 * @return the output of the command
299 */
300def runHeadNodeCmd(command, sshArgs="") {
301 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"")
302}
303
304/**
305 * Runs a command on a fabric switch.
306 *
307 * @param ip the mgmt IP of the fabric switch, reachable from the head node
308 * @param user the mgmt user name of the fabric switch
309 * @param pass the mgmt password of the fabric switch
310 * @param command the command to run on the fabric switch
311 * @param ssgArgs arguments for the ssh command
312 * @return the output of the command
313 */
314def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") {
315 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
316}
317
318/**
319 * Runs a command on all fabric switches
320 *
321 * @param command the command to run on the fabric switches
322 * @param ssgArgs arguments for the ssh command
323 */
324def runFabricSwitchCmdAll(command, sshArgs="") {
325 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
326 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
327 "${deployment_config.fabric_switches[i].user}",
328 "${deployment_config.fabric_switches[i].pass}",
329 "${command}",
330 "${sshArgs}")
331 }
332}
333
334/**
335 * Runs a command on a compute node.
336 *
337 * @param name the name of the compute node
338 * @param command the command to run on the compute node
339 * @param ssgArgs arguments for the ssh command
340 * @return the output of the command
341 */
342def runComputeNodeCmd(name, command, sshArgs="") {
343 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"")
344}
345
346/**
347 * Runs a command on all compute nodes
348 *
349 * @param command the command to run on the compute nodes
350 * @param ssgArgs arguments for the ssh command
351 */
352def runComputeNodeCmdAll(command, sshArgs="") {
353 computeNamesAll = getComputeNames()
354 for (name in computeNamesAll) {
355 runComputeNodeCmd("${name}", "${command}", "${sshArgs}")
356 }
357}
358
359/**
360 * Runs an ONOS CLI command
361 *
362 * @param name the onos node name, reachable from the head node
363 * @param port the port used to login to ONOS CLI
364 * @param user the user name to login to ONOS CLI
365 * @param pass the password to login to ONOS CLI
366 * @param command the command to run in ONOS CLI
367 * @return the output of the command
368 */
369def runOnosCliCmd(name, port, user, pass, command) {
370 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"")
371}
372
373/**
374 * Returns a list of compute node names. When "role" is specified, returns only
375 * names of compute nodes connected to the switch
376 *
377 * @param role the switch role, i.e. "leaf-1"
378 */
379def getComputeNames(role="") {
380 computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim()
381 computeNamesAll = "${computeNamesAll}".split()
382 computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"")
Kailash Khalasi85d82142018-04-18 11:02:04 -0700383 computeNamesAll = new JsonSlurper().parseText("${computeNamesAll}")
Kailash Khalasideeead02018-04-11 13:27:27 -0700384 if ("${role}" == "") return computeNamesAll
385 computeNames = []
386 switchMac = ""
387 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
388 if(deployment_config.fabric_switches[i].containsKey("role")) {
389 if ("${deployment_config.fabric_switches[i].role}" == "${role}")
390 switchMac = "${deployment_config.fabric_switches[i].mac}"
391 }
392 }
393 if ("${switchMac}" != "") {
394 switchMac = switchMac.toLowerCase().replaceAll(':','')
395 // Get fabric IPs of compute nodes connected to the switch
396 try {
397 computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.locations[].elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'",
398 "-q").trim()
399 }catch (exception) {
400 return computeNames
401 }
402 computeFabricIps = "${computeFabricIps}".split()
Kailash Khalasi85d82142018-04-18 11:02:04 -0700403 computeFabricIps = new JsonSlurper().parseText("${computeFabricIps}")
Kailash Khalasideeead02018-04-11 13:27:27 -0700404 // Figure out which compute node connects to the switch
405 for (name in computeNamesAll) {
406 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim()
407 if (fabricIp in computeFabricIps) {
408 computeNames.add("${name}")
409 }
410 }
411 }
412 return computeNames
413}
414
415/**
416 * Connects all fabric switches and compute nodes to ONOS
417 */
418def connectFabricAndComputeToOnos() {
419 // Kill existing switch connections
420 runFabricSwitchCmdAll("./killit || echo no ofagentapp running")
421 // Clean stale ONOS data
422 runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please")
423 // Connect switches to ONOS
424 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
425 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
426 "${deployment_config.fabric_switches[i].user}",
427 "${deployment_config.fabric_switches[i].pass}",
428 "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log",
429 "-qftn")
430 }
431 // Verify ONOS has recognized the switches
432 timeout(time: 5) {
433 waitUntil {
434 try {
435 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim()
436 return num.toInteger() == deployment_config.fabric_switches.size()
437 } catch (exception) {
438 return false
439 }
440 }
441 }
442 // Connect compute nodes to ONOS
443 runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn")
444 // Verify ONOS has recognized the hosts
445 timeout(time: 5) {
446 waitUntil {
447 try {
448 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim()
449 return num.toInteger() >= deployment_config.compute_nodes.size()
450 } catch (exception) {
451 return false
452 }
453 }
454 }
455}