Gitiles
Code Review Sign In
gerrit.opencord.org / cord / f5d6422f07fba07246f59febfb1b2b4fc70e4150 / . / Jenkinsfile.MCORDBuildSystem
blob: 26b57126921245ec3830250253d03f13740d1b5b [file] [log] [blame]
Suchitra Vemurif5d64222018-05-11 16:09:10 -0700[diff] [blame^]1// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15import groovy.json.JsonSlurperClassic
16
17def filename = 'manifest-${branch}.xml'
18def manifestUrl = 'https://gerrit.opencord.org/manifest'
19deployment_config = null;
20pod_config = null;
21
22node ('master') {
23 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
24
25 stage ("Generate and Copy Manifest file") {
26 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
27 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
28 }
29
30 stage ("Parse deployment configuration file") {
31 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
32 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
33 deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
34 pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}"
35 }
36}
37
38node ("${deployment_config.dev_node.name}") {
39 timeout (time: 360) {
40 stage ('Remove old head node from known hosts') {
41 sh "ssh-keygen -R ${deployment_config.head.ip}"
42 }
43
44 stage ('Checkout cord repo') {
45 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
46 }
47
48 stage ('Copy NG40 license file') {
49 sh "cp ../ng40-license orchestration/xos_services/venb/xos/synchronizer/files/ng40-license"
50 }
51
52 try {
53 dir('build') {
54 stage ("Re-deploy head node") {
55 maasOps: {
56 sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}"
57 sh "maas maas machine release ${deployment_config.maas.head_system_id}"
58
59 timeout(time: 15) {
60 waitUntil {
61 try {
62 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready"
63 return true
64 } catch (exception) {
65 return false
66 }
67 }
68 }
69
70 sh 'maas maas machines allocate'
71 sh "maas maas machine deploy ${deployment_config.maas.head_system_id}"
72
73 timeout(time: 30) {
74 waitUntil {
75 try {
76 sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed"
77 return true
78 } catch (exception) {
79 return false
80 }
81 }
82 }
83 }
84 }
85
86 stage ("Download CORD POD configuration") {
87 sh "cd ../orchestration/profiles; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation"
88 }
89
90 stage ("Generate CORD configuration") {
91 sh "make PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config"
92 }
93
94 if (deployment_config.fabric_switches != null) {
95 stage("Reserve IPs for fabric switches") {
96 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
97 def str = createMACIPbindingStr("fabric", i+1,
98 "${deployment_config.fabric_switches[i].mac}",
99 "${deployment_config.fabric_switches[i].ip}")
100 sh "echo $str >> maas/roles/maas/files/dhcpd.reservations"
101 }
102 }
103 }
104
105 stage ("Deploy") {
106 sh 'sed -i "s/^\\(dev_vm_cpu: \\).*/\\120/" genconfig/config.yml'
107 sh 'sed -i "s/^\\(dev_vm_mem: \\).*/\\140960/" genconfig/config.yml'
108 sh "make build"
109 }
110
111 if (deployment_config.compute_nodes != null) {
112 stage ("Power cycle compute nodes") {
113 for(int i=0; i < deployment_config.compute_nodes.size(); i++) {
114 sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle"
115 }
116 }
117
118 stage ("Wait for compute nodes to get deployed") {
119 sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}"
120 def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}")
121 runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
122 timeout(time: 90) {
123 waitUntil {
124 try {
125 num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim()
126 return num.toInteger() == deployment_config.compute_nodes.size()
127 } catch (exception) {
128 return false
129 }
130 }
131 }
132 }
133
134 stage ("Wait for compute nodes to be provisioned") {
135 timeout(time:90) {
136 waitUntil {
137 try {
138 num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim()
139 return num.toInteger() == deployment_config.compute_nodes.size()
140 } catch (exception) {
141 return false
142 }
143 }
144 }
145 }
146 }
147
148 if (deployment_config.fabric_switches != null) {
149 stage ("Wait for fabric switches to get deployed") {
150 runFabricSwitchCmdAll("sudo onl-onie-boot-mode install")
151 runFabricSwitchCmdAll("sudo reboot")
152 // Ensure that switches get provisioned after ONIE reinstall.
153 // Delete them if they were provisioned earlier. If the switches are not
154 // present in 'cord prov list', this command has no effect.
155 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
156 runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}")
157 }
158 timeout(time: 45) {
159 waitUntil {
160 try {
161 def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim()
162 return harvestCompleted.toInteger() == deployment_config.fabric_switches.size()
163 } catch (exception) {
164 return false
165 }
166 }
167 }
168 }
169
170 stage ("Wait for fabric switches to be provisioned") {
171 timeout(time:45) {
172 waitUntil {
173 try {
174 def provCompleted = 0
175 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
176 def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim()
177 provCompleted = provCompleted + count.toInteger()
178 }
179 return provCompleted == deployment_config.fabric_switches.size()
180 } catch (exception) {
181 return false
182 }
183 }
184 }
185 }
186
187 // Post installation configuration starts here
188 fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0]
189 xosUser = "xosadmin@opencord.org"
190 xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim()
191 stage ("Connect fabric switches and compute nodes to ONOS") {
192 // Configure breakout ports
193 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
194 if (deployment_config.fabric_switches[i].breakout_ports != null) {
195 for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) {
196 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
197 "${deployment_config.fabric_switches[i].user}",
198 "${deployment_config.fabric_switches[i].pass}",
199 "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf")
200 }
201 }
202 }
203 connectFabricAndComputeToOnos()
204 }
205 stage ("Configure the compute nodes") {
206 leafSwitchNum = 0
207 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
208 if(deployment_config.fabric_switches[i].containsKey("role")) {
209 if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) {
210 leafSwitchNum += 1
211 }
212 }
213 }
214 for(int i=1; i<=leafSwitchNum; i++) {
215 // Figure out which compute node connects to which switch
216 leafName = "leaf-" + i.toString()
217 computeNames = getComputeNames(leafName)
218 echo "Compute nodes connnected to " + leafName + " switch:"
219 for(name in computeNames) { echo "${name}" }
220 index = 1
221 for(name in computeNames) {
222 if(i>1) {
223 //Update fabric IP of compute nodes
224 index += 1
225 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim()
226 if (fabricIp != "") {
227 nodeId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/nodes | jq '.[\"items\"][] | select(.dataPlaneIp==\"${fabricIp}\") | .id'").trim()
228 newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24"
229 out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"dataPlaneIp\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/nodes/${nodeId}").trim()
230 // Wait until the new fabric IP gets configured
231 timeout(time: 5) {
232 waitUntil {
233 try {
234 num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim()
235 return num.toInteger() == 1
236 } catch (exception) {
237 return false
238 }
239 }
240 }
241 }
242 else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*"
243 }
244 //Add routes to fabric subnets
245 for(int j=1; j<=leafSwitchNum; j++) {
246 if(j!=i) {
247 runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists")
248 }
249 }
250 }
251 }
252 }
253 stage ("Generate and load network configuration") {
254 // Reconnect compute nodes to update the fabric IP in ONOS
255 connectFabricAndComputeToOnos()
256 // Generate network configuration
257 runHeadNodeCmd("""
258 cd /opt/cord_profile/
259 cp fabric-network-cfg.json fabric-network-cfg.json.\$(date +%Y%m%d-%H%M%S)
260 cord generate > fabric-network-cfg.json
261 """)
262 // Install httpie on the head-node
263 runHeadNodeCmd("sudo pip install httpie")
264 // Delete old ONOS netcfg
265 runHeadNodeCmd("http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/network/configuration/")
266 // Load new configuration
267 runHeadNodeCmd("http -a onos:rocks POST http://onos-fabric:8181/onos/v1/network/configuration/ < /opt/cord_profile/fabric-network-cfg.json")
268 // Restart ONOS apps
269 runHeadNodeCmd("""
270 http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5
271 http -a onos:rocks POST http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5
272 """)
273 }
274 }
275 }
276
277 if (deployment_config.make_release == true) {
278 stage ("Trigger Build") {
279 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
280 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
281 }
282 }
283
284 currentBuild.result = 'SUCCESS'
285 } catch (err) {
286 currentBuild.result = 'FAILURE'
287 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
288 } finally {
289 //sh "make -C build clean-all || true"
290 //sh "rm -rf *"
291 }
292 echo "RESULT: ${currentBuild.result}"
293 }
294}
295
296/**
297 * Returns a string used to bind IPs and MAC addresses, substituting the values
298 * given.
299 *
300 * @param word the word used to generate the host name
301 * @param counter the counter used to generate the host name
302 * @param mac the MAC address to substitute
303 * @param ip the IP address to substitute
304 */
305def createMACIPbindingStr(word, counter, mac, ip) {
306 return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}"""
307}
308
309/**
310 * Runs a command on a remote host using sshpass.
311 *
312 * @param ip the node IP address
313 * @param user the node user name
314 * @param pass the node password
315 * @param command the command to run
316 * @param sshArgs arguments for the ssh command
317 * @return the output of the command
318 */
319def runCmd(ip, user, pass, command, sshArgs="") {
320 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"")
321}
322
323/**
324 * Runs a command on the head node.
325 *
326 * @param command the command to run
327 * @param sshArgs arguments for the ssh command
328 * @return the output of the command
329 */
330def runHeadNodeCmd(command, sshArgs="") {
331 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"")
332}
333
334/**
335 * Runs a command on a fabric switch.
336 *
337 * @param ip the mgmt IP of the fabric switch, reachable from the head node
338 * @param user the mgmt user name of the fabric switch
339 * @param pass the mgmt password of the fabric switch
340 * @param command the command to run on the fabric switch
341 * @param ssgArgs arguments for the ssh command
342 * @return the output of the command
343 */
344def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") {
345 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
346}
347
348/**
349 * Runs a command on all fabric switches
350 *
351 * @param command the command to run on the fabric switches
352 * @param ssgArgs arguments for the ssh command
353 */
354def runFabricSwitchCmdAll(command, sshArgs="") {
355 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
356 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
357 "${deployment_config.fabric_switches[i].user}",
358 "${deployment_config.fabric_switches[i].pass}",
359 "${command}",
360 "${sshArgs}")
361 }
362}
363
364/**
365 * Runs a command on a compute node.
366 *
367 * @param name the name of the compute node
368 * @param command the command to run on the compute node
369 * @param ssgArgs arguments for the ssh command
370 * @return the output of the command
371 */
372def runComputeNodeCmd(name, command, sshArgs="") {
373 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"")
374}
375
376/**
377 * Runs a command on all compute nodes
378 *
379 * @param command the command to run on the compute nodes
380 * @param ssgArgs arguments for the ssh command
381 */
382def runComputeNodeCmdAll(command, sshArgs="") {
383 computeNamesAll = getComputeNames()
384 for (name in computeNamesAll) {
385 runComputeNodeCmd("${name}", "${command}", "${sshArgs}")
386 }
387}
388
389/**
390 * Runs an ONOS CLI command
391 *
392 * @param name the onos node name, reachable from the head node
393 * @param port the port used to login to ONOS CLI
394 * @param user the user name to login to ONOS CLI
395 * @param pass the password to login to ONOS CLI
396 * @param command the command to run in ONOS CLI
397 * @return the output of the command
398 */
399def runOnosCliCmd(name, port, user, pass, command) {
400 return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"")
401}
402
403/**
404 * Returns a list of compute node names. When "role" is specified, returns only
405 * names of compute nodes connected to the switch
406 *
407 * @param role the switch role, i.e. "leaf-1"
408 */
409def getComputeNames(role="") {
410 computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim()
411 computeNamesAll = "${computeNamesAll}".split()
412 computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"")
413 computeNamesAll = new JsonSlurperClassic().parseText("${computeNamesAll}")
414 if ("${role}" == "") return computeNamesAll
415 computeNames = []
416 switchMac = ""
417 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
418 if(deployment_config.fabric_switches[i].containsKey("role")) {
419 if ("${deployment_config.fabric_switches[i].role}" == "${role}")
420 switchMac = "${deployment_config.fabric_switches[i].mac}"
421 }
422 }
423 if ("${switchMac}" != "") {
424 switchMac = switchMac.toLowerCase().replaceAll(':','')
425 // Get fabric IPs of compute nodes connected to the switch
426 try {
427 computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.location.elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'",
428 "-q").trim()
429 }catch (exception) {
430 return computeNames
431 }
432 computeFabricIps = "${computeFabricIps}".split()
433 computeFabricIps = new JsonSlurperClassic().parseText("${computeFabricIps}")
434 // Figure out which compute node connects to the switch
435 for (name in computeNamesAll) {
436 fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim()
437 if (fabricIp in computeFabricIps) {
438 computeNames.add("${name}")
439 }
440 }
441 }
442 return computeNames
443}
444
445/**
446 * Connects all fabric switches and compute nodes to ONOS
447 */
448def connectFabricAndComputeToOnos() {
449 // Kill existing switch connections
450 runFabricSwitchCmdAll("./killit || echo no ofagentapp running")
451 // Clean stale ONOS data
452 runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please")
453 // Connect switches to ONOS
454 for(int i=0; i < deployment_config.fabric_switches.size(); i++) {
455 runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}",
456 "${deployment_config.fabric_switches[i].user}",
457 "${deployment_config.fabric_switches[i].pass}",
458 "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log",
459 "-qftn")
460 }
461 // Verify ONOS has recognized the switches
462 timeout(time: 5) {
463 waitUntil {
464 try {
465 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim()
466 return num.toInteger() == deployment_config.fabric_switches.size()
467 } catch (exception) {
468 return false
469 }
470 }
471 }
472 // Connect compute nodes to ONOS
473 runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn")
474 // Verify ONOS has recognized the hosts
475 timeout(time: 5) {
476 waitUntil {
477 try {
478 num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim()
479 return num.toInteger() >= deployment_config.compute_nodes.size()
480 } catch (exception) {
481 return false
482 }
483 }
484 }
485}