Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Kailash Khalasi | 85d8214 | 2018-04-18 11:02:04 -0700 | [diff] [blame] | 15 | import groovy.json.JsonSlurper |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 16 | |
Kailash Khalasi | b8e174e | 2018-04-17 09:03:59 -0700 | [diff] [blame] | 17 | def filename = 'manifest-${branch}.xml' |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 18 | def manifestUrl = 'https://gerrit.opencord.org/manifest' |
| 19 | deployment_config = null; |
| 20 | pod_config = null; |
| 21 | |
| 22 | node ("${devNodeName}") { |
| 23 | timeout (time: 240) { |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 24 | stage ("Parse deployment configuration file") { |
| 25 | sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}' |
Kailash Khalasi | b8e174e | 2018-04-17 09:03:59 -0700 | [diff] [blame] | 26 | sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}' |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 27 | deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}" |
| 28 | pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}" |
| 29 | } |
Kailash Khalasi | 9f2ba4e | 2018-04-19 14:17:17 -0700 | [diff] [blame] | 30 | stage ("Clean-up previous build") { |
| 31 | sh "make -C build PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config" |
| 32 | sh "make -C build clean-all || true" |
| 33 | sh "rm -rf *" |
| 34 | } |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 35 | stage ('Remove old head node from known hosts') { |
| 36 | sh "ssh-keygen -R ${deployment_config.head.ip}" |
| 37 | } |
| 38 | stage ('Checkout cord repo') { |
| 39 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| 40 | } |
| 41 | |
| 42 | try { |
| 43 | dir('build') { |
| 44 | stage ("Re-deploy head node") { |
| 45 | maasOps: { |
| 46 | sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}" |
| 47 | sh "maas maas machine release ${deployment_config.maas.head_system_id}" |
| 48 | |
| 49 | timeout(time: 15) { |
| 50 | waitUntil { |
| 51 | try { |
| 52 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready" |
| 53 | return true |
| 54 | } catch (exception) { |
| 55 | return false |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | sh 'maas maas machines allocate' |
| 61 | sh "maas maas machine deploy ${deployment_config.maas.head_system_id}" |
| 62 | |
| 63 | timeout(time: 30) { |
| 64 | waitUntil { |
| 65 | try { |
| 66 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed" |
| 67 | return true |
| 68 | } catch (exception) { |
| 69 | return false |
| 70 | } |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | stage ("Download CORD POD configuration") { |
Kailash Khalasi | b8e174e | 2018-04-17 09:03:59 -0700 | [diff] [blame] | 77 | sh "cd ../orchestration/profiles; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation" |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 78 | } |
| 79 | |
| 80 | stage ("Generate CORD configuration") { |
| 81 | sh "make PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config" |
| 82 | } |
| 83 | |
| 84 | if (deployment_config.fabric_switches != null) { |
| 85 | stage("Reserve IPs for fabric switches") { |
| 86 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 87 | def str = createMACIPbindingStr("fabric", i+1, |
| 88 | "${deployment_config.fabric_switches[i].mac}", |
| 89 | "${deployment_config.fabric_switches[i].ip}") |
| 90 | sh "echo $str >> maas/roles/maas/files/dhcpd.reservations" |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | stage ("Deploy") { |
| 96 | sh "make build" |
| 97 | } |
| 98 | |
| 99 | if (deployment_config.compute_nodes != null) { |
| 100 | stage ("Power cycle compute nodes") { |
| 101 | for(int i=0; i < deployment_config.compute_nodes.size(); i++) { |
| 102 | sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle" |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | stage ("Wait for compute nodes to get deployed") { |
| 107 | sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}" |
| 108 | def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}") |
| 109 | runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}") |
| 110 | timeout(time: 90) { |
| 111 | waitUntil { |
| 112 | try { |
| 113 | num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim() |
| 114 | return num.toInteger() == deployment_config.compute_nodes.size() |
| 115 | } catch (exception) { |
| 116 | return false |
| 117 | } |
| 118 | } |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | stage ("Wait for compute nodes to be provisioned") { |
| 123 | timeout(time:45) { |
| 124 | waitUntil { |
| 125 | try { |
| 126 | num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim() |
| 127 | return num.toInteger() == deployment_config.compute_nodes.size() |
| 128 | } catch (exception) { |
| 129 | return false |
| 130 | } |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | if (deployment_config.fabric_switches != null) { |
| 137 | stage ("Wait for fabric switches to get deployed") { |
| 138 | runFabricSwitchCmdAll("sudo onl-onie-boot-mode install") |
| 139 | runFabricSwitchCmdAll("sudo reboot") |
| 140 | // Ensure that switches get provisioned after ONIE reinstall. |
| 141 | // Delete them if they were provisioned earlier. If the switches are not |
| 142 | // present in 'cord prov list', this command has no effect. |
| 143 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 144 | runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}") |
| 145 | } |
| 146 | timeout(time: 45) { |
| 147 | waitUntil { |
| 148 | try { |
| 149 | def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim() |
| 150 | return harvestCompleted.toInteger() == deployment_config.fabric_switches.size() |
| 151 | } catch (exception) { |
| 152 | return false |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | stage ("Wait for fabric switches to be provisioned") { |
| 159 | timeout(time:45) { |
| 160 | waitUntil { |
| 161 | try { |
| 162 | def provCompleted = 0 |
| 163 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 164 | def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim() |
| 165 | provCompleted = provCompleted + count.toInteger() |
| 166 | } |
| 167 | return provCompleted == deployment_config.fabric_switches.size() |
| 168 | } catch (exception) { |
| 169 | return false |
| 170 | } |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // Post installation configuration starts here |
| 176 | fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0] |
| 177 | xosUser = "xosadmin@opencord.org" |
| 178 | xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim() |
| 179 | stage ("Connect fabric switches and compute nodes to ONOS") { |
| 180 | // Configure breakout ports |
| 181 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 182 | if (deployment_config.fabric_switches[i].breakout_ports != null) { |
| 183 | for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) { |
| 184 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 185 | "${deployment_config.fabric_switches[i].user}", |
| 186 | "${deployment_config.fabric_switches[i].pass}", |
| 187 | "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf") |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | connectFabricAndComputeToOnos() |
| 192 | } |
| 193 | stage ("Configure the compute nodes") { |
| 194 | leafSwitchNum = 0 |
| 195 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 196 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
| 197 | if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) { |
| 198 | leafSwitchNum += 1 |
| 199 | } |
| 200 | } |
| 201 | } |
| 202 | for(int i=1; i<=leafSwitchNum; i++) { |
| 203 | // Figure out which compute node connects to which switch |
| 204 | leafName = "leaf-" + i.toString() |
| 205 | computeNames = getComputeNames(leafName) |
| 206 | echo "Compute nodes connnected to " + leafName + " switch:" |
| 207 | for(name in computeNames) { echo "${name}" } |
| 208 | index = 1 |
| 209 | for(name in computeNames) { |
| 210 | if(i>1) { |
| 211 | //Update fabric IP of compute nodes |
| 212 | index += 1 |
| 213 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim() |
| 214 | if (fabricIp != "") { |
| 215 | nodeId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/nodes | jq '.[\"items\"][] | select(.dataPlaneIp==\"${fabricIp}\") | .id'").trim() |
| 216 | newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24" |
| 217 | out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"dataPlaneIp\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/nodes/${nodeId}").trim() |
| 218 | // Wait until the new fabric IP gets configured |
| 219 | timeout(time: 5) { |
| 220 | waitUntil { |
| 221 | try { |
| 222 | num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim() |
| 223 | return num.toInteger() == 1 |
| 224 | } catch (exception) { |
| 225 | return false |
| 226 | } |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*" |
| 231 | } |
| 232 | //Add routes to fabric subnets |
| 233 | for(int j=1; j<=leafSwitchNum; j++) { |
| 234 | if(j!=i) { |
| 235 | runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists") |
| 236 | } |
| 237 | } |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | stage ("Generate and load network configuration") { |
| 242 | // Reconnect compute nodes to update the fabric IP in ONOS |
| 243 | connectFabricAndComputeToOnos() |
| 244 | // Refresh fabric configurations |
| 245 | sh "make fabric-refresh" |
| 246 | } |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | if (deployment_config.make_release == true) { |
| 251 | stage ("Trigger Build") { |
| 252 | url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build' |
| 253 | httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201' |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | currentBuild.result = 'SUCCESS' |
| 258 | } catch (err) { |
| 259 | currentBuild.result = 'FAILURE' |
| 260 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
| 261 | } |
| 262 | echo "RESULT: ${currentBuild.result}" |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | /** |
| 267 | * Returns a string used to bind IPs and MAC addresses, substituting the values |
| 268 | * given. |
| 269 | * |
| 270 | * @param word the word used to generate the host name |
| 271 | * @param counter the counter used to generate the host name |
| 272 | * @param mac the MAC address to substitute |
| 273 | * @param ip the IP address to substitute |
| 274 | */ |
| 275 | def createMACIPbindingStr(word, counter, mac, ip) { |
| 276 | return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}""" |
| 277 | } |
| 278 | |
| 279 | /** |
| 280 | * Runs a command on a remote host using sshpass. |
| 281 | * |
| 282 | * @param ip the node IP address |
| 283 | * @param user the node user name |
| 284 | * @param pass the node password |
| 285 | * @param command the command to run |
| 286 | * @param sshArgs arguments for the ssh command |
| 287 | * @return the output of the command |
| 288 | */ |
| 289 | def runCmd(ip, user, pass, command, sshArgs="") { |
| 290 | return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"") |
| 291 | } |
| 292 | |
| 293 | /** |
| 294 | * Runs a command on the head node. |
| 295 | * |
| 296 | * @param command the command to run |
| 297 | * @param sshArgs arguments for the ssh command |
| 298 | * @return the output of the command |
| 299 | */ |
| 300 | def runHeadNodeCmd(command, sshArgs="") { |
| 301 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"") |
| 302 | } |
| 303 | |
| 304 | /** |
| 305 | * Runs a command on a fabric switch. |
| 306 | * |
| 307 | * @param ip the mgmt IP of the fabric switch, reachable from the head node |
| 308 | * @param user the mgmt user name of the fabric switch |
| 309 | * @param pass the mgmt password of the fabric switch |
| 310 | * @param command the command to run on the fabric switch |
| 311 | * @param ssgArgs arguments for the ssh command |
| 312 | * @return the output of the command |
| 313 | */ |
| 314 | def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") { |
| 315 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"") |
| 316 | } |
| 317 | |
| 318 | /** |
| 319 | * Runs a command on all fabric switches |
| 320 | * |
| 321 | * @param command the command to run on the fabric switches |
| 322 | * @param ssgArgs arguments for the ssh command |
| 323 | */ |
| 324 | def runFabricSwitchCmdAll(command, sshArgs="") { |
| 325 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 326 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 327 | "${deployment_config.fabric_switches[i].user}", |
| 328 | "${deployment_config.fabric_switches[i].pass}", |
| 329 | "${command}", |
| 330 | "${sshArgs}") |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | /** |
| 335 | * Runs a command on a compute node. |
| 336 | * |
| 337 | * @param name the name of the compute node |
| 338 | * @param command the command to run on the compute node |
| 339 | * @param ssgArgs arguments for the ssh command |
| 340 | * @return the output of the command |
| 341 | */ |
| 342 | def runComputeNodeCmd(name, command, sshArgs="") { |
| 343 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"") |
| 344 | } |
| 345 | |
| 346 | /** |
| 347 | * Runs a command on all compute nodes |
| 348 | * |
| 349 | * @param command the command to run on the compute nodes |
| 350 | * @param ssgArgs arguments for the ssh command |
| 351 | */ |
| 352 | def runComputeNodeCmdAll(command, sshArgs="") { |
| 353 | computeNamesAll = getComputeNames() |
| 354 | for (name in computeNamesAll) { |
| 355 | runComputeNodeCmd("${name}", "${command}", "${sshArgs}") |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | /** |
| 360 | * Runs an ONOS CLI command |
| 361 | * |
| 362 | * @param name the onos node name, reachable from the head node |
| 363 | * @param port the port used to login to ONOS CLI |
| 364 | * @param user the user name to login to ONOS CLI |
| 365 | * @param pass the password to login to ONOS CLI |
| 366 | * @param command the command to run in ONOS CLI |
| 367 | * @return the output of the command |
| 368 | */ |
| 369 | def runOnosCliCmd(name, port, user, pass, command) { |
| 370 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"") |
| 371 | } |
| 372 | |
| 373 | /** |
| 374 | * Returns a list of compute node names. When "role" is specified, returns only |
| 375 | * names of compute nodes connected to the switch |
| 376 | * |
| 377 | * @param role the switch role, i.e. "leaf-1" |
| 378 | */ |
| 379 | def getComputeNames(role="") { |
| 380 | computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim() |
| 381 | computeNamesAll = "${computeNamesAll}".split() |
| 382 | computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"") |
Kailash Khalasi | 85d8214 | 2018-04-18 11:02:04 -0700 | [diff] [blame] | 383 | computeNamesAll = new JsonSlurper().parseText("${computeNamesAll}") |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 384 | if ("${role}" == "") return computeNamesAll |
| 385 | computeNames = [] |
| 386 | switchMac = "" |
| 387 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 388 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
| 389 | if ("${deployment_config.fabric_switches[i].role}" == "${role}") |
| 390 | switchMac = "${deployment_config.fabric_switches[i].mac}" |
| 391 | } |
| 392 | } |
| 393 | if ("${switchMac}" != "") { |
| 394 | switchMac = switchMac.toLowerCase().replaceAll(':','') |
| 395 | // Get fabric IPs of compute nodes connected to the switch |
| 396 | try { |
| 397 | computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.locations[].elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'", |
| 398 | "-q").trim() |
| 399 | }catch (exception) { |
| 400 | return computeNames |
| 401 | } |
| 402 | computeFabricIps = "${computeFabricIps}".split() |
Kailash Khalasi | 85d8214 | 2018-04-18 11:02:04 -0700 | [diff] [blame] | 403 | computeFabricIps = new JsonSlurper().parseText("${computeFabricIps}") |
Kailash Khalasi | deeead0 | 2018-04-11 13:27:27 -0700 | [diff] [blame] | 404 | // Figure out which compute node connects to the switch |
| 405 | for (name in computeNamesAll) { |
| 406 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim() |
| 407 | if (fabricIp in computeFabricIps) { |
| 408 | computeNames.add("${name}") |
| 409 | } |
| 410 | } |
| 411 | } |
| 412 | return computeNames |
| 413 | } |
| 414 | |
| 415 | /** |
| 416 | * Connects all fabric switches and compute nodes to ONOS |
| 417 | */ |
| 418 | def connectFabricAndComputeToOnos() { |
| 419 | // Kill existing switch connections |
| 420 | runFabricSwitchCmdAll("./killit || echo no ofagentapp running") |
| 421 | // Clean stale ONOS data |
| 422 | runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please") |
| 423 | // Connect switches to ONOS |
| 424 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 425 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 426 | "${deployment_config.fabric_switches[i].user}", |
| 427 | "${deployment_config.fabric_switches[i].pass}", |
| 428 | "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log", |
| 429 | "-qftn") |
| 430 | } |
| 431 | // Verify ONOS has recognized the switches |
| 432 | timeout(time: 5) { |
| 433 | waitUntil { |
| 434 | try { |
| 435 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim() |
| 436 | return num.toInteger() == deployment_config.fabric_switches.size() |
| 437 | } catch (exception) { |
| 438 | return false |
| 439 | } |
| 440 | } |
| 441 | } |
| 442 | // Connect compute nodes to ONOS |
| 443 | runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn") |
| 444 | // Verify ONOS has recognized the hosts |
| 445 | timeout(time: 5) { |
| 446 | waitUntil { |
| 447 | try { |
| 448 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim() |
| 449 | return num.toInteger() >= deployment_config.compute_nodes.size() |
| 450 | } catch (exception) { |
| 451 | return false |
| 452 | } |
| 453 | } |
| 454 | } |
| 455 | } |