You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Kailash Khalasi | fd36028 | 2018-04-20 09:09:54 -0700 | [diff] [blame] | 15 | import groovy.json.JsonSlurper |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 16 | |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 17 | def filename = 'manifest-${branch}.xml' |
| 18 | def manifestUrl = 'https://gerrit.opencord.org/manifest' |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 19 | deployment_config = null; |
| 20 | pod_config = null; |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 21 | |
Kailash Khalasi | fd36028 | 2018-04-20 09:09:54 -0700 | [diff] [blame] | 22 | node ('${devNodeName}') { |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 23 | timeout (time: 240) { |
Kailash Khalasi | fd36028 | 2018-04-20 09:09:54 -0700 | [diff] [blame] | 24 | stage ("Parse deployment configuration file") { |
| 25 | sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}' |
| 26 | sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}' |
| 27 | deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}" |
| 28 | pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}" |
| 29 | } |
| 30 | stage ("Clean-up previous build") { |
| 31 | sh "make -C build PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config" |
| 32 | sh "make -C build clean-all || true" |
| 33 | sh "rm -rf *" |
| 34 | } |
Luca Prete | 7b4a1e8 | 2017-09-12 10:44:48 -0700 | [diff] [blame] | 35 | stage ('Remove old head node from known hosts') { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 36 | sh "ssh-keygen -R ${deployment_config.head.ip}" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 37 | } |
| 38 | stage ('Checkout cord repo') { |
| 39 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| 40 | } |
| 41 | |
Luca Prete | 9081fcc | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 42 | try { |
| 43 | dir('build') { |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 44 | stage ("Re-deploy head node") { |
| 45 | maasOps: { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 46 | sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}" |
| 47 | sh "maas maas machine release ${deployment_config.maas.head_system_id}" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 48 | |
| 49 | timeout(time: 15) { |
| 50 | waitUntil { |
| 51 | try { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 52 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 53 | return true |
| 54 | } catch (exception) { |
| 55 | return false |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | sh 'maas maas machines allocate' |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 61 | sh "maas maas machine deploy ${deployment_config.maas.head_system_id}" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 62 | |
| 63 | timeout(time: 30) { |
| 64 | waitUntil { |
| 65 | try { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 66 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 67 | return true |
| 68 | } catch (exception) { |
| 69 | return false |
| 70 | } |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | stage ("Download CORD POD configuration") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 77 | sh "cd podconfig; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 78 | } |
| 79 | |
| 80 | stage ("Generate CORD configuration") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 81 | sh "make PODCONFIG=automation/${deployment_config.pod_config.file_name} config" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 82 | } |
| 83 | |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 84 | if (deployment_config.fabric_switches != null) { |
Andy Bavier | 6c2f804 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 85 | stage("Reserve IPs for fabric switches") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 86 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 87 | def str = createMACIPbindingStr("fabric", i+1, |
| 88 | "${deployment_config.fabric_switches[i].mac}", |
| 89 | "${deployment_config.fabric_switches[i].ip}") |
Andy Bavier | 6c2f804 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 90 | sh "echo $str >> maas/roles/maas/files/dhcpd.reservations" |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 95 | stage ("Deploy") { |
Luca Prete | 604563f | 2017-10-18 11:52:01 -0700 | [diff] [blame] | 96 | sh "make build" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 97 | } |
| 98 | |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 99 | if (deployment_config.compute_nodes != null) { |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 100 | stage ("Power cycle compute nodes") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 101 | for(int i=0; i < deployment_config.compute_nodes.size(); i++) { |
| 102 | sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle" |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 103 | } |
| 104 | } |
| 105 | |
| 106 | stage ("Wait for compute nodes to get deployed") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 107 | sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}" |
| 108 | def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}") |
| 109 | runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}") |
Luca Prete | 416574f | 2017-10-24 17:22:09 -0700 | [diff] [blame] | 110 | timeout(time: 90) { |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 111 | waitUntil { |
| 112 | try { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 113 | num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim() |
| 114 | return num.toInteger() == deployment_config.compute_nodes.size() |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 115 | } catch (exception) { |
| 116 | return false |
| 117 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 118 | } |
| 119 | } |
| 120 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 121 | |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 122 | stage ("Wait for compute nodes to be provisioned") { |
| 123 | timeout(time:45) { |
| 124 | waitUntil { |
| 125 | try { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 126 | num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim() |
| 127 | return num.toInteger() == deployment_config.compute_nodes.size() |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 128 | } catch (exception) { |
| 129 | return false |
| 130 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 131 | } |
| 132 | } |
| 133 | } |
Luca Prete | 9b3a667 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 134 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 135 | |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 136 | if (deployment_config.fabric_switches != null) { |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 137 | stage ("Wait for fabric switches to get deployed") { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 138 | runFabricSwitchCmdAll("sudo onl-onie-boot-mode install") |
| 139 | runFabricSwitchCmdAll("sudo reboot") |
Andy Bavier | 6c2f804 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 140 | // Ensure that switches get provisioned after ONIE reinstall. |
| 141 | // Delete them if they were provisioned earlier. If the switches are not |
| 142 | // present in 'cord prov list', this command has no effect. |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 143 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 144 | runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}") |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 145 | } |
| 146 | timeout(time: 45) { |
| 147 | waitUntil { |
| 148 | try { |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 149 | def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim() |
| 150 | return harvestCompleted.toInteger() == deployment_config.fabric_switches.size() |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 151 | } catch (exception) { |
| 152 | return false |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | stage ("Wait for fabric switches to be provisioned") { |
| 159 | timeout(time:45) { |
| 160 | waitUntil { |
| 161 | try { |
| 162 | def provCompleted = 0 |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 163 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 164 | def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim() |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 165 | provCompleted = provCompleted + count.toInteger() |
| 166 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 167 | return provCompleted == deployment_config.fabric_switches.size() |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 168 | } catch (exception) { |
| 169 | return false |
| 170 | } |
| 171 | } |
| 172 | } |
| 173 | } |
Marc De Leenheer | 392cf45 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 174 | |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 175 | // Post installation configuration starts here |
| 176 | fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0] |
| 177 | xosUser = "xosadmin@opencord.org" |
| 178 | xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim() |
| 179 | stage ("Connect fabric switches and compute nodes to ONOS") { |
| 180 | // Configure breakout ports |
| 181 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 182 | if (deployment_config.fabric_switches[i].breakout_ports != null) { |
| 183 | for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) { |
| 184 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 185 | "${deployment_config.fabric_switches[i].user}", |
| 186 | "${deployment_config.fabric_switches[i].pass}", |
You Wang | 43145b3 | 2017-12-13 11:18:54 -0800 | [diff] [blame] | 187 | "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf") |
Marc De Leenheer | 392cf45 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 188 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 189 | } |
| 190 | } |
| 191 | connectFabricAndComputeToOnos() |
| 192 | } |
| 193 | stage ("Configure the compute nodes") { |
| 194 | leafSwitchNum = 0 |
| 195 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
You Wang | 61b00ea | 2017-12-14 15:29:11 -0800 | [diff] [blame] | 196 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
You Wang | 8e84c6a | 2017-12-14 15:03:36 -0800 | [diff] [blame] | 197 | if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) { |
| 198 | leafSwitchNum += 1 |
| 199 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 200 | } |
| 201 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 202 | for(int i=1; i<=leafSwitchNum; i++) { |
| 203 | // Figure out which compute node connects to which switch |
| 204 | leafName = "leaf-" + i.toString() |
| 205 | computeNames = getComputeNames(leafName) |
| 206 | echo "Compute nodes connnected to " + leafName + " switch:" |
| 207 | for(name in computeNames) { echo "${name}" } |
| 208 | index = 1 |
| 209 | for(name in computeNames) { |
| 210 | if(i>1) { |
| 211 | //Update fabric IP of compute nodes |
| 212 | index += 1 |
| 213 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim() |
| 214 | if (fabricIp != "") { |
| 215 | tagId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/tags | jq '.[\"items\"][] | select(.value==\"${fabricIp}\") | .id'").trim() |
| 216 | newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24" |
| 217 | out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"value\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/tags/${tagId}").trim() |
| 218 | // Wait until the new fabric IP gets configured |
| 219 | timeout(time: 5) { |
| 220 | waitUntil { |
| 221 | try { |
| 222 | num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim() |
| 223 | return num.toInteger() == 1 |
| 224 | } catch (exception) { |
| 225 | return false |
| 226 | } |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*" |
| 231 | } |
| 232 | //Add routes to fabric subnets |
| 233 | for(int j=1; j<=leafSwitchNum; j++) { |
| 234 | if(j!=i) { |
| 235 | runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists") |
| 236 | } |
| 237 | } |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | stage ("Generate and load network configuration") { |
| 242 | // Reconnect compute nodes to update the fabric IP in ONOS |
| 243 | connectFabricAndComputeToOnos() |
| 244 | // Generate network configuration |
| 245 | runHeadNodeCmd(""" |
| 246 | cd /opt/cord_profile/ |
| 247 | cp fabric-network-cfg.json fabric-network-cfg.json.\$(date +%Y%m%d-%H%M%S) |
| 248 | cord generate > fabric-network-cfg.json |
| 249 | """) |
| 250 | // Install httpie on the head-node |
| 251 | runHeadNodeCmd("sudo pip install httpie") |
| 252 | // Delete old ONOS netcfg |
| 253 | runHeadNodeCmd("http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/network/configuration/") |
| 254 | // Load new configuration |
You Wang | 40f0934 | 2017-12-22 11:04:29 -0800 | [diff] [blame] | 255 | runHeadNodeCmd("http -a onos:rocks POST http://onos-fabric:8181/onos/v1/network/configuration/ < /opt/cord_profile/fabric-network-cfg.json") |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 256 | // Restart ONOS apps |
| 257 | runHeadNodeCmd(""" |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 258 | http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| 259 | http -a onos:rocks POST http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| 260 | """) |
Marc De Leenheer | 392cf45 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 261 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 262 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 263 | } |
Luca Prete | 9081fcc | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 264 | |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 265 | if (deployment_config.make_release == true) { |
Luca Prete | 9081fcc | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 266 | stage ("Trigger Build") { |
| 267 | url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build' |
| 268 | httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201' |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | currentBuild.result = 'SUCCESS' |
| 273 | } catch (err) { |
| 274 | currentBuild.result = 'FAILURE' |
| 275 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
Kailash Khalasi | 95f3f02 | 2018-04-20 12:22:42 -0700 | [diff] [blame^] | 276 | } finally { |
| 277 | sh "make -C build clean-all || true" |
| 278 | sh "rm -rf *" |
Luca Prete | 9081fcc | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 279 | } |
| 280 | echo "RESULT: ${currentBuild.result}" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 281 | } |
| 282 | } |
| 283 | |
| 284 | /** |
| 285 | * Returns a string used to bind IPs and MAC addresses, substituting the values |
| 286 | * given. |
| 287 | * |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 288 | * @param word the word used to generate the host name |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 289 | * @param counter the counter used to generate the host name |
| 290 | * @param mac the MAC address to substitute |
| 291 | * @param ip the IP address to substitute |
| 292 | */ |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 293 | def createMACIPbindingStr(word, counter, mac, ip) { |
| 294 | return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}""" |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | /** |
| 298 | * Runs a command on a remote host using sshpass. |
| 299 | * |
| 300 | * @param ip the node IP address |
| 301 | * @param user the node user name |
| 302 | * @param pass the node password |
| 303 | * @param command the command to run |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 304 | * @param sshArgs arguments for the ssh command |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 305 | * @return the output of the command |
| 306 | */ |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 307 | def runCmd(ip, user, pass, command, sshArgs="") { |
| 308 | return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"") |
| 309 | } |
| 310 | |
| 311 | /** |
| 312 | * Runs a command on the head node. |
| 313 | * |
| 314 | * @param command the command to run |
| 315 | * @param sshArgs arguments for the ssh command |
| 316 | * @return the output of the command |
| 317 | */ |
| 318 | def runHeadNodeCmd(command, sshArgs="") { |
| 319 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"") |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 320 | } |
| 321 | |
| 322 | /** |
| 323 | * Runs a command on a fabric switch. |
| 324 | * |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 325 | * @param ip the mgmt IP of the fabric switch, reachable from the head node |
| 326 | * @param user the mgmt user name of the fabric switch |
| 327 | * @param pass the mgmt password of the fabric switch |
| 328 | * @param command the command to run on the fabric switch |
| 329 | * @param ssgArgs arguments for the ssh command |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 330 | * @return the output of the command |
| 331 | */ |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 332 | def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") { |
| 333 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"") |
| 334 | } |
| 335 | |
| 336 | /** |
| 337 | * Runs a command on all fabric switches |
| 338 | * |
| 339 | * @param command the command to run on the fabric switches |
| 340 | * @param ssgArgs arguments for the ssh command |
| 341 | */ |
| 342 | def runFabricSwitchCmdAll(command, sshArgs="") { |
| 343 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 344 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 345 | "${deployment_config.fabric_switches[i].user}", |
| 346 | "${deployment_config.fabric_switches[i].pass}", |
| 347 | "${command}", |
| 348 | "${sshArgs}") |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | /** |
| 353 | * Runs a command on a compute node. |
| 354 | * |
| 355 | * @param name the name of the compute node |
| 356 | * @param command the command to run on the compute node |
| 357 | * @param ssgArgs arguments for the ssh command |
| 358 | * @return the output of the command |
| 359 | */ |
| 360 | def runComputeNodeCmd(name, command, sshArgs="") { |
| 361 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"") |
| 362 | } |
| 363 | |
| 364 | /** |
| 365 | * Runs a command on all compute nodes |
| 366 | * |
| 367 | * @param command the command to run on the compute nodes |
| 368 | * @param ssgArgs arguments for the ssh command |
| 369 | */ |
| 370 | def runComputeNodeCmdAll(command, sshArgs="") { |
| 371 | computeNamesAll = getComputeNames() |
| 372 | for (name in computeNamesAll) { |
| 373 | runComputeNodeCmd("${name}", "${command}", "${sshArgs}") |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | /** |
| 378 | * Runs an ONOS CLI command |
| 379 | * |
| 380 | * @param name the onos node name, reachable from the head node |
| 381 | * @param port the port used to login to ONOS CLI |
| 382 | * @param user the user name to login to ONOS CLI |
| 383 | * @param pass the password to login to ONOS CLI |
| 384 | * @param command the command to run in ONOS CLI |
| 385 | * @return the output of the command |
| 386 | */ |
| 387 | def runOnosCliCmd(name, port, user, pass, command) { |
| 388 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"") |
| 389 | } |
| 390 | |
| 391 | /** |
| 392 | * Returns a list of compute node names. When "role" is specified, returns only |
| 393 | * names of compute nodes connected to the switch |
| 394 | * |
| 395 | * @param role the switch role, i.e. "leaf-1" |
| 396 | */ |
| 397 | def getComputeNames(role="") { |
| 398 | computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim() |
| 399 | computeNamesAll = "${computeNamesAll}".split() |
| 400 | computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"") |
Kailash Khalasi | fd36028 | 2018-04-20 09:09:54 -0700 | [diff] [blame] | 401 | computeNamesAll = new JsonSlurper().parseText("${computeNamesAll}") |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 402 | if ("${role}" == "") return computeNamesAll |
| 403 | computeNames = [] |
| 404 | switchMac = "" |
| 405 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
You Wang | 61b00ea | 2017-12-14 15:29:11 -0800 | [diff] [blame] | 406 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
| 407 | if ("${deployment_config.fabric_switches[i].role}" == "${role}") |
| 408 | switchMac = "${deployment_config.fabric_switches[i].mac}" |
| 409 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 410 | } |
| 411 | if ("${switchMac}" != "") { |
| 412 | switchMac = switchMac.toLowerCase().replaceAll(':','') |
| 413 | // Get fabric IPs of compute nodes connected to the switch |
You Wang | 40f0934 | 2017-12-22 11:04:29 -0800 | [diff] [blame] | 414 | try { |
| 415 | computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.location.elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'", |
| 416 | "-q").trim() |
| 417 | }catch (exception) { |
| 418 | return computeNames |
| 419 | } |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 420 | computeFabricIps = "${computeFabricIps}".split() |
Kailash Khalasi | fd36028 | 2018-04-20 09:09:54 -0700 | [diff] [blame] | 421 | computeFabricIps = new JsonSlurper().parseText("${computeFabricIps}") |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 422 | // Figure out which compute node connects to the switch |
| 423 | for (name in computeNamesAll) { |
| 424 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim() |
| 425 | if (fabricIp in computeFabricIps) { |
| 426 | computeNames.add("${name}") |
| 427 | } |
| 428 | } |
| 429 | } |
| 430 | return computeNames |
| 431 | } |
| 432 | |
| 433 | /** |
| 434 | * Connects all fabric switches and compute nodes to ONOS |
| 435 | */ |
| 436 | def connectFabricAndComputeToOnos() { |
| 437 | // Kill existing switch connections |
| 438 | runFabricSwitchCmdAll("./killit || echo no ofagentapp running") |
| 439 | // Clean stale ONOS data |
| 440 | runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please") |
| 441 | // Connect switches to ONOS |
| 442 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 443 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 444 | "${deployment_config.fabric_switches[i].user}", |
| 445 | "${deployment_config.fabric_switches[i].pass}", |
| 446 | "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log", |
| 447 | "-qftn") |
| 448 | } |
| 449 | // Verify ONOS has recognized the switches |
| 450 | timeout(time: 5) { |
| 451 | waitUntil { |
| 452 | try { |
| 453 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim() |
| 454 | return num.toInteger() == deployment_config.fabric_switches.size() |
| 455 | } catch (exception) { |
| 456 | return false |
| 457 | } |
| 458 | } |
| 459 | } |
| 460 | // Connect compute nodes to ONOS |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 461 | runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn") |
| 462 | // Verify ONOS has recognized the hosts |
| 463 | timeout(time: 5) { |
| 464 | waitUntil { |
| 465 | try { |
| 466 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim() |
You Wang | 61b00ea | 2017-12-14 15:29:11 -0800 | [diff] [blame] | 467 | return num.toInteger() == deployment_config.compute_nodes.size() |
You Wang | 0a41438 | 2017-10-16 10:11:48 -0700 | [diff] [blame] | 468 | } catch (exception) { |
| 469 | return false |
| 470 | } |
| 471 | } |
| 472 | } |
You Wang | cebbd31 | 2017-09-11 14:35:24 -0700 | [diff] [blame] | 473 | } |