Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 15 | import groovy.json.JsonSlurperClassic |
| 16 | |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 17 | def filename = 'manifest-${branch}.xml' |
| 18 | def manifestUrl = 'https://gerrit.opencord.org/manifest' |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 19 | deployment_config = null; |
| 20 | pod_config = null; |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 21 | |
| 22 | node ('master') { |
| 23 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| 24 | |
| 25 | stage ("Generate and Copy Manifest file") { |
| 26 | sh returnStdout: true, script: 'repo manifest -r -o ' + filename |
| 27 | sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp' |
| 28 | } |
| 29 | |
| 30 | stage ("Parse deployment configuration file") { |
| 31 | sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}' |
| 32 | sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}' |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 33 | deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}" |
| 34 | pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 35 | } |
| 36 | } |
| 37 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 38 | node ("${deployment_config.dev_node.name}") { |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 39 | timeout (time: 240) { |
Luca Prete | 225d965 | 2017-09-12 10:44:48 -0700 | [diff] [blame] | 40 | stage ('Remove old head node from known hosts') { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 41 | sh "ssh-keygen -R ${deployment_config.head.ip}" |
You Wang | f9cb2f3 | 2017-09-11 13:26:42 -0700 | [diff] [blame] | 42 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 43 | stage ('Checkout cord repo') { |
| 44 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| 45 | } |
| 46 | |
Luca Prete | fcbc54b | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 47 | try { |
| 48 | dir('build') { |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 49 | stage ("Re-deploy head node") { |
| 50 | maasOps: { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 51 | sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}" |
| 52 | sh "maas maas machine release ${deployment_config.maas.head_system_id}" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 53 | |
| 54 | timeout(time: 15) { |
| 55 | waitUntil { |
| 56 | try { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 57 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 58 | return true |
| 59 | } catch (exception) { |
| 60 | return false |
| 61 | } |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | sh 'maas maas machines allocate' |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 66 | sh "maas maas machine deploy ${deployment_config.maas.head_system_id}" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 67 | |
| 68 | timeout(time: 30) { |
| 69 | waitUntil { |
| 70 | try { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 71 | sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 72 | return true |
| 73 | } catch (exception) { |
| 74 | return false |
| 75 | } |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | stage ("Download CORD POD configuration") { |
You Wang | e6fbdae | 2018-01-02 11:35:10 -0800 | [diff] [blame] | 82 | sh "cd ../orchestration/profiles; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | stage ("Generate CORD configuration") { |
Andy Bavier | 4775df2 | 2018-01-05 10:52:30 -0700 | [diff] [blame] | 86 | sh "make PODCONFIG_PATH=../orchestration/profiles/automation/${deployment_config.pod_config.file_name} config" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 87 | } |
| 88 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 89 | if (deployment_config.fabric_switches != null) { |
Andy Bavier | 700f266 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 90 | stage("Reserve IPs for fabric switches") { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 91 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 92 | def str = createMACIPbindingStr("fabric", i+1, |
| 93 | "${deployment_config.fabric_switches[i].mac}", |
| 94 | "${deployment_config.fabric_switches[i].ip}") |
Andy Bavier | 700f266 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 95 | sh "echo $str >> maas/roles/maas/files/dhcpd.reservations" |
| 96 | } |
| 97 | } |
| 98 | } |
| 99 | |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 100 | stage ("Deploy") { |
Luca Prete | 614d670 | 2017-10-18 11:52:55 -0700 | [diff] [blame] | 101 | sh "make build" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 102 | } |
| 103 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 104 | if (deployment_config.compute_nodes != null) { |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 105 | stage ("Power cycle compute nodes") { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 106 | for(int i=0; i < deployment_config.compute_nodes.size(); i++) { |
| 107 | sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle" |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 108 | } |
| 109 | } |
| 110 | |
| 111 | stage ("Wait for compute nodes to get deployed") { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 112 | sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}" |
| 113 | def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}") |
| 114 | runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}") |
Luca Prete | 49c818c | 2017-10-24 17:22:09 -0700 | [diff] [blame] | 115 | timeout(time: 90) { |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 116 | waitUntil { |
| 117 | try { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 118 | num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim() |
| 119 | return num.toInteger() == deployment_config.compute_nodes.size() |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 120 | } catch (exception) { |
| 121 | return false |
| 122 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 123 | } |
| 124 | } |
| 125 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 126 | |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 127 | stage ("Wait for compute nodes to be provisioned") { |
| 128 | timeout(time:45) { |
| 129 | waitUntil { |
| 130 | try { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 131 | num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim() |
| 132 | return num.toInteger() == deployment_config.compute_nodes.size() |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 133 | } catch (exception) { |
| 134 | return false |
| 135 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 136 | } |
| 137 | } |
| 138 | } |
Luca Prete | e209527 | 2017-10-17 11:47:26 -0700 | [diff] [blame] | 139 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 140 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 141 | if (deployment_config.fabric_switches != null) { |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 142 | stage ("Wait for fabric switches to get deployed") { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 143 | runFabricSwitchCmdAll("sudo onl-onie-boot-mode install") |
| 144 | runFabricSwitchCmdAll("sudo reboot") |
Andy Bavier | 700f266 | 2017-09-19 13:52:30 -0700 | [diff] [blame] | 145 | // Ensure that switches get provisioned after ONIE reinstall. |
| 146 | // Delete them if they were provisioned earlier. If the switches are not |
| 147 | // present in 'cord prov list', this command has no effect. |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 148 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 149 | runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}") |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 150 | } |
| 151 | timeout(time: 45) { |
| 152 | waitUntil { |
| 153 | try { |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 154 | def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim() |
| 155 | return harvestCompleted.toInteger() == deployment_config.fabric_switches.size() |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 156 | } catch (exception) { |
| 157 | return false |
| 158 | } |
| 159 | } |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | stage ("Wait for fabric switches to be provisioned") { |
| 164 | timeout(time:45) { |
| 165 | waitUntil { |
| 166 | try { |
| 167 | def provCompleted = 0 |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 168 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 169 | def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim() |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 170 | provCompleted = provCompleted + count.toInteger() |
| 171 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 172 | return provCompleted == deployment_config.fabric_switches.size() |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 173 | } catch (exception) { |
| 174 | return false |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | } |
Marc De Leenheer | 672afa8 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 179 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 180 | // Post installation configuration starts here |
| 181 | fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0] |
| 182 | xosUser = "xosadmin@opencord.org" |
| 183 | xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim() |
| 184 | stage ("Connect fabric switches and compute nodes to ONOS") { |
| 185 | // Configure breakout ports |
| 186 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 187 | if (deployment_config.fabric_switches[i].breakout_ports != null) { |
| 188 | for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) { |
| 189 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 190 | "${deployment_config.fabric_switches[i].user}", |
| 191 | "${deployment_config.fabric_switches[i].pass}", |
You Wang | d1d67aa | 2017-12-13 11:18:54 -0800 | [diff] [blame] | 192 | "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf") |
Marc De Leenheer | 672afa8 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 193 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 194 | } |
| 195 | } |
| 196 | connectFabricAndComputeToOnos() |
| 197 | } |
| 198 | stage ("Configure the compute nodes") { |
| 199 | leafSwitchNum = 0 |
| 200 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
You Wang | d37a173 | 2017-12-14 15:29:11 -0800 | [diff] [blame] | 201 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
You Wang | dd8793e | 2017-12-14 15:03:36 -0800 | [diff] [blame] | 202 | if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) { |
| 203 | leafSwitchNum += 1 |
| 204 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 205 | } |
| 206 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 207 | for(int i=1; i<=leafSwitchNum; i++) { |
| 208 | // Figure out which compute node connects to which switch |
| 209 | leafName = "leaf-" + i.toString() |
| 210 | computeNames = getComputeNames(leafName) |
| 211 | echo "Compute nodes connnected to " + leafName + " switch:" |
| 212 | for(name in computeNames) { echo "${name}" } |
| 213 | index = 1 |
| 214 | for(name in computeNames) { |
| 215 | if(i>1) { |
| 216 | //Update fabric IP of compute nodes |
| 217 | index += 1 |
| 218 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim() |
| 219 | if (fabricIp != "") { |
| 220 | nodeId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/nodes | jq '.[\"items\"][] | select(.dataPlaneIp==\"${fabricIp}\") | .id'").trim() |
| 221 | newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24" |
| 222 | out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"dataPlaneIp\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/nodes/${nodeId}").trim() |
| 223 | // Wait until the new fabric IP gets configured |
| 224 | timeout(time: 5) { |
| 225 | waitUntil { |
| 226 | try { |
| 227 | num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim() |
| 228 | return num.toInteger() == 1 |
| 229 | } catch (exception) { |
| 230 | return false |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | } |
| 235 | else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*" |
| 236 | } |
| 237 | //Add routes to fabric subnets |
| 238 | for(int j=1; j<=leafSwitchNum; j++) { |
| 239 | if(j!=i) { |
| 240 | runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists") |
| 241 | } |
| 242 | } |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | stage ("Generate and load network configuration") { |
| 247 | // Reconnect compute nodes to update the fabric IP in ONOS |
| 248 | connectFabricAndComputeToOnos() |
| 249 | // Generate network configuration |
| 250 | runHeadNodeCmd(""" |
| 251 | cd /opt/cord_profile/ |
| 252 | cp fabric-network-cfg.json fabric-network-cfg.json.\$(date +%Y%m%d-%H%M%S) |
| 253 | cord generate > fabric-network-cfg.json |
| 254 | """) |
| 255 | // Install httpie on the head-node |
| 256 | runHeadNodeCmd("sudo pip install httpie") |
| 257 | // Delete old ONOS netcfg |
| 258 | runHeadNodeCmd("http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/network/configuration/") |
| 259 | // Load new configuration |
You Wang | 22e4886 | 2017-12-22 11:04:29 -0800 | [diff] [blame] | 260 | runHeadNodeCmd("http -a onos:rocks POST http://onos-fabric:8181/onos/v1/network/configuration/ < /opt/cord_profile/fabric-network-cfg.json") |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 261 | // Restart ONOS apps |
| 262 | runHeadNodeCmd(""" |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 263 | http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| 264 | http -a onos:rocks POST http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| 265 | """) |
Marc De Leenheer | 672afa8 | 2017-11-27 17:25:09 -0800 | [diff] [blame] | 266 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 267 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 268 | } |
Luca Prete | fcbc54b | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 269 | |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 270 | if (deployment_config.make_release == true) { |
Luca Prete | fcbc54b | 2017-09-12 14:37:33 -0700 | [diff] [blame] | 271 | stage ("Trigger Build") { |
| 272 | url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build' |
| 273 | httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201' |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | currentBuild.result = 'SUCCESS' |
| 278 | } catch (err) { |
| 279 | currentBuild.result = 'FAILURE' |
| 280 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
| 281 | } finally { |
| 282 | sh "make -C build clean-all || true" |
| 283 | sh "rm -rf *" |
| 284 | } |
| 285 | echo "RESULT: ${currentBuild.result}" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 286 | } |
| 287 | } |
| 288 | |
| 289 | /** |
| 290 | * Returns a string used to bind IPs and MAC addresses, substituting the values |
| 291 | * given. |
| 292 | * |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 293 | * @param word the word used to generate the host name |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 294 | * @param counter the counter used to generate the host name |
| 295 | * @param mac the MAC address to substitute |
| 296 | * @param ip the IP address to substitute |
| 297 | */ |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 298 | def createMACIPbindingStr(word, counter, mac, ip) { |
| 299 | return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}""" |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 300 | } |
| 301 | |
| 302 | /** |
| 303 | * Runs a command on a remote host using sshpass. |
| 304 | * |
| 305 | * @param ip the node IP address |
| 306 | * @param user the node user name |
| 307 | * @param pass the node password |
| 308 | * @param command the command to run |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 309 | * @param sshArgs arguments for the ssh command |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 310 | * @return the output of the command |
| 311 | */ |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 312 | def runCmd(ip, user, pass, command, sshArgs="") { |
| 313 | return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"") |
| 314 | } |
| 315 | |
| 316 | /** |
| 317 | * Runs a command on the head node. |
| 318 | * |
| 319 | * @param command the command to run |
| 320 | * @param sshArgs arguments for the ssh command |
| 321 | * @return the output of the command |
| 322 | */ |
| 323 | def runHeadNodeCmd(command, sshArgs="") { |
| 324 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"") |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 325 | } |
| 326 | |
| 327 | /** |
| 328 | * Runs a command on a fabric switch. |
| 329 | * |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 330 | * @param ip the mgmt IP of the fabric switch, reachable from the head node |
| 331 | * @param user the mgmt user name of the fabric switch |
| 332 | * @param pass the mgmt password of the fabric switch |
| 333 | * @param command the command to run on the fabric switch |
| 334 | * @param ssgArgs arguments for the ssh command |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 335 | * @return the output of the command |
| 336 | */ |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 337 | def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") { |
| 338 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"") |
| 339 | } |
| 340 | |
| 341 | /** |
| 342 | * Runs a command on all fabric switches |
| 343 | * |
| 344 | * @param command the command to run on the fabric switches |
| 345 | * @param ssgArgs arguments for the ssh command |
| 346 | */ |
| 347 | def runFabricSwitchCmdAll(command, sshArgs="") { |
| 348 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 349 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 350 | "${deployment_config.fabric_switches[i].user}", |
| 351 | "${deployment_config.fabric_switches[i].pass}", |
| 352 | "${command}", |
| 353 | "${sshArgs}") |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | /** |
| 358 | * Runs a command on a compute node. |
| 359 | * |
| 360 | * @param name the name of the compute node |
| 361 | * @param command the command to run on the compute node |
| 362 | * @param ssgArgs arguments for the ssh command |
| 363 | * @return the output of the command |
| 364 | */ |
| 365 | def runComputeNodeCmd(name, command, sshArgs="") { |
| 366 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"") |
| 367 | } |
| 368 | |
| 369 | /** |
| 370 | * Runs a command on all compute nodes |
| 371 | * |
| 372 | * @param command the command to run on the compute nodes |
| 373 | * @param ssgArgs arguments for the ssh command |
| 374 | */ |
| 375 | def runComputeNodeCmdAll(command, sshArgs="") { |
| 376 | computeNamesAll = getComputeNames() |
| 377 | for (name in computeNamesAll) { |
| 378 | runComputeNodeCmd("${name}", "${command}", "${sshArgs}") |
| 379 | } |
| 380 | } |
| 381 | |
| 382 | /** |
| 383 | * Runs an ONOS CLI command |
| 384 | * |
| 385 | * @param name the onos node name, reachable from the head node |
| 386 | * @param port the port used to login to ONOS CLI |
| 387 | * @param user the user name to login to ONOS CLI |
| 388 | * @param pass the password to login to ONOS CLI |
| 389 | * @param command the command to run in ONOS CLI |
| 390 | * @return the output of the command |
| 391 | */ |
| 392 | def runOnosCliCmd(name, port, user, pass, command) { |
| 393 | return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"") |
| 394 | } |
| 395 | |
| 396 | /** |
| 397 | * Returns a list of compute node names. When "role" is specified, returns only |
| 398 | * names of compute nodes connected to the switch |
| 399 | * |
| 400 | * @param role the switch role, i.e. "leaf-1" |
| 401 | */ |
| 402 | def getComputeNames(role="") { |
| 403 | computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim() |
| 404 | computeNamesAll = "${computeNamesAll}".split() |
| 405 | computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"") |
| 406 | computeNamesAll = new JsonSlurperClassic().parseText("${computeNamesAll}") |
| 407 | if ("${role}" == "") return computeNamesAll |
| 408 | computeNames = [] |
| 409 | switchMac = "" |
| 410 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
You Wang | d37a173 | 2017-12-14 15:29:11 -0800 | [diff] [blame] | 411 | if(deployment_config.fabric_switches[i].containsKey("role")) { |
| 412 | if ("${deployment_config.fabric_switches[i].role}" == "${role}") |
| 413 | switchMac = "${deployment_config.fabric_switches[i].mac}" |
| 414 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 415 | } |
| 416 | if ("${switchMac}" != "") { |
| 417 | switchMac = switchMac.toLowerCase().replaceAll(':','') |
| 418 | // Get fabric IPs of compute nodes connected to the switch |
You Wang | 22e4886 | 2017-12-22 11:04:29 -0800 | [diff] [blame] | 419 | try { |
| 420 | computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.location.elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'", |
| 421 | "-q").trim() |
| 422 | }catch (exception) { |
| 423 | return computeNames |
| 424 | } |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 425 | computeFabricIps = "${computeFabricIps}".split() |
| 426 | computeFabricIps = new JsonSlurperClassic().parseText("${computeFabricIps}") |
| 427 | // Figure out which compute node connects to the switch |
| 428 | for (name in computeNamesAll) { |
| 429 | fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim() |
| 430 | if (fabricIp in computeFabricIps) { |
| 431 | computeNames.add("${name}") |
| 432 | } |
| 433 | } |
| 434 | } |
| 435 | return computeNames |
| 436 | } |
| 437 | |
| 438 | /** |
| 439 | * Connects all fabric switches and compute nodes to ONOS |
| 440 | */ |
| 441 | def connectFabricAndComputeToOnos() { |
| 442 | // Kill existing switch connections |
| 443 | runFabricSwitchCmdAll("./killit || echo no ofagentapp running") |
| 444 | // Clean stale ONOS data |
| 445 | runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please") |
| 446 | // Connect switches to ONOS |
| 447 | for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| 448 | runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| 449 | "${deployment_config.fabric_switches[i].user}", |
| 450 | "${deployment_config.fabric_switches[i].pass}", |
| 451 | "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log", |
| 452 | "-qftn") |
| 453 | } |
| 454 | // Verify ONOS has recognized the switches |
| 455 | timeout(time: 5) { |
| 456 | waitUntil { |
| 457 | try { |
| 458 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim() |
| 459 | return num.toInteger() == deployment_config.fabric_switches.size() |
| 460 | } catch (exception) { |
| 461 | return false |
| 462 | } |
| 463 | } |
| 464 | } |
| 465 | // Connect compute nodes to ONOS |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 466 | runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn") |
| 467 | // Verify ONOS has recognized the hosts |
| 468 | timeout(time: 5) { |
| 469 | waitUntil { |
| 470 | try { |
| 471 | num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim() |
You Wang | 97bf12e | 2018-01-24 12:05:39 -0800 | [diff] [blame] | 472 | return num.toInteger() >= deployment_config.compute_nodes.size() |
You Wang | 1abf2bf | 2017-12-12 15:21:07 -0800 | [diff] [blame] | 473 | } catch (exception) { |
| 474 | return false |
| 475 | } |
| 476 | } |
| 477 | } |
Luca Prete | e2d51fd | 2017-08-31 16:19:21 -0700 | [diff] [blame] | 478 | } |