| // Copyright 2017-present Open Networking Foundation |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| import groovy.json.JsonSlurperClassic |
| |
| def filename = 'manifest-${branch}.xml' |
| def manifestUrl = 'https://gerrit.opencord.org/manifest' |
| deployment_config = null; |
| pod_config = null; |
| |
| node ('master') { |
| checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| |
| stage ("Generate and Copy Manifest file") { |
| sh returnStdout: true, script: 'repo manifest -r -o ' + filename |
| sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp' |
| } |
| |
| stage ("Parse deployment configuration file") { |
| sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}' |
| sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}' |
| deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}" |
| pod_config = readYaml file: "${configRepoBaseDir}${deployment_config.pod_config.file_name}" |
| } |
| } |
| |
| node ("${deployment_config.dev_node.name}") { |
| timeout (time: 240) { |
| stage ('Remove old head node from known hosts') { |
| sh "ssh-keygen -R ${deployment_config.head.ip}" |
| } |
| stage ('Checkout cord repo') { |
| checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
| } |
| |
| try { |
| dir('build') { |
| stage ("Re-deploy head node") { |
| maasOps: { |
| sh "maas login maas http://${deployment_config.maas.ip}/MAAS/api/2.0 ${deployment_config.maas.api_key}" |
| sh "maas maas machine release ${deployment_config.maas.head_system_id}" |
| |
| timeout(time: 15) { |
| waitUntil { |
| try { |
| sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Ready" |
| return true |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| |
| sh 'maas maas machines allocate' |
| sh "maas maas machine deploy ${deployment_config.maas.head_system_id}" |
| |
| timeout(time: 30) { |
| waitUntil { |
| try { |
| sh "maas maas machine read ${deployment_config.maas.head_system_id} | grep Deployed" |
| return true |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| } |
| |
| stage ("Download CORD POD configuration") { |
| sh "cd podconfig; git clone -b ${branch} ${deployment_config.pod_config.repo_url} automation" |
| } |
| |
| stage ("Generate CORD configuration") { |
| sh "make PODCONFIG=automation/${deployment_config.pod_config.file_name} config" |
| } |
| |
| if (deployment_config.fabric_switches != null) { |
| stage("Reserve IPs for fabric switches") { |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| def str = createMACIPbindingStr("fabric", i+1, |
| "${deployment_config.fabric_switches[i].mac}", |
| "${deployment_config.fabric_switches[i].ip}") |
| sh "echo $str >> maas/roles/maas/files/dhcpd.reservations" |
| } |
| } |
| } |
| |
| stage ("Deploy") { |
| sh "make build" |
| } |
| |
| if (deployment_config.compute_nodes != null) { |
| stage ("Power cycle compute nodes") { |
| for(int i=0; i < deployment_config.compute_nodes.size(); i++) { |
| sh "ipmitool -U ${deployment_config.compute_nodes[i].ipmi.user} -P ${deployment_config.compute_nodes[i].ipmi.pass} -H ${deployment_config.compute_nodes[i].ipmi.ip} power cycle" |
| } |
| } |
| |
| stage ("Wait for compute nodes to get deployed") { |
| sh "ssh-keygen -f /home/${deployment_config.dev_node.user}/.ssh/known_hosts -R ${deployment_config.head.ip}" |
| def cordApiKey = runHeadNodeCmd("sudo maas-region-admin apikey --username ${deployment_config.head.user}") |
| runHeadNodeCmd("maas login pod-maas http://${deployment_config.head.ip}/MAAS/api/1.0 ${cordApiKey}") |
| timeout(time: 90) { |
| waitUntil { |
| try { |
| num = runHeadNodeCmd("maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim() |
| return num.toInteger() == deployment_config.compute_nodes.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| |
| stage ("Wait for compute nodes to be provisioned") { |
| timeout(time:45) { |
| waitUntil { |
| try { |
| num = runHeadNodeCmd("cord prov list | grep -i node | grep -i complete | wc -l").trim() |
| return num.toInteger() == deployment_config.compute_nodes.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| } |
| |
| if (deployment_config.fabric_switches != null) { |
| stage ("Wait for fabric switches to get deployed") { |
| runFabricSwitchCmdAll("sudo onl-onie-boot-mode install") |
| runFabricSwitchCmdAll("sudo reboot") |
| // Ensure that switches get provisioned after ONIE reinstall. |
| // Delete them if they were provisioned earlier. If the switches are not |
| // present in 'cord prov list', this command has no effect. |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| runHeadNodeCmd("cord prov delete ${deployment_config.fabric_switches[i].mac}") |
| } |
| timeout(time: 45) { |
| waitUntil { |
| try { |
| def harvestCompleted = runHeadNodeCmd("cord harvest list | grep -i fabric | wc -l").trim() |
| return harvestCompleted.toInteger() == deployment_config.fabric_switches.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| |
| stage ("Wait for fabric switches to be provisioned") { |
| timeout(time:45) { |
| waitUntil { |
| try { |
| def provCompleted = 0 |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| def count = runHeadNodeCmd("cord prov list | grep -i ${deployment_config.fabric_switches[i].ip} | grep -i complete | wc -l").trim() |
| provCompleted = provCompleted + count.toInteger() |
| } |
| return provCompleted == deployment_config.fabric_switches.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| |
| // Post installation configuration starts here |
| fabricIpPrefix = pod_config.fabric_ip.split(/\.\d+\.\d+\/24/)[0] |
| xosUser = "xosadmin@opencord.org" |
| xosPass = runHeadNodeCmd("cat /opt/credentials/xosadmin@opencord.org").trim() |
| stage ("Connect fabric switches and compute nodes to ONOS") { |
| // Configure breakout ports |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| if (deployment_config.fabric_switches[i].breakout_ports != null) { |
| for(int j=0; j < deployment_config.fabric_switches[i].breakout_ports.size(); j++) { |
| runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| "${deployment_config.fabric_switches[i].user}", |
| "${deployment_config.fabric_switches[i].pass}", |
| "sed -i -e 's/#port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/port_mode_${deployment_config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf") |
| } |
| } |
| } |
| connectFabricAndComputeToOnos() |
| } |
| stage ("Configure the compute nodes") { |
| leafSwitchNum = 0 |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| if(deployment_config.fabric_switches[i].containsKey("role")) { |
| if(deployment_config.fabric_switches[i].role.toLowerCase().contains("leaf")) { |
| leafSwitchNum += 1 |
| } |
| } |
| } |
| for(int i=1; i<=leafSwitchNum; i++) { |
| // Figure out which compute node connects to which switch |
| leafName = "leaf-" + i.toString() |
| computeNames = getComputeNames(leafName) |
| echo "Compute nodes connnected to " + leafName + " switch:" |
| for(name in computeNames) { echo "${name}" } |
| index = 1 |
| for(name in computeNames) { |
| if(i>1) { |
| //Update fabric IP of compute nodes |
| index += 1 |
| fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*/24'").trim() |
| if (fabricIp != "") { |
| tagId = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X GET http://${deployment_config.head.ip}/xosapi/v1/core/tags | jq '.[\"items\"][] | select(.value==\"${fabricIp}\") | .id'").trim() |
| newFabricIp = "${fabricIpPrefix}." + i.toString() + "." + index.toString() + "/24" |
| out = sh(returnStdout: true, script: "curl -u ${xosUser}:${xosPass} -X PUT -d '{\"value\":\"${newFabricIp}\"}' http://${deployment_config.head.ip}/xosapi/v1/core/tags/${tagId}").trim() |
| // Wait until the new fabric IP gets configured |
| timeout(time: 5) { |
| waitUntil { |
| try { |
| num = runComputeNodeCmd("${name}", "ip a | grep " + newFabricIp + " | wc -l").trim() |
| return num.toInteger() == 1 |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |
| else echo "Cannot find fabric IP matching pattern ${fabricIpPrefix}.[1-9][0-9]*.[0-9]*" |
| } |
| //Add routes to fabric subnets |
| for(int j=1; j<=leafSwitchNum; j++) { |
| if(j!=i) { |
| runComputeNodeCmd("${name}", "sudo ip route add ${fabricIpPrefix}." + j.toString() + ".0/24 via ${fabricIpPrefix}." + i.toString() + ".254 || echo route already exists") |
| } |
| } |
| } |
| } |
| } |
| stage ("Generate and load network configuration") { |
| // Reconnect compute nodes to update the fabric IP in ONOS |
| connectFabricAndComputeToOnos() |
| // Generate network configuration |
| runHeadNodeCmd(""" |
| cd /opt/cord_profile/ |
| cp fabric-network-cfg.json fabric-network-cfg.json.\$(date +%Y%m%d-%H%M%S) |
| cord generate > fabric-network-cfg.json |
| """) |
| // Install httpie on the head-node |
| runHeadNodeCmd("sudo pip install httpie") |
| // Delete old ONOS netcfg |
| runHeadNodeCmd("http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/network/configuration/") |
| // Load new configuration |
| runHeadNodeCmd("http -a onos:rocks POST http://onos-fabric:8181/onos/v1/network/configuration/ < /opt/cord_profile/fabric-network-cfg.json") |
| // Restart ONOS apps |
| runHeadNodeCmd(""" |
| http -a onos:rocks DELETE http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| http -a onos:rocks POST http://onos-fabric:8181/onos/v1/applications/org.onosproject.segmentrouting/active; sleep 5 |
| """) |
| } |
| } |
| } |
| |
| if (deployment_config.make_release == true) { |
| stage ("Trigger Build") { |
| url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build' |
| httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201' |
| } |
| } |
| |
| currentBuild.result = 'SUCCESS' |
| } catch (err) { |
| currentBuild.result = 'FAILURE' |
| step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
| } finally { |
| sh "make -C build clean-all || true" |
| sh "rm -rf *" |
| } |
| echo "RESULT: ${currentBuild.result}" |
| } |
| } |
| |
| /** |
| * Returns a string used to bind IPs and MAC addresses, substituting the values |
| * given. |
| * |
| * @param word the word used to generate the host name |
| * @param counter the counter used to generate the host name |
| * @param mac the MAC address to substitute |
| * @param ip the IP address to substitute |
| */ |
| def createMACIPbindingStr(word, counter, mac, ip) { |
| return """host ${word}-${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}""" |
| } |
| |
| /** |
| * Runs a command on a remote host using sshpass. |
| * |
| * @param ip the node IP address |
| * @param user the node user name |
| * @param pass the node password |
| * @param command the command to run |
| * @param sshArgs arguments for the ssh command |
| * @return the output of the command |
| */ |
| def runCmd(ip, user, pass, command, sshArgs="") { |
| return sh(returnStdout: true, script: "sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} \"${command}\"") |
| } |
| |
| /** |
| * Runs a command on the head node. |
| * |
| * @param command the command to run |
| * @param sshArgs arguments for the ssh command |
| * @return the output of the command |
| */ |
| def runHeadNodeCmd(command, sshArgs="") { |
| return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"${command}\"") |
| } |
| |
| /** |
| * Runs a command on a fabric switch. |
| * |
| * @param ip the mgmt IP of the fabric switch, reachable from the head node |
| * @param user the mgmt user name of the fabric switch |
| * @param pass the mgmt password of the fabric switch |
| * @param command the command to run on the fabric switch |
| * @param ssgArgs arguments for the ssh command |
| * @return the output of the command |
| */ |
| def runFabricSwitchCmd(ip, user, pass, command, sshArgs="") { |
| return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"") |
| } |
| |
| /** |
| * Runs a command on all fabric switches |
| * |
| * @param command the command to run on the fabric switches |
| * @param ssgArgs arguments for the ssh command |
| */ |
| def runFabricSwitchCmdAll(command, sshArgs="") { |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| "${deployment_config.fabric_switches[i].user}", |
| "${deployment_config.fabric_switches[i].pass}", |
| "${command}", |
| "${sshArgs}") |
| } |
| } |
| |
| /** |
| * Runs a command on a compute node. |
| * |
| * @param name the name of the compute node |
| * @param command the command to run on the compute node |
| * @param ssgArgs arguments for the ssh command |
| * @return the output of the command |
| */ |
| def runComputeNodeCmd(name, command, sshArgs="") { |
| return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh ${sshArgs} -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"ssh ${sshArgs} ubuntu@${name} ${command}\"") |
| } |
| |
| /** |
| * Runs a command on all compute nodes |
| * |
| * @param command the command to run on the compute nodes |
| * @param ssgArgs arguments for the ssh command |
| */ |
| def runComputeNodeCmdAll(command, sshArgs="") { |
| computeNamesAll = getComputeNames() |
| for (name in computeNamesAll) { |
| runComputeNodeCmd("${name}", "${command}", "${sshArgs}") |
| } |
| } |
| |
| /** |
| * Runs an ONOS CLI command |
| * |
| * @param name the onos node name, reachable from the head node |
| * @param port the port used to login to ONOS CLI |
| * @param user the user name to login to ONOS CLI |
| * @param pass the password to login to ONOS CLI |
| * @param command the command to run in ONOS CLI |
| * @return the output of the command |
| */ |
| def runOnosCliCmd(name, port, user, pass, command) { |
| return sh(returnStdout: true, script: "sshpass -p ${deployment_config.head.pass} ssh -oStrictHostKeyChecking=no -l ${deployment_config.head.user} ${deployment_config.head.ip} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} -p ${port} ${name} ${command}\"") |
| } |
| |
| /** |
| * Returns a list of compute node names. When "role" is specified, returns only |
| * names of compute nodes connected to the switch |
| * |
| * @param role the switch role, i.e. "leaf-1" |
| */ |
| def getComputeNames(role="") { |
| computeNamesAll = runHeadNodeCmd("cord prov list | grep node | awk '{print \\\$2}' | sed -e \\\"s/.*/'&'/\\\"").trim() |
| computeNamesAll = "${computeNamesAll}".split() |
| computeNamesAll = "${computeNamesAll}".replaceAll("'", "\"") |
| computeNamesAll = new JsonSlurperClassic().parseText("${computeNamesAll}") |
| if ("${role}" == "") return computeNamesAll |
| computeNames = [] |
| switchMac = "" |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| if(deployment_config.fabric_switches[i].containsKey("role")) { |
| if ("${deployment_config.fabric_switches[i].role}" == "${role}") |
| switchMac = "${deployment_config.fabric_switches[i].mac}" |
| } |
| } |
| if ("${switchMac}" != "") { |
| switchMac = switchMac.toLowerCase().replaceAll(':','') |
| // Get fabric IPs of compute nodes connected to the switch |
| try { |
| computeFabricIps = runHeadNodeCmd("sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts -j | jq '.[] | select(.location.elementId | contains(\\\"${switchMac}\\\")) | .ipAddresses' | grep -o '\\\"${fabricIpPrefix}.[1-9][0-9]*.[0-9]*\\\"'", |
| "-q").trim() |
| }catch (exception) { |
| return computeNames |
| } |
| computeFabricIps = "${computeFabricIps}".split() |
| computeFabricIps = new JsonSlurperClassic().parseText("${computeFabricIps}") |
| // Figure out which compute node connects to the switch |
| for (name in computeNamesAll) { |
| fabricIp = runComputeNodeCmd("${name}", "ip a | grep -o '${fabricIpPrefix}.[1-9][0-9]*.[0-9]*'").trim() |
| if (fabricIp in computeFabricIps) { |
| computeNames.add("${name}") |
| } |
| } |
| } |
| return computeNames |
| } |
| |
| /** |
| * Connects all fabric switches and compute nodes to ONOS |
| */ |
| def connectFabricAndComputeToOnos() { |
| // Kill existing switch connections |
| runFabricSwitchCmdAll("./killit || echo no ofagentapp running") |
| // Clean stale ONOS data |
| runOnosCliCmd("onos-fabric", "8101", "onos", "rocks", "wipe-out -r -j please") |
| // Connect switches to ONOS |
| for(int i=0; i < deployment_config.fabric_switches.size(); i++) { |
| runFabricSwitchCmd("${deployment_config.fabric_switches[i].ip}", |
| "${deployment_config.fabric_switches[i].user}", |
| "${deployment_config.fabric_switches[i].pass}", |
| "./connect -bg 2>&1 > ${deployment_config.fabric_switches[i].ip}.log", |
| "-qftn") |
| } |
| // Verify ONOS has recognized the switches |
| timeout(time: 5) { |
| waitUntil { |
| try { |
| num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric devices | grep available=true | wc -l\"").trim() |
| return num.toInteger() == deployment_config.fabric_switches.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| // Connect compute nodes to ONOS |
| runComputeNodeCmdAll("ping -c 1 ${fabricIpPrefix}.1.254", "-qftn") |
| // Verify ONOS has recognized the hosts |
| timeout(time: 5) { |
| waitUntil { |
| try { |
| num = runHeadNodeCmd("\"sshpass -p rocks ssh -q -oStrictHostKeyChecking=no -l onos -p 8101 onos-fabric hosts | grep id= | wc -l\"").trim() |
| return num.toInteger() == deployment_config.compute_nodes.size() |
| } catch (exception) { |
| return false |
| } |
| } |
| } |
| } |