alshabib | 08d0a1f | 2017-01-05 15:04:28 -0800 | [diff] [blame] | 1 | def filename = 'manifest-${branch}.xml' |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 2 | def manifestUrl = 'https://gerrit.opencord.org/manifest' |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 3 | |
alshabib | 41f74fd | 2017-01-07 09:48:00 -0800 | [diff] [blame] | 4 | node ('master') { |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 5 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
alshabib | 41f74fd | 2017-01-07 09:48:00 -0800 | [diff] [blame] | 6 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 7 | stage ("Generate and Copy Manifest file") { |
| 8 | sh returnStdout: true, script: 'repo manifest -r -o ' + filename |
| 9 | sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp' |
| 10 | } |
alshabib | 41f74fd | 2017-01-07 09:48:00 -0800 | [diff] [blame] | 11 | } |
| 12 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 13 | node ("${devNodeJenkinsName}") { |
| 14 | timeout (time: 240) { |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 15 | stage 'Checkout cord repo' |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 16 | checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true] |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 17 | |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 18 | dir('build') { |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 19 | try { |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 20 | stage ("Re-deploy head node and Build Vagrant box") { |
| 21 | parallel( |
| 22 | maasOps: { |
| 23 | sh "maas login maas http://${maasHeadIP}/MAAS/api/2.0 ${apiKey}" |
| 24 | sh "maas maas machine release ${headNodeMAASSystemId}" |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 25 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 26 | timeout(time: 15) { |
| 27 | waitUntil { |
| 28 | try { |
| 29 | sh "maas maas machine read ${headNodeMAASSystemId} | grep Ready" |
| 30 | return true |
| 31 | } catch (exception) { |
| 32 | return false |
| 33 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 34 | } |
alshabib | ef06994 | 2016-09-09 17:08:36 -0700 | [diff] [blame] | 35 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 36 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 37 | sh 'maas maas machines allocate' |
| 38 | sh "maas maas machine deploy ${headNodeMAASSystemId}" |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 39 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 40 | timeout(time: 30) { |
| 41 | waitUntil { |
| 42 | try { |
| 43 | sh "maas maas machine read ${headNodeMAASSystemId} | grep Deployed" |
| 44 | return true |
| 45 | } catch (exception) { |
| 46 | return false |
| 47 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 48 | } |
alshabib | ef06994 | 2016-09-09 17:08:36 -0700 | [diff] [blame] | 49 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 50 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 51 | }, vagrantOps: { |
| 52 | sh 'vagrant up corddev' |
| 53 | }, failFast : true |
| 54 | ) |
| 55 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 56 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 57 | stage ("Fetch CORD packages") { |
| 58 | sh 'vagrant ssh -c "cd /cord/build; ./gradlew fetch" corddev' |
| 59 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 60 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 61 | stage ("Build CORD Images") { |
| 62 | sh 'vagrant ssh -c "cd /cord/build; ./gradlew buildImages" corddev' |
| 63 | } |
alshabib | 61509fb | 2016-09-09 02:43:20 -0700 | [diff] [blame] | 64 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 65 | stage ("Downloading CORD POD configuration") { |
| 66 | sh 'vagrant ssh -c "cd /cord/build/config; git clone ${podConfigRepoUrl}" corddev' |
| 67 | } |
Jonathan Hart | 1719e9a | 2017-03-13 21:53:47 -0700 | [diff] [blame] | 68 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 69 | stage ("Publish to headnode") { |
| 70 | sh 'vagrant ssh -c "cd /cord/build; ./gradlew -PtargetReg=${headNodeIP}:5000 -PdeployConfig=config/pod-configs/${podConfigFileName} publish" corddev' |
| 71 | } |
alshabib | 61509fb | 2016-09-09 02:43:20 -0700 | [diff] [blame] | 72 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 73 | stage ("Deploy") { |
| 74 | sh 'vagrant ssh -c "cd /cord/build; ./gradlew -PtargetReg=${headNodeIP}:5000 -PdeployConfig=config/pod-configs/${podConfigFileName} deploy" corddev' |
| 75 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 76 | |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 77 | stage ("Power cycle compute nodes") { |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 78 | parallel( |
| 79 | compute_1: { |
| 80 | sh 'ipmitool -U ${computeNode1IPMIUser} -P ${computeNode1IPMIPass} -H ${computeNode1IPMIIP} power cycle' |
| 81 | }, compute_2: { |
| 82 | sh 'ipmitool -U ${computeNode2IPMIUser} -P ${computeNode2IPMIPass} -H ${computeNode2IPMIIP} power cycle' |
| 83 | }, failFast : true |
| 84 | ) |
| 85 | } |
alshabib | 7f3be8d | 2016-09-27 18:04:56 -0700 | [diff] [blame] | 86 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 87 | stage ("Wait for compute nodes to get deployed") { |
| 88 | sh 'ssh-keygen -f /home/${devNodeUser}/.ssh/known_hosts -R ${headNodeIP}' |
| 89 | def cordapikey = runHeadCmd("sudo maas-region-admin apikey --username ${headNodeUser}") |
| 90 | runHeadCmd("maas login pod-maas http://${headNodeIP}/MAAS/api/1.0 $cordapikey") |
| 91 | timeout(time: 45) { |
| 92 | waitUntil { |
| 93 | try { |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 94 | num = runHeadCmd("maas pod-maas nodes list | grep -i deployed | wc -l").trim() |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 95 | return num == '2' |
| 96 | } catch (exception) { |
| 97 | return false |
| 98 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 99 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 100 | } |
| 101 | } |
alshabib | 4e1c96e | 2016-09-28 16:26:24 -0700 | [diff] [blame] | 102 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 103 | stage ("Wait for computes nodes to be provisioned") { |
| 104 | ip = runHeadCmd("docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner").trim() |
| 105 | timeout(time:45) { |
| 106 | waitUntil { |
| 107 | try { |
| 108 | out = runHeadCmd("curl -sS http://$ip:4243/provision/ | jq -c '.[] | select(.status | contains(2))'").trim() |
| 109 | return out != "" |
| 110 | } catch (exception) { |
| 111 | return false |
| 112 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 113 | } |
alshabib | 7f3be8d | 2016-09-27 18:04:56 -0700 | [diff] [blame] | 114 | } |
| 115 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 116 | |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 117 | def fabricConfExists = true |
| 118 | try { |
| 119 | print "${fabricMACs}" |
| 120 | } catch (err) { |
| 121 | fabricConfExists = false |
| 122 | } |
| 123 | if (fabricConfExists) { |
| 124 | def fabric_macs = multiStringToArray("${fabricMACs}") |
| 125 | def fabric_ips = multiStringToArray("${fabricIPs}") |
| 126 | |
| 127 | stage("Reserve IPs for fabric switches and restart maas-dhcp service") { |
| 128 | for(int i=0; i < fabric_macs.length; i++) { |
| 129 | def append = ""; |
| 130 | if (i!=0) { |
| 131 | append = "-a"; |
| 132 | } |
| 133 | def str = createMACIPbindingStr(i+1, fabric_macs[i], fabric_ips[i]) |
| 134 | runHeadCmd("echo -e $str '|' sudo tee $append /etc/dhcp/dhcpd.reservations > /dev/null") |
| 135 | } |
| 136 | runHeadCmd("sudo restart maas-dhcpd") |
| 137 | runHeadCmd("cord harvest go") |
| 138 | } |
| 139 | |
| 140 | stage ("Wait for fabric switches to get deployed") { |
| 141 | for(int i=0; i < fabric_ips.length; i++) { |
| 142 | runFabricCmd("${fabric_ips[i]}", "sudo onl-onie-boot-mode install") |
| 143 | runFabricCmd("${fabric_ips[i]}", "sudo reboot") |
| 144 | } |
| 145 | timeout(time: 45) { |
| 146 | waitUntil { |
| 147 | try { |
| 148 | def harvestCompleted = runHeadCmd("cord harvest list '|' grep -i fabric '|' wc -l").trim() |
| 149 | return harvestCompleted == fabric_macs.length.toString() |
| 150 | } catch (exception) { |
| 151 | return false |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | stage ("Wait for fabric switches to be provisioned") { |
| 158 | timeout(time:45) { |
| 159 | waitUntil { |
| 160 | try { |
| 161 | def provCompleted = 0 |
| 162 | for(int i=0; i < fabric_ips.length; i++) { |
| 163 | def count = runHeadCmd("cord prov list '|' grep -i ${fabric_ips[i]} '|' grep -i complete '|' wc -l").trim() |
| 164 | print "Count: ${count}" |
| 165 | provCompleted = provCompleted + count.toInteger() |
| 166 | print "New prov completed: ${provCompleted}" |
| 167 | } |
| 168 | return provCompleted == fabric_ips.length |
| 169 | } catch (exception) { |
| 170 | return false |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 | |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 177 | stage ("Trigger Build") { |
| 178 | url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build' |
| 179 | httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201' |
| 180 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 181 | |
| 182 | currentBuild.result = 'SUCCESS' |
| 183 | } catch (err) { |
| 184 | currentBuild.result = 'FAILURE' |
Jonathan Hart | 1719e9a | 2017-03-13 21:53:47 -0700 | [diff] [blame] | 185 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 186 | } finally { |
| 187 | sh 'vagrant destroy -f corddev' |
Jonathan Hart | 1719e9a | 2017-03-13 21:53:47 -0700 | [diff] [blame] | 188 | sh 'rm -rf config/pod-configs' |
alshabib | 7f3be8d | 2016-09-27 18:04:56 -0700 | [diff] [blame] | 189 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 190 | echo "RESULT: ${currentBuild.result}" |
| 191 | } |
alshabib | 1930266 | 2017-01-05 14:27:41 -0800 | [diff] [blame] | 192 | } |
alshabib | 152823c | 2016-09-07 23:49:12 -0700 | [diff] [blame] | 193 | } |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 194 | |
| 195 | /** |
| 196 | * Transforms a multiline string into an array. |
| 197 | * Each line is a value of the array. |
| 198 | * |
| 199 | * @param the multiline string to transform |
| 200 | * @return the computed array |
| 201 | */ |
| 202 | def multiStringToArray(str) { |
| 203 | return str.split("\\r?\\n") as String[] |
| 204 | } |
| 205 | |
| 206 | /** |
| 207 | * Returns a string used to bind IPs and MAC addresses, substituting the values |
| 208 | * given. |
| 209 | * |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 210 | * @param counter the counter used to generate the host name |
| 211 | * @param mac the MAC address to substitute |
| 212 | * @param ip the IP address to substitute |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 213 | */ |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 214 | def createMACIPbindingStr(counter, mac, ip) { |
| 215 | return """\\'host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}\\'""" |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 216 | } |
| 217 | |
| 218 | /** |
| 219 | * Runs a command on the head node. |
| 220 | * |
| 221 | * @param command the command to run on the head node |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 222 | * @return the output of the command |
Luca Prete | 283f80a | 2017-05-18 15:21:24 -0700 | [diff] [blame] | 223 | */ |
| 224 | def runHeadCmd(command) { |
| 225 | return sh(returnStdout: true, script: "sshpass -p ${headNodePass} ssh -oStrictHostKeyChecking=no -l ${headNodeUser} ${headNodeIP} ${command}") |
| 226 | } |
Luca Prete | 607c2d5 | 2017-05-24 09:56:06 -0700 | [diff] [blame] | 227 | |
| 228 | /** |
| 229 | * Runs a command on a fabric switch. |
| 230 | * |
| 231 | * @param ip the mgmt IP of the fabric switch, reachable from the head node |
| 232 | * @param fabric_command the command to run on the fabric switch |
| 233 | * @return the output of the command |
| 234 | */ |
| 235 | def runFabricCmd(ip, command) { |
| 236 | return runHeadCmd("sshpass -p ${fabricPass} ssh -oStrictHostKeyChecking=no -l ${fabricUser} ${ip} ${command}") |
| 237 | } |