blob: 96d22877bd06f8f29e133a3b3ff07fcc51eb2966 [file] [log] [blame]
alshabib08d0a1f2017-01-05 15:04:28 -08001def filename = 'manifest-${branch}.xml'
Luca Prete283f80a2017-05-18 15:21:24 -07002def manifestUrl = 'https://gerrit.opencord.org/manifest'
alshabib4e1c96e2016-09-28 16:26:24 -07003
alshabib41f74fd2017-01-07 09:48:00 -08004node ('master') {
Luca Prete283f80a2017-05-18 15:21:24 -07005 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
alshabib41f74fd2017-01-07 09:48:00 -08006
Luca Prete283f80a2017-05-18 15:21:24 -07007 stage ("Generate and Copy Manifest file") {
8 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
9 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
10 }
alshabib41f74fd2017-01-07 09:48:00 -080011}
12
Luca Prete283f80a2017-05-18 15:21:24 -070013node ("${devNodeJenkinsName}") {
14 timeout (time: 240) {
alshabib19302662017-01-05 14:27:41 -080015 stage 'Checkout cord repo'
Luca Prete283f80a2017-05-18 15:21:24 -070016 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
alshabib19302662017-01-05 14:27:41 -080017
alshabib19302662017-01-05 14:27:41 -080018 dir('build') {
alshabib19302662017-01-05 14:27:41 -080019 try {
Luca Prete283f80a2017-05-18 15:21:24 -070020 stage ("Re-deploy head node and Build Vagrant box") {
21 parallel(
22 maasOps: {
23 sh "maas login maas http://${maasHeadIP}/MAAS/api/2.0 ${apiKey}"
24 sh "maas maas machine release ${headNodeMAASSystemId}"
alshabib19302662017-01-05 14:27:41 -080025
Luca Prete283f80a2017-05-18 15:21:24 -070026 timeout(time: 15) {
27 waitUntil {
28 try {
29 sh "maas maas machine read ${headNodeMAASSystemId} | grep Ready"
30 return true
31 } catch (exception) {
32 return false
33 }
alshabib19302662017-01-05 14:27:41 -080034 }
alshabibef069942016-09-09 17:08:36 -070035 }
alshabib4e1c96e2016-09-28 16:26:24 -070036
Luca Prete283f80a2017-05-18 15:21:24 -070037 sh 'maas maas machines allocate'
38 sh "maas maas machine deploy ${headNodeMAASSystemId}"
alshabib4e1c96e2016-09-28 16:26:24 -070039
Luca Prete283f80a2017-05-18 15:21:24 -070040 timeout(time: 30) {
41 waitUntil {
42 try {
43 sh "maas maas machine read ${headNodeMAASSystemId} | grep Deployed"
44 return true
45 } catch (exception) {
46 return false
47 }
alshabib19302662017-01-05 14:27:41 -080048 }
alshabibef069942016-09-09 17:08:36 -070049 }
alshabib4e1c96e2016-09-28 16:26:24 -070050
Luca Prete283f80a2017-05-18 15:21:24 -070051 }, vagrantOps: {
52 sh 'vagrant up corddev'
53 }, failFast : true
54 )
55 }
alshabib4e1c96e2016-09-28 16:26:24 -070056
Luca Prete283f80a2017-05-18 15:21:24 -070057 stage ("Fetch CORD packages") {
58 sh 'vagrant ssh -c "cd /cord/build; ./gradlew fetch" corddev'
59 }
alshabib4e1c96e2016-09-28 16:26:24 -070060
Luca Prete283f80a2017-05-18 15:21:24 -070061 stage ("Build CORD Images") {
62 sh 'vagrant ssh -c "cd /cord/build; ./gradlew buildImages" corddev'
63 }
alshabib61509fb2016-09-09 02:43:20 -070064
Luca Prete283f80a2017-05-18 15:21:24 -070065 stage ("Downloading CORD POD configuration") {
66 sh 'vagrant ssh -c "cd /cord/build/config; git clone ${podConfigRepoUrl}" corddev'
67 }
Jonathan Hart1719e9a2017-03-13 21:53:47 -070068
Luca Prete283f80a2017-05-18 15:21:24 -070069 stage ("Publish to headnode") {
70 sh 'vagrant ssh -c "cd /cord/build; ./gradlew -PtargetReg=${headNodeIP}:5000 -PdeployConfig=config/pod-configs/${podConfigFileName} publish" corddev'
71 }
alshabib61509fb2016-09-09 02:43:20 -070072
Luca Prete283f80a2017-05-18 15:21:24 -070073 stage ("Deploy") {
74 sh 'vagrant ssh -c "cd /cord/build; ./gradlew -PtargetReg=${headNodeIP}:5000 -PdeployConfig=config/pod-configs/${podConfigFileName} deploy" corddev'
75 }
alshabib4e1c96e2016-09-28 16:26:24 -070076
Luca Prete607c2d52017-05-24 09:56:06 -070077 stage ("Power cycle compute nodes") {
Luca Prete283f80a2017-05-18 15:21:24 -070078 parallel(
79 compute_1: {
80 sh 'ipmitool -U ${computeNode1IPMIUser} -P ${computeNode1IPMIPass} -H ${computeNode1IPMIIP} power cycle'
81 }, compute_2: {
82 sh 'ipmitool -U ${computeNode2IPMIUser} -P ${computeNode2IPMIPass} -H ${computeNode2IPMIIP} power cycle'
83 }, failFast : true
84 )
85 }
alshabib7f3be8d2016-09-27 18:04:56 -070086
Luca Prete283f80a2017-05-18 15:21:24 -070087 stage ("Wait for compute nodes to get deployed") {
88 sh 'ssh-keygen -f /home/${devNodeUser}/.ssh/known_hosts -R ${headNodeIP}'
89 def cordapikey = runHeadCmd("sudo maas-region-admin apikey --username ${headNodeUser}")
90 runHeadCmd("maas login pod-maas http://${headNodeIP}/MAAS/api/1.0 $cordapikey")
91 timeout(time: 45) {
92 waitUntil {
93 try {
Luca Prete607c2d52017-05-24 09:56:06 -070094 num = runHeadCmd("maas pod-maas nodes list | grep -i deployed | wc -l").trim()
Luca Prete283f80a2017-05-18 15:21:24 -070095 return num == '2'
96 } catch (exception) {
97 return false
98 }
alshabib19302662017-01-05 14:27:41 -080099 }
alshabib4e1c96e2016-09-28 16:26:24 -0700100 }
101 }
alshabib4e1c96e2016-09-28 16:26:24 -0700102
Luca Prete283f80a2017-05-18 15:21:24 -0700103 stage ("Wait for computes nodes to be provisioned") {
104 ip = runHeadCmd("docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner").trim()
105 timeout(time:45) {
106 waitUntil {
107 try {
108 out = runHeadCmd("curl -sS http://$ip:4243/provision/ | jq -c '.[] | select(.status | contains(2))'").trim()
109 return out != ""
110 } catch (exception) {
111 return false
112 }
alshabib19302662017-01-05 14:27:41 -0800113 }
alshabib7f3be8d2016-09-27 18:04:56 -0700114 }
115 }
alshabib19302662017-01-05 14:27:41 -0800116
Luca Prete607c2d52017-05-24 09:56:06 -0700117 def fabricConfExists = true
118 try {
119 print "${fabricMACs}"
120 } catch (err) {
121 fabricConfExists = false
122 }
123 if (fabricConfExists) {
124 def fabric_macs = multiStringToArray("${fabricMACs}")
125 def fabric_ips = multiStringToArray("${fabricIPs}")
126
127 stage("Reserve IPs for fabric switches and restart maas-dhcp service") {
128 for(int i=0; i < fabric_macs.length; i++) {
129 def append = "";
130 if (i!=0) {
131 append = "-a";
132 }
133 def str = createMACIPbindingStr(i+1, fabric_macs[i], fabric_ips[i])
134 runHeadCmd("echo -e $str '|' sudo tee $append /etc/dhcp/dhcpd.reservations > /dev/null")
135 }
136 runHeadCmd("sudo restart maas-dhcpd")
137 runHeadCmd("cord harvest go")
138 }
139
140 stage ("Wait for fabric switches to get deployed") {
141 for(int i=0; i < fabric_ips.length; i++) {
142 runFabricCmd("${fabric_ips[i]}", "sudo onl-onie-boot-mode install")
143 runFabricCmd("${fabric_ips[i]}", "sudo reboot")
144 }
145 timeout(time: 45) {
146 waitUntil {
147 try {
148 def harvestCompleted = runHeadCmd("cord harvest list '|' grep -i fabric '|' wc -l").trim()
149 return harvestCompleted == fabric_macs.length.toString()
150 } catch (exception) {
151 return false
152 }
153 }
154 }
155 }
156
157 stage ("Wait for fabric switches to be provisioned") {
158 timeout(time:45) {
159 waitUntil {
160 try {
161 def provCompleted = 0
162 for(int i=0; i < fabric_ips.length; i++) {
163 def count = runHeadCmd("cord prov list '|' grep -i ${fabric_ips[i]} '|' grep -i complete '|' wc -l").trim()
164 print "Count: ${count}"
165 provCompleted = provCompleted + count.toInteger()
166 print "New prov completed: ${provCompleted}"
167 }
168 return provCompleted == fabric_ips.length
169 } catch (exception) {
170 return false
171 }
172 }
173 }
174 }
175 }
176
Luca Prete283f80a2017-05-18 15:21:24 -0700177 stage ("Trigger Build") {
178 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
179 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
180 }
alshabib19302662017-01-05 14:27:41 -0800181
182 currentBuild.result = 'SUCCESS'
183 } catch (err) {
184 currentBuild.result = 'FAILURE'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700185 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
alshabib19302662017-01-05 14:27:41 -0800186 } finally {
187 sh 'vagrant destroy -f corddev'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700188 sh 'rm -rf config/pod-configs'
alshabib7f3be8d2016-09-27 18:04:56 -0700189 }
alshabib19302662017-01-05 14:27:41 -0800190 echo "RESULT: ${currentBuild.result}"
191 }
alshabib19302662017-01-05 14:27:41 -0800192 }
alshabib152823c2016-09-07 23:49:12 -0700193}
Luca Prete283f80a2017-05-18 15:21:24 -0700194
195/**
196 * Transforms a multiline string into an array.
197 * Each line is a value of the array.
198 *
199 * @param the multiline string to transform
200 * @return the computed array
201 */
202def multiStringToArray(str) {
203 return str.split("\\r?\\n") as String[]
204}
205
206/**
207 * Returns a string used to bind IPs and MAC addresses, substituting the values
208 * given.
209 *
Luca Prete607c2d52017-05-24 09:56:06 -0700210 * @param counter the counter used to generate the host name
211 * @param mac the MAC address to substitute
212 * @param ip the IP address to substitute
Luca Prete283f80a2017-05-18 15:21:24 -0700213 */
Luca Prete607c2d52017-05-24 09:56:06 -0700214def createMACIPbindingStr(counter, mac, ip) {
215 return """\\'host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}\\'"""
Luca Prete283f80a2017-05-18 15:21:24 -0700216}
217
218/**
219 * Runs a command on the head node.
220 *
221 * @param command the command to run on the head node
Luca Prete607c2d52017-05-24 09:56:06 -0700222 * @return the output of the command
Luca Prete283f80a2017-05-18 15:21:24 -0700223 */
224def runHeadCmd(command) {
225 return sh(returnStdout: true, script: "sshpass -p ${headNodePass} ssh -oStrictHostKeyChecking=no -l ${headNodeUser} ${headNodeIP} ${command}")
226}
Luca Prete607c2d52017-05-24 09:56:06 -0700227
228/**
229 * Runs a command on a fabric switch.
230 *
231 * @param ip the mgmt IP of the fabric switch, reachable from the head node
232 * @param fabric_command the command to run on the fabric switch
233 * @return the output of the command
234 */
235def runFabricCmd(ip, command) {
236 return runHeadCmd("sshpass -p ${fabricPass} ssh -oStrictHostKeyChecking=no -l ${fabricUser} ${ip} ${command}")
237}