blob: f41b31f1a7682d3b105af547442ad0710d612ac1 [file] [log] [blame]
alshabib08d0a1f2017-01-05 15:04:28 -08001def filename = 'manifest-${branch}.xml'
Luca Prete283f80a2017-05-18 15:21:24 -07002def manifestUrl = 'https://gerrit.opencord.org/manifest'
Luca Prete782d9412017-06-29 10:21:43 +02003def config = null;
alshabib4e1c96e2016-09-28 16:26:24 -07004
alshabib41f74fd2017-01-07 09:48:00 -08005node ('master') {
Luca Prete283f80a2017-05-18 15:21:24 -07006 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
alshabib41f74fd2017-01-07 09:48:00 -08007
Luca Prete283f80a2017-05-18 15:21:24 -07008 stage ("Generate and Copy Manifest file") {
9 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
10 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
11 }
Luca Prete782d9412017-06-29 10:21:43 +020012
Jonathan Hart9bd4a822017-08-10 09:08:12 -070013 stage ("Parse deployment configuration file") {
Luca Prete782d9412017-06-29 10:21:43 +020014 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
Luca Prete1125f602017-06-30 14:39:53 +020015 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
Luca Prete782d9412017-06-29 10:21:43 +020016 config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
17 }
alshabib41f74fd2017-01-07 09:48:00 -080018}
19
Luca Prete782d9412017-06-29 10:21:43 +020020node ("${config.dev_node.name}") {
Luca Prete283f80a2017-05-18 15:21:24 -070021 timeout (time: 240) {
Luca Prete782d9412017-06-29 10:21:43 +020022 stage ('Checkout cord repo') {
23 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
24 }
alshabib19302662017-01-05 14:27:41 -080025
Luca Prete782d9412017-06-29 10:21:43 +020026 dir('build') {
alshabib19302662017-01-05 14:27:41 -080027 try {
Luca Prete283f80a2017-05-18 15:21:24 -070028 stage ("Re-deploy head node and Build Vagrant box") {
29 parallel(
30 maasOps: {
Luca Prete782d9412017-06-29 10:21:43 +020031 sh "maas login maas http://${config.maas.ip}/MAAS/api/2.0 ${config.maas.api_key}"
32 sh "maas maas machine release ${config.maas.head_system_id}"
alshabib19302662017-01-05 14:27:41 -080033
Luca Prete283f80a2017-05-18 15:21:24 -070034 timeout(time: 15) {
35 waitUntil {
36 try {
Luca Prete782d9412017-06-29 10:21:43 +020037 sh "maas maas machine read ${config.maas.head_system_id} | grep Ready"
Luca Prete283f80a2017-05-18 15:21:24 -070038 return true
39 } catch (exception) {
40 return false
41 }
alshabib19302662017-01-05 14:27:41 -080042 }
alshabibef069942016-09-09 17:08:36 -070043 }
alshabib4e1c96e2016-09-28 16:26:24 -070044
Luca Prete283f80a2017-05-18 15:21:24 -070045 sh 'maas maas machines allocate'
Luca Prete782d9412017-06-29 10:21:43 +020046 sh "maas maas machine deploy ${config.maas.head_system_id}"
alshabib4e1c96e2016-09-28 16:26:24 -070047
Luca Prete283f80a2017-05-18 15:21:24 -070048 timeout(time: 30) {
49 waitUntil {
50 try {
Luca Prete782d9412017-06-29 10:21:43 +020051 sh "maas maas machine read ${config.maas.head_system_id} | grep Deployed"
Luca Prete283f80a2017-05-18 15:21:24 -070052 return true
53 } catch (exception) {
54 return false
55 }
alshabib19302662017-01-05 14:27:41 -080056 }
alshabibef069942016-09-09 17:08:36 -070057 }
alshabib4e1c96e2016-09-28 16:26:24 -070058
Luca Prete283f80a2017-05-18 15:21:24 -070059 }, vagrantOps: {
60 sh 'vagrant up corddev'
61 }, failFast : true
62 )
63 }
alshabib4e1c96e2016-09-28 16:26:24 -070064
Luca Prete283f80a2017-05-18 15:21:24 -070065 stage ("Fetch CORD packages") {
Luca Prete782d9412017-06-29 10:21:43 +020066 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew fetch\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070067 }
alshabib4e1c96e2016-09-28 16:26:24 -070068
Luca Prete283f80a2017-05-18 15:21:24 -070069 stage ("Build CORD Images") {
Luca Prete782d9412017-06-29 10:21:43 +020070 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew buildImages\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070071 }
alshabib61509fb2016-09-09 02:43:20 -070072
Luca Prete283f80a2017-05-18 15:21:24 -070073 stage ("Downloading CORD POD configuration") {
Luca Prete1125f602017-06-30 14:39:53 +020074 sh "vagrant ssh -c \"cd /opt/cord/build/config; git clone -b ${branch} ${config.pod_config.repo_url}\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070075 }
Jonathan Hart1719e9a2017-03-13 21:53:47 -070076
Luca Prete283f80a2017-05-18 15:21:24 -070077 stage ("Publish to headnode") {
Luca Prete782d9412017-06-29 10:21:43 +020078 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} publish\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070079 }
alshabib61509fb2016-09-09 02:43:20 -070080
Luca Prete283f80a2017-05-18 15:21:24 -070081 stage ("Deploy") {
Luca Prete782d9412017-06-29 10:21:43 +020082 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} deploy\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070083 }
alshabib4e1c96e2016-09-28 16:26:24 -070084
Luca Prete607c2d52017-05-24 09:56:06 -070085 stage ("Power cycle compute nodes") {
Luca Prete782d9412017-06-29 10:21:43 +020086 for(int i=0; i < config.compute_nodes.size(); i++) {
87 sh "ipmitool -U ${config.compute_nodes[i].ipmi.user} -P ${config.compute_nodes[i].ipmi.pass} -H ${config.compute_nodes[i].ipmi.ip} power cycle"
88 }
Luca Prete283f80a2017-05-18 15:21:24 -070089 }
alshabib7f3be8d2016-09-27 18:04:56 -070090
Luca Prete283f80a2017-05-18 15:21:24 -070091 stage ("Wait for compute nodes to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +020092 sh "ssh-keygen -f /home/${config.dev_node.user}/.ssh/known_hosts -R ${config.head.ip}"
93 def cordApiKey = runCmd("${config.head.ip}",
94 "${config.head.user}",
95 "${config.head.pass}",
96 "sudo maas-region-admin apikey --username ${config.head.user}")
97 runCmd("${config.head.ip}",
98 "${config.head.user}",
99 "${config.head.pass}",
100 "maas login pod-maas http://${config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
Luca Prete283f80a2017-05-18 15:21:24 -0700101 timeout(time: 45) {
102 waitUntil {
103 try {
Luca Prete782d9412017-06-29 10:21:43 +0200104 num = runCmd("${config.head.ip}",
105 "${config.head.user}",
106 "${config.head.pass}",
107 "maas pod-maas nodes list | grep -i deployed | wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200108 return num.toInteger() == config.compute_nodes.size()
Luca Prete283f80a2017-05-18 15:21:24 -0700109 } catch (exception) {
110 return false
111 }
alshabib19302662017-01-05 14:27:41 -0800112 }
alshabib4e1c96e2016-09-28 16:26:24 -0700113 }
114 }
alshabib4e1c96e2016-09-28 16:26:24 -0700115
Luca Prete283f80a2017-05-18 15:21:24 -0700116 stage ("Wait for computes nodes to be provisioned") {
Luca Prete782d9412017-06-29 10:21:43 +0200117 ip = runCmd("${config.head.ip}",
118 "${config.head.user}",
119 "${config.head.pass}",
120 "docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner").trim()
Luca Prete283f80a2017-05-18 15:21:24 -0700121 timeout(time:45) {
122 waitUntil {
123 try {
Luca Prete782d9412017-06-29 10:21:43 +0200124 out = runCmd("${config.head.ip}",
125 "${config.head.user}",
126 "${config.head.pass}",
Luca Prete512389a2017-07-07 12:33:29 +0200127 "curl -sS http://$ip:4243/provision/ | jq -c \".[] | select(.status | contains(${config.compute_nodes.size()}))\"".trim())
Luca Prete283f80a2017-05-18 15:21:24 -0700128 return out != ""
129 } catch (exception) {
130 return false
131 }
alshabib19302662017-01-05 14:27:41 -0800132 }
alshabib7f3be8d2016-09-27 18:04:56 -0700133 }
134 }
alshabib19302662017-01-05 14:27:41 -0800135
Luca Prete782d9412017-06-29 10:21:43 +0200136 if (config.fabric_switches != null) {
Luca Prete607c2d52017-05-24 09:56:06 -0700137 stage("Reserve IPs for fabric switches and restart maas-dhcp service") {
Luca Prete782d9412017-06-29 10:21:43 +0200138 for(int i=0; i < config.fabric_switches.size(); i++) {
Luca Prete607c2d52017-05-24 09:56:06 -0700139 def append = "";
140 if (i!=0) {
141 append = "-a";
142 }
Luca Prete782d9412017-06-29 10:21:43 +0200143 def str = createMACIPbindingStr(i+1,
144 "${config.fabric_switches[i].mac}",
145 "${config.fabric_switches[i].ip}")
146 runCmd("${config.head.ip}",
147 "${config.head.user}",
148 "${config.head.pass}",
149 "echo -e $str '|' sudo tee $append /etc/dhcp/dhcpd.reservations > /dev/null")
Luca Prete607c2d52017-05-24 09:56:06 -0700150 }
Luca Prete782d9412017-06-29 10:21:43 +0200151 runCmd("${config.head.ip}",
152 "${config.head.user}",
153 "${config.head.pass}",
154 "sudo restart maas-dhcpd")
155
156 runCmd("${config.head.ip}",
157 "${config.head.user}",
158 "${config.head.pass}",
159 "cord harvest go")
Luca Prete607c2d52017-05-24 09:56:06 -0700160 }
161
162 stage ("Wait for fabric switches to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +0200163 for(int i=0; i < config.fabric_switches.size(); i++) {
164 runFabricCmd("${config.head.ip}",
165 "${config.head.user}",
166 "${config.head.pass}",
167 "${config.fabric_switches[i].ip}",
168 "${config.fabric_switches[i].user}",
169 "${config.fabric_switches[i].pass}",
170 "sudo onl-onie-boot-mode install")
171
172 runFabricCmd("${config.head.ip}",
173 "${config.head.user}",
174 "${config.head.pass}",
175 "${config.fabric_switches[i].ip}",
176 "${config.fabric_switches[i].user}",
177 "${config.fabric_switches[i].pass}",
178 "sudo reboot")
Luca Prete607c2d52017-05-24 09:56:06 -0700179 }
180 timeout(time: 45) {
181 waitUntil {
182 try {
Luca Prete782d9412017-06-29 10:21:43 +0200183 def harvestCompleted = runCmd("${config.head.ip}",
184 "${config.head.user}",
185 "${config.head.pass}",
186 "cord harvest list '|' grep -i fabric '|' wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200187 return harvestCompleted.toInteger() == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700188 } catch (exception) {
189 return false
190 }
191 }
192 }
193 }
194
195 stage ("Wait for fabric switches to be provisioned") {
196 timeout(time:45) {
197 waitUntil {
198 try {
199 def provCompleted = 0
Luca Prete782d9412017-06-29 10:21:43 +0200200 for(int i=0; i < config.fabric_switches.size(); i++) {
201 def count = runCmd("${config.head.ip}",
202 "${config.head.user}",
203 "${config.head.pass}",
204 "cord prov list '|' grep -i ${config.fabric_switches[i].ip} '|' grep -i complete '|' wc -l").trim()
Luca Prete607c2d52017-05-24 09:56:06 -0700205 provCompleted = provCompleted + count.toInteger()
Luca Prete607c2d52017-05-24 09:56:06 -0700206 }
Luca Prete782d9412017-06-29 10:21:43 +0200207 return provCompleted == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700208 } catch (exception) {
209 return false
210 }
211 }
212 }
213 }
214 }
215
Luca Prete782d9412017-06-29 10:21:43 +0200216 if (config.make_release == true) {
217 stage ("Trigger Build") {
218 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
219 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
220 }
Luca Prete283f80a2017-05-18 15:21:24 -0700221 }
alshabib19302662017-01-05 14:27:41 -0800222
223 currentBuild.result = 'SUCCESS'
224 } catch (err) {
225 currentBuild.result = 'FAILURE'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700226 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
alshabib19302662017-01-05 14:27:41 -0800227 } finally {
228 sh 'vagrant destroy -f corddev'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700229 sh 'rm -rf config/pod-configs'
alshabib7f3be8d2016-09-27 18:04:56 -0700230 }
alshabib19302662017-01-05 14:27:41 -0800231 echo "RESULT: ${currentBuild.result}"
232 }
alshabib19302662017-01-05 14:27:41 -0800233 }
alshabib152823c2016-09-07 23:49:12 -0700234}
Luca Prete283f80a2017-05-18 15:21:24 -0700235
236/**
Luca Prete283f80a2017-05-18 15:21:24 -0700237 * Returns a string used to bind IPs and MAC addresses, substituting the values
238 * given.
239 *
Luca Prete607c2d52017-05-24 09:56:06 -0700240 * @param counter the counter used to generate the host name
241 * @param mac the MAC address to substitute
242 * @param ip the IP address to substitute
Luca Prete283f80a2017-05-18 15:21:24 -0700243 */
Luca Prete607c2d52017-05-24 09:56:06 -0700244def createMACIPbindingStr(counter, mac, ip) {
245 return """\\'host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}\\'"""
Luca Prete283f80a2017-05-18 15:21:24 -0700246}
247
248/**
Luca Prete782d9412017-06-29 10:21:43 +0200249 * Runs a command on a remote host using sshpass.
Luca Prete283f80a2017-05-18 15:21:24 -0700250 *
Luca Prete782d9412017-06-29 10:21:43 +0200251 * @param ip the node IP address
252 * @param user the node user name
253 * @param pass the node password
254 * @param command the command to run
Luca Prete607c2d52017-05-24 09:56:06 -0700255 * @return the output of the command
Luca Prete283f80a2017-05-18 15:21:24 -0700256 */
Luca Prete782d9412017-06-29 10:21:43 +0200257def runCmd(ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200258 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}")
Luca Prete283f80a2017-05-18 15:21:24 -0700259}
Luca Prete607c2d52017-05-24 09:56:06 -0700260
261/**
262 * Runs a command on a fabric switch.
263 *
Luca Prete782d9412017-06-29 10:21:43 +0200264 * @param headIp the head node IP address
265 * @param headUser the head node user name
266 * @param headPass the head node password
Luca Prete607c2d52017-05-24 09:56:06 -0700267 * @param ip the mgmt IP of the fabric switch, reachable from the head node
Luca Prete782d9412017-06-29 10:21:43 +0200268 * @param user the mgmt user name of the fabric switch
269 * @param pass the mgmt password of the fabric switch
270 * @param command the command to run on the fabric switch
Luca Prete607c2d52017-05-24 09:56:06 -0700271 * @return the output of the command
272 */
Luca Preted642a592017-06-30 11:11:35 +0200273def runFabricCmd(headIp, headUser, headPass, ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200274 return sh(returnStdout: true, script: "sshpass -p ${headPass} ssh -oStrictHostKeyChecking=no -l ${headUser} ${headIp} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
Jonathan Hart9bd4a822017-08-10 09:08:12 -0700275}