blob: 4c542b77376bdb45778680f3db95a981a50a3172 [file] [log] [blame]
alshabib08d0a1f2017-01-05 15:04:28 -08001def filename = 'manifest-${branch}.xml'
Luca Prete283f80a2017-05-18 15:21:24 -07002def manifestUrl = 'https://gerrit.opencord.org/manifest'
Luca Prete782d9412017-06-29 10:21:43 +02003def config = null;
alshabib4e1c96e2016-09-28 16:26:24 -07004
alshabib41f74fd2017-01-07 09:48:00 -08005node ('master') {
Luca Prete283f80a2017-05-18 15:21:24 -07006 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
alshabib41f74fd2017-01-07 09:48:00 -08007
Luca Prete283f80a2017-05-18 15:21:24 -07008 stage ("Generate and Copy Manifest file") {
9 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
10 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
11 }
Luca Prete782d9412017-06-29 10:21:43 +020012
Jonathan Hart9bd4a822017-08-10 09:08:12 -070013 stage ("Parse deployment configuration file") {
Luca Prete782d9412017-06-29 10:21:43 +020014 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
Luca Prete1125f602017-06-30 14:39:53 +020015 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
Luca Prete782d9412017-06-29 10:21:43 +020016 config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
17 }
alshabib41f74fd2017-01-07 09:48:00 -080018}
19
Luca Prete782d9412017-06-29 10:21:43 +020020node ("${config.dev_node.name}") {
Luca Prete283f80a2017-05-18 15:21:24 -070021 timeout (time: 240) {
Luca Prete782d9412017-06-29 10:21:43 +020022 stage ('Checkout cord repo') {
23 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
24 }
alshabib19302662017-01-05 14:27:41 -080025
Luca Prete782d9412017-06-29 10:21:43 +020026 dir('build') {
alshabib19302662017-01-05 14:27:41 -080027 try {
Luca Prete283f80a2017-05-18 15:21:24 -070028 stage ("Re-deploy head node and Build Vagrant box") {
29 parallel(
30 maasOps: {
Luca Prete782d9412017-06-29 10:21:43 +020031 sh "maas login maas http://${config.maas.ip}/MAAS/api/2.0 ${config.maas.api_key}"
32 sh "maas maas machine release ${config.maas.head_system_id}"
alshabib19302662017-01-05 14:27:41 -080033
Luca Prete283f80a2017-05-18 15:21:24 -070034 timeout(time: 15) {
35 waitUntil {
36 try {
Luca Prete782d9412017-06-29 10:21:43 +020037 sh "maas maas machine read ${config.maas.head_system_id} | grep Ready"
Luca Prete283f80a2017-05-18 15:21:24 -070038 return true
39 } catch (exception) {
40 return false
41 }
alshabib19302662017-01-05 14:27:41 -080042 }
alshabibef069942016-09-09 17:08:36 -070043 }
alshabib4e1c96e2016-09-28 16:26:24 -070044
Luca Prete283f80a2017-05-18 15:21:24 -070045 sh 'maas maas machines allocate'
Luca Prete782d9412017-06-29 10:21:43 +020046 sh "maas maas machine deploy ${config.maas.head_system_id}"
alshabib4e1c96e2016-09-28 16:26:24 -070047
Luca Prete283f80a2017-05-18 15:21:24 -070048 timeout(time: 30) {
49 waitUntil {
50 try {
Luca Prete782d9412017-06-29 10:21:43 +020051 sh "maas maas machine read ${config.maas.head_system_id} | grep Deployed"
Luca Prete283f80a2017-05-18 15:21:24 -070052 return true
53 } catch (exception) {
54 return false
55 }
alshabib19302662017-01-05 14:27:41 -080056 }
alshabibef069942016-09-09 17:08:36 -070057 }
alshabib4e1c96e2016-09-28 16:26:24 -070058
Luca Prete283f80a2017-05-18 15:21:24 -070059 }, vagrantOps: {
60 sh 'vagrant up corddev'
61 }, failFast : true
62 )
63 }
alshabib4e1c96e2016-09-28 16:26:24 -070064
Luca Prete283f80a2017-05-18 15:21:24 -070065 stage ("Fetch CORD packages") {
Luca Prete782d9412017-06-29 10:21:43 +020066 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew fetch\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070067 }
alshabib4e1c96e2016-09-28 16:26:24 -070068
Luca Prete283f80a2017-05-18 15:21:24 -070069 stage ("Build CORD Images") {
Luca Prete782d9412017-06-29 10:21:43 +020070 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew buildImages\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070071 }
alshabib61509fb2016-09-09 02:43:20 -070072
Luca Prete283f80a2017-05-18 15:21:24 -070073 stage ("Downloading CORD POD configuration") {
Luca Prete1125f602017-06-30 14:39:53 +020074 sh "vagrant ssh -c \"cd /opt/cord/build/config; git clone -b ${branch} ${config.pod_config.repo_url}\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070075 }
Jonathan Hart1719e9a2017-03-13 21:53:47 -070076
Luca Prete283f80a2017-05-18 15:21:24 -070077 stage ("Publish to headnode") {
Luca Prete782d9412017-06-29 10:21:43 +020078 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} publish\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070079 }
alshabib61509fb2016-09-09 02:43:20 -070080
Luca Prete283f80a2017-05-18 15:21:24 -070081 stage ("Deploy") {
Luca Prete782d9412017-06-29 10:21:43 +020082 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} deploy\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070083 }
alshabib4e1c96e2016-09-28 16:26:24 -070084
Luca Prete607c2d52017-05-24 09:56:06 -070085 stage ("Power cycle compute nodes") {
Luca Prete782d9412017-06-29 10:21:43 +020086 for(int i=0; i < config.compute_nodes.size(); i++) {
87 sh "ipmitool -U ${config.compute_nodes[i].ipmi.user} -P ${config.compute_nodes[i].ipmi.pass} -H ${config.compute_nodes[i].ipmi.ip} power cycle"
88 }
Luca Prete283f80a2017-05-18 15:21:24 -070089 }
alshabib7f3be8d2016-09-27 18:04:56 -070090
Luca Prete283f80a2017-05-18 15:21:24 -070091 stage ("Wait for compute nodes to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +020092 sh "ssh-keygen -f /home/${config.dev_node.user}/.ssh/known_hosts -R ${config.head.ip}"
93 def cordApiKey = runCmd("${config.head.ip}",
94 "${config.head.user}",
95 "${config.head.pass}",
96 "sudo maas-region-admin apikey --username ${config.head.user}")
97 runCmd("${config.head.ip}",
98 "${config.head.user}",
99 "${config.head.pass}",
100 "maas login pod-maas http://${config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
Luca Prete283f80a2017-05-18 15:21:24 -0700101 timeout(time: 45) {
102 waitUntil {
103 try {
Luca Prete782d9412017-06-29 10:21:43 +0200104 num = runCmd("${config.head.ip}",
105 "${config.head.user}",
106 "${config.head.pass}",
107 "maas pod-maas nodes list | grep -i deployed | wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200108 return num.toInteger() == config.compute_nodes.size()
Luca Prete283f80a2017-05-18 15:21:24 -0700109 } catch (exception) {
110 return false
111 }
alshabib19302662017-01-05 14:27:41 -0800112 }
alshabib4e1c96e2016-09-28 16:26:24 -0700113 }
114 }
alshabib4e1c96e2016-09-28 16:26:24 -0700115
You Wang4e0c2ed2017-09-11 13:46:43 -0700116 stage ("Wait for compute nodes to be provisioned") {
Luca Prete283f80a2017-05-18 15:21:24 -0700117 timeout(time:45) {
118 waitUntil {
119 try {
You Wang4e0c2ed2017-09-11 13:46:43 -0700120 num = runCmd("${config.head.ip}",
Luca Prete782d9412017-06-29 10:21:43 +0200121 "${config.head.user}",
122 "${config.head.pass}",
You Wang4e0c2ed2017-09-11 13:46:43 -0700123 "cord prov list '|' grep -i node '|' grep -i complete '|' wc -l").trim()
124 return num.toInteger() == config.compute_nodes.size()
Luca Prete283f80a2017-05-18 15:21:24 -0700125 } catch (exception) {
126 return false
127 }
alshabib19302662017-01-05 14:27:41 -0800128 }
alshabib7f3be8d2016-09-27 18:04:56 -0700129 }
130 }
alshabib19302662017-01-05 14:27:41 -0800131
Luca Prete782d9412017-06-29 10:21:43 +0200132 if (config.fabric_switches != null) {
Luca Prete607c2d52017-05-24 09:56:06 -0700133 stage("Reserve IPs for fabric switches and restart maas-dhcp service") {
Luca Prete782d9412017-06-29 10:21:43 +0200134 for(int i=0; i < config.fabric_switches.size(); i++) {
Luca Prete607c2d52017-05-24 09:56:06 -0700135 def append = "";
136 if (i!=0) {
137 append = "-a";
138 }
Luca Prete782d9412017-06-29 10:21:43 +0200139 def str = createMACIPbindingStr(i+1,
140 "${config.fabric_switches[i].mac}",
141 "${config.fabric_switches[i].ip}")
142 runCmd("${config.head.ip}",
143 "${config.head.user}",
144 "${config.head.pass}",
145 "echo -e $str '|' sudo tee $append /etc/dhcp/dhcpd.reservations > /dev/null")
Luca Prete607c2d52017-05-24 09:56:06 -0700146 }
Luca Prete782d9412017-06-29 10:21:43 +0200147 runCmd("${config.head.ip}",
148 "${config.head.user}",
149 "${config.head.pass}",
150 "sudo restart maas-dhcpd")
151
152 runCmd("${config.head.ip}",
153 "${config.head.user}",
154 "${config.head.pass}",
155 "cord harvest go")
Luca Prete607c2d52017-05-24 09:56:06 -0700156 }
157
158 stage ("Wait for fabric switches to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +0200159 for(int i=0; i < config.fabric_switches.size(); i++) {
160 runFabricCmd("${config.head.ip}",
161 "${config.head.user}",
162 "${config.head.pass}",
163 "${config.fabric_switches[i].ip}",
164 "${config.fabric_switches[i].user}",
165 "${config.fabric_switches[i].pass}",
166 "sudo onl-onie-boot-mode install")
167
168 runFabricCmd("${config.head.ip}",
169 "${config.head.user}",
170 "${config.head.pass}",
171 "${config.fabric_switches[i].ip}",
172 "${config.fabric_switches[i].user}",
173 "${config.fabric_switches[i].pass}",
174 "sudo reboot")
Luca Prete607c2d52017-05-24 09:56:06 -0700175 }
176 timeout(time: 45) {
177 waitUntil {
178 try {
Luca Prete782d9412017-06-29 10:21:43 +0200179 def harvestCompleted = runCmd("${config.head.ip}",
180 "${config.head.user}",
181 "${config.head.pass}",
182 "cord harvest list '|' grep -i fabric '|' wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200183 return harvestCompleted.toInteger() == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700184 } catch (exception) {
185 return false
186 }
187 }
188 }
189 }
190
191 stage ("Wait for fabric switches to be provisioned") {
192 timeout(time:45) {
193 waitUntil {
194 try {
195 def provCompleted = 0
Luca Prete782d9412017-06-29 10:21:43 +0200196 for(int i=0; i < config.fabric_switches.size(); i++) {
197 def count = runCmd("${config.head.ip}",
198 "${config.head.user}",
199 "${config.head.pass}",
200 "cord prov list '|' grep -i ${config.fabric_switches[i].ip} '|' grep -i complete '|' wc -l").trim()
Luca Prete607c2d52017-05-24 09:56:06 -0700201 provCompleted = provCompleted + count.toInteger()
Luca Prete607c2d52017-05-24 09:56:06 -0700202 }
Luca Prete782d9412017-06-29 10:21:43 +0200203 return provCompleted == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700204 } catch (exception) {
205 return false
206 }
207 }
208 }
209 }
210 }
211
Luca Prete782d9412017-06-29 10:21:43 +0200212 if (config.make_release == true) {
213 stage ("Trigger Build") {
214 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
215 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
216 }
Luca Prete283f80a2017-05-18 15:21:24 -0700217 }
alshabib19302662017-01-05 14:27:41 -0800218
219 currentBuild.result = 'SUCCESS'
220 } catch (err) {
221 currentBuild.result = 'FAILURE'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700222 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
alshabib19302662017-01-05 14:27:41 -0800223 } finally {
224 sh 'vagrant destroy -f corddev'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700225 sh 'rm -rf config/pod-configs'
alshabib7f3be8d2016-09-27 18:04:56 -0700226 }
alshabib19302662017-01-05 14:27:41 -0800227 echo "RESULT: ${currentBuild.result}"
228 }
alshabib19302662017-01-05 14:27:41 -0800229 }
alshabib152823c2016-09-07 23:49:12 -0700230}
Luca Prete283f80a2017-05-18 15:21:24 -0700231
232/**
Luca Prete283f80a2017-05-18 15:21:24 -0700233 * Returns a string used to bind IPs and MAC addresses, substituting the values
234 * given.
235 *
Luca Prete607c2d52017-05-24 09:56:06 -0700236 * @param counter the counter used to generate the host name
237 * @param mac the MAC address to substitute
238 * @param ip the IP address to substitute
Luca Prete283f80a2017-05-18 15:21:24 -0700239 */
Luca Prete607c2d52017-05-24 09:56:06 -0700240def createMACIPbindingStr(counter, mac, ip) {
241 return """\\'host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}\\'"""
Luca Prete283f80a2017-05-18 15:21:24 -0700242}
243
244/**
Luca Prete782d9412017-06-29 10:21:43 +0200245 * Runs a command on a remote host using sshpass.
Luca Prete283f80a2017-05-18 15:21:24 -0700246 *
Luca Prete782d9412017-06-29 10:21:43 +0200247 * @param ip the node IP address
248 * @param user the node user name
249 * @param pass the node password
250 * @param command the command to run
Luca Prete607c2d52017-05-24 09:56:06 -0700251 * @return the output of the command
Luca Prete283f80a2017-05-18 15:21:24 -0700252 */
Luca Prete782d9412017-06-29 10:21:43 +0200253def runCmd(ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200254 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}")
Luca Prete283f80a2017-05-18 15:21:24 -0700255}
Luca Prete607c2d52017-05-24 09:56:06 -0700256
257/**
258 * Runs a command on a fabric switch.
259 *
Luca Prete782d9412017-06-29 10:21:43 +0200260 * @param headIp the head node IP address
261 * @param headUser the head node user name
262 * @param headPass the head node password
Luca Prete607c2d52017-05-24 09:56:06 -0700263 * @param ip the mgmt IP of the fabric switch, reachable from the head node
Luca Prete782d9412017-06-29 10:21:43 +0200264 * @param user the mgmt user name of the fabric switch
265 * @param pass the mgmt password of the fabric switch
266 * @param command the command to run on the fabric switch
Luca Prete607c2d52017-05-24 09:56:06 -0700267 * @return the output of the command
268 */
Luca Preted642a592017-06-30 11:11:35 +0200269def runFabricCmd(headIp, headUser, headPass, ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200270 return sh(returnStdout: true, script: "sshpass -p ${headPass} ssh -oStrictHostKeyChecking=no -l ${headUser} ${headIp} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
Jonathan Hart9bd4a822017-08-10 09:08:12 -0700271}