blob: 013662a945939ea77da5a737f21cd793eee7b092 [file] [log] [blame]
alshabib08d0a1f2017-01-05 15:04:28 -08001def filename = 'manifest-${branch}.xml'
Luca Prete283f80a2017-05-18 15:21:24 -07002def manifestUrl = 'https://gerrit.opencord.org/manifest'
Luca Prete782d9412017-06-29 10:21:43 +02003def config = null;
alshabib4e1c96e2016-09-28 16:26:24 -07004
alshabib41f74fd2017-01-07 09:48:00 -08005node ('master') {
Luca Prete283f80a2017-05-18 15:21:24 -07006 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
alshabib41f74fd2017-01-07 09:48:00 -08007
Luca Prete283f80a2017-05-18 15:21:24 -07008 stage ("Generate and Copy Manifest file") {
9 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
10 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
11 }
Luca Prete782d9412017-06-29 10:21:43 +020012
Jonathan Hart9bd4a822017-08-10 09:08:12 -070013 stage ("Parse deployment configuration file") {
Luca Prete782d9412017-06-29 10:21:43 +020014 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
Luca Prete1125f602017-06-30 14:39:53 +020015 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
Luca Prete782d9412017-06-29 10:21:43 +020016 config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
17 }
alshabib41f74fd2017-01-07 09:48:00 -080018}
19
Luca Prete782d9412017-06-29 10:21:43 +020020node ("${config.dev_node.name}") {
Luca Prete283f80a2017-05-18 15:21:24 -070021 timeout (time: 240) {
Luca Prete782d9412017-06-29 10:21:43 +020022 stage ('Checkout cord repo') {
23 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
24 }
alshabib19302662017-01-05 14:27:41 -080025
Luca Prete782d9412017-06-29 10:21:43 +020026 dir('build') {
alshabib19302662017-01-05 14:27:41 -080027 try {
Luca Prete283f80a2017-05-18 15:21:24 -070028 stage ("Re-deploy head node and Build Vagrant box") {
29 parallel(
30 maasOps: {
Luca Prete782d9412017-06-29 10:21:43 +020031 sh "maas login maas http://${config.maas.ip}/MAAS/api/2.0 ${config.maas.api_key}"
32 sh "maas maas machine release ${config.maas.head_system_id}"
alshabib19302662017-01-05 14:27:41 -080033
Luca Prete283f80a2017-05-18 15:21:24 -070034 timeout(time: 15) {
35 waitUntil {
36 try {
Luca Prete782d9412017-06-29 10:21:43 +020037 sh "maas maas machine read ${config.maas.head_system_id} | grep Ready"
Luca Prete283f80a2017-05-18 15:21:24 -070038 return true
39 } catch (exception) {
40 return false
41 }
alshabib19302662017-01-05 14:27:41 -080042 }
alshabibef069942016-09-09 17:08:36 -070043 }
alshabib4e1c96e2016-09-28 16:26:24 -070044
Luca Prete283f80a2017-05-18 15:21:24 -070045 sh 'maas maas machines allocate'
Luca Prete782d9412017-06-29 10:21:43 +020046 sh "maas maas machine deploy ${config.maas.head_system_id}"
alshabib4e1c96e2016-09-28 16:26:24 -070047
Luca Prete283f80a2017-05-18 15:21:24 -070048 timeout(time: 30) {
49 waitUntil {
50 try {
Luca Prete782d9412017-06-29 10:21:43 +020051 sh "maas maas machine read ${config.maas.head_system_id} | grep Deployed"
Luca Prete283f80a2017-05-18 15:21:24 -070052 return true
53 } catch (exception) {
54 return false
55 }
alshabib19302662017-01-05 14:27:41 -080056 }
alshabibef069942016-09-09 17:08:36 -070057 }
alshabib4e1c96e2016-09-28 16:26:24 -070058
Luca Prete283f80a2017-05-18 15:21:24 -070059 }, vagrantOps: {
60 sh 'vagrant up corddev'
61 }, failFast : true
62 )
63 }
alshabib4e1c96e2016-09-28 16:26:24 -070064
Andy Bavier0d6cd9e2017-09-25 13:47:10 -070065 if (config.fabric_switches != null) {
66 stage("Reserve IPs for fabric switches") {
67 for(int i=0; i < config.fabric_switches.size(); i++) {
68 def str = createMACIPbindingStr(i+1,
69 "${config.fabric_switches[i].mac}",
70 "${config.fabric_switches[i].ip}")
71 sh "echo $str >> maas/roles/maas/files/dhcpd.reservations"
72 }
73 }
74 }
75
Luca Prete283f80a2017-05-18 15:21:24 -070076 stage ("Fetch CORD packages") {
Luca Prete782d9412017-06-29 10:21:43 +020077 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew fetch\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070078 }
alshabib4e1c96e2016-09-28 16:26:24 -070079
Luca Prete283f80a2017-05-18 15:21:24 -070080 stage ("Build CORD Images") {
Luca Prete782d9412017-06-29 10:21:43 +020081 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew buildImages\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070082 }
alshabib61509fb2016-09-09 02:43:20 -070083
Luca Prete283f80a2017-05-18 15:21:24 -070084 stage ("Downloading CORD POD configuration") {
Luca Prete1125f602017-06-30 14:39:53 +020085 sh "vagrant ssh -c \"cd /opt/cord/build/config; git clone -b ${branch} ${config.pod_config.repo_url}\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070086 }
Jonathan Hart1719e9a2017-03-13 21:53:47 -070087
Luca Prete283f80a2017-05-18 15:21:24 -070088 stage ("Publish to headnode") {
Luca Prete782d9412017-06-29 10:21:43 +020089 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} publish\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070090 }
alshabib61509fb2016-09-09 02:43:20 -070091
Luca Prete283f80a2017-05-18 15:21:24 -070092 stage ("Deploy") {
Luca Prete782d9412017-06-29 10:21:43 +020093 sh "vagrant ssh -c \"cd /opt/cord/build; ./gradlew -PtargetReg=${config.head.ip}:5000 -PdeployConfig=config/pod-configs/${config.pod_config.file_name} deploy\" corddev"
Luca Prete283f80a2017-05-18 15:21:24 -070094 }
alshabib4e1c96e2016-09-28 16:26:24 -070095
Luca Prete607c2d52017-05-24 09:56:06 -070096 stage ("Power cycle compute nodes") {
Luca Prete782d9412017-06-29 10:21:43 +020097 for(int i=0; i < config.compute_nodes.size(); i++) {
98 sh "ipmitool -U ${config.compute_nodes[i].ipmi.user} -P ${config.compute_nodes[i].ipmi.pass} -H ${config.compute_nodes[i].ipmi.ip} power cycle"
99 }
Luca Prete283f80a2017-05-18 15:21:24 -0700100 }
alshabib7f3be8d2016-09-27 18:04:56 -0700101
Luca Prete283f80a2017-05-18 15:21:24 -0700102 stage ("Wait for compute nodes to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +0200103 sh "ssh-keygen -f /home/${config.dev_node.user}/.ssh/known_hosts -R ${config.head.ip}"
104 def cordApiKey = runCmd("${config.head.ip}",
105 "${config.head.user}",
106 "${config.head.pass}",
107 "sudo maas-region-admin apikey --username ${config.head.user}")
108 runCmd("${config.head.ip}",
109 "${config.head.user}",
110 "${config.head.pass}",
111 "maas login pod-maas http://${config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
Luca Prete283f80a2017-05-18 15:21:24 -0700112 timeout(time: 45) {
113 waitUntil {
114 try {
Luca Prete782d9412017-06-29 10:21:43 +0200115 num = runCmd("${config.head.ip}",
116 "${config.head.user}",
117 "${config.head.pass}",
118 "maas pod-maas nodes list | grep -i deployed | wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200119 return num.toInteger() == config.compute_nodes.size()
Luca Prete283f80a2017-05-18 15:21:24 -0700120 } catch (exception) {
121 return false
122 }
alshabib19302662017-01-05 14:27:41 -0800123 }
alshabib4e1c96e2016-09-28 16:26:24 -0700124 }
125 }
alshabib4e1c96e2016-09-28 16:26:24 -0700126
You Wangb364dce2017-09-11 13:46:43 -0700127 stage ("Wait for compute nodes to be provisioned") {
Luca Prete283f80a2017-05-18 15:21:24 -0700128 timeout(time:45) {
129 waitUntil {
130 try {
You Wangb364dce2017-09-11 13:46:43 -0700131 num = runCmd("${config.head.ip}",
Luca Prete782d9412017-06-29 10:21:43 +0200132 "${config.head.user}",
133 "${config.head.pass}",
You Wangb364dce2017-09-11 13:46:43 -0700134 "cord prov list '|' grep -i node '|' grep -i complete '|' wc -l").trim()
135 return num.toInteger() == config.compute_nodes.size()
Luca Prete283f80a2017-05-18 15:21:24 -0700136 } catch (exception) {
137 return false
138 }
alshabib19302662017-01-05 14:27:41 -0800139 }
alshabib7f3be8d2016-09-27 18:04:56 -0700140 }
141 }
alshabib19302662017-01-05 14:27:41 -0800142
Luca Prete782d9412017-06-29 10:21:43 +0200143 if (config.fabric_switches != null) {
Luca Prete607c2d52017-05-24 09:56:06 -0700144 stage ("Wait for fabric switches to get deployed") {
Luca Prete782d9412017-06-29 10:21:43 +0200145 for(int i=0; i < config.fabric_switches.size(); i++) {
146 runFabricCmd("${config.head.ip}",
147 "${config.head.user}",
148 "${config.head.pass}",
149 "${config.fabric_switches[i].ip}",
150 "${config.fabric_switches[i].user}",
151 "${config.fabric_switches[i].pass}",
152 "sudo onl-onie-boot-mode install")
153
154 runFabricCmd("${config.head.ip}",
155 "${config.head.user}",
156 "${config.head.pass}",
157 "${config.fabric_switches[i].ip}",
158 "${config.fabric_switches[i].user}",
159 "${config.fabric_switches[i].pass}",
160 "sudo reboot")
Luca Prete607c2d52017-05-24 09:56:06 -0700161 }
162 timeout(time: 45) {
163 waitUntil {
164 try {
Luca Prete782d9412017-06-29 10:21:43 +0200165 def harvestCompleted = runCmd("${config.head.ip}",
166 "${config.head.user}",
167 "${config.head.pass}",
168 "cord harvest list '|' grep -i fabric '|' wc -l").trim()
Luca Prete512389a2017-07-07 12:33:29 +0200169 return harvestCompleted.toInteger() == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700170 } catch (exception) {
171 return false
172 }
173 }
174 }
175 }
176
177 stage ("Wait for fabric switches to be provisioned") {
178 timeout(time:45) {
179 waitUntil {
180 try {
181 def provCompleted = 0
Luca Prete782d9412017-06-29 10:21:43 +0200182 for(int i=0; i < config.fabric_switches.size(); i++) {
183 def count = runCmd("${config.head.ip}",
184 "${config.head.user}",
185 "${config.head.pass}",
186 "cord prov list '|' grep -i ${config.fabric_switches[i].ip} '|' grep -i complete '|' wc -l").trim()
Luca Prete607c2d52017-05-24 09:56:06 -0700187 provCompleted = provCompleted + count.toInteger()
Luca Prete607c2d52017-05-24 09:56:06 -0700188 }
Luca Prete782d9412017-06-29 10:21:43 +0200189 return provCompleted == config.fabric_switches.size()
Luca Prete607c2d52017-05-24 09:56:06 -0700190 } catch (exception) {
191 return false
192 }
193 }
194 }
195 }
196 }
197
Luca Prete782d9412017-06-29 10:21:43 +0200198 if (config.make_release == true) {
199 stage ("Trigger Build") {
200 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
201 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
202 }
Luca Prete283f80a2017-05-18 15:21:24 -0700203 }
alshabib19302662017-01-05 14:27:41 -0800204
205 currentBuild.result = 'SUCCESS'
206 } catch (err) {
207 currentBuild.result = 'FAILURE'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700208 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
alshabib19302662017-01-05 14:27:41 -0800209 } finally {
210 sh 'vagrant destroy -f corddev'
Jonathan Hart1719e9a2017-03-13 21:53:47 -0700211 sh 'rm -rf config/pod-configs'
alshabib7f3be8d2016-09-27 18:04:56 -0700212 }
alshabib19302662017-01-05 14:27:41 -0800213 echo "RESULT: ${currentBuild.result}"
214 }
alshabib19302662017-01-05 14:27:41 -0800215 }
alshabib152823c2016-09-07 23:49:12 -0700216}
Luca Prete283f80a2017-05-18 15:21:24 -0700217
218/**
Luca Prete283f80a2017-05-18 15:21:24 -0700219 * Returns a string used to bind IPs and MAC addresses, substituting the values
220 * given.
221 *
Luca Prete607c2d52017-05-24 09:56:06 -0700222 * @param counter the counter used to generate the host name
223 * @param mac the MAC address to substitute
224 * @param ip the IP address to substitute
Luca Prete283f80a2017-05-18 15:21:24 -0700225 */
Luca Prete607c2d52017-05-24 09:56:06 -0700226def createMACIPbindingStr(counter, mac, ip) {
Luca Prete6e7f5ab2017-10-03 14:42:31 -0700227 return """host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}"""
Luca Prete283f80a2017-05-18 15:21:24 -0700228}
229
230/**
Luca Prete782d9412017-06-29 10:21:43 +0200231 * Runs a command on a remote host using sshpass.
Luca Prete283f80a2017-05-18 15:21:24 -0700232 *
Luca Prete782d9412017-06-29 10:21:43 +0200233 * @param ip the node IP address
234 * @param user the node user name
235 * @param pass the node password
236 * @param command the command to run
Luca Prete607c2d52017-05-24 09:56:06 -0700237 * @return the output of the command
Luca Prete283f80a2017-05-18 15:21:24 -0700238 */
Luca Prete782d9412017-06-29 10:21:43 +0200239def runCmd(ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200240 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}")
Luca Prete283f80a2017-05-18 15:21:24 -0700241}
Luca Prete607c2d52017-05-24 09:56:06 -0700242
243/**
244 * Runs a command on a fabric switch.
245 *
Luca Prete782d9412017-06-29 10:21:43 +0200246 * @param headIp the head node IP address
247 * @param headUser the head node user name
248 * @param headPass the head node password
Luca Prete607c2d52017-05-24 09:56:06 -0700249 * @param ip the mgmt IP of the fabric switch, reachable from the head node
Luca Prete782d9412017-06-29 10:21:43 +0200250 * @param user the mgmt user name of the fabric switch
251 * @param pass the mgmt password of the fabric switch
252 * @param command the command to run on the fabric switch
Luca Prete607c2d52017-05-24 09:56:06 -0700253 * @return the output of the command
254 */
Luca Preted642a592017-06-30 11:11:35 +0200255def runFabricCmd(headIp, headUser, headPass, ip, user, pass, command) {
Luca Prete512389a2017-07-07 12:33:29 +0200256 return sh(returnStdout: true, script: "sshpass -p ${headPass} ssh -oStrictHostKeyChecking=no -l ${headUser} ${headIp} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
Jonathan Hart9bd4a822017-08-10 09:08:12 -0700257}