blob: 2e53c1a1068e933210a6fc30870639472d071eed [file] [log] [blame]
You Wangcebbd312017-09-11 14:35:24 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15def filename = 'manifest-${branch}.xml'
16def manifestUrl = 'https://gerrit.opencord.org/manifest'
17def config = null;
18
19node ('master') {
20 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
21
22 stage ("Generate and Copy Manifest file") {
23 sh returnStdout: true, script: 'repo manifest -r -o ' + filename
24 sh returnStdout: true, script: 'cp ' + filename + ' ' + env.JENKINS_HOME + '/tmp'
25 }
26
27 stage ("Parse deployment configuration file") {
28 sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
29 sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
30 config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
31 }
32}
33
34node ("${config.dev_node.name}") {
35 timeout (time: 240) {
Luca Prete7b4a1e82017-09-12 10:44:48 -070036 stage ('Remove old head node from known hosts') {
You Wangcebbd312017-09-11 14:35:24 -070037 sh "ssh-keygen -R ${config.head.ip}"
38 }
39 stage ('Checkout cord repo') {
40 checkout changelog: false, poll: false, scm: [$class: 'RepoScm', currentBranch: true, manifestBranch: params.branch, manifestRepositoryUrl: "${manifestUrl}", quiet: true]
41 }
42
Luca Prete9081fcc2017-09-12 14:37:33 -070043 try {
44 dir('build') {
You Wangcebbd312017-09-11 14:35:24 -070045 stage ("Re-deploy head node") {
46 maasOps: {
47 sh "maas login maas http://${config.maas.ip}/MAAS/api/2.0 ${config.maas.api_key}"
48 sh "maas maas machine release ${config.maas.head_system_id}"
49
50 timeout(time: 15) {
51 waitUntil {
52 try {
53 sh "maas maas machine read ${config.maas.head_system_id} | grep Ready"
54 return true
55 } catch (exception) {
56 return false
57 }
58 }
59 }
60
61 sh 'maas maas machines allocate'
62 sh "maas maas machine deploy ${config.maas.head_system_id}"
63
64 timeout(time: 30) {
65 waitUntil {
66 try {
67 sh "maas maas machine read ${config.maas.head_system_id} | grep Deployed"
68 return true
69 } catch (exception) {
70 return false
71 }
72 }
73 }
74 }
75 }
76
77 stage ("Download CORD POD configuration") {
78 sh "cd podconfig; git clone -b ${branch} ${config.pod_config.repo_url} automation"
79 }
80
81 stage ("Generate CORD configuration") {
82 sh "make PODCONFIG=automation/${config.pod_config.file_name} config"
83 }
84
Andy Bavier6c2f8042017-09-19 13:52:30 -070085 if (config.fabric_switches != null) {
86 stage("Reserve IPs for fabric switches") {
87 for(int i=0; i < config.fabric_switches.size(); i++) {
88 def str = createMACIPbindingStr(i+1,
89 "${config.fabric_switches[i].mac}",
90 "${config.fabric_switches[i].ip}")
91 sh "echo $str >> maas/roles/maas/files/dhcpd.reservations"
92 }
93 }
94 }
95
You Wangcebbd312017-09-11 14:35:24 -070096 stage ("Deploy") {
Luca Pretee92458d2017-10-18 11:52:30 -070097 sh "make build"
You Wangcebbd312017-09-11 14:35:24 -070098 }
99
Luca Prete2339b182017-10-17 11:47:26 -0700100 if (config.compute_nodes != null) {
You Wangcebbd312017-09-11 14:35:24 -0700101
Luca Prete2339b182017-10-17 11:47:26 -0700102 stage ("Power cycle compute nodes") {
103 for(int i=0; i < config.compute_nodes.size(); i++) {
104 sh "ipmitool -U ${config.compute_nodes[i].ipmi.user} -P ${config.compute_nodes[i].ipmi.pass} -H ${config.compute_nodes[i].ipmi.ip} power cycle"
105 }
106 }
107
108 stage ("Wait for compute nodes to get deployed") {
109 sh "ssh-keygen -f /home/${config.dev_node.user}/.ssh/known_hosts -R ${config.head.ip}"
110 def cordApiKey = runCmd("${config.head.ip}",
111 "${config.head.user}",
112 "${config.head.pass}",
113 "sudo maas-region-admin apikey --username ${config.head.user}")
114 runCmd("${config.head.ip}",
115 "${config.head.user}",
116 "${config.head.pass}",
117 "maas login pod-maas http://${config.head.ip}/MAAS/api/1.0 ${cordApiKey}")
Luca Pretebe198b82017-10-24 17:22:09 -0700118 timeout(time: 90) {
Luca Prete2339b182017-10-17 11:47:26 -0700119 waitUntil {
120 try {
121 num = runCmd("${config.head.ip}",
122 "${config.head.user}",
123 "${config.head.pass}",
124 "maas pod-maas nodes list | grep substatus_name | grep -i deployed | wc -l").trim()
125 return num.toInteger() == config.compute_nodes.size()
126 } catch (exception) {
127 return false
128 }
You Wangcebbd312017-09-11 14:35:24 -0700129 }
130 }
131 }
You Wangcebbd312017-09-11 14:35:24 -0700132
Luca Prete2339b182017-10-17 11:47:26 -0700133 stage ("Wait for compute nodes to be provisioned") {
134 timeout(time:45) {
135 waitUntil {
136 try {
137 num = runCmd("${config.head.ip}",
138 "${config.head.user}",
139 "${config.head.pass}",
140 "cord prov list '|' grep -i node '|' grep -i complete '|' wc -l").trim()
141 return num.toInteger() == config.compute_nodes.size()
142 } catch (exception) {
143 return false
144 }
You Wangcebbd312017-09-11 14:35:24 -0700145 }
146 }
147 }
You Wangcebbd312017-09-11 14:35:24 -0700148
Luca Prete2339b182017-10-17 11:47:26 -0700149 }
You Wangcebbd312017-09-11 14:35:24 -0700150 if (config.fabric_switches != null) {
You Wangcebbd312017-09-11 14:35:24 -0700151
152 stage ("Wait for fabric switches to get deployed") {
153 for(int i=0; i < config.fabric_switches.size(); i++) {
154 runFabricCmd("${config.head.ip}",
155 "${config.head.user}",
156 "${config.head.pass}",
157 "${config.fabric_switches[i].ip}",
158 "${config.fabric_switches[i].user}",
159 "${config.fabric_switches[i].pass}",
160 "sudo onl-onie-boot-mode install")
161
162 runFabricCmd("${config.head.ip}",
163 "${config.head.user}",
164 "${config.head.pass}",
165 "${config.fabric_switches[i].ip}",
166 "${config.fabric_switches[i].user}",
167 "${config.fabric_switches[i].pass}",
168 "sudo reboot")
Andy Bavier6c2f8042017-09-19 13:52:30 -0700169
170 // Ensure that switches get provisioned after ONIE reinstall.
171 // Delete them if they were provisioned earlier. If the switches are not
172 // present in 'cord prov list', this command has no effect.
173 runCmd("${config.head.ip}",
174 "${config.head.user}",
175 "${config.head.pass}",
176 "cord prov delete ${config.fabric_switches[i].mac}")
You Wangcebbd312017-09-11 14:35:24 -0700177 }
178 timeout(time: 45) {
179 waitUntil {
180 try {
181 def harvestCompleted = runCmd("${config.head.ip}",
182 "${config.head.user}",
183 "${config.head.pass}",
184 "cord harvest list '|' grep -i fabric '|' wc -l").trim()
185 return harvestCompleted.toInteger() == config.fabric_switches.size()
186 } catch (exception) {
187 return false
188 }
189 }
190 }
191 }
192
193 stage ("Wait for fabric switches to be provisioned") {
194 timeout(time:45) {
195 waitUntil {
196 try {
197 def provCompleted = 0
198 for(int i=0; i < config.fabric_switches.size(); i++) {
199 def count = runCmd("${config.head.ip}",
200 "${config.head.user}",
201 "${config.head.pass}",
202 "cord prov list '|' grep -i ${config.fabric_switches[i].ip} '|' grep -i complete '|' wc -l").trim()
203 provCompleted = provCompleted + count.toInteger()
204 }
205 return provCompleted == config.fabric_switches.size()
206 } catch (exception) {
207 return false
208 }
209 }
210 }
211 }
Marc De Leenheerc7687dc2017-11-27 17:25:09 -0800212
213 stage ("Configure fabric switches") {
214 for(int i=0; i < config.fabric_switches.size(); i++) {
215 if (config.fabric_switches[i].breakout_ports != null) {
216 for(int j=0; j < config.fabric_switches[i].breakout_ports.size(); j++) {
217 runFabricCmd("${config.head.ip}",
218 "${config.head.user}",
219 "${config.head.pass}",
220 "${config.fabric_switches[i].ip}",
221 "${config.fabric_switches[i].user}",
222 "${config.fabric_switches[i].pass}",
223 "sed -i -e 's/#port_mode_${config.fabric_switches[i].breakout_ports[j]}=/port_mode_${config.fabric_switches[i].breakout_ports[j]}=/g' /etc/accton/ofdpa.conf")
224 }
225 runFabricCmd("${config.head.ip}",
226 "${config.head.user}",
227 "${config.head.pass}",
228 "${config.fabric_switches[i].ip}",
229 "${config.fabric_switches[i].user}",
230 "${config.fabric_switches[i].pass}",
Marc De Leenheer8c1c91d2017-12-01 17:35:57 -0800231 "/root/killit")
232 runFabricCmd("${config.head.ip}",
233 "${config.head.user}",
234 "${config.head.pass}",
235 "${config.fabric_switches[i].ip}",
236 "${config.fabric_switches[i].user}",
237 "${config.fabric_switches[i].pass}",
238 "/root/connect -bg")
Marc De Leenheerc7687dc2017-11-27 17:25:09 -0800239 }
240 }
241 }
You Wangcebbd312017-09-11 14:35:24 -0700242 }
You Wangcebbd312017-09-11 14:35:24 -0700243 }
Luca Prete9081fcc2017-09-12 14:37:33 -0700244
245 if (config.make_release == true) {
246 stage ("Trigger Build") {
247 url = 'https://jenkins.opencord.org/job/release-build/job/' + params.branch + '/build'
248 httpRequest authentication: 'auto-release', httpMode: 'POST', url: url, validResponseCodes: '201'
249 }
250 }
251
252 currentBuild.result = 'SUCCESS'
253 } catch (err) {
254 currentBuild.result = 'FAILURE'
255 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
256 } finally {
257 sh "make -C build clean-all || true"
258 sh "rm -rf *"
259 }
260 echo "RESULT: ${currentBuild.result}"
You Wangcebbd312017-09-11 14:35:24 -0700261 }
262}
263
264/**
265 * Returns a string used to bind IPs and MAC addresses, substituting the values
266 * given.
267 *
268 * @param counter the counter used to generate the host name
269 * @param mac the MAC address to substitute
270 * @param ip the IP address to substitute
271 */
272def createMACIPbindingStr(counter, mac, ip) {
Andy Bavier6c2f8042017-09-19 13:52:30 -0700273 return """host fabric${counter} {'\n'hardware ethernet ${mac}';''\n'fixed-address ${ip}';''\n'}"""
You Wangcebbd312017-09-11 14:35:24 -0700274}
275
276/**
277 * Runs a command on a remote host using sshpass.
278 *
279 * @param ip the node IP address
280 * @param user the node user name
281 * @param pass the node password
282 * @param command the command to run
283 * @return the output of the command
284 */
285def runCmd(ip, user, pass, command) {
286 return sh(returnStdout: true, script: "sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}")
287}
288
289/**
290 * Runs a command on a fabric switch.
291 *
292 * @param headIp the head node IP address
293 * @param headUser the head node user name
294 * @param headPass the head node password
295 * @param ip the mgmt IP of the fabric switch, reachable from the head node
296 * @param user the mgmt user name of the fabric switch
297 * @param pass the mgmt password of the fabric switch
298 * @param command the command to run on the fabric switch
299 * @return the output of the command
300 */
301def runFabricCmd(headIp, headUser, headPass, ip, user, pass, command) {
302 return sh(returnStdout: true, script: "sshpass -p ${headPass} ssh -oStrictHostKeyChecking=no -l ${headUser} ${headIp} \"sshpass -p ${pass} ssh -oStrictHostKeyChecking=no -l ${user} ${ip} ${command}\"")
303}