blob: 0d22dd41d47a6c8fb5029023dd93e03c6ac10460 [file] [log] [blame]
Andy Bavier99c11d32016-09-14 17:21:20 -04001#!/usr/bin/env bash
2
3set -e
Andy Bavier99c11d32016-09-14 17:21:20 -04004
Zack Williams0620c932017-01-25 14:36:31 -07005CORDDIR=~/cord
Andy Bavier99c11d32016-09-14 17:21:20 -04006VMDIR=/cord/build/
7CONFIG=config/cord_in_a_box.yml
Andy Baviera69ee722016-11-17 07:26:01 -08008SSHCONFIG=~/.ssh/config
Andy Bavier99c11d32016-09-14 17:21:20 -04009
Andy Bavierd1165102017-01-13 16:38:57 -050010# For CORD version
11REPO_BRANCH="master"
12VERSION_STRING="CiaB development version"
13
Andy Bavier0f07bb32017-01-17 10:20:26 -050014function add_box() {
15 vagrant box list | grep $1 | grep virtualbox || vagrant box add $1
16 vagrant box list | grep $1 | grep libvirt || vagrant mutate $1 libvirt --input-provider virtualbox
17}
18
Andy Bavier99c11d32016-09-14 17:21:20 -040019function cleanup_from_previous_test() {
Andy Bavier99c11d32016-09-14 17:21:20 -040020 echo "## Cleanup ##"
21
Andy Baviera4883ba2016-12-21 15:46:58 -050022 if [ -d $CORDDIR/build ]
23 then
24 echo "Destroying all Vagrant VMs"
25 cd $CORDDIR/build
26 sudo su $USER -c 'vagrant destroy'
27 fi
Andy Bavier99c11d32016-09-14 17:21:20 -040028
Andy Bavier2505f592016-11-11 15:58:55 -050029 echo "Removing $CORDDIR"
30 cd ~
Andy Bavier99c11d32016-09-14 17:21:20 -040031 rm -rf $CORDDIR
Andy Bavier99c11d32016-09-14 17:21:20 -040032}
33
34function bootstrap() {
35 cd ~
36 sudo apt-get update
37 [ -e vagrant_1.8.5_x86_64.deb ] || wget https://releases.hashicorp.com/vagrant/1.8.5/vagrant_1.8.5_x86_64.deb
Andy Bavierad3ea842016-12-21 16:19:07 -050038 dpkg -l vagrant || sudo dpkg -i vagrant_1.8.5_x86_64.deb
Andy Bavier1a35ae62016-11-15 19:58:13 -080039 sudo apt-get -y install qemu-kvm libvirt-bin libvirt-dev curl nfs-kernel-server git build-essential
Andy Bavier99c11d32016-09-14 17:21:20 -040040
41 [ -e ~/.ssh/id_rsa ] || ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
Andy Bavier99c11d32016-09-14 17:21:20 -040042
Andy Bavier99c11d32016-09-14 17:21:20 -040043 sudo adduser $USER libvirtd
44
45 sudo curl -o /usr/local/bin/repo https://storage.googleapis.com/git-repo-downloads/repo
46 sudo chmod a+x /usr/local/bin/repo
47
48 if [ ! -d "$CORDDIR" ]
49 then
50 mkdir $CORDDIR && cd $CORDDIR
51 git config --global user.name 'Test User'
52 git config --global user.email 'test@null.com'
53 git config --global color.ui false
54
Scott Baker0ab86822017-02-01 14:33:40 -080055 repo init -u https://gerrit.opencord.org/manifest -b $REPO_BRANCH -g build,onos,orchestration,voltha
Andy Bavier99c11d32016-09-14 17:21:20 -040056 repo sync
57
Zack Williamsdae7ff62016-11-14 15:20:06 -070058 # check out gerrit branches using repo
59 for gerrit_branch in ${GERRIT_BRANCHES[@]}; do
60 echo "checking out opencord gerrit branch: $gerrit_branch"
61 repo download ${gerrit_branch/:/ }
62 done
Andy Bavier99c11d32016-09-14 17:21:20 -040063 fi
64
65 cd $CORDDIR/build
Andy Bavierad3ea842016-12-21 16:19:07 -050066 vagrant plugin list | grep vagrant-libvirt || vagrant plugin install vagrant-libvirt --plugin-version 0.0.35
67 vagrant plugin list | grep vagrant-mutate || vagrant plugin install vagrant-mutate
Andy Bavier0f07bb32017-01-17 10:20:26 -050068 add_box ubuntu/trusty64
Andy Bavier99c11d32016-09-14 17:21:20 -040069}
70
71function cloudlab_setup() {
72 if [ -e /usr/testbed/bin/mkextrafs ]
73 then
Andy Bavier9c0835e2016-11-11 15:56:47 -050074 sudo mkdir -p /mnt/extra
Scott Baker8b9182d2016-10-27 17:53:52 -070075
Andy Bavier8ce395f2016-10-14 12:51:30 -040076 # Sometimes this command fails on the first try
Andy Bavier9c0835e2016-11-11 15:56:47 -050077 sudo /usr/testbed/bin/mkextrafs -r /dev/sdb -qf "/mnt/extra/" || sudo /usr/testbed/bin/mkextrafs -r /dev/sdb -qf "/mnt/extra/"
78
79 # Check that the mount succeeded (sometimes mkextrafs succeeds but device not mounted)
Andy Baviera69ee722016-11-17 07:26:01 -080080 mount | grep sdb || (echo "ERROR: mkextrafs failed, exiting!" && exit 1)
Scott Baker8b9182d2016-10-27 17:53:52 -070081
82 # we'll replace /var/lib/libvirt/images with a symlink below
83 [ -d /var/lib/libvirt/images/ ] && [ ! -h /var/lib/libvirt/images ] && sudo rmdir /var/lib/libvirt/images
84
Andy Bavier9c0835e2016-11-11 15:56:47 -050085 sudo mkdir -p /mnt/extra/libvirt_images
Andy Bavier2505f592016-11-11 15:58:55 -050086 if [ ! -e /var/lib/libvirt/images ]
87 then
88 sudo ln -s /mnt/extra/libvirt_images /var/lib/libvirt/images
89 fi
Andy Bavier99c11d32016-09-14 17:21:20 -040090 fi
91}
92
Andy Bavier2505f592016-11-11 15:58:55 -050093function vagrant_vms_up() {
Andy Bavier99c11d32016-09-14 17:21:20 -040094 cd $CORDDIR/build
95
Andy Bavier65c8e152016-12-22 16:01:27 -050096 sudo su $USER -c 'vagrant up corddev prod --provider libvirt'
Andy Baviera69ee722016-11-17 07:26:01 -080097
98 # This is a workaround for a weird issue with ARP cache timeout breaking 'vagrant ssh'
99 # It allows SSH'ing to the machine via 'ssh corddev'
Andy Bavier2505f592016-11-11 15:58:55 -0500100 sudo su $USER -c "vagrant ssh-config corddev prod > $SSHCONFIG"
101
102 scp ~/.ssh/id_rsa* corddev:.ssh
103 ssh corddev "chmod go-r ~/.ssh/id_rsa"
Andy Bavier99c11d32016-09-14 17:21:20 -0400104}
105
106function install_head_node() {
107 cd $CORDDIR/build
108
Andy Baviera69ee722016-11-17 07:26:01 -0800109 # SSH config saved earlier allows us to connect to VM without running 'vagrant'
Zack Williams0620c932017-01-25 14:36:31 -0700110 ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG fetch"
111 ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG buildImages"
Andy Bavier2505f592016-11-11 15:58:55 -0500112 ssh corddev "cd /cord/build; ping -c 3 prod; ./gradlew -PdeployConfig=$VMDIR/$CONFIG -PtargetReg=10.100.198.201:5000 publish"
Andy Baviera69ee722016-11-17 07:26:01 -0800113 ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG deploy"
Andy Bavier99c11d32016-09-14 17:21:20 -0400114}
115
116function set_up_maas_user() {
Andy Bavier2505f592016-11-11 15:58:55 -0500117 # Set up MAAS user on server to restart nodes via libvirt
118 grep maas /etc/passwd || sudo useradd -m maas
Andy Bavier99c11d32016-09-14 17:21:20 -0400119 sudo adduser maas libvirtd
120
Andy Bavier2505f592016-11-11 15:58:55 -0500121 # Copy generated public key to maas user's authorized_keys
122 sudo su maas -c "mkdir -p ~/.ssh"
123 sudo cp $HOME/.ssh/id_rsa.pub ~maas/.ssh/authorized_keys
124 sudo chown maas:maas ~maas/.ssh/authorized_keys
Andy Bavier99c11d32016-09-14 17:21:20 -0400125
Andy Bavier2505f592016-11-11 15:58:55 -0500126 # Copy generated private key to maas user's home dir in prod VM
127 scp $HOME/.ssh/id_rsa prod:/tmp
128 ssh prod "sudo mkdir -p ~maas/.ssh"
129 ssh prod "sudo cp /tmp/id_rsa ~maas/.ssh/id_rsa"
130 ssh prod "sudo chown -R maas:maas ~maas/.ssh"
131}
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600132
Andy Bavier0f07bb32017-01-17 10:20:26 -0500133function turn_off_learning () {
134 NET=$1
135 BRIDGE=`sudo virsh net-info $NET|grep "Bridge:"|awk '{print $2}'`
136 sudo brctl setageing $BRIDGE 0
137 sudo brctl stp $BRIDGE off
138}
139
140function leaf_spine_up() {
141 cd $CORDDIR/build
142
143 if [[ $FABRIC -ne 0 ]]
144 then
145 sudo su $USER -c "FABRIC=$FABRIC vagrant up leaf-1 leaf-2 spine-1 spine-2 --provider libvirt"
146 else
147 # Linux bridging seems to be having issues with two spine switches
148 sudo su $USER -c "FABRIC=$FABRIC vagrant up leaf-1 leaf-2 spine-1 --provider libvirt"
149 fi
150
151 # Turn off MAC learning on "links" -- i.e., bridges created by libvirt.
152 # Without this, sometimes packets are dropped because the bridges
153 # think they are not local -- this needs further investigation.
154 # A better solution might be to replace the bridges with UDP tunnels, but this
155 # is not supported with the version of libvirt available on Ubuntu 14.04.
156 turn_off_learning head-node-leaf-1
157 turn_off_learning compute-node-1-leaf-1
158 turn_off_learning compute-node-2-leaf-2
159 turn_off_learning compute-node-3-leaf-2
160 turn_off_learning leaf-1-spine-1
161 turn_off_learning leaf-1-spine-2
162 turn_off_learning leaf-2-spine-1
163 turn_off_learning leaf-2-spine-2
164}
165
Andy Bavier99c11d32016-09-14 17:21:20 -0400166function add_compute_node() {
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600167 echo add_compute_node: $1 $2
Andy Bavier2505f592016-11-11 15:58:55 -0500168
Andy Bavier99c11d32016-09-14 17:21:20 -0400169 cd $CORDDIR/build
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600170 sudo su $USER -c "vagrant up $1 --provider libvirt"
Andy Bavier99c11d32016-09-14 17:21:20 -0400171
Andy Bavier2505f592016-11-11 15:58:55 -0500172 # Set up power cycling for the compute node and wait for it to be provisioned
173 ssh prod "cd /cord/build/ansible; ansible-playbook maas-provision.yml --extra-vars \"maas_user=maas vagrant_name=$2\""
Andy Bavier99c11d32016-09-14 17:21:20 -0400174
175 echo ""
Andy Bavier0f07bb32017-01-17 10:20:26 -0500176 echo "$1 is fully provisioned!"
177}
178
179function initialize_fabric() {
180 echo "Initializing fabric"
181 ssh prod "cd /cord/build/platform-install; ansible-playbook -i /etc/maas/ansible/pod-inventory cord-refresh-fabric.yml"
182
183 echo "Fabric ping test"
184 ssh prod "cd /cord/build/platform-install; ansible-playbook -i /etc/maas/ansible/pod-inventory cord-fabric-pingtest.yml"
Andy Bavier99c11d32016-09-14 17:21:20 -0400185}
186
187function run_e2e_test () {
188 cd $CORDDIR/build
189
Andy Bavier2505f592016-11-11 15:58:55 -0500190 # User has been added to the lbvirtd group, but su $USER to be safe
Andy Baviera69ee722016-11-17 07:26:01 -0800191 ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG postDeployTests"
Andy Bavier99c11d32016-09-14 17:21:20 -0400192}
193
194function run_diagnostics() {
Zack Williams4fd3dcc2017-02-08 20:46:14 -0700195 ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG PIrunDiag"
Andy Bavier99c11d32016-09-14 17:21:20 -0400196}
197
198# Parse options
Zack Williamsdae7ff62016-11-14 15:20:06 -0700199GERRIT_BRANCHES=
Andy Bavier99c11d32016-09-14 17:21:20 -0400200RUN_TEST=0
Andy Bavier5c2e4fa2016-10-31 13:50:52 -0400201SETUP_ONLY=0
Andy Bavier99c11d32016-09-14 17:21:20 -0400202DIAGNOSTICS=0
203CLEANUP=0
Andy Bavier0f07bb32017-01-17 10:20:26 -0500204FABRIC=0
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600205#By default, cord-in-a-box creates 1 compute node. If more than one compute is
206#needed, use -n option
207NUM_COMPUTE_NODES=1
Andy Bavier99c11d32016-09-14 17:21:20 -0400208
Andy Bavier0f07bb32017-01-17 10:20:26 -0500209while getopts "b:cdfhn:stv" opt; do
Andy Bavier99c11d32016-09-14 17:21:20 -0400210 case ${opt} in
Zack Williamsdae7ff62016-11-14 15:20:06 -0700211 b ) GERRIT_BRANCHES+=("$OPTARG")
Andy Bavier99c11d32016-09-14 17:21:20 -0400212 ;;
213 c ) CLEANUP=1
214 ;;
215 d ) DIAGNOSTICS=1
216 ;;
Andy Bavier0f07bb32017-01-17 10:20:26 -0500217 f ) FABRIC=1
218 ;;
Andy Bavier99c11d32016-09-14 17:21:20 -0400219 h ) echo "Usage:"
220 echo " $0 install OpenStack and prep XOS and ONOS VMs [default]"
Zack Williamsdae7ff62016-11-14 15:20:06 -0700221 echo " $0 -b <project:changeset/revision> checkout a changesets from gerrit. Can"
222 echo " be used multiple times."
Andy Bavier99c11d32016-09-14 17:21:20 -0400223 echo " $0 -c cleanup from previous test"
224 echo " $0 -d run diagnostic collector"
Andy Bavier0f07bb32017-01-17 10:20:26 -0500225 echo " $0 -f use ONOS fabric (EXPERIMENTAL)"
Andy Bavier99c11d32016-09-14 17:21:20 -0400226 echo " $0 -h display this help message"
Zack Williamsa9e76ec2017-01-20 16:17:35 -0700227 echo " $0 -n # number of compute nodes to setup. Currently max 2 nodes can be supported"
Andy Bavier5c2e4fa2016-10-31 13:50:52 -0400228 echo " $0 -s run initial setup phase only (don't start building CORD)"
Andy Bavier99c11d32016-09-14 17:21:20 -0400229 echo " $0 -t do install, bring up cord-pod configuration, run E2E test"
Andy Bavierd1165102017-01-13 16:38:57 -0500230 echo " $0 -v print CiaB version and exit"
Andy Bavier99c11d32016-09-14 17:21:20 -0400231 exit 0
232 ;;
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600233 n ) NUM_COMPUTE_NODES=$OPTARG
234 ;;
Andy Bavier5c2e4fa2016-10-31 13:50:52 -0400235 s ) SETUP_ONLY=1
236 ;;
Andy Bavier99c11d32016-09-14 17:21:20 -0400237 t ) RUN_TEST=1
238 ;;
Andy Bavierd1165102017-01-13 16:38:57 -0500239 v ) echo "$VERSION_STRING ($REPO_BRANCH branch)"
240 exit 0
241 ;;
Andy Bavier99c11d32016-09-14 17:21:20 -0400242 \? ) echo "Invalid option: -$OPTARG"
243 exit 1
244 ;;
245 esac
246done
247
248# What to do
249if [[ $CLEANUP -eq 1 ]]
250then
251 cleanup_from_previous_test
252fi
253
Andy Bavierd1165102017-01-13 16:38:57 -0500254echo ""
255echo "Preparing to install $VERSION_STRING ($REPO_BRANCH branch)"
256echo ""
257
Andy Bavier99c11d32016-09-14 17:21:20 -0400258bootstrap
259cloudlab_setup
Andy Bavier2505f592016-11-11 15:58:55 -0500260vagrant_vms_up
Andy Bavier5c2e4fa2016-10-31 13:50:52 -0400261
262if [[ $SETUP_ONLY -ne 0 ]]
263then
264 echo "Finished build environment setup, exiting..."
265 exit 0
266fi
267
Andy Bavier99c11d32016-09-14 17:21:20 -0400268install_head_node
269set_up_maas_user
Andy Bavier0f07bb32017-01-17 10:20:26 -0500270leaf_spine_up
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600271
Andy Bavier0f07bb32017-01-17 10:20:26 -0500272if [[ $NUM_COMPUTE_NODES -gt 3 ]]
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600273then
Andy Bavier0f07bb32017-01-17 10:20:26 -0500274 echo "currently max only three compute nodes can be supported..."
275 NUM_COMPUTE_NODES=3
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600276fi
277
278for i in `seq 1 $NUM_COMPUTE_NODES`;
279do
280 echo adding the compute node: compute-node-$i
Andy Bavier0f07bb32017-01-17 10:20:26 -0500281 add_compute_node compute-node-$i build_compute-node-$i
Srikanth Vavilapallib1c4da02016-11-17 15:24:28 -0600282done
Andy Bavier99c11d32016-09-14 17:21:20 -0400283
Zack Williams4fd3dcc2017-02-08 20:46:14 -0700284# run diagnostics both before/after the fabric/e2e tests
285if [[ $DIAGNOSTICS -eq 1 ]]
286then
287 run_diagnostics
288fi
289
290
Andy Bavier0f07bb32017-01-17 10:20:26 -0500291if [[ $FABRIC -ne 0 ]]
292then
293 initialize_fabric
294fi
295
Andy Bavier99c11d32016-09-14 17:21:20 -0400296if [[ $RUN_TEST -eq 1 ]]
297then
298 run_e2e_test
299fi
300
301if [[ $DIAGNOSTICS -eq 1 ]]
302then
303 run_diagnostics
304fi
305
306exit 0