Install the CORD-in-a-Box head node into the 'prod' VM
Change-Id: I121130e21ae4268a9dbc9d0a6fb8ff8095ffdcc7
diff --git a/Vagrantfile b/Vagrantfile
index ffa826e..691c3a1 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -10,7 +10,7 @@
end
#By default, this variable is set to 2, such that Vagrantfile allows creation
- #of compute nodes up to 2. If the number of compute nodes to be supported more
+ #of compute nodes up to 2. If the number of compute nodes to be supported more
#than 2, set the environment variable NUM_COMPUTE_NODES to the desired value
#before executing this Vagrantfile.
num_compute_nodes = (ENV['NUM_COMPUTE_NODES'] || 2).to_i
@@ -38,13 +38,32 @@
d.vm.box = "ubuntu/trusty64"
d.vm.synced_folder '.', '/vagrant', disabled: true
d.vm.hostname = "prod"
+ d.vm.network "forwarded_port", guest: 80, host: 80, host_ip: '*'
d.vm.network "private_network", ip: "10.100.198.201"
- d.vm.network "private_network", ip: "0.0.0.0", virtualbox__intnet: "cord-test-network"
+ d.vm.network "private_network",
+ ip: "0.0.0.0",
+ auto_config: false,
+ virtualbox__intnet: "cord-mgmt-network",
+ libvirt__network_name: "cord-mgmt-network",
+ libvirt__forward_mode: "none",
+ libvirt__dhcp_enabled: false
+ d.vm.network "private_network",
+ ip: "10.6.1.201", # To set up the 10.6.1.1 IP address on bridge
+ virtualbox__intnet: "cord-fabric-network",
+ libvirt__network_name: "cord-fabric-network",
+ libvirt__forward_mode: "nat",
+ libvirt__dhcp_enabled: false
d.vm.provision :shell, path: "scripts/bootstrap_ansible.sh"
d.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /cord/build/ansible/prod.yml -c local"
d.vm.provider "virtualbox" do |v|
v.memory = 2048
end
+ d.vm.provider :libvirt do |v, override|
+ v.memory = 16384
+ v.cpus = 8
+ v.storage :file, :size => '100G', :type => 'qcow2'
+ override.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /cord/build/ansible/add-extra-drive.yml -c local"
+ end
end
config.vm.define "switch" do |s|
@@ -53,7 +72,8 @@
s.vm.network "private_network", ip: "10.100.198.253"
s.vm.network "private_network",
type: "dhcp",
- virtualbox__intnet: "cord-test-network",
+ virtualbox__intnet: "cord-fabric-network",
+ libvirt__network_name: "cord-fabric-network",
mac: "cc37ab000001"
s.vm.provision :shell, path: "scripts/bootstrap_ansible.sh"
s.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /cord/build/ansible/fakeswitch.yml -c local"
@@ -79,53 +99,40 @@
end
end
- (1..3).each do |i|
- # Defining VM properties
- config.vm.define "compute_node#{i}" do |c|
- c.vm.box = "clink15/pxe"
- c.vm.synced_folder '.', '/vagrant', disabled: true
- c.vm.communicator = "none"
- c.vm.hostname = "computenode"
- c.vm.network "private_network",
- adapter: "1",
- type: "dhcp",
- auto_config: false,
- virtualbox__intnet: "cord-test-network"
- c.vm.provider "virtualbox" do |v|
- v.name = "compute_node#{i}"
- v.memory = 1048
- v.gui = "true"
- end
- end
- end
-
num_compute_nodes.times do |n|
- # Libvirt compute node
- # Not able to merge with virtualbox config for compute nodes above
- # Issue is that here no box and no private network are specified
config.vm.define "compute_node-#{n+1}" do |c|
compute_ip = compute_ips[n]
compute_index = n+1
c.vm.synced_folder '.', '/vagrant', disabled: true
c.vm.communicator = "none"
c.vm.hostname = "computenode-#{compute_index}"
- c.vm.network "public_network",
+ c.vm.network "private_network",
adapter: 1,
+ ip: "0.0.0.0",
auto_config: false,
- dev: "mgmtbr",
- mode: "bridge",
- type: "bridge"
+ virtualbox__intnet: "cord-mgmt-network",
+ libvirt__network_name: "cord-mgmt-network"
c.vm.network "private_network",
adapter: 2,
- ip: "#{compute_ip}"
- c.vm.provider :libvirt do |domain|
- domain.memory = 8192
- domain.cpus = 4
- domain.machine_virtual_size = 100
- domain.storage :file, :size => '100G', :type => 'qcow2'
- domain.boot 'network'
- domain.boot 'hd'
- domain.nested = true
+ ip: "#{compute_ip}",
+ auto_config: false,
+ virtualbox__intnet: "cord-fabric-network",
+ libvirt__network_name: "cord-fabric-network",
+ libvirt__forward_mode: "nat",
+ libvirt__dhcp_enabled: false
+ c.vm.provider :libvirt do |v|
+ v.memory = 8192
+ v.cpus = 4
+ v.machine_virtual_size = 100
+ v.storage :file, :size => '100G', :type => 'qcow2'
+ v.boot 'network'
+ v.boot 'hd'
+ v.nested = true
+ end
+ c.vm.provider "virtualbox" do |v, override|
+ override.vm.box = "clink15/pxe"
+ v.memory = 1048
+ v.gui = "true"
end
end
end
diff --git a/ansible/add-extra-drive.yml b/ansible/add-extra-drive.yml
new file mode 100644
index 0000000..22a3a1b
--- /dev/null
+++ b/ansible/add-extra-drive.yml
@@ -0,0 +1,5 @@
+- hosts: localhost
+ remote_user: vagrant
+ serial: 1
+ roles:
+ - extra-drive
diff --git a/ansible/maas-provision.yml b/ansible/maas-provision.yml
new file mode 100644
index 0000000..9b180c5
--- /dev/null
+++ b/ansible/maas-provision.yml
@@ -0,0 +1,5 @@
+- hosts: localhost
+ remote_user: vagrant
+ serial: 1
+ roles:
+ - maas-provision
diff --git a/ansible/roles/extra-drive/defaults/main.yml b/ansible/roles/extra-drive/defaults/main.yml
new file mode 100644
index 0000000..1d16dac
--- /dev/null
+++ b/ansible/roles/extra-drive/defaults/main.yml
@@ -0,0 +1,6 @@
+extra_disk_dev: /dev/vda
+
+extra_disk_links:
+ - { src: /mnt/lxd, dest: /var/lib/lxd }
+ - { src: /mnt/docker-registry, dest: /docker-registry }
+ - { src: /mnt/lxcfs, dest: /var/lib/lxcfs }
diff --git a/ansible/roles/extra-drive/tasks/main.yml b/ansible/roles/extra-drive/tasks/main.yml
new file mode 100644
index 0000000..edf6bde
--- /dev/null
+++ b/ansible/roles/extra-drive/tasks/main.yml
@@ -0,0 +1,36 @@
+- name: Check if the disk is partitioned
+ stat: path={{ extra_disk_dev }}1
+ register: device_stat
+
+- name: Set disk label
+ command: parted {{ extra_disk_dev }} mklabel msdos
+ when: device_stat.stat.exists == false
+
+- name: Create primary partition
+ command: parted {{ extra_disk_dev }} mkpart primary 1 100%
+ when: device_stat.stat.exists == false
+
+- name: Make filesystem
+ filesystem:
+ fstype: ext4
+ dev: "{{ extra_disk_dev }}1"
+
+- name: Mount extra disk
+ mount:
+ name: /mnt
+ src: "{{ extra_disk_dev }}1"
+ fstype: ext4
+ state: mounted
+
+- name: Create directories
+ file:
+ path: "{{ item.src }}"
+ state: directory
+ with_items: "{{ extra_disk_links }}"
+
+- name: Set up links
+ file:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ state: link
+ with_items: "{{ extra_disk_links }}"
diff --git a/ansible/roles/maas-provision/tasks/main.yml b/ansible/roles/maas-provision/tasks/main.yml
new file mode 100644
index 0000000..32e43ea
--- /dev/null
+++ b/ansible/roles/maas-provision/tasks/main.yml
@@ -0,0 +1,37 @@
+# Doesn't seem to be a MAAS module for Ansible yet
+- name: Get MAAS credentials
+ become: yes
+ command: maas-region-admin apikey --username=cord
+ register: maas_key
+ tags:
+ - skip_ansible_lint
+
+- name: Login to MAAS
+ command: maas login cord http://localhost/MAAS/api/1.0 {{ maas_key.stdout }}
+ tags:
+ - skip_ansible_lint
+
+- name: Wait for node to become ready
+ shell: maas cord nodes list|jq -r '.[] | select(.status == 0).system_id'
+ register: nodeid
+ until: nodeid.stdout
+ retries: 40
+ delay: 15
+ tags:
+ - skip_ansible_lint
+
+# We need to be able to reboot the VM on the physical server running CiaB
+- name: Add remote power state
+ command: maas cord node update {{ nodeid.stdout }} power_type="virsh" power_parameters_power_address="qemu+ssh://{{ maas_user }}@10.100.198.1/system" power_parameters_power_id="{{ vagrant_name }}"
+ tags:
+ - skip_ansible_lint
+
+- name: Wait for node to be fully provisioned
+ become: yes
+ shell: /usr/local/bin/get-node-prov-state |jq '.[] | select(.id == "{{ nodeid.stdout }}").status'
+ register: prov_state
+ until: prov_state.stdout == "2"
+ retries: 80
+ delay: 30
+ tags:
+ - skip_ansible_lint
diff --git a/ansible/roles/vlan-stag-ctag/defaults/main.yml b/ansible/roles/vlan-stag-ctag/defaults/main.yml
new file mode 100644
index 0000000..575532f
--- /dev/null
+++ b/ansible/roles/vlan-stag-ctag/defaults/main.yml
@@ -0,0 +1,3 @@
+olt_if: eth1
+s_tag: 222
+c_tag: 111
diff --git a/ansible/roles/vlan-stag-ctag/handlers/main.yml b/ansible/roles/vlan-stag-ctag/handlers/main.yml
new file mode 100644
index 0000000..fdeefb2
--- /dev/null
+++ b/ansible/roles/vlan-stag-ctag/handlers/main.yml
@@ -0,0 +1,2 @@
+- name: Reboot node
+ command: shutdown -r now
diff --git a/ansible/roles/vlan-stag-ctag/tasks/main.yml b/ansible/roles/vlan-stag-ctag/tasks/main.yml
new file mode 100644
index 0000000..c4c325c
--- /dev/null
+++ b/ansible/roles/vlan-stag-ctag/tasks/main.yml
@@ -0,0 +1,8 @@
+- name: Set up /etc/rc.local (creates VLAN interfaces)
+ template:
+ src: templates/rc.local.j2
+ dest: /etc/rc.local
+ mode: 0755
+ owner: root
+ notify:
+ - Reboot node
diff --git a/ansible/roles/vlan-stag-ctag/templates/rc.local.j2 b/ansible/roles/vlan-stag-ctag/templates/rc.local.j2
new file mode 100644
index 0000000..4632053
--- /dev/null
+++ b/ansible/roles/vlan-stag-ctag/templates/rc.local.j2
@@ -0,0 +1,13 @@
+#!/bin/sh -e
+
+# Set up VLAN tagging interface ("simulated OLT")
+ip link add link {{ olt_if }} name {{ olt_if }}.{{ s_tag }} type vlan id {{ s_tag }}
+ip link add link {{ olt_if }}.{{ s_tag }} name {{ olt_if }}.{{ s_tag }}.{{ c_tag }} type vlan id {{ c_tag }}
+ifconfig {{ olt_if }} up
+ifconfig {{ olt_if }}.{{ s_tag }} up
+ifconfig {{ olt_if }}.{{ s_tag }}.{{ c_tag }} up
+
+# Get IP address from vSG
+dhclient {{ olt_if }}.{{ s_tag }}.{{ c_tag }}
+
+exit 0
diff --git a/ansible/testclient.yml b/ansible/testclient.yml
new file mode 100644
index 0000000..404cecc
--- /dev/null
+++ b/ansible/testclient.yml
@@ -0,0 +1,5 @@
+- hosts: localhost
+ remote_user: vagrant
+ serial: 1
+ roles:
+ - vlan-stag-ctag
diff --git a/config/cord_in_a_box.yml b/config/cord_in_a_box.yml
index 9fe0a19..edc4bca 100644
--- a/config/cord_in_a_box.yml
+++ b/config/cord_in_a_box.yml
@@ -10,8 +10,8 @@
# User name and password used by Ansible to connect to the host for remote
# provisioning
- user: 'ubuntu'
- password: 'foo'
+ user: 'vagrant'
+ password: 'vagrant'
# Specifies tasks within the head node provisioning not to execute, including:
#
@@ -33,8 +33,8 @@
#
# virtualbox_support - install support for managing virtual box based
# compute nodes
- virtualbox_support: 1
- power_helper_user: 'cord'
+ # virtualbox_support: 1
+ # power_helper_user: 'cord'
extraVars:
#- 'on_cloudlab=True'
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 31dd925..5d52291 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -19,9 +19,9 @@
Target server requirements:
* 64-bit server, with
- * 48GB+ RAM
- * 12+ CPU cores
- * 1TB+ disk
+ * 32GB+ RAM
+ * 8+ CPU cores
+ * 200GB+ disk
* Access to the Internet
* Ubuntu 14.04 LTS freshly installed (see [TBF]() for instruction on how to install Ubuntu 14.04).
* User account used to install CORD-in-a-Box has password-less *sudo* capability (e.g., like the `ubuntu` user)
@@ -81,40 +81,32 @@
## Inspecting CORD-in-a-Box
-CORD-in-a-Box installs the target server as a CORD head node, with OpenStack,
-ONOS, and XOS services running inside VMs. An OpenStack compute node
-is also brought up inside a virtual machine. You can see all the virtual
-machines by running `virsh list` on the target server:
+CORD-in-a-Box creates a virtual CORD POD running inside Vagrant VMs.
+You can inspect their current status as follows:
```
-$ virsh list
- Id Name State
-----------------------------------------------------
- 2 build_corddev running
- 3 juju-1 running
- 4 ceilometer-1 running
- 5 glance-1 running
- 6 keystone-1 running
- 7 percona-cluster-1 running
- 8 nagios-1 running
- 9 neutron-api-1 running
- 10 nova-cloud-controller-1 running
- 11 openstack-dashboard-1 running
- 12 rabbitmq-server-1 running
- 13 onos-cord-1 running
- 14 onos-fabric-1 running
- 15 xos-1 running
- 18 build_compute_node running
+~$ cd opencord/build/
+~/opencord/build$ vagrant status
+Current machine states:
+
+corddev running (libvirt)
+prod running (libvirt)
+switch not created (libvirt)
+testbox not created (libvirt)
+compute_node-1 running (libvirt)
+compute_node-2 not created (libvirt)
```
-The `build_corddev` VM is the Vagrant development machine that executes
-the build process. It download and build Docker containers and publish them
-to the target server. It then installs MaaS on the target server (for bare-metal
-provisioning) and the ONOS, XOS, and OpenStack services in VMs. This VM
+### corddev VM
+
+The `corddev` VM is a development machine used by the `cord-in-a-box.sh` script to drive the
+installation. It downloads and builds Docker containers and publishes them
+to the virtal head node (see below). It then installs MaaS on the virtual head node (for bare-metal
+provisioning) and the ONOS, XOS, and OpenStack services in containers. This VM
can be entered as follows:
```
-cd ~/opencord/build; vagrant ssh corddev
+$ ssh corddev
```
The CORD build environment is located in `/cord/build` inside this VM. It is
@@ -122,44 +114,139 @@
[quickstart_physical.md](./quickstart_physical.md) for more information on
how to run build steps.
-The VMs ending with names ending with `-1` are running the various CORD
-head node services. Two instances of ONOS are running, in
-the `onos-cord-1` and `onos-fabric-1` VMs, though only `onos-cord-1` is used in
-the CORD-in-a-Box. XOS is running inside the `xos-1`
-VM and is controlling ONOS and OpenStack. You can get a deeper understanding of
-the configuration of the target server by visiting [head_node_services.md](./head_node_services.md).
-These VMs can be entered as follows:
+### prod VM
+
+The `prod` VM is the virtual head node of the POD. It runs the OpenStack,
+ONOS, and XOS services inside containers. It also simulates a subscriber
+devices using a container. To enter it, simply type:
```
-ssh ubuntu@<vm-name>
+$ ssh prod
```
-The `build_compute_node` VM is the virtual compute node controlled by OpenStack.
-This VM can be entered as follows:
+Inside the VM, a number of services run in Docker and LXD containers.
+
+NOTE: NO ONOS CONTAINERS HERE YET!
```
-ssh ubuntu@$( cord prov list | tail -1 | awk '{print $2}' )
+vagrant@prod:~$ docker ps
+CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+649bb35c54aa xosproject/xos-ui "python /opt/xos/mana" About an hour ago Up About an hour 8000/tcp, 0.0.0.0:8888->8888/tcp cordpod_xos_ui_1
+700f23298686 xosproject/xos-synchronizer-exampleservice "bash -c 'sleep 120; " About an hour ago Up About an hour 8000/tcp cordpod_xos_synchronizer_exampleservice_1
+05266a97d245 xosproject/xos-synchronizer-vtr "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_vtr_1
+1fcd23a4b0b5 xosproject/xos-synchronizer-vsg "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_vsg_1
+72741c1d431a xosproject/xos-synchronizer-onos "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_onos_1
+b31eb4a938ff xosproject/xos-synchronizer-fabric "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_fabric_1
+b14213da2ae2 xosproject/xos-synchronizer-openstack "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_openstack_1
+159c937b84d5 xosproject/xos-synchronizer-vtn "bash -c 'sleep 120; " 2 hours ago Up 2 hours 8000/tcp cordpod_xos_synchronizer_vtn_1
+741c5465bd69 xosproject/xos "python /opt/xos/mana" 2 hours ago Up 2 hours 0.0.0.0:81->81/tcp, 8000/tcp cordpodbs_xos_bootstrap_ui_1
+90452f1e31e9 xosproject/xos "bash -c 'cd /opt/xos" 2 hours ago Up 2 hours 8000/tcp cordpodbs_xos_synchronizer_onboarding_1
+5d744962e86c redis "docker-entrypoint.sh" 2 hours ago Up 2 hours 6379/tcp cordpodbs_xos_redis_1
+bd29494712dc xosproject/xos-postgres "/usr/lib/postgresql/" 2 hours ago Up 2 hours 5432/tcp cordpodbs_xos_db_1
+c127d493f194 docker-registry:5000/mavenrepo:candidate "nginx -g 'daemon off" 3 hours ago Up 3 hours 443/tcp, 0.0.0.0:8080->80/tcp mavenrepo
+9d4b98d49e69 docker-registry:5000/cord-maas-automation:candidate "/go/bin/cord-maas-au" 3 hours ago Up 3 hours automation
+2f0f8bba4c4e docker-registry:5000/cord-maas-switchq:candidate "/go/bin/switchq" 3 hours ago Up 3 hours 0.0.0.0:4244->4244/tcp switchq
+53e3d81ddb56 docker-registry:5000/cord-provisioner:candidate "/go/bin/cord-provisi" 3 hours ago Up 3 hours 0.0.0.0:4243->4243/tcp provisioner
+5853b72e0f99 docker-registry:5000/config-generator:candidate "/go/bin/config-gener" 3 hours ago Up 3 hours 1337/tcp, 0.0.0.0:4245->4245/tcp generator
+3605c5208cb9 docker-registry:5000/cord-ip-allocator:candidate "/go/bin/cord-ip-allo" 3 hours ago Up 3 hours 0.0.0.0:4242->4242/tcp allocator
+dda7030d7028 docker-registry:5000/consul:candidate "docker-entrypoint.sh" 3 hours ago Up 3 hours storage
+775dbcf4c719 docker-registry:5000/cord-dhcp-harvester:candidate "/go/bin/harvester" 3 hours ago Up 3 hours 0.0.0.0:8954->8954/tcp harvester
+97a6c43fb405 registry:2.4.0 "/bin/registry serve " 4 hours ago Up 3 hours 0.0.0.0:5000->5000/tcp registry
+5a768a06e913 registry:2.4.0 "/bin/registry serve " 4 hours ago Up 3 hours 0.0.0.0:5001->5000/tcp registry-mirror
```
-### Docker Containers
-
-The target server runs a Docker image registry, a Maven repository containing
+The above shows Docker containers launched by XOS (image names starting with
+`xosproject`). There is also a Docker image registry, a Maven repository containing
the CORD ONOS apps, and a number of microservices used in bare-metal provisioning.
-You can see these by running `docker ps`:
```
-$ docker ps
-CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
-adfe0a0b68e8 docker-registry:5000/mavenrepo:candidate "nginx -g 'daemon off" 3 hours ago Up 3 hours 443/tcp, 0.0.0.0:8080->80/tcp mavenrepo
-da6bdd4ca322 docker-registry:5000/cord-dhcp-harvester:candidate "python /dhcpharveste" 3 hours ago Up 3 hours 0.0.0.0:8954->8954/tcp harvester
-b6fe30f03f73 docker-registry:5000/cord-maas-switchq:candidate "/go/bin/switchq" 3 hours ago Up 3 hours switchq
-a1a7d4c7589f docker-registry:5000/cord-maas-automation:candidate "/go/bin/cord-maas-au" 3 hours ago Up 3 hours automation
-628fb3725abf docker-registry:5000/cord-provisioner:candidate "/go/bin/cord-provisi" 3 hours ago Up 3 hours provisioner
-fe7b3414cf88 docker-registry:5000/config-generator:candidate "/go/bin/config-gener" 3 hours ago Up 3 hours 1337/tcp generator
-c7159495f9b4 docker-registry:5000/cord-ip-allocator:candidate "/go/bin/cord-ip-allo" 3 hours ago Up 3 hours allocator
-33bf33214d98 docker-registry:5000/consul:candidate "docker-entrypoint.sh" 3 hours ago Up 3 hours storage
-b44509b3314e registry:2.4.0 "/bin/registry serve " 3 hours ago Up 3 hours 0.0.0.0:5000->5000/tcp registry
-79060bba9994 registry:2.4.0 "/bin/registry serve " 3 hours ago Up 3 hours 0.0.0.0:5001->5000/tcp registry-mirror
+vagrant@prod:~$ sudo lxc list
++-------------------------+---------+------------------------------+------+------------+-----------+
+| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| ceilometer-1 | RUNNING | 10.1.0.4 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| glance-1 | RUNNING | 10.1.0.5 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| juju-1 | RUNNING | 10.1.0.3 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| keystone-1 | RUNNING | 10.1.0.6 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| mongodb-1 | RUNNING | 10.1.0.13 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| nagios-1 | RUNNING | 10.1.0.8 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| neutron-api-1 | RUNNING | 10.1.0.9 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| nova-cloud-controller-1 | RUNNING | 10.1.0.10 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| openstack-dashboard-1 | RUNNING | 10.1.0.11 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| percona-cluster-1 | RUNNING | 10.1.0.7 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| rabbitmq-server-1 | RUNNING | 10.1.0.12 (eth0) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+| testclient | RUNNING | 192.168.0.244 (eth0.222.111) | | PERSISTENT | 0 |
++-------------------------+---------+------------------------------+------+------------+-----------+
+```
+
+The LXD containers ending with names ending with `-1` are running
+OpenStack-related services. These containers can be
+entered as follows:
+
+```
+$ ssh ubuntu@<container-name>
+```
+
+The `testclient` container runs the simulated subscriber device used
+for running simple end-to-end connectivity tests. Its only connectivity is
+to the vSG, but it can be entered using:
+
+```
+$ sudo lxc exec testclient bash
+```
+
+### compute_node-1 VM
+
+The `compute_node-1` VM is the virtual compute node controlled by OpenStack.
+This VM can be entered from the `prod` VM. Run `cord prov list` to get the
+node name (assigned by MaaS) and then run:
+
+```
+$ ssh ubuntu@<compute-node-name>
+```
+
+Virtual machines created via XOS/OpenStack will be instantiated inside the `compute_node`
+VM. To login to an OpenStack VM, first get the management IP address (172.27.0.x):
+
+```
+vagrant@prod:~$ nova list --all-tenants
++--------------------------------------+-------------------------+--------+------------+-------------+---------------------------------------------------+
+| ID | Name | Status | Task State | Power State | Networks |
++--------------------------------------+-------------------------+--------+------------+-------------+---------------------------------------------------+
+| 3ba837a0-81ff-47b5-8f03-020175eed6b3 | mysite_exampleservice-2 | ACTIVE | - | Running | management=172.27.0.3; public=10.6.1.194 |
+| 549ffc1e-c454-4ef8-9df7-b02ab692eb36 | mysite_vsg-1 | ACTIVE | - | Running | management=172.27.0.2; mysite_vsg-access=10.0.2.2 |
++--------------------------------------+-------------------------+--------+------------+-------------+---------------------------------------------------+
+```
+
+Then run `ssh-agent` and add the default key -- this key loaded into the VM when
+it was created:
+
+```
+vagrant@prod:~$ ssh-agent bash
+vagrant@prod:~$ ssh-add
+```
+
+SSH to the compute node with the `-A` option and then to the VM using
+the management IP obtained above. So if the compute node name is `bony-alley`
+and the management IP is 172.27.0.2:
+
+```
+vagrant@prod:~$ ssh -A ubuntu@bony-alley
+ubuntu@bony-alley:~$ ssh ubuntu@172.27.0.2
+
+# Now you're inside the mysite-vsg-1 VM
+ubuntu@mysite-vsg-1:~$
```
### MaaS GUI
@@ -186,6 +273,7 @@
## Test results
+
After CORD-in-a-Box was set up, a couple of basic health
tests were executed on the platform. The results of these tests can be
found near the end of `~/install.out`.
diff --git a/scripts/cord-in-a-box.sh b/scripts/cord-in-a-box.sh
index 5eecc1c..7c475da 100755
--- a/scripts/cord-in-a-box.sh
+++ b/scripts/cord-in-a-box.sh
@@ -8,48 +8,15 @@
SSHCONFIG=~/.ssh/config
function cleanup_from_previous_test() {
- set +e
-
echo "## Cleanup ##"
- echo "Shutting down all Vagrant VMs"
+ echo "Destroying all Vagrant VMs"
cd $CORDDIR/build
vagrant destroy
- echo "Destroying juju environment"
- juju destroy-environment --force -y manual
-
- VMS=$( sudo uvt-kvm list )
- for VM in $VMS
- do
- echo "Destroying $VM"
- sudo uvt-kvm destroy $VM
- done
-
- echo "Cleaning up files"
- rm -rf ~/.juju
- rm -f ~/.ssh/known_hosts
- rm -rf ~/platform-install
- rm -rf ~/cord_apps
- rm -rf ~/.ansible_async
-
- echo "Removing MAAS"
- [ -e /usr/local/bin/remove-maas-components ] && /usr/local/bin/remove-maas-components
-
- echo "Remove apt-cacher-ng"
- sudo apt-get remove -y apt-cacher-ng
- sudo rm -f /etc/apt/apt.conf.d/02apt-cacher-ng
-
- echo "Removing mgmtbr"
- ifconfig mgmtbr && sudo ip link set dev mgmtbr down && sudo brctl delbr mgmtbr
-
- echo "Removing Juju packages"
- sudo apt-get remove --purge -y $(dpkg --get-selections | grep "juju\|nova\|neutron\|keystone\|glance" | awk '{print $1}')
- sudo apt-get autoremove -y
-
+ echo "Removing $CORDDIR"
+ cd ~
rm -rf $CORDDIR
-
- set -e
}
function bootstrap() {
@@ -60,10 +27,6 @@
sudo apt-get -y install qemu-kvm libvirt-bin libvirt-dev curl nfs-kernel-server git build-essential
[ -e ~/.ssh/id_rsa ] || ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
- cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-
- # Log into the local node once to get host key
- ssh -o StrictHostKeyChecking=no localhost "ls > /dev/null"
USER=$(whoami)
sudo adduser $USER libvirtd
@@ -86,15 +49,6 @@
echo "checking out opencord gerrit branch: $gerrit_branch"
repo download ${gerrit_branch/:/ }
done
-
- cd $CORDDIR/build
- sed -i "s/user: 'ubuntu'/user: \"$USER\"/" $CONFIG
-
- # Set external interface in config file
- IFACE=$(route | grep default | awk '{print $8}' )
- SRC="'eth0'"
- DST="'"$IFACE"'"
- sed -i "s/$SRC/$DST/" $CONFIG
fi
cd $CORDDIR/build
@@ -119,116 +73,70 @@
[ -d /var/lib/libvirt/images/ ] && [ ! -h /var/lib/libvirt/images ] && sudo rmdir /var/lib/libvirt/images
sudo mkdir -p /mnt/extra/libvirt_images
- sudo mkdir -p /mnt/extra/docker
- sudo mkdir -p /mnt/extra/docker-registry
- [ ! -e /var/lib/libvirt/images ] && sudo ln -s /mnt/extra/libvirt_images /var/lib/libvirt/images
- [ ! -e /var/lib/docker ] && sudo ln -s /mnt/extra/docker /var/lib/docker
- [ ! -e /docker-registry ] && sudo ln -s /mnt/extra/docker-registry /docker-registry
-
- cd $CORDDIR/build
- SRC="#- 'on_cloudlab=True'"
- DST="- 'on_cloudlab=True'"
- sed -i "s/$SRC/$DST/" config/cord_in_a_box.yml
+ if [ ! -e /var/lib/libvirt/images ]
+ then
+ sudo ln -s /mnt/extra/libvirt_images /var/lib/libvirt/images
+ fi
fi
}
-function unfortunate_hacks() {
- cd $CORDDIR/build
-
- # Allow compute nodes to PXE boot from mgmtbr
- sed -i "s/@type='udp']/@type='udp' or @type='bridge']/" \
- ~/.vagrant.d/gems/gems/vagrant-libvirt-0.0.35/lib/vagrant-libvirt/action/set_boot_order.rb
-}
-
-function corddev_up() {
+function vagrant_vms_up() {
cd $CORDDIR/build
sudo su $USER -c 'vagrant up corddev --provider libvirt'
+ sudo su $USER -c 'vagrant up prod --provider libvirt'
# This is a workaround for a weird issue with ARP cache timeout breaking 'vagrant ssh'
# It allows SSH'ing to the machine via 'ssh corddev'
- sudo su $USER -c "grep corddev $SSHCONFIG || vagrant ssh-config corddev >> $SSHCONFIG"
+ sudo su $USER -c "vagrant ssh-config corddev prod > $SSHCONFIG"
+
+ scp ~/.ssh/id_rsa* corddev:.ssh
+ ssh corddev "chmod go-r ~/.ssh/id_rsa"
}
function install_head_node() {
cd $CORDDIR/build
- # Network setup to install physical server as head node
- BRIDGE=$( route -n | grep 10.100.198.0 | awk '{print $8}' )
- ip addr list dev $BRIDGE | grep 10.100.198.201 || sudo ip addr add dev $BRIDGE 10.100.198.201
- ifconfig mgmtbr || sudo brctl addbr mgmtbr
- sudo ifconfig mgmtbr 10.1.0.1/24 up
-
# SSH config saved earlier allows us to connect to VM without running 'vagrant'
- scp ~/.ssh/id_rsa* corddev:.ssh
ssh corddev "cd /cord/build; ./gradlew fetch"
ssh corddev "cd /cord/build; ./gradlew buildImages"
- ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG -PtargetReg=10.100.198.201:5000 publish"
+ ssh corddev "cd /cord/build; ping -c 3 prod; ./gradlew -PdeployConfig=$VMDIR/$CONFIG -PtargetReg=10.100.198.201:5000 publish"
ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG deploy"
-
- # SSH config was overwritten by the deploy step
- # Will go away when head node runs in 'prod' VM
- sudo su $USER -c "grep corddev $SSHCONFIG || vagrant ssh-config corddev >> $SSHCONFIG"
}
function set_up_maas_user() {
- # Set up MAAS user to restart nodes via libvirt
- sudo mkdir -p /home/maas
- sudo chown maas:maas /home/maas
- sudo chsh -s /bin/bash maas
+ # Set up MAAS user on server to restart nodes via libvirt
+ grep maas /etc/passwd || sudo useradd -m maas
sudo adduser maas libvirtd
- sudo su maas -c 'cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys'
-}
+ # Copy generated public key to maas user's authorized_keys
+ sudo su maas -c "mkdir -p ~/.ssh"
+ sudo cp $HOME/.ssh/id_rsa.pub ~maas/.ssh/authorized_keys
+ sudo chown maas:maas ~maas/.ssh/authorized_keys
-FIRST_COMPUTE_NODE=true
+ # Copy generated private key to maas user's home dir in prod VM
+ scp $HOME/.ssh/id_rsa prod:/tmp
+ ssh prod "sudo mkdir -p ~maas/.ssh"
+ ssh prod "sudo cp /tmp/id_rsa ~maas/.ssh/id_rsa"
+ ssh prod "sudo chown -R maas:maas ~maas/.ssh"
+}
function add_compute_node() {
echo add_compute_node: $1 $2
-
+
cd $CORDDIR/build
sudo su $USER -c "vagrant up $1 --provider libvirt"
- if [ "$FIRST_COMPUTE_NODE" == true ]; then
- # Change MAC address of bridge to match cord-pod service profile
- # This change won't survive a reboot
- BRIDGE=$( route -n | grep 10.6.1.0 | awk '{print $8}' )
- sudo ifconfig $BRIDGE hw ether 02:42:0a:06:01:01
+ # Change MAC address of bridge to match cord-pod service profile
+ # This change won't survive a reboot
+ BRIDGE=$( route -n | grep 10.6.1.0 | awk '{print $8}' )
+ sudo ifconfig $BRIDGE hw ether 02:42:0a:06:01:01
- # Add gateway IP addresses to $BRIDGE for vsg and exampleservice tests
- # This change won't survive a reboot
- sudo ip address add 10.6.1.129 dev $BRIDGE
- sudo ip address add 10.6.1.193 dev $BRIDGE
+ ip addr list | grep 10.6.1.129 || sudo ip address add 10.6.1.129 dev $BRIDGE
+ ip addr list | grep 10.6.1.193 || sudo ip address add 10.6.1.193 dev $BRIDGE
- FIRST_COMPUTE_NODE=false
- fi
-
- # Sign into MAAS
- KEY=$(sudo maas-region-admin apikey --username=cord)
- maas login cord http://localhost/MAAS/api/1.0 $KEY
-
- NODEID=$(maas cord nodes list|jq -r '.[] | select(.status == 0).system_id')
- until [ "$NODEID" ]; do
- echo "Waiting for the compute node to transition to NEW state"
- sleep 15
- NODEID=$(maas cord nodes list|jq -r '.[] | select(.status == 0).system_id')
- done
-
- # Add remote power state
- maas cord node update $NODEID power_type="virsh" \
- power_parameters_power_address="qemu+ssh://maas@localhost/system" \
- power_parameters_power_id="$2"
-
- STATUS=$(sudo /usr/local/bin/get-node-prov-state |jq ".[] | select(.id == \"$NODEID\").status")
- until [ "$STATUS" == "2" ]; do
- if [ "$STATUS" == "3" ]; then
- echo "*** [WARNING] Possible error in node provisioning process"
- echo "*** [WARNING] Check /etc/maas/ansible/logs/$NODEID.log"
- fi
- echo "Waiting for the compute node to be fully provisioned"
- sleep 60
- STATUS=$(sudo /usr/local/bin/get-node-prov-state |jq ".[] | select(.id == \"$NODEID\").status")
- done
+ # Set up power cycling for the compute node and wait for it to be provisioned
+ ssh prod "cd /cord/build/ansible; ansible-playbook maas-provision.yml --extra-vars \"maas_user=maas vagrant_name=$2\""
echo ""
echo "compute_node is fully provisioned!"
@@ -237,7 +145,7 @@
function run_e2e_test () {
cd $CORDDIR/build
- # User has been added to the libvirtd group, but su $USER to be safe
+ # User has been added to the lbvirtd group, but su $USER to be safe
ssh corddev "cd /cord/build; ./gradlew -PdeployConfig=$VMDIR/$CONFIG postDeployTests"
}
@@ -297,12 +205,9 @@
cleanup_from_previous_test
fi
-set -e
-
bootstrap
cloudlab_setup
-unfortunate_hacks
-corddev_up
+vagrant_vms_up
if [[ $SETUP_ONLY -ne 0 ]]
then
@@ -313,7 +218,7 @@
install_head_node
set_up_maas_user
-#Limiting the maximum number of compute nodes that can be supported to 2. If
+#Limiting the maximum number of compute nodes that can be supported to 2. If
#more than 2 compute nodes are needed, this script need to be enhanced to set
#the environment variable NUM_COMPUTE_NODES before invoking the Vagrant commands
if [[ $NUM_COMPUTE_NODES -gt 2 ]]