update the quick start guides and add an onos image to the fetch
Change-Id: I1a374c5a96ebadfe18b157d369c5b201b5fceea4
diff --git a/build.gradle b/build.gradle
index 3b62215..b09b492 100644
--- a/build.gradle
+++ b/build.gradle
@@ -76,6 +76,12 @@
'tag': null,
'componentDir': './components/platform-install'
],
+ 'onosproject/onos': [
+ 'type': 'image',
+ 'upstream': upstreamReg,
+ 'name': 'onosproject/onos',
+ 'digest': 'sha256:6c310b6bc798f745977973c8c883d3dd1eb250fd124ae4d627fd98a69efb5afc'
+ ],
'nginx': [
'type': 'image',
'upstream': upstreamReg,
diff --git a/config/sample.yml b/config/sample.yml
new file mode 100644
index 0000000..7ae0372
--- /dev/null
+++ b/config/sample.yml
@@ -0,0 +1,70 @@
+# Deployment configuration for a phyical hardware POD
+---
+seedServer:
+ # IP address of the head node
+ ip: '10.90.0.2'
+
+ # If the head node is being accessed via a tunnel or reponds to SSH on a non-
+ # standard port, the port value should be set.
+ # port: '2022'
+
+ # User name and password used by Ansible to connect to the host for remote
+ # provisioning
+ user: 'ubuntu'
+ password: 'ubuntu'
+
+ # Network address information for the head node:
+ #
+ # fabric_ip - the IP address and mask bits to be used to configure the network
+ # interface connected to the leaf - spine fabric
+ #
+ # management_ip - the IP address and mask bits to be used to configure the network
+ # interface connecting the head node to the POD internal
+ # management network. The head node will deliver DHCP addresses to
+ # the other compute nodes over this interface
+ #
+ # external_ip - the IP address and mask bits to be used to configure the network
+ # interface connecting the head node (and the POD) to the
+ # Internet. All traffic in the POD to external hosts will be
+ # NAT-ed through this interface
+ # management_iface - the name of the interface that connects the head node to the POD
+ # internal network.
+ # external_iface - the name of the interface that connects the head node to the
+ # Internet
+ # management_network - the network and mask bits to used for hosts on the management
+ # network
+ fabric_ip: '10.6.1.1/24'
+ management_ip: '10.6.0.1/24'
+ external_ip: '47.135.132.21/24'
+ #management_iface: 'eth3'
+ #external_iface: 'eth2'
+ management_network: 10.1.0.0/24
+
+ # the skipTags options allow various part of the deployment to be skipped
+ # switch_support - does not deploy switch boot images to the PXE server
+ #
+ # reboot - will not reboot the head node after updating its network configuration
+ # this may mean the network configuration will not take effect, but will
+ # also prevent you from being locked out of the server if there is a
+ # network configuration error.
+ #
+ # interface_config - will not modify the network configuration of the head node,
+ # including the consistent naming of the network interfaces
+ skipTags:
+ # - 'switch_support'
+ # - 'reboot'
+ # - 'interface_config'
+
+docker:
+ imageVersion: candidate
+
+otherNodes:
+ # Experimental
+ #
+ # Specifies the subnet and address range that will be used to allocate IP addresses
+ # to the compute nodes as they are deployed into the POD.
+ fabric:
+ network: 10.1.1.1/24
+ management: 10.1.0.0/24
+ range_low: 10.1.1.2
+ range_high: 10.6.1.253
diff --git a/docs/images/physical.graffle b/docs/images/physical.graffle
new file mode 100644
index 0000000..0cecc95
--- /dev/null
+++ b/docs/images/physical.graffle
Binary files differ
diff --git a/docs/images/physical.png b/docs/images/physical.png
new file mode 100644
index 0000000..7584944
--- /dev/null
+++ b/docs/images/physical.png
Binary files differ
diff --git a/docs/quickstart_physical.md b/docs/quickstart_physical.md
new file mode 100644
index 0000000..7aadc30
--- /dev/null
+++ b/docs/quickstart_physical.md
@@ -0,0 +1,446 @@
+# CORD Quick Start Guide using Physical Nodes
+
+This guide is meant to enable the user to utilize the artifacts of this
+repository to to deploy CORD on to a physical hardware rack. The artifacts in
+this repository will deploy CORD against a standard physical rack wired
+according to the **best practices** as defined in this document.
+
+## Physical configuration
+
+
+As depicted in the diagram above the base model for the CORD POD deployment
+contains:
+- 4 OF switches comprising the leaf - spine fabric utilized for data traffic
+- 4 compute nodes with with 2 40G ports and 2 1G ports
+- 1 top of rack (TOR) switch utilized for management communications
+
+The best practices in terms of connecting the components of the CORD POD
+include:
+- Leaf nodes are connected to the spines nodes starting at the highest port
+number on the leaf.
+- For a given leaf node, its connection to the spine nodes terminate on the
+same port number on each spine.
+- Leaf *n* connections to spine nodes terminate at port *n* on each spine
+node.
+- Leaf spine switches are connected into the management TOR starting from the
+highest port number.
+- Compute nodes 40G interfaces are named *eth0* and *eth1*.
+- Compute nodes 10G interfaces are named *eth2* and *eth3*.
+- Compute node *n* is connected to the management TOR switch on port *n*,
+egressing from the compute node at *eth2*.
+- Compute node *n* is connected to its primary leaf, egressing at *eth0* and terminating on the leaf at port *n*.
+- Compute node *n* is connected to its secondary leaf, egressing at *eth1* and
+terminating on the leaf at port *n*.
+- *eth3* on the head node is the uplink from the POD to the Internet.
+
+The following assumptions are made about the phyical CORD POD being deployed:
+- The leaf - spine switchs are Accton 6712s
+- The compute nodes are using 40G Intel NIC cards
+- The compute node that is to be designated the *head node* has
+Ubuntu 14.04 LTS installed.
+
+**Prerequisite: Vagrant is installed and operationally.**
+**Note:** *This quick start guide has only been tested against Vagrant and
+VirtualBox, specially on MacOS.*
+
+## Bootstrapping the Head Node
+The head node is the key to the physical deployment of a CORD POD. The
+automated deployment of the physical POD is designed such that the head node is
+manually deployed, with the aid of automation tools, such as Ansible and from
+this head node the rest of the POD deployment is automated.
+
+The head node can be deployed either from a node outside the CORD POD or by
+deploying from the head to the head node. The procedure in each scenario is
+slightly different because during the bootstrapping of the head node it is
+possible that the interfaces needed to be renamed and the system to be
+rebooted. This guide assumes that the head node is being bootstrapped from a
+host outside of the POD.
+
+## Create Development Environment
+The development environment is required for the other tasks in this repository.
+The other tasks could technically be done outside this Vagrant based development
+environment, but it would be left to the user to ensure connectivity and
+required tools are installed. It is far easier to leverage the Vagrant based
+environment.
+
+### Cloning the Repository
+To clone the repository select a location on the outside the POD (OtP) host and
+issue the `git` command to download (clone) the repository.
+```
+git clone http://gerrit.opencord.org/cord
+```
+When this is complete, a listing (`ls`) of this directory should yield output
+similar to:
+```
+ls
+LICENSE.txt ansible/ components/ gradle/ gradlew.bat utils/
+README.md build.gradle config/ gradle.properties scripts/
+Vagrantfile buildSrc/ docs/ gradlew* settings.gradle
+```
+### Create Development Machine and Head Node Production Server
+To create the development machine the following single Vagrant command can be
+used. This will create an Ubuntu 14.04 LTS based virtual machine and install
+some basic required packages, such as Docker, Docker Compose, and
+Oracle Java 8.
+```
+vagrant up corddev
+```
+**NOTE:** *It may have several minutes for the first command `vagrant up
+corddev` to complete as it will include creating the VM as well as downloading
+and installing various software packages.*
+
+### Connect to the Development Machine
+To connect to the development machine the following vagrant command can be used.
+```
+vagrant ssh corddev
+```
+
+Once connected to the Vagrant machine, you can find the deployment artifacts
+in the `/cord` directory on the VM.
+```
+cd /cord
+```
+
+### Gradle
+[Gradle](https://gradle.org/) is the build tool that is used to help
+orchestrate the build and deployment of a POD. A *launch* script is included
+in the vagrant machine that will automatically download and install `gradle`.
+The script is called `gradlew` and the download / install will be invoked on
+the first use of this script; thus the first use may take a little longer
+than subsequent invocations and requires a connection to the internet.
+
+### Complete
+Once you have created and connected to the development environment this task is
+complete. The `cord` repository files can be found on the development machine
+under `/cord`. This directory is mounted from the host machine so changes
+made to files in this directory will be reflected on the host machine and
+vice-versa.
+
+## Fetch
+The fetching phase of the deployment pulls Docker images from the public
+repository down to the local machine as well as clones any `git` submodules
+that are part of the project. This phase can be initiated with the following
+command:
+```
+./gradlew fetch
+```
+
+### complete
+Once the fetch command has successfully been run, this step is complete. After
+this command completes you should be able to see the Docker images that were
+downloaded using the `docker images` command on the development machine:
+```
+docker images
+REPOSITORY TAG IMAGE ID CREATED SIZE
+python 2.7-alpine 836fa7aed31d 5 days ago 56.45 MB
+consul <none> 62f109a3299c 2 weeks ago 41.05 MB
+registry 2.4.0 8b162eee2794 9 weeks ago 171.1 MB
+abh1nav/dockerui latest 6e4d05915b2a 19 months ago 469.5 MB
+```
+
+## Build Images
+Bare metal provisioning leverages utilities built and packaged as Docker
+container images. These utilities are:
+
+ - cord-maas-bootstrap - (directory: `bootstrap`) run at MAAS installation
+ time to customize the MAAS instance via REST interfaces
+ - cord-maas-automation - (directory: `automation`) daemon on the head node to
+ automate PXE booted servers through the MAAS bare metal deployment work flow
+ - cord-maas-switchq - (directory: `switchq`) daemon on the head
+ node that watches for new switches being added to the POD and triggers
+ provisioning when a switch is identified (via the OUI on MAC address).
+ - cord-maas-provisioner - (directory: `provisioner`) daemon on the head node
+ to managing the execution of ansible playbooks against switches and compute
+ nodes as they are added to the POD.
+ - cord-ip-allocator - (directr: `ip-allocator`) daemon on the head node used
+ to allocate IP address for the fabric interfaces.
+ - cord-dhcp-harvester - (directory: `harvester`) run on the head node to
+ facilitate CORD / DHCP / DNS integration so that all hosts can be resolved
+ via DNS
+ - opencord/mavenrepo
+ - cord-test/nose
+ - cord-test/quagga
+ - cord-test/radius
+ - onosproject/onos
+
+The images can be built by using the following command. This will build all
+the images.
+```
+./gradlew buildImages
+```
+
+**NOTE:** *The first time you run `./gradlew` it will download from the Internet
+the `gradle` binary and install it locally. This is a one time operation.*
+
+### Complete
+Once the `buildImages` command successfully runs this task is complete. The
+CORD artifacts have been built and the Docker images can be viewed by using the
+`docker images` command on the development machine.
+```
+docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.ID}}'
+REPOSITORY TAG SIZE IMAGE ID
+cord-maas-switchq latest 781 MB 4736cc8c4f71
+cord-provisioner latest 814.6 MB 50ab479e4b52
+cord-dhcp-harvester latest 60.67 MB 88f900d74f19
+cord-maas-bootstrap latest 367.5 MB 19bde768c786
+cord-maas-automation latest 366.8 MB 1e2ab7242060
+cord-ip-allocator latest 324.3 MB f8f2849107f6
+opencord/mavenrepo latest 434.2 MB 9d1ad7214262
+cord-test/nose latest 1.028 GB 67b996f2ad19
+cord-test/quagga latest 454.4 MB b46f7dd20bdf
+cord-test/radius latest 312.1 MB e09d78aef295
+onosproject/onos <none> 825.6 MB 309088c647cf
+python 2.7-alpine 56.45 MB 836fa7aed31d
+golang 1.6-alpine 282.9 MB d688f409d292
+golang alpine 282.9 MB d688f409d292
+ubuntu 14.04 196.6 MB 38c759202e30
+consul <none> 41.05 MB 62f109a3299c
+nginx latest 182.7 MB 0d409d33b27e
+registry 2.4.0 171.1 MB 8b162eee2794
+swarm <none> 19.32 MB 47dc182ea74b
+nginx <none> 182.7 MB 3c69047c6034
+hbouvier/docker-radius latest 280.9 MB 5d5d3c0a91b0
+abh1nav/dockerui latest 469.5 MB 6e4d05915b2a
+```
+**NOTE:** *Not all the above Docker images were built by the `buildImages`
+command. Some of them, list golang, are used as a base for other Docker
+images; and some, like `abh1nav/dockerui` were downloaded when the development
+machine was created with `vagrant up`.*
+
+## Deployment Configuration File
+The commands to deploy the POD can be customized via a *deployment configuration
+file*. The file is in [YAML](http://yaml.org/).
+
+To construct a configuration file for yoru physical POD you should copy the
+sample deployment configuration found in `config/sample.yml` and modify the
+values to fit your physical deployment.
+
+## Prime the Target server
+The target server is the server that will assume the role of the head node in
+the cord POD. Priming this server consists of deploying some base software that
+is required to deploy the base software, such as a docker registry. Having the
+docker registry on the target server allows the deployment process to push
+images to the target server that are used in the reset of the process, thus
+making the head node a self contained deployment.
+```
+./gradew -PdeployConfig=config/podX.yml prime
+```
+
+### Complete
+Once the `prime` command successfully runs this task is complete. When this
+step is complete a Docker registry and Docker registry mirror. It can be
+verified that these are running by using the `docker ps` command.
+```
+docker ps --format 'table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.CreatedAt}}'
+CONTAINER ID IMAGE COMMAND CREATED AT
+5f1cbebe7e61 registry:2.4.0 "/bin/registry serve " 2016-07-13 17:03:08 +0000 UTC
+6d3a911e5323 registry:2.4.0 "/bin/registry serve " 2016-07-13 17:03:08 +0000 UTC
+```
+
+## Publish
+Publishing consists of *pushing* the build docker images to the Docker
+repository on the target head node. This step can take a while as it has to
+transfer all the image from the development machine to the target head node.
+This step is started with the following command:
+```
+./gradew -PtargetReg=<head-node-ip-address>:5000 publish
+```
+
+### Complete
+Once the `publish` command successfully runs this task is complete. When this
+step is complete it can be verified by performing a query on the target
+server's Docker registry using the following command on the development
+machine.
+```
+curl -sS http://head-node-ip-address:5000/v2/_catalog | jq .
+{
+ "repositories": [
+ "consul",
+ "cord-dhcp-harvester",
+ "cord-ip-allocator",
+ "cord-maas-automation",
+ "cord-maas-bootstrap",
+ "cord-maas-switchq",
+ "cord-provisioner",
+ "mavenrepo",
+ "nginx",
+ "onosproject/onos",
+ "swarm"
+ ]
+}
+```
+
+## Deploy Bare Metal Provisioning Capabilities
+There are three parts to deploying bare metal: deploying the head node PXE
+server (`MAAS`), PXE booting a compute node, and post deployment provisioning
+of the compute node. These tasks are accomplished utilizing additionally
+Vagrant machines as well as executing `gradle` tasks in the Vagrant
+development machine.
+
+### Deploy MAAS
+Canonical MAAS provides the PXE and other bare metal provisioning services for
+CORD and will be deployed on the head node.
+```
+./gradlew deployBase
+```
+
+This task can take some time so be patient. It should complete without errors,
+so if an error is encountered something went horrible wrong (tm).
+
+### Complete
+This step is complete when the command successfully runs. The Web UI for MAAS
+can be viewed by browsing to the target machine using a URL of the form
+`http://head-node-ip-address/MAAS`.
+
+After the `deployBase` command install `MAAS`, it initiates the download of
+an Ubuntu 14.04 boot image that will be used to boot the other POD servers.
+This download can take some time and the process cannot continue until the
+download is complete. The status of the download can be verified through
+the UI by visiting the URL `http://head-node-ip-address/MAAS/images/`,
+or via the command line from head node via the following commands:
+```
+APIKEY=$(sudo maas-region-admin apikey --user=cord)
+maas login cord http://localhost/MAAS/api/1.0 "$APIKEY"
+maas cord boot-resources read | jq 'map(select(.type != "Synced"))'
+```
+
+It the output of of the above commands is not a empty list, `[]`, then the
+images have not yet been completely downloaded. depending on your network speed
+this could take several minutes. Please wait and then attempt the last command
+again until the returned list is empty, `[]`. When the list is empty you can
+proceed.
+
+Browse around the UI and get familiar with MAAS via documentation at `http://maas.io`
+
+## Deploy XOS
+XOS provides service provisioning and orchestration for the CORD POD. To deploy
+XOS to the head node use the following command:
+```
+./gradlew deployPlatform
+```
+
+This task can take some time so be patient. It should complete without errors,
+so if an error is encountered something went horrible wrong (tm).
+
+### Complete
+This step is complete when the command successfully runs. The deployment of XOS
+includes a deployment of Open Stack.
+
+## Booting Compute Nodes
+
+### Network configuration
+The proposed configuration for a CORD POD is has the following network configuration on the head node:
+
+ - eth0 / eth1 - 40G interfaces, not relevant for the test environment.
+ - eth2 - the interface on which the head node supports PXE boots and is an internally interface to which all
+ the compute nodes connected
+ - eth3 - WAN link. the head node will NAT from eth2 to eth3
+ - mgmtbr - Not associated with a physical network and used to connect in the VM created by the openstack
+ install that is part of XOS
+
+The Ansible scripts configure MAAS to support DHCP/DNS/PXE on the eth2 and mgmtbr interfaces.
+
+Once it has been verified that the ubuntu boot image has been downloaded the
+compute nodes may be PXE booted.
+
+**Note:** *In order to ensure that the compute node PXE boot the bios settings
+may have to be adjusted. Additionally, the remote power management on the
+compute nodes must be enabled.*
+
+The compute node will boot, register with MAAS, and then be shut off. After this
+is complete an entry for the node will be in the MAAS UI at
+`http://head-node-ip-address/MAAS/#/nodes`. It will be given a random hostname
+made up, in the Canonical way, of a adjective and an noun, such as
+`popular-feast.cord.lab`. *The name will be different for every deployment.* The
+new node will be in the `New` state.
+
+As the machines boot they should be automatically transitioned from `New`
+through the states of `Commissioning` and `Acquired` to `Deployed`.
+
+### Post Deployment Provisioning of the Compute Node
+Once the node is in the `Deployed` state, it will be provisioned for use in a
+CORD POD by the execution of an `Ansible` playbook.
+
+### Complete
+Once the compute node is in the `Deployed` state and post deployment provisioning on the compute node is
+complete, this task is complete.
+
+Logs of the post deployment provisioning of the compute nodes can be found
+in `/etc/maas/ansible/logs` on the head node.
+
+Assitionally, the post deployment provisioning of the compute nodes can be
+queried from the provision service using curl
+```
+curl -sS http://$(docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner):4243/provision/ | jq '[.[] | { "status": .status, "name": .request.Info.name}]'
+[
+ {
+ "message": "",
+ "name": "steel-ghost.cord.lab",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "feline-shirt.cord.lab",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "yellow-plot.cord.lab",
+ "status": 2
+ }
+]
+```
+In the above a "status" of 2 means that the provisioning is complete. The
+other values that status might hold are:
+ - `0` - Pending, the request has been accepted by the provisioner but not yet
+ started
+ - `1` - Running, the request is being processed and the node is being
+ provisioned
+ - `2` - Complete, the provisioning has been completed successfully
+ - `3` - Failed, the provisioning has failed and the `message` will be
+ populated with the exit message from provisioning.
+
+# Booting OpenFlow switches
+Once the compute nodes have begun their boot process you may also boot the
+switches that support the leaf spine fabric. These switches should ONIE install
+boot and download their boot image from MAAS.
+
+### Complete
+This step is complete when the command completes successfully. You can verify
+the provisioning of the false switch by querying the provisioning service
+using curl.
+```
+curl -sS http://$(docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner):4243/provision/ | jq '[.[] | { "status": .status, "name": .request.Info.name, "message": .message}]'
+[
+ {
+ "message": "",
+ "name": "leaf-1",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "leaf-2",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "spine-1",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "spine-2",
+ "status": 2
+ }
+]
+
+```
+In the above a "status" of 2 means that the provisioning is complete. The
+other values that status might hold are:
+ - `0` - Pending, the request has been accepted by the provisioner but not yet
+ started
+ - `1` - Running, the request is being processed and the node is being
+ provisioned
+ - `2` - Complete, the provisioning has been completed successfully
+ - `3` - Failed, the provisioning has failed and the `message` will be
+ populated with the exit message from provisioning.
diff --git a/docs/quickstart_vm.md b/docs/quickstart_vm.md
index 959f664..81ab69c 100644
--- a/docs/quickstart_vm.md
+++ b/docs/quickstart_vm.md
@@ -5,278 +5,451 @@
is to give the user an experience of bring up a CORD on bare
metal.*]
-This tutorial walks you through the steps to bring up a CORD "POD"
-on a single server using multiple virtual machines which
-represent the physical nodes of a production deployment.
+This guide is meant to enable the user to quickly exercise the capabilities
+provided by the artifacts of this repository. There are three high level tasks
+that can be exercised:
+ - Create development environment
+ - Build and Publish Docker images that support bare metal provisioning
+ - Deploy the bare metal provisioning capabilities to a virtual machine (head
+ node) and PXE boot a compute node
-Specifically, the tutorial covers the following:
+**Prerequisite: Vagrant is installed and operationally.**
+**Note:** *This quick start guide has only been tested against Vagrant and
+VirtualBox, specially on MacOS.*
-1. Bring up a build environment
-1. Fetch and build the binary artifacts
-1. Deploy the software to the target *seed* server
-1. Bring up and auto configure / provision compute nodes and
-switches in the POD
-1. Run some tests on the platform
-1. Clean up
+## Create Development Environment
+The development environment is required for the other tasks in this repository.
+The other tasks could technically be done outside this Vagrant based development
+environment, but it would be left to the user to ensure connectivity and
+required tools are installed. It is far easier to leverage the Vagrant based
+environment.
-### What you need (Prerequisites)
-
-You will need a build machine (can be your developer laptop) and a target
-server.
-
-Build host:
-
-* Mac OS X, Linux, or Windows with a 64-bit OS
-* Git (2.5.4 or later) - [TODO add pointers and/or instructions]
-* Vagrant (1.8.1 or later) - [TODO add pointers and/or instructions]
-* Access to the Internet (at least through the fetch phase)
-* SSH access to the target server
-
-### Bring up and Use the CORD Build Environment
-
-On the build host, clone the CORD integration repository and switch into its
-top directory:
-
-For now you can clone the repository anonymously:
-
+### Cloning the Repository
+To clone the repository select a location on the outside the POD (OtP) host and
+issue the `git` command to download (clone) the repository.
```
-git clone https://gerrit.opencord.org/cord
-cd cord
+git clone http://gerrit.opencord.org/cord
+```
+When this is complete, a listing (`ls`) of this directory should yield output
+similar to:
+```
+ls
+LICENSE.txt ansible/ components/ gradle/ gradlew.bat utils/
+README.md build.gradle config/ gradle.properties scripts/
+Vagrantfile buildSrc/ docs/ gradlew* settings.gradle
+```
+### Create Development Machine and Head Node Production Server
+To create the development machine the following single Vagrant command can be
+used. This will create an Ubuntu 14.04 LTS based virtual machine and install
+some basic required packages, such as Docker, Docker Compose, and
+Oracle Java 8.
+```
+vagrant up corddev
+```
+**NOTE:** *It may have several minutes for the first command `vagrant up
+corddev` to complete as it will include creating the VM as well as downloading
+and installing various software packages.*
+
+To create the VM that represents the POD head node the following vagrant
+command can be used. This will create a basic Ubuntu 14.04 LTS server with
+no additional software installed.
+```
+vagrant up prod
```
-If -- for whatever reason -- you decide to clone the repo as a private
-repository using ssh, then you should add your private key to your local ssh
-forwarding agent, because we will clone additional CORD repositories within
-the Vagrant environment using the same git access mode, and this will require
-your local agent to know your identity:
-
+### Connect to the Development Machine
+To connect to the development machine the following vagrant command can be used.
```
-ssh-add ~/.ssh/id_rsa
+vagrant ssh corddev
```
-Bring up the standardized CORD build and development environment (VM) and the
-VM that will play the role of the head end in the sample CORD POD. This will
-take a few minutes, depending on your connection speed:
-
-```
-vagrant up corddev prod
-```
-
-Login to the build environment:
-
-```
-vagrant ssh corddev -- -L 8080:localhost:80
-```
-
-Switch to the CORD integration directory, which is shared from your host:
-
+Once connected to the Vagrant machine, you can find the deployment artifacts
+in the `/cord` directory on the VM.
```
cd /cord
```
-### Fetch and Build the CORD Artifacts
+### Gradle
+[Gradle](https://gradle.org/) is the build tool that is used to help
+orchestrate the build and deployment of a POD. A *launch* script is included
+in the vagrant machine that will automatically download and install `gradle`.
+The script is called `gradlew` and the download / install will be invoked on
+the first use of this script; thus the first use may take a little longer
+than subsequent invocations and requires a connection to the internet.
-Pre-fetch all pre-requisites needed to build all components for CORD. This is
-the step that takes the longest time as a large number of images and files
-need to be downloaded.
+### Complete
+Once you have created and connected to the development environment this task is
+complete. The `cord` repository files can be found on the development machine
+under `/cord`. This directory is mounted from the host machine so changes
+made to files in this directory will be reflected on the host machine and
+vice-versa.
+## Fetch
+The fetching phase of the deployment pulls Docker images from the public
+repository down to the local machine as well as clones any `git` submodules
+that are part of the project. This phase can be initiated with the following
+command:
```
./gradlew fetch
```
-> *What this does:*
->
-> This command pulls the Docker registry image down from the
-> Internet as well as performs a `git submodule update
-> --recursive` on the source tree to pull down from the GIT
-> repositories all the sub projects referenced by the `cord`
-> repository.
-_NOTE: The first time you run `./gradlew` it will download from the Internet
-the `gradle` binary and install it locally. This is a one time operation._
+### complete
+Once the fetch command has successfully been run, this step is complete. After
+this command completes you should be able to see the Docker images that were
+downloaded using the `docker images` command on the development machine:
+```
+docker images
+REPOSITORY TAG IMAGE ID CREATED SIZE
+python 2.7-alpine 836fa7aed31d 5 days ago 56.45 MB
+consul <none> 62f109a3299c 2 weeks ago 41.05 MB
+registry 2.4.0 8b162eee2794 9 weeks ago 171.1 MB
+abh1nav/dockerui latest 6e4d05915b2a 19 months ago 469.5 MB
+```
-Time to build artifacts needed for deployment:
+## Build Images
+Bare metal provisioning leverages utilities built and packaged as Docker
+container images. These utilities are:
+ - cord-maas-bootstrap - (directory: `bootstrap`) run at MAAS installation
+ time to customize the MAAS instance via REST interfaces
+ - cord-maas-automation - (directory: `automation`) daemon on the head node to
+ automate PXE booted servers through the MAAS bare metal deployment work flow
+ - cord-maas-switchq - (directory: `switchq`) daemon on the head
+ node that watches for new switches being added to the POD and triggers
+ provisioning when a switch is identified (via the OUI on MAC address).
+ - cord-maas-provisioner - (directory: `provisioner`) daemon on the head node
+ to managing the execution of ansible playbooks against switches and compute
+ nodes as they are added to the POD.
+ - cord-ip-allocator - (directr: `ip-allocator`) daemon on the head node used
+ to allocate IP address for the fabric interfaces.
+ - cord-dhcp-harvester - (directory: `harvester`) run on the head node to
+ facilitate CORD / DHCP / DNS integration so that all hosts can be resolved
+ via DNS
+ - opencord/mavenrepo
+ - cord-test/nose
+ - cord-test/quagga
+ - cord-test/radius
+ - onosproject/onos
+
+The images can be built by using the following command. This will build all
+the images.
```
./gradlew buildImages
```
-> *What this does:*
->
-> This command visits each submodule in the project and
-> build any Docker images that are part of the project. These
-> Docker images are stored in the local Docker daemon's
-> data area.
-Prime the target server so that it is minimally ready to
-receive the images. This includes starting a *Docker
-Registry* on the target server (`prod`).
+**NOTE:** *The first time you run `./gradlew` it will download from the Internet
+the `gradle` binary and install it locally. This is a one time operation.*
-### Prepare the Target Server and Push Images to Target Server
-
+### Complete
+Once the `buildImages` command successfully runs this task is complete. The
+CORD artifacts have been built and the Docker images can be viewed by using the
+`docker images` command on the development machine.
```
-./gradlew prime
+docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.ID}}'
+REPOSITORY TAG SIZE IMAGE ID
+cord-maas-switchq latest 781 MB 4736cc8c4f71
+cord-provisioner latest 814.6 MB 50ab479e4b52
+cord-dhcp-harvester latest 60.67 MB 88f900d74f19
+cord-maas-bootstrap latest 367.5 MB 19bde768c786
+cord-maas-automation latest 366.8 MB 1e2ab7242060
+cord-ip-allocator latest 324.3 MB f8f2849107f6
+opencord/mavenrepo latest 434.2 MB 9d1ad7214262
+cord-test/nose latest 1.028 GB 67b996f2ad19
+cord-test/quagga latest 454.4 MB b46f7dd20bdf
+cord-test/radius latest 312.1 MB e09d78aef295
+onosproject/onos <none> 825.6 MB 309088c647cf
+python 2.7-alpine 56.45 MB 836fa7aed31d
+golang 1.6-alpine 282.9 MB d688f409d292
+golang alpine 282.9 MB d688f409d292
+ubuntu 14.04 196.6 MB 38c759202e30
+consul <none> 41.05 MB 62f109a3299c
+nginx latest 182.7 MB 0d409d33b27e
+registry 2.4.0 171.1 MB 8b162eee2794
+swarm <none> 19.32 MB 47dc182ea74b
+nginx <none> 182.7 MB 3c69047c6034
+hbouvier/docker-radius latest 280.9 MB 5d5d3c0a91b0
+abh1nav/dockerui latest 469.5 MB 6e4d05915b2a
```
-> *What this does:*
->
-> This uses and Ansible playbook to install docker on the
-> target server as well as start a docker registry on that
-> server.
+**NOTE:** *Not all the above Docker images were built by the `buildImages`
+command. Some of them, list golang, are used as a base for other Docker
+images; and some, like `abh1nav/dockerui` were downloaded when the development
+machine was created with `vagrant up`.*
-Let's publish all artifacts to the staging registry (Docker registry). Here we
-must specify the IP address of the target server (`prod`).
+## Deployment Configuration File
+The commands to deploy the POD can be customized via a *deployment
+configuration file*. The file is in [YAML](http://yaml.org/). For
+the purposes of the quick start using vagrant VMs for the POD nodes a
+deployment configuration file has been provide in
+[`config/default.yml`](config/default.yml). This default configuration
+specifies the target server as the vagrant machine named `prod` that was
+created earlier.
+## Prime the Target server
+The target server is the server that will assume the role of the head node in
+the cord POD. Priming this server consists of deploying some base software that
+is required to deploy the base software, such as a docker registry. Having the
+docker registry on the target server allows the deployment process to push
+images to the target server that are used in the reset of the process, thus
+making the head node a self contained deployment.
```
-./gradlew -PtargetReg=10.100.198.201:5000 publish
-```
-> *What this does:*
->
-> This command pushes all the images that have been built to
-> the registry specified using the `-PtargetReg` command line
-> option. The address used above, `10.100.198.201:5000`,
-> refers to the `prod` VM created at the start of this guide
-> using `vagrant up`.
-
-### Deploy and Configure Bare Metal Services on Target Server
-
-Deploy the head node software to the the target server (`prod`).
-
-**NOTE:** The default MAAS deployment does not support power management for
-virtual box based hosts. As part of the MAAS installation support was added for
-power management, but it does require some additional configuration. This
-additional configuration is detailed in the #virtual-box-power-management
-section below, but is mentioned here because when deploying the head node an
-additional parameter must be set. This parameter specifies the username on the
-host machine that should be used when SSHing from the head node to the host
-machine to remotely execute the `vboxmanage` command. This is typically the
-username used when logging into your laptop or desktop development machine.
-This should be specified in the `config/default.yml` file as the
-`power_helper_user` value, this values defaults to `cord`.
-```
-./gradlew Deploy
-```
-> *What this does:*
->
-> This command uses and Ansible playbook to configure networking
-> on the target server and then installs Canonical's MAAS (Metal
-> as a Service) software on the target server. This software
-> provides PXE boot, DHCP, DNS, and HTTP image download for the
-> CORD POD.
->
-> After installing the base software component, the Ansible
-> playbook configures the MAAS server including default users
-> and DHCP/DNS, as well as starts the boot image downloaded
-> from the Internet.
->
-> Lastly, after the software is configure, the Ansible script
-> starts several Docker containers that help manage automation
-> of the POD's host and switch resources as they become
-> available.
-
-At this point you can view the MAAS web UI by visiting the following URL
-```
-http://localhost:8080/MAAS
+./gradew prime
```
-To authenticate to the UI you can use the user name `cord` with
-the password `cord`.
-
-Browse around the UI and get familiar with MAAS via documentation at
-`http://maas.io`
-
-**NOTE**: Before moving on the the next step it is import to ensure that MAAS
-has completed the download and processing of the operating system images
-required to PXE boot additional servers. This can be done either by viewing the
-`Images` tab in the MAAS UI or by the following commands run on the target
-(head) node.
+### Complete
+Once the `prime` command successfully runs this task is complete. When this
+step is complete a Docker registry and Docker registry mirror. It can be
+verified that these are running by using the `docker ps` command on the
+producation head node VM.
```
-sudo apt-get install -y jq
+docker ps --format 'table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.CreatedAt}}'
+CONTAINER ID IMAGE COMMAND CREATED AT
+5f1cbebe7e61 registry:2.4.0 "/bin/registry serve " 2016-07-13 17:03:08 +0000 UTC
+6d3a911e5323 registry:2.4.0 "/bin/registry serve " 2016-07-13 17:03:08 +0000 UTC
+```
+
+## Publish
+Publishing consists of *pushing* the build docker images to the Docker
+repository on the target head node. This step can take a while as it has to
+transfer all the image from the development machine to the target head node.
+This step is started with the following command:
+```
+./gradew -PtargetReg=10.100.198.201:5000 publish
+```
+
+### Complete
+Once the `publish` command successfully runs this task is complete. When this
+step is complete it can be verified by performing a query on the target
+server's Docker registry using the following command on the development
+machine.
+```
+curl -sS http://10.100.198.201:5000/v2/_catalog | jq .
+{
+ "repositories": [
+ "consul",
+ "cord-dhcp-harvester",
+ "cord-ip-allocator",
+ "cord-maas-automation",
+ "cord-maas-bootstrap",
+ "cord-maas-switchq",
+ "cord-provisioner",
+ "mavenrepo",
+ "nginx",
+ "onosproject/onos",
+ "swarm"
+ ]
+}
+```
+
+## Deploy Bare Metal Provisioning Capabilities
+There are three parts to deploying bare metal: deploying the head node PXE
+server (`MAAS`), PXE booting a compute node, and post deployment provisioning
+of the compute node. These tasks are accomplished utilizing additionally
+Vagrant machines as well as executing `gradle` tasks in the Vagrant
+development machine.
+
+### VirtualBox Power Management
+The default MAAS deployment does not support power management for virtual box
+based hosts. As part of the MAAS installation support was added for power
+management, but it does require some additional configuration. This additional
+configuration is detailed at the end of this document, but is mentioned here
+because when deploying the head node an additional parameter must be set. This
+parameter specifies the username on the host machine that should be used when
+SSHing from the head node to the host machine to remotely execute the
+`vboxmanage` command. This is typically the username used when logging into your
+laptop or desktop development machine. This value is set by editing the
+`config/default.yml` file and replacing the default value of
+`seedServer.power_helper_user` with the approriate username. The default value
+is `cord`
+
+### Deploy MAAS
+Canonical MAAS provides the PXE and other bare metal provisioning services for
+CORD and will be deployed on the head node.
+```
+./gradlew deployBase
+```
+
+This task can take some time so be patient. It should complete without errors,
+so if an error is encountered something went horrible wrong (tm).
+
+### Complete
+This step is complete when the command successfully runs. The Web UI for MAAS
+can be viewed by browsing to the vagrant machine named `prod`. Because this
+machine is on a host internal network it can't be directly reached from the
+host machine, typically your laptop. In order to expose the UI, from the VM host
+machine you will issue the following command:
+```
+vagrant ssh prod -- -L 8080:localhost:80
+```
+This command will create a SSH tunnel from the VM host machine to the head
+node so that from the VM host you can view the MAAS UI by visiting the URL
+`http://localhost:8080/MAAS`. The default authentication credentials are a
+username of `cord` and a password of `cord`.
+
+After the `deployBase` command install `MAAS`, it initiates the download of
+an Ubuntu 14.04 boot image that will be used to boot the other POD servers.
+This download can take some time and the process cannot continue until the
+download is complete. The status of the download can be verified through
+the UI by visiting the URL `http://localhost:8888/MAAS/images/`, or via the
+command line from head node via the following commands:
+```
APIKEY=$(sudo maas-region-admin apikey --user=cord)
-maas login cord http://localhost/MAAS/api/1.0 $APIKEY
+maas login cord http://localhost/MAAS/api/1.0 "$APIKEY"
maas cord boot-resources read | jq 'map(select(.type != "Synced"))'
```
-It the output of of the above commands is not a empty list, `[]`, then
-the images have not yet been completely downloaded. depending on your
-network speed this could take several minutes. Please wait and then
-attempt the last command again until the returned list is empty, `[]`.
-When the list is empty you can proceed.
+It the output of of the above commands is not a empty list, `[]`, then the
+images have not yet been completely downloaded. depending on your network speed
+this could take several minutes. Please wait and then attempt the last command
+again until the returned list is empty, `[]`. When the list is empty you can
+proceed.
-### Create and Boot Compute Nodes
-To create a compute node you use the following vagrant command. This command
-will create a VM that PXE boots to the interface on which the MAAS server is
-listening. **This task is executed on your host machine and not in the
-development virtual machine.**
+Browse around the UI and get familiar with MAAS via documentation at `http://maas.io`
+
+## Deploy XOS
+XOS provides service provisioning and orchestration for the CORD POD. To deploy
+XOS to the head node use the following command:
```
-vagrant up compute_node1
+./gradlew deployPlatform
```
-Vagrant will create a UI, which will popup on your screen, so that the PXE boot
-process of the compute node can be visually monitored. After an initial PXE
-boot of the compute node it will automatically be shutdown.
+This task can take some time so be patient. It should complete without errors,
+so if an error is encountered something went horrible wrong (tm).
-The compute node Vagrant machine it s bit different that most Vagrant machine
-because it is not created with a user account to which Vagrant can connect,
-which is the normal behavior of Vagrant. Instead the Vagrant files is
-configured with a _dummy_ `communicator` which will fail causing the following
-error to be displayed, but the compute node Vagrant machine will still have
-been created correctly.
-```
-The requested communicator 'none' could not be found.
-Please verify the name is correct and try again.
-```
+### Complete
+This step is complete when the command successfully runs. The deployment of XOS
+includes a deployment of Open Stack.
-The compute node VM will boot, register with MAAS, and then be shut off. After
-this is complete an entry for the node will be in the MAAS UI at
-`http://localhost:8888/MAAS/#/nodes`. It will be given a random hostname made
-up, in the Canonical way, of a adjective and an noun, such as
-`popular-feast.cord.lab`. _The name will be different for everyone._ The new
-node will be in the `New` state.
-
-If you have properly configured power management for virtualbox (see below)
-the host will be automatically transitioned from `New` through the start of
-`Commissioning` and `Acquired` to `Deployed`.
-
-The `Vagrant` file defines 3 compute node machines you can start using the
-process as defined above. The three machines can be started using the commands
+## Create and Boot Compute Node
+The sample vagrant VM based POD is configured to support the creation of 3
+compute nodes. These nodes will PXE boot from the head node and are created
+using the `vagrant up` command as follows:
```
vagrant up compute_node1
vagrant up compute_node2
vagrant up compute_node3
```
-> *What this does:*
->
-> As each compute node is booted, it downloads and installs an Ubuntu OS images
-> and is automated through the MAAS deployment workflow. This is the same
-> workflow that will be executed if the hosts were not virtual hosts. During
-> this process the hosts will be rebooted several times.
->
-> After a host has reached the `Deployed` state in MAAS an additional
-> provisioning step will be executed that customizes the compute node for
-> use in a CORD POD. This step includes updating the network configuration of
-> the compute node as well as deploying some standard software, such as
-> `Docker`.
+**NOTE:** *This task is executed on your host machine and not in the
+development virtual machine*
-### Create and Boot Example (False) Switch
-Included with the virtual compute nodes is an additional vagrant machine VM
-that can be used to test the provisioning of a switch. This VM is configured so
-that the CORD POD automation will assume it is a switch and run additional
-provisioning on this *false* switch to deploy software and utility scripts.
-**This task is executed on your host machine and not in the development virtual
-machine.**
+When starting the compute node VMs the console (UI) for each will be displayed
+so you are able to watch the boot process if you like.
-The following command can be used to start the *false* switch.
+As vagrant starts these machines, you will see the following error:
+```
+==> compute_node1: Waiting for machine to boot. This may take a few minutes...
+The requested communicator 'none' could not be found.
+Please verify the name is correct and try again.
+```
+
+This error is normal and is because vagrant attempts to `SSH` to a server
+after it is started. However, because the process is PXE booting these servers
+this is not possible. To work around (with) vagrant the vagrant `communicator`
+setting for each of the compute nodes is set to "none", thus vagrant complains,
+but the machines will PXE boot.
+
+The compute node VM will boot, register with MAAS, and then be shut off. After
+this is complete an entry for the node will be in the MAAS UI at
+`http://localhost:8888/MAAS/#/nodes`. It will be given a random hostname made
+up, in the Canonical way, of a adjective and an noun, such as
+`popular-feast.cord.lab`. *The name will be different for every deployment.*
+The new node will be in the `New` state.
+
+If you have properly configured power management for virtualbox (see below) the
+host will be automatically transitioned from `New` through the states of
+`Commissioning` and `Acquired` to `Deployed`.
+
+### Post Deployment Provisioning of the Compute Node
+Once the node is in the `Deployed` state, it will be provisioned for use in a
+CORD POD by the execution of an `Ansible` playbook.
+
+### Complete
+Once the compute node is in the `Deployed` state and post deployment provisioning on the compute node is
+complete, this task is complete.
+
+Logs of the post deployment provisioning of the compute nodes can be found
+in `/etc/maas/ansible/logs` on the head node.
+
+Assitionally, the post deployment provisioning of the compute nodes can be
+queried from the provision service using curl
+```
+curl -sS http://$(docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner):4243/provision/ | jq '[.[] | { "status": .status, "name": .request.Info.name}]'
+[
+ {
+ "message": "",
+ "name": "steel-ghost.cord.lab",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "feline-shirt.cord.lab",
+ "status": 2
+ },
+ {
+ "message": "",
+ "name": "yellow-plot.cord.lab",
+ "status": 2
+ }
+]
+```
+In the above a "status" of 2 means that the provisioning is complete. The
+other values that status might hold are:
+ - `0` - Pending, the request has been accepted by the provisioner but not yet
+ started
+ - `1` - Running, the request is being processed and the node is being
+ provisioned
+ - `2` - Complete, the provisioning has been completed successfully
+ - `3` - Failed, the provisioning has failed and the `message` will be
+ populated with the exit message from provisioning.
+
+## Create and Boot False switch
+The VM based deployment includes the definition of a vagrant machine that will
+exercise some of the automation used to boot OpenFlow switches in the CORD POD.
+It accomplishes this by forcing the MAC address on the vagrant VM to be a MAC
+that is recognized as a supported OpenFlow switch.
+
+The POD automation will thus perform post provisioning on the this VM to
+download and install software to the device. This vagrant VM can be created
+with the following command:
```
vagrant up switch
```
-#### Virtual Box Power Management
-Virtual box power management is implemented via helper scripts that SSH to the virtual box host and
-execute `vboxmanage` commands. For this to work The scripts must be configured with a username and host
-to utilize when SSHing and that account must allow SSH from the head node guest to the host using
-SSH keys such that no password entry is required.
+### Complete
+This step is complete when the command completes successfully. You can verify
+the provisioning of the false switch by querying the provisioning service
+using curl.
+```
+curl -sS http://$(docker inspect --format '{{.NetworkSettings.Networks.maas_default.IPAddress}}' provisioner):4243/provision/cc:37:ab:00:00:01 | jq '[{ "status": .status, "name": .request.Info.name, "message": .message}]'
+[
+ {
+ "message": "",
+ "name": "fakeswitch",
+ "status": 2
+ }
+]
+```
+In the above a "status" of 2 means that the provisioning is complete. The
+other values that status might hold are:
+ - `0` - Pending, the request has been accepted by the provisioner but not yet
+ started
+ - `1` - Running, the request is being processed and the node is being
+ provisioned
+ - `2` - Complete, the provisioning has been completed successfully
+ - `3` - Failed, the provisioning has failed and the `message` will be
+ populated with the exit message from provisioning.
-To enable SSH key based login, assuming that VirtualBox is running on a Linux based system, you can copy
-the MAAS ssh public key from `/var/lib/maas/.ssh/id_rsa.pub` on the head known to your accounts `authorized_keys`
-files. You can verify that this is working by issuing the following commands from your host machine:
+#### Virtual Box Power Management
+Virtual box power management is implemented via helper scripts that SSH to the
+virtual box host and execute `vboxmanage` commands. For this to work The scripts
+must be configured with a username and host to utilize when SSHing and that
+account must allow SSH from the head node guest to the host using SSH keys such
+that no password entry is required.
+
+To enable SSH key based login, assuming that VirtualBox is running on a Linux
+based system, you can copy the MAAS ssh public key from
+`/var/lib/maas/.ssh/id_rsa.pub` on the head known to your accounts
+`authorized_keys` files. You can verify that this is working by issuing the
+following commands from your host machine:
```
vagrant ssh headnode
sudo su - maas
