VOL-642: Installation of VOLT-HA in a Kubernetes cluster

- Changed existing installer to support swarm and kubernetes
- Using kubespray as the base framework to deploy kubernetes
- Implemented config/deploy/teardown tasks for voltha

Amendments:

- Changed from python to python3 when running the inventory builder

Change-Id: I059fd348b9a749397b373e333b5602944c817745
diff --git a/install/ansible/group_vars/all b/install/ansible/group_vars/all
index 9ab3722..af887c2 100644
--- a/install/ansible/group_vars/all
+++ b/install/ansible/group_vars/all
@@ -14,3 +14,5 @@
 registry_volume_size: 5
 consul_volume_size: 5
 logger_volume_size: 20
+install_source: /home/vinstall
+voltha_namespace: voltha
diff --git a/install/ansible/roles/cluster-host/tasks/main.yml b/install/ansible/roles/cluster-host/tasks/main.yml
index 7358e6e..5d0b39e 100644
--- a/install/ansible/roles/cluster-host/tasks/main.yml
+++ b/install/ansible/roles/cluster-host/tasks/main.yml
@@ -37,7 +37,7 @@
 
 #- name: Required configuration directories are copied
 #  copy:
-#    src: "/home/vinstall/{{ item }}"
+#    src: "{{ install_source }}/{{ item }}"
 #    dest: "{{ target_voltha_home }}"
 #    owner: voltha
 #    group: voltha
@@ -50,7 +50,7 @@
 
 - name: Required configuration directories are copied
   synchronize:
-    src: "/home/vinstall/{{ item }}"
+    src: "{{ install_source }}/{{ item }}"
     dest: "{{ target_voltha_home }}"
     archive: no
     owner: no
@@ -65,7 +65,7 @@
 
 - name: Required configuration scripts are copied
   synchronize:
-    src: "/home/vinstall/{{ item }}"
+    src: "{{ install_source }}/{{ item }}"
     dest: "{{ target_voltha_home }}"
     archive: no
     owner: no
@@ -104,7 +104,7 @@
 
 #- name: upstart barrier filesystem loop mount script is installed
 #  copy:
-#    src: "/home/vinstall/losetup.conf"
+#    src: "{{ install_source }}/losetup.conf"
 #    dest: /etc/init
 #    owner: root
 #    group: root
diff --git a/install/ansible/roles/kubernetes-host/files/ssh_config b/install/ansible/roles/kubernetes-host/files/ssh_config
new file mode 100644
index 0000000..990a43d
--- /dev/null
+++ b/install/ansible/roles/kubernetes-host/files/ssh_config
@@ -0,0 +1,3 @@
+Host *
+   StrictHostKeyChecking no
+   UserKnownHostsFile=/dev/null
diff --git a/install/ansible/roles/kubernetes-host/tasks/main.yml b/install/ansible/roles/kubernetes-host/tasks/main.yml
new file mode 100644
index 0000000..8d5a564
--- /dev/null
+++ b/install/ansible/roles/kubernetes-host/tasks/main.yml
@@ -0,0 +1,56 @@
+# Note: When the target == "cluster" the installer
+# is running to install voltha in the cluster hosts.
+# Whe the target == "installer" the installer is being
+# created.
+- name: A .ssh directory for the voltha user exists
+  file:
+    #path: "{{ ansible_env['HOME'] }}/.ssh"
+    path: "{{ target_voltha_home }}/.ssh"
+    state: directory
+    owner: voltha
+    group: voltha
+  tags: [cluster_host]
+
+- name: known_hosts file is absent for the voltha user
+  file:
+    path: "{{ target_voltha_home }}/.ssh/known_hosts"
+    state: absent
+  tags: [cluster_host]
+
+- name: Known host checking is disabled
+  copy:
+    src: files/ssh_config
+    dest: "{{ target_voltha_home }}/.ssh/config"
+    owner: voltha
+    group: voltha
+    mode: 0600
+  tags: [cluster_host]
+
+- name: Cluster host keys are propagated to all hosts in the cluster
+  copy:
+    src: files/.keys
+    dest: "{{ target_voltha_home }}"
+    owner: voltha
+    group: voltha
+    mode: 0600
+  tags: [cluster_host]
+
+- name: A voltha directory under /var/log for voltha logs exists
+  file:
+    path: "/var/log/voltha/logging_volume"
+    state: directory
+  tags: [cluster_host]
+
+- name: Directories for voltha processes are created
+  file:
+    path: "{{ target_voltha_dir }}/{{ item }}"
+    state: directory
+    mode: 0755
+    owner: voltha
+    group: voltha
+  with_items:
+    - registry_data
+    - consul/data
+    - consul/config
+  when: target == "cluster"
+  tags: [cluster_host]
diff --git a/install/ansible/roles/voltha-kubernetes/tasks/config.yml b/install/ansible/roles/voltha-kubernetes/tasks/config.yml
new file mode 100644
index 0000000..87e5ab2
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/tasks/config.yml
@@ -0,0 +1,83 @@
+# Note: When the target == "cluster" the installer
+# is running to install voltha in the cluster hosts.
+# Whe the target == "installer" the installer is being
+# created.
+- name: VOLT-HA Config | The environment is properly set on login
+  template:
+    src: bashrc.j2
+    dest: "{{ target_voltha_home }}/.bashrc"
+    owner: voltha
+    group: voltha
+    mode: "u=rw,g=r,o=r"
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | The .bashrc file is executed on ssh login
+  template:
+    src: bash_profile.j2
+    dest: "{{ target_voltha_home }}/.bash_profile"
+    owner: voltha
+    group: voltha
+    mode: "u=rw,g=r,o=r"
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Required directory exists
+  file:
+    path: "{{ target_voltha_dir }}"
+    state: directory
+    owner: voltha
+    group: voltha
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Configuration files and directories are copied
+  synchronize:
+    src: "{{ install_source }}/{{ item }}"
+    dest: "{{ target_voltha_dir }}"
+    archive: no
+    owner: no
+    perms: no
+    recursive: yes
+    links: yes
+  with_items:
+    - k8s
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Configuration directories are owned by voltha
+  file:
+    path: "{{ target_voltha_dir }}/{{ item }}"
+    owner: voltha
+    group: voltha
+    recurse: yes
+    follow: no
+  with_items:
+    - k8s
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Docker containers for Voltha are pulled
+  command: docker pull {{ docker_registry }}/{{ item }}
+  with_items: "{{ voltha_containers }}"
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Docker images are re-tagged to expected names
+  command: docker tag {{ docker_registry }}/{{ item }} {{ item }}
+  with_items: "{{ voltha_containers }}"
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Docker images are re-tagged to cluster registry names
+  command: docker tag {{ docker_registry }}/{{ item }} {{ inventory_hostname }}:5001/{{ item }}
+  with_items: "{{ voltha_containers }}"
+  when: target == "cluster"
+  tags: [voltha]
+
+- name: VOLT-HA Config | Add CNI bridge for PONSIM layer 2 support
+  template:
+    src: ponsim_bridge.j2
+    dest: "/etc/cni/net.d/10-pon0.conf"
+  when: target == "cluster"
+  tags: [voltha]
\ No newline at end of file
diff --git a/install/ansible/roles/voltha-kubernetes/tasks/deploy.yml b/install/ansible/roles/voltha-kubernetes/tasks/deploy.yml
new file mode 100644
index 0000000..f5a2929
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/tasks/deploy.yml
@@ -0,0 +1,159 @@
+- name: "VOLT-HA Deploy | Add Namespace"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/namespace.yml
+  run_once: true
+
+# Ingress
+- name: "VOLT-HA Deploy | Start Ingress Controller"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/ingress
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for Default HTTP backend to be ready"
+  command: kubectl rollout status deployment/default-http-backend -w -n {{ voltha_namespace }}
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for Ingress Controller to be ready"
+  command: kubectl rollout status deployment/nginx-ingress-controller -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Zookeeper
+
+- name: "VOLT-HA Deploy | Start zookeeper"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/zookeeper.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for zookeeper to be ready"
+  command: kubectl rollout status statefulset/{{ item }} -w -n {{ voltha_namespace }}
+  with_items:
+    - zookeeper1
+    - zookeeper2
+    - zookeeper3
+  run_once: true
+
+# Kafka
+- name: "VOLT-HA Deploy | Start kafka"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/kafka.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for kafka to be ready"
+  command: kubectl rollout status statefulset/kafka -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Fluentd
+
+- name: "VOLT-HA Deploy | Start fluentd"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/fluentd.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for fluentd to be ready"
+  command: kubectl rollout status deployment/{{ item }} -w -n {{ voltha_namespace }}
+  with_items:
+    - fluentdactv
+    - fluentdstby
+    - fluentd
+  run_once: true
+
+# Consul
+- name: "VOLT-HA Deploy | Start consul"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/consul.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for consul to be ready"
+  command: kubectl rollout status statefulset/consul -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Voltha Core (for consul)
+- name: "VOLT-HA Deploy | Start VOLT-HA core (for consul)"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/vcore_for_consul.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for VOLT-HA core (for consul) to be ready"
+  command: kubectl rollout status deployment/vcore -w -n {{ voltha_namespace }}
+  run_once: true
+
+# OFagent
+- name: "VOLT-HA Deploy | Start OpenFlow Agent"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/ofagent.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for OpenFlow Agent to be ready"
+  command: kubectl rollout status deployment/ofagent -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Envoy (for consul)
+- name: "VOLT-HA Deploy | Start Envoy (for consul)"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/envoy_for_consul.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for Envoy (for consul) to be ready"
+  command: kubectl rollout status deployment/voltha -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Voltha CLI
+- name: "VOLT-HA Deploy | Start VOLT-HA CLI"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/vcli.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for VOLT-HA CLI to be ready"
+  command: kubectl rollout status deployment/vcli -w -n {{ voltha_namespace }}
+  run_once: true
+
+# NETCONF
+- name: "VOLT-HA Deploy | Start NETCONF"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/netconf.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for NETCONF to be ready"
+  command: kubectl rollout status deployment/netconf -w -n {{ voltha_namespace }}
+  run_once: true
+
+# Grafana
+- name: "VOLT-HA Deploy | Start Grafana"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/grafana.yml
+  run_once: true
+  when: monitor_voltha_stats
+
+- name: "VOLT-HA Deploy | Wait for Grafana to be ready"
+  command: kubectl rollout status deployment/grafana -w -n {{ voltha_namespace }}
+  run_once: true
+  when: monitor_voltha_stats
+
+# Dashd and Shovel
+- name: "VOLT-HA Deploy | Start DASHD and SHOVEL"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/stats.yml
+  run_once: true
+  when: monitor_voltha_stats
+
+- name: "VOLT-HA Deploy | Wait for DASHD and SHOVEL to be ready"
+  command: kubectl rollout status deployment/{{ item }} -w -n {{ voltha_namespace }}
+  with_items:
+    - dashd
+    - shovel
+  run_once: true
+  when: monitor_voltha_stats
+
+# FreeRADIUS Config
+- name: "VOLT-HA Deploy | Add FreeRADIUS Configuration"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/freeradius-config.yml
+  run_once: true
+
+# FreeRADIUS
+- name: "VOLT-HA Deploy | Start FreeRADIUS"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/freeradius.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for FreeRADIUS to be ready"
+  command: kubectl rollout status deployment/freeradius -w -n {{ voltha_namespace }}
+  run_once: true
+
+# ONOS
+- name: "VOLT-HA Deploy | Start ONOS"
+  command: kubectl apply -f {{ target_voltha_dir }}/k8s/onos.yml
+  run_once: true
+
+- name: "VOLT-HA Deploy | Wait for ONOS to be ready"
+  command: kubectl rollout status deployment/onos -w -n {{ voltha_namespace }}
+  run_once: true
+
+
+
+
diff --git a/install/ansible/roles/voltha-kubernetes/tasks/main.yml b/install/ansible/roles/voltha-kubernetes/tasks/main.yml
new file mode 100644
index 0000000..6053343
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/tasks/main.yml
@@ -0,0 +1,8 @@
+- import_tasks: config.yml
+  when: config_voltha is defined
+
+- import_tasks: deploy.yml
+  when: deploy_voltha is defined
+
+- import_tasks: teardown.yml
+  when: teardown_voltha is defined
diff --git a/install/ansible/roles/voltha-kubernetes/tasks/teardown.yml b/install/ansible/roles/voltha-kubernetes/tasks/teardown.yml
new file mode 100644
index 0000000..fdd0485
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/tasks/teardown.yml
@@ -0,0 +1,82 @@
+# ONOS
+- name: "VOLT-HA Teardown | Stop ONOS"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/onos.yml
+  run_once: true
+
+# FreeRADIUS
+- name: "VOLT-HA Teardown | Stop FreeRADIUS"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/freeradius.yml
+  run_once: true
+
+# FreeRADIUS Config
+- name: "VOLT-HA Teardown | Remove FreeRADIUS Configuration"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/freeradius-config.yml
+  run_once: true
+
+# Dashd and Shovel
+- name: "VOLT-HA Teardown | Stop DASHD and SHOVEL"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/stats.yml
+  run_once: true
+
+# Grafana
+- name: "VOLT-HA Teardown | Stop Grafana"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/grafana.yml
+  run_once: true
+
+# NETCONF
+- name: "VOLT-HA Teardown | Stop NETCONF"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/netconf.yml
+  run_once: true
+
+# Voltha CLI
+- name: "VOLT-HA Teardown | Stop VOLT-HA CLI"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/vcli.yml
+  run_once: true
+
+# Envoy (for consul)
+- name: "VOLT-HA Teardown | Stop Envoy (for consul)"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/envoy_for_consul.yml
+  run_once: true
+
+# OFagent
+- name: "VOLT-HA Teardown | Stop OpenFlow Agent"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/ofagent.yml
+  run_once: true
+
+# Voltha Core (for consul)
+- name: "VOLT-HA Teardown | Stop VOLT-HA core (for consul)"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/vcore_for_consul.yml
+  run_once: true
+
+# Consul
+- name: "VOLT-HA Teardown | Stop consul"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/consul.yml
+  run_once: true
+
+# Fluentd
+- name: "VOLT-HA Teardown | Stop fluentd"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/fluentd.yml
+  run_once: true
+
+# Kafka
+- name: "VOLT-HA Teardown | Stop kafka"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/kafka.yml
+  run_once: true
+
+# Zookeeper
+- name: "VOLT-HA Teardown | Stop zookeeper"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/zookeeper.yml
+  run_once: true
+
+# Ingress
+- name: "VOLT-HA Teardown | Stop Ingress Controller"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/ingress
+  run_once: true
+
+# Namespace
+- name: "VOLT-HA Teardown | Remove Namespace"
+  command: kubectl delete --ignore-not-found=true -f {{ target_voltha_dir }}/k8s/namespace.yml
+  run_once: true
+
+
+
diff --git a/install/ansible/roles/voltha-kubernetes/templates/bash_profile.j2 b/install/ansible/roles/voltha-kubernetes/templates/bash_profile.j2
new file mode 100644
index 0000000..45cb87e
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/templates/bash_profile.j2
@@ -0,0 +1,3 @@
+if [ -f ~/.bashrc ]; then
+  . ~/.bashrc
+fi
diff --git a/install/ansible/roles/voltha-kubernetes/templates/bashrc.j2 b/install/ansible/roles/voltha-kubernetes/templates/bashrc.j2
new file mode 100644
index 0000000..73bcc88
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/templates/bashrc.j2
@@ -0,0 +1,2 @@
+DOCKER_HOST_IP={{ ansible_default_ipv4.address }}
+export DOCKER_HOST_IP
diff --git a/install/ansible/roles/voltha-kubernetes/templates/ponsim_bridge.j2 b/install/ansible/roles/voltha-kubernetes/templates/ponsim_bridge.j2
new file mode 100644
index 0000000..6ddcfbd
--- /dev/null
+++ b/install/ansible/roles/voltha-kubernetes/templates/ponsim_bridge.j2
@@ -0,0 +1,15 @@
+{
+    "name": "pon0",
+    "type": "bridge",
+    "bridge": "pon0",
+    "isGateway": true,
+    "ipMask": true,
+    "ipam": {
+        "type": "host-local",
+        "subnet": "10.22.0.0/16",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ]
+    }
+}
+
diff --git a/install/ansible/voltha-k8s.yml b/install/ansible/voltha-k8s.yml
new file mode 100644
index 0000000..1761d04
--- /dev/null
+++ b/install/ansible/voltha-k8s.yml
@@ -0,0 +1,23 @@
+- hosts: cluster
+  remote_user: voltha
+  serial: 1
+  vars:
+    target: cluster
+  roles:
+    - { role: common, when: config_voltha is defined }
+    - { role: kubernetes-host, when: config_voltha is defined }
+    - { role: voltha-kubernetes, when: config_voltha is defined }
+
+- hosts:  k8s-master
+  remote_user: voltha
+  serial: 1
+  roles:
+    - {
+        role: voltha-kubernetes, when: deploy_voltha is defined,
+        monitor_voltha_stats: true,
+        use_ponsim: true
+      }
+    - {
+        role: voltha-kubernetes,
+        when: teardown_voltha is defined
+      }
diff --git a/install/install.cfg b/install/install.cfg
index 964800f..4bf0b66 100644
--- a/install/install.cfg
+++ b/install/install.cfg
@@ -10,3 +10,15 @@
 # The space reserved for Consul's storage
 # consulLimit=5
 
+# Specify the cluster framework type (swarm or kubernetes)
+cluster_framework="swarm"
+
+# Docker registry address
+#cluster_registry="localhost:5000"
+
+# Address range for kubernetes services
+#cluster_service_subnet="192.168.0.0\/18"
+
+# Address range for kubernetes pods
+#cluster_pod_subnet="192.168.64.0\/18"
+
diff --git a/install/installer.sh b/install/installer.sh
index f1f9732..f3a5aa3 100755
--- a/install/installer.sh
+++ b/install/installer.sh
@@ -35,13 +35,13 @@
 # Configure barrier file sizes but only if a value was provided in the config file
 
 if [ -v logLimit ]; then
-	sed -i -e "/logger_volume_size/s/.*/logger_volume_size: ${logLimit}/" ansible/group_vars/all
+    sed -i -e "/logger_volume_size/s/.*/logger_volume_size: ${logLimit}/" ansible/group_vars/all
 fi
 if [ -v regLimit ]; then
-	sed -i -e "/registry_volume_size/s/.*/registry_volume_size: ${regLimit}/" ansible/group_vars/all
+    sed -i -e "/registry_volume_size/s/.*/registry_volume_size: ${regLimit}/" ansible/group_vars/all
 fi
 if [ -v consulLimit ]; then
-	sed -i -e "/consul_volume_size/s/.*/consul_volume_size: ${consulLimit}/" ansible/group_vars/all
+    sed -i -e "/consul_volume_size/s/.*/consul_volume_size: ${consulLimit}/" ansible/group_vars/all
 fi
 
 # Create the key directory
@@ -60,156 +60,215 @@
 
 for i in $hosts
 do
-	# Generate the key for the host
-	echo -e "${lBlue}Generating the key-pair for communication with host ${yellow}$i${NC}"
-	ssh-keygen -f ./$i -t rsa -N ''
-	mv $i .keys
+    # Generate the key for the host
+    echo -e "${lBlue}Generating the key-pair for communication with host ${yellow}$i${NC}"
+    ssh-keygen -f ./$i -t rsa -N ''
+    mv $i .keys
 
-	# Generate the pre-configuration script
-	echo -e "${lBlue}Creating the pre-configuration script${NC}"
-	head -n +1 BashLoginTarget.sh > bash_login.sh
-	echo "" >> bash_login.sh
-	echo -n 'key="' >> bash_login.sh
-	sed -i -e 's/$/"/' $i.pub
-	cat $i.pub >> bash_login.sh
-	tail -n +2 BashLoginTarget.sh | grep -v "{{ key }}" >> bash_login.sh
-	rm $i.pub
+    # Generate the pre-configuration script
+    echo -e "${lBlue}Creating the pre-configuration script${NC}"
+    head -n +1 BashLoginTarget.sh > bash_login.sh
+    echo "" >> bash_login.sh
+    echo -n 'key="' >> bash_login.sh
+    sed -i -e 's/$/"/' $i.pub
+    cat $i.pub >> bash_login.sh
+    tail -n +2 BashLoginTarget.sh | grep -v "{{ key }}" >> bash_login.sh
+    rm $i.pub
 
-	# Copy the pre-config file to the VM
-	echo -e "${lBlue}Transfering pre-configuration script to ${yellow}$i${NC}"
-	if [ -d ".test" ]; then
-		echo -e "${red}Test mode set!!${lBlue} Using pre-populated ssh key for ${yellow}$i${NC}"
-		scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .test/$i bash_login.sh $iUser@$i:.bash_login
-	else
-		scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no bash_login.sh $iUser@$i:.bash_login
-	fi
-	rm bash_login.sh
+    # Copy the pre-config file to the VM
+    echo -e "${lBlue}Transfering pre-configuration script to ${yellow}$i${NC}"
+    if [ -d ".test" ]; then
+        echo -e "${red}Test mode set!!${lBlue} Using pre-populated ssh key for ${yellow}$i${NC}"
+        scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .test/$i bash_login.sh $iUser@$i:.bash_login
+    else
+        scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no bash_login.sh $iUser@$i:.bash_login
+    fi
+    rm bash_login.sh
 
-	# Run the pre-config file on the VM
-	echo -e "${lBlue}Running the pre-configuration script on ${yellow}$i${NC}"
-	if [ -d ".test" ]; then
-		echo -e "${red}Test mode set!!${lBlue} Using pre-populated ssh key for ${yellow}$i${NC}"
-		ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .test/$i $iUser@$i
-	else
-		ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $iUser@$i
-	fi
+    # Run the pre-config file on the VM
+    echo -e "${lBlue}Running the pre-configuration script on ${yellow}$i${NC}"
+    if [ -d ".test" ]; then
+        echo -e "${red}Test mode set!!${lBlue} Using pre-populated ssh key for ${yellow}$i${NC}"
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .test/$i $iUser@$i
+    else
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $iUser@$i
+    fi
 
-	# Configure ansible and ssh for silent operation
-	echo -e "${lBlue}Configuring ansible${NC}"
-	echo $i >> ansible/hosts/cluster
-	echo "ansible_ssh_private_key_file: $wd/.keys/$i" > ansible/host_vars/$i
+    # Configure ansible and ssh for silent operation
+    echo -e "${lBlue}Configuring ansible${NC}"
+    echo $i >> ansible/hosts/cluster
+    echo "ansible_ssh_private_key_file: $wd/.keys/$i" > ansible/host_vars/$i
 
-	# Create the tunnel to the registry to allow pulls from localhost
-	echo -e "${lBlue}Creating a secure shell tunnel to the registry for ${yellow}$i${NC}"
-	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i -f voltha@$i -R 5000:localhost:5000 -N
-	
+    # Create the tunnel to the registry to allow pulls from localhost
+    echo -e "${lBlue}Creating a secure shell tunnel to the registry for ${yellow}$i${NC}"
+    ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i -f voltha@$i -R 5000:localhost:5000 -N
+
 done
-# Add the dependent software list to the cluster variables
-echo -e "${lBlue}Setting up dependent software${NC}"
-# Delete any grub updates since the boot disk is almost
-# guaranteed not to be the same device as the installer.
-mkdir grub_updates
-sudo mv deb_files/*grub* grub_updates
-# Sort the packages in dependency order to get rid of scary non-errors
-# that are issued by ansible.
-#echo -e "${lBlue}Dependency sorting dependent software${NC}"
-#./sort_packages.sh
-#echo "deb_files:" >> ansible/group_vars/all
-#for i in `cat sortedDebs.txt`
-#do
-#echo "  - $i" >> ansible/group_vars/all
-#done
 
-# Make sure the ssh keys propagate to all hosts allowing passwordless logins between them
-echo -e "${lBlue}Propagating ssh keys${NC}"
 cp -r .keys ansible/roles/cluster-host/files
 
-# Install python on all the 3 servers since python is required for
-for i in $hosts
-do
-	echo -e "${lBlue}Installing ${lCyan}Python${lBlue}${NC}"
-	scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i -r python-deb voltha@$i:.
-	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i "sudo dpkg -i /home/voltha/python-deb/*minimal*"
-	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i sudo dpkg -i -R /home/voltha/python-deb
-	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i rm -fr python-deb
 
-done
+if [  "$cluster_framework" == "kubernetes" ]; then
 
-# Create the daemon.json file for the swarm
-echo "{" > daemon.json
-echo -n '  "insecure-registries" : [' >> daemon.json
-first=""
-for i in .keys/*
-do
-	if [ -z "$first" ]; then
-		echo -n '"'`basename $i`':5001"' >> daemon.json
-		first="not"
-	else
-		echo -n ' , "'`basename $i`':5001"' >> daemon.json
-	fi
-done
-echo "]" >> daemon.json
-echo "}" >> daemon.json
-unset first
+	echo -e "${green}Deploying kubernetes${NC}"
 
-# Running ansible
-echo -e "${lBlue}Running ansible${NC}"
-cp ansible/ansible.cfg .ansible.cfg
-ansible-playbook ansible/voltha.yml -i ansible/hosts/cluster
+    rm -rf kubespray
 
-# Now all 3 servers need to be rebooted because of software installs.
-# Reboot them and wait patiently until they all come back.
-# Note this destroys the registry tunnel wich is no longer needed.
-hList=""
-for i in $hosts
-do
-	echo -e "${lBlue}Rebooting cluster hosts${NC}"
-	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i sudo telinit 6
-	hList="$i $hList"
-done
+    git clone https://github.com/kubernetes-incubator/kubespray.git
 
-# Give the hosts time to shut down so that pings stop working or the
-# script just falls through the next loop and the rest fails.
-echo -e "${lBlue}Waiting for shutdown${NC}"
-sleep 5
+    cp -rfp kubespray/inventory/sample kubespray/inventory/voltha
 
+    # TODO: Replace with variables taken from config file
+    # bootstrap_os: ubuntu
+    # network_plugin: weave
 
-while [ ! -z "$hList" ];
-do
-	# Attempt to ping the VMs on the list one by one.
-	echo -e "${lBlue}Waiting for hosts to reboot ${yellow}$hList${NC}"
-	for i in $hList
-	do
-		ping -q -c 1 $i > /dev/null 2>&1
-		ret=$?
-		if [ $ret -eq 0 ]; then
-			ipExpr=`echo $i | sed -e "s/\./[.]/g"`
-			hList=`echo $hList | sed -e "s/$ipExpr//" | sed -e "s/^ //" | sed -e "s/ $//"`
-		fi
-	done
-	
-done
+    sed -i -e "/bootstrap_os: none/s/.*/bootstrap_os: ubuntu/" \
+        kubespray/inventory/voltha/group_vars/all.yml
+    sed -i -e "/kube_network_plugin: calico/s/.*/kube_network_plugin: weave/" \
+        kubespray/inventory/voltha/group_vars/k8s-cluster.yml
+    sed -i -e "/kube_service_addresses: 10.233.0.0\/18/s/.*/kube_service_addresses: $cluster_service_subnet/" \
+        kubespray/inventory/voltha/group_vars/k8s-cluster.yml
+    sed -i -e "/kube_pods_subnet: 10.233.64.0\/18/s/.*/kube_pods_subnet: $cluster_pod_subnet/" \
+        kubespray/inventory/voltha/group_vars/k8s-cluster.yml
+    sed -i -e "s/docker_options: \"/&--insecure-registry=$cluster_registry /" \
+        kubespray/inventory/voltha/group_vars/k8s-cluster.yml
 
-# Now initialize the the docker swarm cluster with managers.
-# The first server needs to be the primary swarm manager
-# the other nodes are backup mangers that join the swarm.
-# In the future, worker nodes will likely be added.
+    CONFIG_FILE=kubespray/inventory/voltha/hosts.ini python3 \
+        kubespray/contrib/inventory_builder/inventory.py $hosts
 
-echo "[swarm-master]" > ansible/hosts/swarm-master
-echo "[swarm-master-backup]" > ansible/hosts/swarm-master-backup
+    # The inventory defines
+    sed -i -e '/\[kube-master\]/a\
+    node3
+    ' kubespray/inventory/voltha/hosts.ini
 
-ctr=1
-for i in $hosts
-do
+    ansible-playbook -u root -i kubespray/inventory/voltha/hosts.ini kubespray/cluster.yml
+
+    echo "[k8s-master]" > ansible/hosts/k8s-master
+
+    ctr=1
+    for i in $hosts
+    do
         if [ $ctr -eq 1 ]; then
-                echo  $i >> ansible/hosts/swarm-master
-		echo "swarm_master_addr: \"$i\"" >> ansible/group_vars/all
-		ctr=0
-        else
-                echo  $i >> ansible/hosts/swarm-master-backup
+            echo  $i >> ansible/hosts/k8s-master
+            ctr=0
         fi
-done
-ansible-playbook ansible/swarm.yml -i ansible/hosts/swarm-master
-ansible-playbook ansible/swarm.yml -i ansible/hosts/swarm-master-backup
-ansible-playbook ansible/voltha.yml -i ansible/hosts/swarm-master
+    done
 
+    ansible-playbook ansible/voltha-k8s.yml -i ansible/hosts/cluster -e 'config_voltha=true'
+    ansible-playbook ansible/voltha-k8s.yml -i ansible/hosts/k8s-master -e 'deploy_voltha=true'
+
+else
+    #
+    # Legacy swarm instructions
+    #
+
+    # Add the dependent software list to the cluster variables
+    echo -e "${lBlue}Setting up dependent software${NC}"
+    # Delete any grub updates since the boot disk is almost
+    # guaranteed not to be the same device as the installer.
+    mkdir grub_updates
+    sudo mv deb_files/*grub* grub_updates
+    # Sort the packages in dependency order to get rid of scary non-errors
+    # that are issued by ansible.
+    #echo -e "${lBlue}Dependency sorting dependent software${NC}"
+    #./sort_packages.sh
+    #echo "deb_files:" >> ansible/group_vars/all
+    #for i in `cat sortedDebs.txt`
+    #do
+    #echo "  - $i" >> ansible/group_vars/all
+    #done
+
+    # Make sure the ssh keys propagate to all hosts allowing passwordless logins between them
+    echo -e "${lBlue}Propagating ssh keys${NC}"
+    cp -r .keys ansible/roles/cluster-host/files
+
+    # Install python on all the 3 servers since python is required for
+    for i in $hosts
+    do
+        echo -e "${lBlue}Installing ${lCyan}Python${lBlue}${NC}"
+        scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i -r python-deb voltha@$i:.
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i "sudo dpkg -i /home/voltha/python-deb/*minimal*"
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i sudo dpkg -i -R /home/voltha/python-deb
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i rm -fr python-deb
+
+    done
+
+    # Create the daemon.json file for the swarm
+    echo "{" > daemon.json
+    echo -n '  "insecure-registries" : [' >> daemon.json
+    first=""
+    for i in .keys/*
+    do
+        if [ -z "$first" ]; then
+            echo -n '"'`basename $i`':5001"' >> daemon.json
+            first="not"
+        else
+            echo -n ' , "'`basename $i`':5001"' >> daemon.json
+        fi
+    done
+    echo "]" >> daemon.json
+    echo "}" >> daemon.json
+    unset first
+
+    # Running ansible
+    echo -e "${lBlue}Running ansible${NC}"
+    cp ansible/ansible.cfg .ansible.cfg
+    ansible-playbook ansible/voltha.yml -i ansible/hosts/cluster
+
+    # Now all 3 servers need to be rebooted because of software installs.
+    # Reboot them and wait patiently until they all come back.
+    # Note this destroys the registry tunnel wich is no longer needed.
+    hList=""
+    for i in $hosts
+    do
+        echo -e "${lBlue}Rebooting cluster hosts${NC}"
+        ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i .keys/$i  voltha@$i sudo telinit 6
+        hList="$i $hList"
+    done
+
+    # Give the hosts time to shut down so that pings stop working or the
+    # script just falls through the next loop and the rest fails.
+    echo -e "${lBlue}Waiting for shutdown${NC}"
+    sleep 5
+
+
+    while [ ! -z "$hList" ];
+    do
+        # Attempt to ping the VMs on the list one by one.
+        echo -e "${lBlue}Waiting for hosts to reboot ${yellow}$hList${NC}"
+        for i in $hList
+        do
+            ping -q -c 1 $i > /dev/null 2>&1
+            ret=$?
+            if [ $ret -eq 0 ]; then
+                ipExpr=`echo $i | sed -e "s/\./[.]/g"`
+                hList=`echo $hList | sed -e "s/$ipExpr//" | sed -e "s/^ //" | sed -e "s/ $//"`
+            fi
+        done
+
+    done
+
+    # Now initialize the the docker swarm cluster with managers.
+    # The first server needs to be the primary swarm manager
+    # the other nodes are backup mangers that join the swarm.
+    # In the future, worker nodes will likely be added.
+
+    echo "[swarm-master]" > ansible/hosts/swarm-master
+    echo "[swarm-master-backup]" > ansible/hosts/swarm-master-backup
+
+    ctr=1
+    for i in $hosts
+    do
+            if [ $ctr -eq 1 ]; then
+                    echo  $i >> ansible/hosts/swarm-master
+            echo "swarm_master_addr: \"$i\"" >> ansible/group_vars/all
+            ctr=0
+            else
+                    echo  $i >> ansible/hosts/swarm-master-backup
+            fi
+    done
+    ansible-playbook ansible/swarm.yml -i ansible/hosts/swarm-master
+    ansible-playbook ansible/swarm.yml -i ansible/hosts/swarm-master-backup
+    ansible-playbook ansible/voltha.yml -i ansible/hosts/swarm-master
+
+fi