[CORD-2466]
Create a multi-node k8s-compatible scenario
Change-Id: I618a66323672d646847e438f1e0c320962e75642
diff --git a/.gitignore b/.gitignore
index 94514c5..5b1df61 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,3 +59,5 @@
ib_actions.yml
ib_graph.dot
+# kubespray
+kubespray
diff --git a/Makefile b/Makefile
index c3c3038..77cebbc 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@
BUILD ?= .
CORD ?= ..
PI ?= $(BUILD)/platform-install
+KUBESPRAY ?= $(BUILD)/kubespray
MAAS ?= $(BUILD)/maas
ONOS_APPS ?= $(CORD)/onos-apps
@@ -20,14 +21,11 @@
endif
PROFILE_D ?= $(CORD)/orchestration/profiles/$(USECASE)
-
-PODCONFIG_D ?= $(PROFILE_D)/podconfig
-PODCONFIG_PATH ?= $(PODCONFIG_D)/$(PODCONFIG)
+PODCONFIG_PATH ?= $(PROFILE_D)/podconfig/$(PODCONFIG)
SCENARIOS_D ?= $(BUILD)/scenarios
GENCONFIG_D ?= $(BUILD)/genconfig
-
CONFIG_CORD_PROFILE_DIR ?= $(CORD)/../cord_profile
# Milestones/logs paths
@@ -35,13 +33,14 @@
LOGS ?= $(BUILD)/logs
PREP_MS ?= $(M)/prereqs-check $(M)/build-local-bootstrap $(M)/ciab-ovs $(M)/vagrant-up $(M)/vagrant-ssh-install $(M)/copy-cord $(M)/cord-config $(M)/copy-config $(M)/prep-buildnode $(M)/prep-headnode $(M)/deploy-elasticstack $(M)/prep-computenode
+KS_MS ?= $(M)/prep-kubespray $(M)/deploy-kubespray
MAAS_MS ?= $(M)/build-maas-images $(M)/maas-prime $(M)/publish-maas-images $(M)/deploy-maas
OPENSTACK_MS ?= $(M)/glance-images $(M)/deploy-openstack $(M)/deploy-computenode $(M)/onboard-openstack
XOS_MS ?= $(M)/docker-images $(M)/core-image $(M)/publish-docker-images $(M)/start-xos $(M)/onboard-profile
ONOS_MS ?= $(M)/build-onos-apps $(M)/publish-onos-apps $(M)/deploy-onos $(M)/deploy-mavenrepo
POST_INSTALL_MS ?= $(M)/setup-automation $(M)/setup-ciab-pcu $(M)/compute1-up $(M)/compute2-up $(M)/compute3-up
LOCAL_MILESTONES ?= $(M)/local-cord-config $(M)/local-docker-images $(M)/local-core-image $(M)/local-start-xos $(M)/local-onboard-profile
-ALL_MILESTONES ?= $(PREP_MS) $(MAAS_MS) $(OPENSTACK_MS) $(XOS_MS) $(ONOS_MS) $(POST_INSTALL_MS) $(LOCAL_MILESTONES)
+ALL_MILESTONES ?= $(PREP_MS) $(KS_MS) $(MAAS_MS) $(OPENSTACK_MS) $(XOS_MS) $(ONOS_MS) $(POST_INSTALL_MS) $(LOCAL_MILESTONES)
# Configuration files
MASTER_CONFIG ?= $(GENCONFIG_D)/config.yml
@@ -80,10 +79,11 @@
VAGRANT ?= VAGRANT_CWD=$(VAGRANT_CWD) vagrant
ANSIBLE ?= ansible -i $(INVENTORY)
ANSIBLE_PB ?= ansible-playbook $(ANSIBLE_ARGS) -i $(INVENTORY) --extra-vars @$(MASTER_CONFIG)
+ANSIBLE_PB_KS ?= ANSIBLE_CONFIG=../ansible.cfg ansible-playbook $(ANSIBLE_ARGS) -b -i inventory/inventory.cord --extra-vars @../$(MASTER_CONFIG)
ANSIBLE_PB_LOCAL ?= ansible-playbook $(ANSIBLE_ARGS) -i $(PI)/inventory/head-localhost $(EXTRA_VARS)
ANSIBLE_PB_MAAS ?= ansible-playbook $(ANSIBLE_ARGS) -i /etc/maas/ansible/pod-inventory $(EXTRA_VARS)
IMAGEBUILDER ?= python $(BUILD)/scripts/imagebuilder.py
-LOGCMD ?= 2>&1 | tee -a $(LOGS)/$(TS)_$(@F)
+LOGCMD ?= 2>&1 | tee -a $(abspath $(LOGS)/$(TS)_$(@F))
SSH_HEAD ?= ssh $(HEADNODE)
SSH_BUILD ?= ssh $(BUILDNODE)
@@ -140,6 +140,10 @@
$(SSH_HEAD) "/opt/cord/build/platform-install/scripts/clean_openstack.sh" $(LOGCMD)
rm -f $(M)/onboard-openstack
+clean-kubespray:
+ rm -f $(KS_MS)
+ rm -rf $(KUBESPRAY)
+
collect-diag:
$(ANSIBLE_PB) $(PI)/collect-diag-playbook.yml $(LOGCMD)
@@ -184,6 +188,7 @@
CONFIG_SSH_KEY_PREREQS ?=
PREP_BUILDNODE_PREREQS ?=
PREP_HEADNODE_PREREQS ?=
+PREP_KUBESPRAY_PREREQS ?=
DOCKER_IMAGES_PREREQS ?=
START_XOS_PREREQS ?=
BUILD_ONOS_APPS_PREREQS ?=
@@ -254,6 +259,16 @@
touch $@
+# kubespray targets
+$(M)/prep-kubespray: | $(M)/vagrant-ssh-install $(PREP_KUBESPRAY_PREREQS)
+ $(ANSIBLE_PB) $(BUILD)/ansible/prep-kubespray.yml $(LOGCMD)
+ touch $@
+
+$(M)/deploy-kubespray: | $(M)/prep-kubespray
+ cd $(KUBESPRAY); $(ANSIBLE_PB_KS) cluster.yml $(LOGCMD)
+ touch $@
+
+
# MaaS targets
$(M)/build-maas-images: | $(M)/prep-buildnode $(BUILD_MAAS_IMAGES_PREREQS)
$(SSH_BUILD) "cd $(BUILD_CORD_DIR)/build/maas; rm -f consul.image; make MAKE_CONFIG=../$(MAKEFILE_CONFIG) build" $(LOGCMD)
diff --git a/ansible/prep-kubespray.yml b/ansible/prep-kubespray.yml
new file mode 100644
index 0000000..9521b19
--- /dev/null
+++ b/ansible/prep-kubespray.yml
@@ -0,0 +1,35 @@
+---
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ansible/prep-kubespray.yml
+# prepare to run kubespray on the system
+
+- name: Prep systems for kubespray
+ hosts: head, compute
+ tasks:
+ - name: Disable swap
+ become: yes
+ command: "swapoff -a"
+ tags:
+ - skip_ansible_lint # there isn't a module for this, command is idempotent
+
+- name: Download and configure kubespray
+ hosts: localhost
+ connection: local
+ gather_facts: False
+
+ roles:
+ - prep-kubespray
+
diff --git a/ansible/roles/genconfig/templates/config.mk.j2 b/ansible/roles/genconfig/templates/config.mk.j2
index caf108b..11b614f 100644
--- a/ansible/roles/genconfig/templates/config.mk.j2
+++ b/ansible/roles/genconfig/templates/config.mk.j2
@@ -67,6 +67,9 @@
{% if prep_headnode_prereqs is defined %}
PREP_HEADNODE_PREREQS = $(M)/{{ prep_headnode_prereqs | join(" $(M)/") }}
{% endif %}
+{% if prep_kubespray_prereqs is defined %}
+PREP_KUBESPRAY_PREREQS = $(M)/{{ prep_kubespray_prereqs | join(" $(M)/") }}
+{% endif %}
{% if docker_images_prereqs is defined %}
DOCKER_IMAGES_PREREQS = $(M)/{{ docker_images_prereqs | join(" $(M)/") }}
{% endif %}
diff --git a/ansible/roles/prep-kubespray/defaults/main.yml b/ansible/roles/prep-kubespray/defaults/main.yml
new file mode 100644
index 0000000..6c8266d
--- /dev/null
+++ b/ansible/roles/prep-kubespray/defaults/main.yml
@@ -0,0 +1,20 @@
+---
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# prep-kubespray/defaults/main.yml
+
+kubespray_version: "master"
+kubespray_dir: "../kubespray"
+
diff --git a/ansible/roles/prep-kubespray/tasks/main.yml b/ansible/roles/prep-kubespray/tasks/main.yml
new file mode 100644
index 0000000..43a5128
--- /dev/null
+++ b/ansible/roles/prep-kubespray/tasks/main.yml
@@ -0,0 +1,30 @@
+---
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# prep-kubespray/tasks/main.yml
+
+- name: Checkout kubespray repo
+ git:
+ repo: "https://github.com/kubernetes-incubator/kubespray"
+ dest: "{{ kubespray_dir }}"
+ version: "{{ kubespray_version }}"
+
+- name: Create kubespray inventory
+ template:
+ src: kubespray_inventory.j2
+ dest: "{{ kubespray_dir }}/inventory/inventory.cord"
+ mode: 0644
+
+
diff --git a/ansible/roles/prep-kubespray/templates/kubespray_inventory.j2 b/ansible/roles/prep-kubespray/templates/kubespray_inventory.j2
new file mode 100644
index 0000000..bb70db4
--- /dev/null
+++ b/ansible/roles/prep-kubespray/templates/kubespray_inventory.j2
@@ -0,0 +1,30 @@
+# created by prep-kubespray/templates/kubespray_inventory.j2
+
+[kube-master]
+{% for node in groups['head'] %}
+{{ node }}
+{% endfor %}
+{% for node in groups['compute'] %}
+{{ node }}
+{% endfor %}
+
+[etcd]
+{% for node in groups['head'] %}
+{{ node }}
+{% endfor %}
+{% for node in groups['compute'] %}
+{{ node }}
+{% endfor %}
+
+[kube-node]
+{% for node in groups['head'] %}
+{{ node }}
+{% endfor %}
+{% for node in groups['compute'] %}
+{{ node }}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
diff --git a/podconfig/rcord-controlkube.yml b/podconfig/rcord-controlkube.yml
new file mode 100644
index 0000000..70c8d79
--- /dev/null
+++ b/podconfig/rcord-controlkube.yml
@@ -0,0 +1,25 @@
+---
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# rcord-controlcube Pod Config
+# Creates a kubernetes
+
+cord_scenario: controlkube
+cord_profile: rcord
+
+# copy kubespray config and install kubect on node running playbook
+kubeconfig_localhost: true
+kubectl_localhost: true
+
diff --git a/scenarios/controlkube/Vagrantfile b/scenarios/controlkube/Vagrantfile
new file mode 100644
index 0000000..3686e76
--- /dev/null
+++ b/scenarios/controlkube/Vagrantfile
@@ -0,0 +1,77 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+#
+# controlkube Scenario Vagrantfile
+
+require 'yaml'
+settings = YAML.load_file('genconfig/config.yml')
+
+Vagrant.configure("2") do |config|
+
+ config.vm.box = settings["vagrant_box"]
+
+ config.vm.synced_folder '.', '/vagrant', disabled: true
+
+ # set the headnode VM
+ config.vm.define "head1" do |h|
+ h.vm.hostname = "head1"
+ h.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: '0.0.0.0'
+ h.vm.provider :virtualbox do |vb|
+ vb.memory = settings['head_vm_mem']
+ vb.cpus = settings['head_vm_cpu']
+ vb.linked_clone = true
+ end
+ h.vm.provider :libvirt do |v, override|
+ v.memory = settings['head_vm_mem']
+ v.cpus = settings['head_vm_cpu']
+ end
+ h.vm.network "private_network", # management network, eth1
+ ip: "0.1.1.0", # unused IP address (setting required)
+ auto_config: false,
+ virtualbox__intnet: settings['vm_management_network_name'],
+ libvirt__network_name: settings['vm_management_network_name'],
+ libvirt__forward_mode: "none",
+ libvirt__dhcp_enabled: false
+ end
+
+ config.vm.define "compute1" do |c|
+ c.vm.hostname = "compute1"
+ c.vm.provider :virtualbox do |vb|
+ vb.memory = settings['compute_vm_mem']
+ vb.cpus = settings['compute_vm_cpu']
+ vb.linked_clone = true
+ end
+ c.vm.provider :libvirt do |v|
+ v.memory = settings['compute_vm_mem']
+ v.cpus = settings['compute_vm_cpu']
+ end
+ c.vm.network "private_network", # management network, eth1
+ ip: "0.1.1.0", # unused IP address (setting required)
+ auto_config: false,
+ virtualbox__intnet: settings['vm_management_network_name'],
+ libvirt__network_name: settings['vm_management_network_name'],
+ libvirt__forward_mode: "none",
+ libvirt__dhcp_enabled: false
+ end
+
+ config.vm.define "compute2" do |c|
+ c.vm.hostname = "compute2"
+ c.vm.provider :virtualbox do |vb|
+ vb.memory = settings['compute_vm_mem']
+ vb.cpus = settings['compute_vm_cpu']
+ vb.linked_clone = true
+ end
+ c.vm.provider :libvirt do |v|
+ v.memory = settings['compute_vm_mem']
+ v.cpus = settings['compute_vm_cpu']
+ end
+ c.vm.network "private_network", # management network, eth1
+ ip: "0.1.1.0", # unused IP address (setting required)
+ auto_config: false,
+ virtualbox__intnet: settings['vm_management_network_name'],
+ libvirt__network_name: settings['vm_management_network_name'],
+ libvirt__forward_mode: "none",
+ libvirt__dhcp_enabled: false
+ end
+end
+
diff --git a/scenarios/controlkube/config.yml b/scenarios/controlkube/config.yml
new file mode 100644
index 0000000..a277318
--- /dev/null
+++ b/scenarios/controlkube/config.yml
@@ -0,0 +1,127 @@
+---
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# controlkube Scenario - installs XOS/ONOS on kubernetes, similar to controlpod
+
+frontend_only: False
+use_maas: False
+
+# create a cord_profile dir next to the cord checkout
+config_cord_dir: "{{ ( playbook_dir ~ '/../..' ) | realpath }}"
+config_cord_profile_dir: "{{ ( playbook_dir ~ '/../../../cord_profile' ) | realpath }}"
+
+build_cord_dir: /opt/cord
+
+buildnode: head1
+headnode: head1
+
+# Make build config
+build_targets:
+ - deploy-kubespray
+# - prep-computenode
+# - prep-headnode
+# - core-image
+
+config_ssh_key_prereqs:
+ - vagrant-ssh-install
+
+copy_cord_prereqs:
+ - vagrant-ssh-install
+
+cord_config_prereqs:
+ - vagrant-ssh-install
+ - copy-cord
+
+prep_buildnode_prereqs:
+ - copy-cord
+ - copy-config
+
+# Run build prep before head prep, when build == head
+prep_headnode_prereqs:
+ - prep-buildnode
+
+# Wait until headnode prepped before building containers, for consistent DNS
+docker_images_prereqs:
+ - prep-headnode
+
+# node topology, used to bring up management interfaces
+physical_node_list:
+ - name: head1
+ ipv4_last_octet: 1
+ aliases:
+ - head
+ - name: compute1
+ ipv4_last_octet: 17
+ - name: compute2
+ ipv4_last_octet: 18
+
+# Vagrant VM configuration
+vagrant_vms:
+ - head1
+ - compute1
+ - compute2
+
+# Vagrant VM configuration
+vagrant_box: "bento/ubuntu-16.04"
+
+head_vm_mem: 2048
+head_vm_cpu: 4
+
+compute_vm_mem: 2048
+compute_vm_cpu: 4
+
+vm_management_network_name: cordmgmt
+vm_public_network_name: cordpub
+vm_public_network_cidr: "10.230.100.0/24"
+
+# images for imagebuilder to build/pull (tagged elsewhere)
+docker_image_whitelist:
+ - "xosproject/xos-base"
+ - "xosproject/xos"
+ - "xosproject/xos-client"
+ - "xosproject/xos-corebuilder"
+ - "xosproject/xos-gui"
+ - "xosproject/xos-gui-builder"
+ - "xosproject/xos-libraries"
+ - "xosproject/xos-postgres"
+ - "xosproject/xos-tosca"
+ - "xosproject/xos-ws"
+ - "xosproject/chameleon"
+ - "xosproject/xos-synchronizer-base"
+ - "gliderlabs/consul-server"
+ - "gliderlabs/registrator"
+ - "nginx"
+ - "onosproject/onos"
+ - "redis"
+ - "node"
+ - "sebp/elk"
+
+# Ansible Inventory
+inventory_groups:
+
+ config:
+ localhost:
+ ansible_connection: local
+
+ build:
+ head1:
+
+ head:
+ head1:
+
+ compute:
+ compute1:
+ compute2:
+
diff --git a/scenarios/preppedpod/config.yml b/scenarios/preppedpod/config.yml
index 9e570c4..12967f9 100644
--- a/scenarios/preppedpod/config.yml
+++ b/scenarios/preppedpod/config.yml
@@ -16,8 +16,8 @@
# preppedpod Scenario - install CORD on prepared systems with OS installed
# Only used with a virtual install, physical does not use VMs
-# opencloud profile config
frontend_only: False
+use_maas: False
# create a cord_profile dir next to the cord checkout
config_cord_dir: "{{ ( playbook_dir ~ '/../..' ) | realpath }}"
@@ -36,7 +36,12 @@
vagrant_up_prereqs:
- prereqs-check
-# Copy cord and config to physical/virtual nodes
+config_ssh_key_prereqs:
+ - vagrant-ssh-install
+
+copy_cord_prereqs:
+ - vagrant-ssh-install
+
cord_config_prereqs:
- vagrant-ssh-install
- copy-cord
diff --git a/scripts/cord-bootstrap.sh b/scripts/cord-bootstrap.sh
index 34ac885..b4dd77c 100755
--- a/scripts/cord-bootstrap.sh
+++ b/scripts/cord-bootstrap.sh
@@ -55,10 +55,10 @@
sudo apt-get update
sudo apt-get -y install apt-transport-https build-essential curl git python-dev \
python-netaddr python-pip software-properties-common sshpass
+ sudo pip install gitpython graphviz Jinja2>=2.9
sudo apt-add-repository -y ppa:ansible/ansible # latest supported version
sudo apt-get update
sudo apt-get install -y ansible
- sudo pip install gitpython graphviz
fi
}