update the network design to make support easier and cleaner
Change-Id: Iddf25266e3fd534ba88887acb7116faa04224c39
(cherry picked from commit c15f8e1231905cb83e0db014b8c1b3d7c72b23f0)
diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py
index 4c00853..c7a0826 100755
--- a/bootstrap/bootstrap.py
+++ b/bootstrap/bootstrap.py
@@ -290,7 +290,7 @@
hostIfc = ifc if ifc['name'] == config['bridge'] else hostIfc
add_or_update_node_group_interface(client, ng, config['gw'], localIfc, config['interface'], config['network'])
- add_or_update_node_group_interface(client, ng, config['gw'], hostIfc, config['bridge'], config['bridge-subnet'])
+ #add_or_update_node_group_interface(client, ng, config['gw'], hostIfc, config['bridge'], config['bridge-subnet'])
# Update the server settings to upstream DNS request to Google
# POST /api/2.0/maas/ op=set_config
diff --git a/build.gradle b/build.gradle
index 2bc1e16..0f50779 100644
--- a/build.gradle
+++ b/build.gradle
@@ -286,9 +286,11 @@
.p(config.seedServer.sudoPassword, "ansible_sudo_pass")
.p(config.seedServer.fabric_ip, "fabric_ip")
.p(config.seedServer.management_ip, "management_ip")
+ .p(config.seedServer.management_gw, "management_gw")
.p(config.seedServer.management_network, "management_network")
.p(config.seedServer.management_iface, "management_iface")
.p(config.seedServer.external_ip, "external_ip")
+ .p(config.seedServer.external_gw, "external_gw")
.p(config.seedServer.external_network, "external_network")
.p(config.seedServer.external_iface, "external_iface")
.p(config.seedServer.fabric_ip, "fabric_ip")
@@ -342,9 +344,11 @@
.p(config.seedServer.sudoPassword, "ansible_sudo_pass")
.p(config.seedServer.fabric_ip, "fabric_ip")
.p(config.seedServer.management_ip, "management_ip")
+ .p(config.seedServer.management_gw, "management_gw")
.p(config.seedServer.management_network, "management_network")
.p(config.seedServer.management_iface, "management_iface")
.p(config.seedServer.external_ip, "external_ip")
+ .p(config.seedServer.external_gw, "external_gw")
.p(config.seedServer.external_network, "external_network")
.p(config.seedServer.external_iface, "external_iface")
.p(config.seedServer.fabric_ip, "fabric_ip")
diff --git a/roles/compute-node/files/pull-latest-docker-images b/roles/compute-node/files/pull-latest-docker-images
new file mode 100755
index 0000000..abcdf02
--- /dev/null
+++ b/roles/compute-node/files/pull-latest-docker-images
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+for i in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep docker-registry | grep -v "<none>" | sort); do
+ docker pull $i
+done
diff --git a/roles/compute-node/files/rename_ifaces.sh b/roles/compute-node/files/rename_ifaces.sh
index b355579..a490717 100755
--- a/roles/compute-node/files/rename_ifaces.sh
+++ b/roles/compute-node/files/rename_ifaces.sh
@@ -132,10 +132,10 @@
for i in $(cat $2); do
if [ "eth$IDX" == "$4" ]; then
echo "auto eth$IDX" >> $OUT
- if [ "$EXT_IP" == "dhcp" ]; then
+ if [ "$EXT_ADDR" == "dhcp" ]; then
echo "iface eth$IDX inet dhcp" >> $OUT
- elif [ "$EXT_IP" == "manual" ]; then
- echo "iface eth$IDX inet manual" >> $OUT
+ elif [ "$EXT_ADDR" == "manual" ]; then
+ echo "iface eth$IDX inet manual" >> $OUT
else
echo "iface eth$IDX inet static" >> $OUT
echo " address $EXT_IP" >> $OUT
@@ -143,22 +143,8 @@
echo " netmask $EXT_NETMASK" >> $OUT
echo " broadcast $EXT_BROADCAST" >> $OUT
echo " gateway $EXT_GW" >> $OUT
- echo " dns-nameservers 8.8.8.8 8.8.4.4" >> $OUT
- echo " dns-search cord.lab" >> $OUT
- fi
- elif [ "eth$IDX" == "$5" ]; then
- echo "auto eth$IDX" >> $OUT
- if [ "$MGT_IP" == "dhcp" ]; then
- echo "iface eth$IDX inet dhcp" >> $OUT
- elif [ "$MGT_IP" == "manual" ]; then
- echo "iface eth$IDX inet manual" >> $OUT
- else
- echo "iface eth$IDX inet static" >> $OUT
- echo " address $MGT_IP" >> $OUT
- echo " network $MGT_NETWORK" >> $OUT
- echo " netmask $MGT_NETMASK" >> $OUT
- echo " broadcast $MGT_BROADCAST" >> $OUT
- echo " gateway $MGT_GW" >> $OUT
+ echo " dns-nameservers 8.8.8.8 8.8.4.4" >> $OUT
+ echo " dns-search cord.lab" >> $OUT
fi
else
echo "auto eth$IDX" >> $OUT
@@ -172,12 +158,23 @@
local BRNAME=$5
local F=$(echo $5 | grep "^eth[0-9]*$" | wc -l)
if [ $F -ne 0 ]; then
- BRNAME="br-mgmt"
+ BRNAME="mgmtbr"
fi
local BPCNT=$(echo $BRIDGE_PORTS | wc -w)
if [ $BPCNT -ne 0 ]; then
echo "auto $BRNAME" >> $OUT
- echo "iface $BRNAME inet dhcp" >> $OUT
+ if [ "$MGT_ADDR" == "dhcp" ]; then
+ echo "iface $BRNAME inet dhcp" >> $OUT
+ elif [ "$MGT_ADDR" == "manual" ]; then
+ echo "iface $BRNAME inet manual" >> $OUT
+ else
+ echo "iface $BRNAME inet static" >> $OUT
+ echo " address $MGT_IP" >> $OUT
+ echo " network $MGT_NETWORK" >> $OUT
+ echo " netmask $MGT_NETMASK" >> $OUT
+ echo " broadcast $MGT_BROADCAST" >> $OUT
+ echo " gateway $MGT_GW" >> $OUT
+ fi
echo " bridge_ports $BRIDGE_PORTS" >> $OUT
fi
}
@@ -191,23 +188,25 @@
EXT_IFACE=$3
EXT_ADDR=$4
+EXT_GW=$5
if [ "$EXT_ADDR" != "dhcp" ]; then
EXT_IP=$(echo $EXT_ADDR | cut -d/ -f1)
EXT_MASKBITS=$(echo $EXT_ADDR | cut -d/ -f2)
EXT_NETWORK=$(network $EXT_IP $EXT_MASKBITS)
EXT_NETMASK=$(netmask $EXT_MASKBITS)
EXT_BROADCAST=$(broadcast $EXT_IP $EXT_MASKBITS)
- EXT_GW=$(first $EXT_ADDR)
+ test -z $EXT_GW && EXT_GW=$(first $EXT_ADDR)
fi
-MGT_IFACE=$5
-MGT_ADDR=$6
+MGT_IFACE=$6
+MGT_ADDR=$7
+MGT_GW=$8
if [ "$MGT_ADDR" != "dhcp" ]; then
MGT_IP=$(echo $MGT_ADDR | cut -d/ -f1)
MGT_MASKBITS=$(echo $MGT_ADDR | cut -d/ -f2)
MGT_NETWORK=$(network $MGT_IP $MGT_MASKBITS)
MGT_NETMASK=$(netmask $MGT_MASKBITS)
MGT_BROADCAST=$(broadcast $MGT_IP $MGT_MASKBITS)
- MGT_GW=$(first $MGT_ADDR)
+ test -z $MGT_GW && MGT_GW=$(first $MGT_ADDR)
fi
LIST_ETH=$(mktemp -u)
@@ -236,23 +235,23 @@
diff /etc/network/interfaces $IFACES_FILE 2>&1 > /dev/null
if [ $? -ne 0 ]; then
- CHANGED="true"
- cp /etc/network/interfaces /etc/network/interfaces.1
- cp $IFACES_FILE /etc/network/interfaces
+ CHANGED="true"
+ cp /etc/network/interfaces /etc/network/interfaces.1
+ cp $IFACES_FILE /etc/network/interfaces
fi
generate_persistent_names $LIST_FABRIC $LIST_ETH "$FAB_IFACE" "$EXT_IFACE"
if [ -r /etc/udev/rules.d/70-persistent-net.rules ]; then
- diff /etc/udev/rules.d/70-persistent-net.rules $NAMES_FILE 2>&1 > /dev/null
- if [ $? -ne 0 ]; then
- CHANGED="true"
- cp /etc/udev/rules.d/70-persistent-net.rules /etc/udev/rules.d/70-persistent-net.rules.1
- cp $NAMES_FILE /etc/udev/rules.d/70-persistent-net.rules
- fi
+ diff /etc/udev/rules.d/70-persistent-net.rules $NAMES_FILE 2>&1 > /dev/null
+ if [ $? -ne 0 ]; then
+ CHANGED="true"
+ cp /etc/udev/rules.d/70-persistent-net.rules /etc/udev/rules.d/70-persistent-net.rules.1
+ cp $NAMES_FILE /etc/udev/rules.d/70-persistent-net.rules
+ fi
else
- CHANGED="true"
- cp $NAMES_FILE /etc/udev/rules.d/70-persistent-net.rules
+ CHANGED="true"
+ cp $NAMES_FILE /etc/udev/rules.d/70-persistent-net.rules
fi
rm -rf $IFACES_FILE
diff --git a/roles/compute-node/tasks/main.yml b/roles/compute-node/tasks/main.yml
index 1be091a..236fae4 100644
--- a/roles/compute-node/tasks/main.yml
+++ b/roles/compute-node/tasks/main.yml
@@ -108,6 +108,7 @@
- get-node-prov-state
- remove-xos-components
- post-fabric-config
+ - pull-latest-docker-images
- name: Verify Mellanox 40Gb NIC
shell: /usr/bin/lspci | grep "Ethernet controller" | grep -c ConnectX-3 || true
@@ -151,7 +152,7 @@
- name: Consistent Interface Naming
become: yes
- script: files/rename_ifaces.sh "{{ compute_node.interfaces.fabric }}" "{{ compute_node.addresses.fabric }}" "{{ compute_node.interfaces.external }}" "{{ compute_node.addresses.external }}" "{{ compute_node.interfaces.management }}" "{{ compute_node.addresses.management }}"
+ script: files/rename_ifaces.sh "{{ compute_node.interfaces.fabric }}" "{{ compute_node.addresses.fabric }}" "{{ compute_node.interfaces.external }}" "{{ compute_node.addresses.external }}" "{{ compute_node.gateway.external }}" "{{ compute_node.interfaces.management }}" "{{ compute_node.addresses.management }}" "{{ compute_node.gateway.management }}"
register: ifaces_changed
changed_when: ifaces_changed.stdout.find("true") != -1
tags:
diff --git a/roles/compute-node/vars/main.yml b/roles/compute-node/vars/main.yml
index 64316c3..05425ea 100644
--- a/roles/compute-node/vars/main.yml
+++ b/roles/compute-node/vars/main.yml
@@ -3,9 +3,12 @@
compute_node:
interfaces:
fabric: "{{ fabric_iface | default('fabric') }}"
- management: "{{ management_iface | default('br-mgmt') }}"
+ management: "{{ management_iface | default('mgmtbr') }}"
external: "{{ external_iface | default('') }}"
addresses:
fabric: "{{ fabric_ip | mandatory }}"
management: "{{ management_ip | default('dhcp') }}"
external: "{{ external_ip | default('manual') }}"
+ gateway:
+ external: "{{ external_gw | default('') }}"
+ management: "{{ management_gw | default('') }}"
diff --git a/roles/maas/files/generate_network_config.sh b/roles/maas/files/generate_network_config.sh
deleted file mode 100755
index eb8f7a2..0000000
--- a/roles/maas/files/generate_network_config.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-
-IFACE_MGMT=$1
-NET_MGMT=$2
-NET_BRIDGE=$3
-MGMTBR=$4
-
-ip2int()
-{
- local a b c d
- { IFS=. read a b c d; } <<< $1
- echo $(((((((a << 8) | b) << 8) | c) << 8) | d))
-}
-
-int2ip()
-{
- local ui32=$1; shift
- local ip n
- for n in 1 2 3 4; do
- ip=$((ui32 & 0xff))${ip:+.}$ip
- ui32=$((ui32 >> 8))
- done
- echo $ip
-}
-
-netmask()
-{
- local mask=$((0xffffffff << (32 - $1))); shift
- int2ip $mask
-}
-
-
-broadcast()
-{
- local addr=$(ip2int $1); shift
- local mask=$((0xffffffff << (32 -$1))); shift
- int2ip $((addr | ~mask))
-}
-
-network()
-{
- local addr=$(ip2int $1); shift
- local mask=$((0xffffffff << (32 -$1))); shift
- int2ip $((addr & mask))
-}
-
-first()
-{
- local addr=$(ip2int $1)
- addr=`expr $addr + 1`
- int2ip $addr
-}
-
-MBITS=`echo "$NET_MGMT" | cut -d/ -f2`
-MNETW=`echo "$NET_MGMT" | cut -d/ -f1`
-MMASK=`netmask $MBITS`
-MHOST=`first $MNETW`
-
-BBITS=`echo "$NET_BRIDGE" | cut -d/ -f2`
-BNETW=`echo "$NET_BRIDGE" | cut -d/ -f1`
-BMASK=`netmask $BBITS`
-BHOST=`first $BNETW`
-
-OUT=$(mktemp -u)
-cat /etc/network/interfaces | awk '/## CORD - DO NOT EDIT BELOW THIS LINE/{exit};1' | awk "/^auto / { if (\$2 == \"${IFACE_MGMT}\") { IN=1 } else {IN=0} } /^iface / { if (\$2 == \"${IFACE_MGMT}\") { IN=1 } else {IN=0}} /^#/ || /^\s*\$/ { IN=0 } IN==0 {print} IN==1 { print \"#\" \$0 }" > $OUT
-
-cat <<EOT >> $OUT
-## CORD - DO NOT EDIT BELOW THIS LINE
-
-auto ${IFACE_MGMT}
-iface ${IFACE_MGMT} inet static
- address ${MHOST}
- network ${MNETW}
- netmask ${MMASK}
- gateway ${MHOST}
-
-auto ${MGMTBR}
-iface ${MGMTBR} inet static
- address ${BHOST}
- network ${BNETW}
- netmask ${BMASK}
- gateway ${BHOST}
-EOT
-
-diff /etc/network/interfaces $OUT 2>&1 > /dev/null
-if [ $? -ne 0 ]; then
- cp /etc/network/interfaces /etc/network/interfaces.last
- cp $OUT /etc/network/interfaces
- echo -n "true"
-else
- echo -n "false"
-fi
-
-rm $OUT
diff --git a/roles/maas/tasks/main.yml b/roles/maas/tasks/main.yml
index c6bde70..d247b4a 100644
--- a/roles/maas/tasks/main.yml
+++ b/roles/maas/tasks/main.yml
@@ -1,4 +1,14 @@
---
+- name: Ensure Management Bridge Interface
+ become: yes
+ command: ifconfig mgmtbr
+ register: have_mgmtbr
+ failed_when: false
+ changed_when: false
+
+- fail: msg="The head node must have a network bridge named 'mgmtbr' to continue the deployment. Please see docuemataiton at 'http://github.com/opencord/cord`."
+ when: have_mgmtbr.rc != 0
+
- name: Install Prerequisites
become: yes
apt: name={{ item }} state=latest force=true
@@ -17,40 +27,20 @@
with_items:
- docker-py
-- name: Stop MAAS Switchq Container
+- name: Stop Containers
become: yes
docker:
- name: switchq
- image: opencord/cord-maas-switchq:latest
+ name: "{{ item.name }}"
+ image: "{{ item.image }}"
state: absent
-
-- name: Stop MAAS Automation Container
- become: yes
- docker:
- name: automation
- image: opencord/cord-maas-automation:latest
- state: absent
-
-- name: Stop Provisioner Container
- become: yes
- docker:
- name: provisioner
- image: opencord/cord-provisioner:latest
- state: absent
-
-- name: Stop DHCP Harvester Container
- become: yes
- docker:
- name: harvester
- image: opencord/cord-dhcp-harvester:latest
- state: absent
-
-- name: Stop IP Allocator Container
- become: yes
- docker:
- name: allocator
- image: opencord/cord-ip-allocator:latest
- state: absent
+ with_items:
+ - { name: "storage", image: "docker-registry:5000/consul:{{ docker.image_version }}" }
+ - { name: "allocator", image: "docker-registry:5000/cord-ip-allocator:{{ docker.image_version }}" }
+ - { name: "provisioner", image: "docker-registry:5000/cord-provisioner:{{ docker.image_version }}" }
+ - { name: "switchq", image: "docker-registry:5000/cord-maas-switchq:{{ docker.image_version }}" }
+ - { name: "automation", image: "docker-registry:5000/cord-maas-automation:{{ docker.image_version }}" }
+ - { name: "generator", image: "docker-registry:5000/config-generator:{{ docker.image_version }}" }
+ - { name: "harvester", image: "docker-registry:5000/cord-dhcp-harvester:{{ docker.image_version }}" }
- name: MAAS Repository
become: yes
@@ -133,12 +123,6 @@
command: virsh net-destroy default
when: virsh_default_network_exists.stdout != '0'
-- name: Network Configuration
- become: yes
- script: files/generate_network_config.sh {{ interfaces.management }} {{ networks.management }} {{ networks.bridge }} {{ networks.bridge_name }}
- register: network_config_changed
- changed_when: network_config_changed.stdout == 'true'
-
- name: Network Masquerading (NAT)
become: yes
template:
@@ -154,26 +138,6 @@
register: masq_changed
changed_when: masq_changed.stdout == 'true'
-- name: VM Bridge
- become: yes
- template:
- src: templates/create_bridge.j2
- dest: /etc/network/if-pre-up.d/create_bridge_{{ networks.bridge_name }}
- owner: root
- group: root
- mode: 0755
-
-- name: Activate VM Bridge
- become: yes
- command: /etc/network/if-pre-up.d/create_bridge_{{ networks.bridge_name }} report-changed
- register: bridge_changed
- changed_when: bridge_changed.stdout == 'true'
-
-- name: Management Interface
- become: yes
- shell: ifdown {{ interfaces.management }} && ifup {{ interfaces.management }}
- when: network_config_changed.stdout == 'true'
-
- name: Management Interface IP Address
shell: ifconfig {{ interfaces.management }} 2>&1 | grep "inet addr:" | sed -e 's/.*:\([.0-9]*\)[ ]*Bcast.*/\1/g'
register: mgmt_ip_address
diff --git a/roles/maas/vars/main.yml b/roles/maas/vars/main.yml
index 062c00c..b2ac1d3 100644
--- a/roles/maas/vars/main.yml
+++ b/roles/maas/vars/main.yml
@@ -28,7 +28,7 @@
# 'management' specifies the interface on which the head node will
# service DHCP and PXE boot requests
fabric: "{{ fabric_iface | default('fabric') }}"
- management: "{{ management_iface | default('br-mgmt') }}"
+ management: "{{ management_iface | default('mgmtbr') }}"
external: "{{ external_iface | default('') }}"
networks:
@@ -42,7 +42,7 @@
# 'fabric' specifies the network that will be assigned to
# the leaf - spine fabric
management: "{{ management_network | default('10.6.0.0/24') }}"
- bridge: "{{ bridge_network | default('172.18.0.0/24') }}"
+ bridge: "{{ bridge_network | default('172.42.0.0/24') }}"
fabric: "{{ fabric_network | default('10.6.1.0/24') }}"
# CHANGE: