[EDGEPOD-186] Add Ansible playbook for Aether Edge
Change-Id: I71d73a5a4fdf7e5ccf12df924597af6ab90a68b4
diff --git a/aether-playbook/ansible.cfg b/aether-playbook/ansible.cfg
new file mode 100644
index 0000000..497536b
--- /dev/null
+++ b/aether-playbook/ansible.cfg
@@ -0,0 +1,16 @@
+[ssh_connection]
+pipelining=True
+ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+
+[defaults]
+force_valid_group_names = ignore
+host_key_checking=False
+gathering = smart
+fact_caching = jsonfile
+fact_caching_connection = /tmp
+stdout_callback = skippy
+deprecation_warnings=False
+inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg
+
+[inventory]
+ignore_patterns = artifacts, credentials
diff --git a/aether-playbook/inventory/sample/group_vars/all.yml b/aether-playbook/inventory/sample/group_vars/all.yml
new file mode 100644
index 0000000..c625f7f
--- /dev/null
+++ b/aether-playbook/inventory/sample/group_vars/all.yml
@@ -0,0 +1,63 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ntp_enabled: true
+ntp_servers:
+ - time1.google.com
+ - time2.google.com
+ntp_timezone: Etc/UTC
+
+# Internal service domains to add to /etc/hosts (DO NOT CHANGE)
+etc_hosts_entries:
+ - ip: 10.168.0.200
+ name: registry.central.aetherproject.net
+ - ip: 10.168.0.3
+ name: rancher.central.aetherproject.net
+
+# Additional Docker daemon options
+docker_daemon_options:
+ insecure-registries:
+ - registry.central.aetherproject.net
+
+docker_version: 18.06.3~ce~3-0~ubuntu
+
+upf_sriov_enabled: true
+upf_sriov_pf: enp94s0f0
+
+# SGI and S1U addresses for spgwu
+spgwu_sgi_ip: 192.168.250.4/24
+spgwu_s1u_ip: 192.168.251.4/24
+
+# UE pool (DO NOT CHANGE)
+ue_pool: 10.250.0.0/16
+
+# Enable to configure router to handle UE traffic to the Internet
+# Applicable only when using Ubuntu machine as a router now (VyOS is coming soon)
+router_setup_enabled: true
+router_type: linux
+sgi_gateway_ip: 192.168.250.254/24
+s1u_gateway_ip: 192.168.251.254/24
+sgi_s1u_gateway_iface: enp94s0f0
+netplan_config_file: /etc/netplan/sgi-s1u-gateway.yaml
+
+rancher_cluster_token: #HIDDEN
+rancher_ca_checksum: #HIDDEN
+
+# Refer to these values when configuring Accelleran
+accelleran_ru_ip: 192.168.251.5/24
+accelleran_tac: 101
+accelleran_mme_ip: 10.168.0.204
+accelleran_redis_host: 10.212.73.3
+accelleran_instance_filter: onfcell101
diff --git a/aether-playbook/inventory/sample/inventory.ini b/aether-playbook/inventory/sample/inventory.ini
new file mode 100644
index 0000000..1a8b175
--- /dev/null
+++ b/aether-playbook/inventory/sample/inventory.ini
@@ -0,0 +1,36 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of all nodes
+menlo-edge ansible_host=10.212.73.3
+
+# Specify a node to run k8s control plane and etcd
+# Note that the number of the nodes must be odd
+[kube-master]
+menlo-edge
+
+# Specify k8s worker nodes
+[kube-worker]
+menlo-edge
+
+# Specify a node to run UPF(SPGWU)
+# Make sure the node has SRIOV supported NIC
+[upf]
+menlo-edge
+
+# Specify a router that will serve SGI/S1U network gateway and the Internet connectivity to UE
+[router]
+menlo-edge
+
+#[accelleran]
diff --git a/aether-playbook/inventory/template/group_vars/all.yml b/aether-playbook/inventory/template/group_vars/all.yml
new file mode 100644
index 0000000..fe33ebd
--- /dev/null
+++ b/aether-playbook/inventory/template/group_vars/all.yml
@@ -0,0 +1,63 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ntp_enabled: true
+ntp_servers:
+ - time1.google.com
+ - time2.google.com
+ntp_timezone: Etc/UTC
+
+# Internal service domains to add to /etc/hosts (DO NOT CHANGE)
+etc_hosts_entries:
+ - ip: 10.168.0.200
+ name: registry.central.aetherproject.net
+ - ip: 10.168.0.3
+ name: rancher.central.aetherproject.net
+
+# Additional Docker daemon options
+docker_daemon_options:
+ insecure-registries:
+ - registry.central.aetherproject.net
+ iptables: false
+
+upf_sriov_enabled: true
+upf_sriov_pf: #SET_VALUE
+
+# SGI and S1U addresses for spgwu
+spgwu_sgi_ip: #SET_VALUE
+spgwu_s1u_ip: #SET_VALUE
+
+# UE pool (DO NOT CHANGE)
+ue_pool: 10.250.0.0/16
+
+# Enable to configure router to handle UE traffic to the Internet
+# Applicable only when using Ubuntu machine as a router now (VyOS is coming soon)
+router_setup_enabled: #SET_VALUE (true or false)
+router_type: linux
+sgi_gateway_ip: #SET_VALUE
+s1u_gateway_ip: #SET_VALUE
+sgi_s1u_gateway_iface: #SET_VALUE
+netplan_config_file: /etc/netplan/sgi-s1u-gateway.yaml
+
+# Rancher cluster token and checksum (DO NOT CHANGE)
+rancher_cluster_token: #GET_VALUE_FROM_AETHER_TEAM
+rancher_ca_checksum: "7f7858afaa621e304d0d17fa22fd2005aa2f1acd0637f4026cab7bcc2fa43cd9"
+
+# Refer to these values when configuring Accelleran
+accelleran_tac: #GET_VALUE_FROM_AETHER_TEAM
+accelleran_instance_filter: #GET_VALUE_FROM_AETHER_TEAM
+accelleran_mme_ip: 10.168.0.204
+accelleran_ru_ip: #SET_VALUE (S1U network address)
+accelleran_redis_host: #SET_VALUE (mgmt IP of any node)
diff --git a/aether-playbook/inventory/template/inventory.ini b/aether-playbook/inventory/template/inventory.ini
new file mode 100644
index 0000000..aaa3417
--- /dev/null
+++ b/aether-playbook/inventory/template/inventory.ini
@@ -0,0 +1,38 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of all nodes
+# example)
+# menlo-edge ansible_host=10.92.1.50
+#SET_VALUE
+
+# Specify a node to run k8s control plane and etcd
+# Note that the number of the nodes must be odd
+[kube-master]
+#SET_VALUE
+
+# Specify k8s worker nodes
+[kube-worker]
+#SET_VALUE
+
+# Specify a node to run UPF(SPGWU)
+# Make sure the node has SRIOV supported NIC
+[upf]
+#SET_VALUE
+
+# Specify a router that will serve SGI/S1U network gateways and the Internet connectivity to UE
+[router]
+#SET_VALUE (optional)
+
+#[accelleran]
diff --git a/aether-playbook/requirements.txt b/aether-playbook/requirements.txt
new file mode 100644
index 0000000..045cbeb
--- /dev/null
+++ b/aether-playbook/requirements.txt
@@ -0,0 +1 @@
+ansible==2.9.5
diff --git a/aether-playbook/reset.yml b/aether-playbook/reset.yml
new file mode 100644
index 0000000..c0a0ffe
--- /dev/null
+++ b/aether-playbook/reset.yml
@@ -0,0 +1,43 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Check ansible version
+- hosts: localhost
+ become: no
+ gather_facts: False
+ tasks:
+ - name: "Check ansible version ==2.9.5"
+ assert:
+ msg: "Ansible must be v2.9.5"
+ that:
+ - ansible_version.string is version("2.9.5", "==")
+ vars:
+ ansible_connection: local
+
+- hosts: kube-master:kube-worker
+ vars_prompt:
+ name: "reset_confirmation"
+ prompt: "Are you sure you want to reset cluster state? Type 'yes' to reset your cluster."
+ default: "no"
+ private: no
+
+ pre_tasks:
+ - name: check confirmation
+ fail:
+ msg: "Reset confirmation failed"
+ when: reset_confirmation != "yes"
+
+ roles:
+ - { role: reset }
diff --git a/aether-playbook/roles/k8s/defaults/main.yml b/aether-playbook/roles/k8s/defaults/main.yml
new file mode 100644
index 0000000..6132d2f
--- /dev/null
+++ b/aether-playbook/roles/k8s/defaults/main.yml
@@ -0,0 +1,32 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ntp_enabled: true
+ntp_servers:
+ - time1.google.com
+ - time2.google.com
+ntp_timezone: Etc/UTC
+
+etc_hosts_entries:
+ - ip: 10.168.0.200
+ name: registry.central.aetherproject.net
+ - ip: 10.168.0.3
+ name: rancher.central.aetherproject.net
+
+docker_daemon_options:
+ insecure-registries:
+ - registry.central.aetherproject.net
+
+docker_version: 18.06.3~ce~3-0~ubuntu
diff --git a/aether-playbook/roles/k8s/handlers/main.yml b/aether-playbook/roles/k8s/handlers/main.yml
new file mode 100644
index 0000000..cd20f52
--- /dev/null
+++ b/aether-playbook/roles/k8s/handlers/main.yml
@@ -0,0 +1,20 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: restart ntp
+ systemd: name=ntp state=restarted
+
+- name: restart docker
+ systemd: name=docker state=restarted
diff --git a/aether-playbook/roles/k8s/tasks/docker.yml b/aether-playbook/roles/k8s/tasks/docker.yml
new file mode 100644
index 0000000..7d90bbb
--- /dev/null
+++ b/aether-playbook/roles/k8s/tasks/docker.yml
@@ -0,0 +1,70 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Install dependencies for Docker
+ apt:
+ name:
+ - apt-transport-https
+ - ca-certificates
+ - curl
+ - software-properties-common
+ state: present
+ tags: docker
+
+- name: Add Docker GPG key
+ shell: |
+ curl -sSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ args:
+ warn: false
+ tags: docker
+
+- name: Add Docker repository
+ shell: |
+ add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+ args:
+ warn: false
+ tags: docker
+
+- name: Install Docker CE
+ apt:
+ name: docker-ce={{ docker_version }}
+ state: present
+ update_cache: yes
+ tags: docker
+
+- name: Add current user to "docker" group
+ user:
+ name: "{{ ansible_user }}"
+ groups: docker
+ append: true
+ tags: docker
+
+- name: Configure Docker daemon options
+ copy:
+ dest: /etc/docker/daemon.json
+ content: "{{ docker_daemon_options | to_json }}"
+ register: docker_register_options
+ notify: restart docker
+ tags: docker
+
+- name: Reload systemd daemon
+ systemd:
+ daemon_reload: true
+ when: docker_register_options is changed
+ tags: docker
diff --git a/aether-playbook/roles/k8s/tasks/k8s.yml b/aether-playbook/roles/k8s/tasks/k8s.yml
new file mode 100644
index 0000000..1561d1a
--- /dev/null
+++ b/aether-playbook/roles/k8s/tasks/k8s.yml
@@ -0,0 +1,48 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Initialize k8s role
+ set_fact:
+ k8s_roles: ""
+ tags: k8s
+
+- name: Add k8s master role
+ set_fact:
+ k8s_roles: "{{ k8s_roles }} --etcd --controlplane"
+ when: "'kube-master' in group_names"
+ tags: k8s
+
+- name: Add k8s worker role
+ set_fact:
+ k8s_roles: "{{ k8s_roles }} --worker"
+ when: "'kube-worker' in group_names"
+ tags: k8s
+
+- name: Print k8s roles
+ debug:
+ msg: k8s_roles {{ k8s_roles }}
+ tags: k8s
+
+- name: Install k8s
+ command: >-
+ docker run -d --privileged --restart=unless-stopped --net=host
+ --volume /etc/kubernetes:/etc/kubernetes
+ --volume /var/run:/var/run
+ rancher/rancher-agent:v2.3.3
+ --server https://rancher.central.aetherproject.net
+ --token {{ rancher_cluster_token }}
+ --ca-checksum {{ rancher_ca_checksum }}
+ {{ k8s_roles }}
+ tags: k8s
diff --git a/aether-playbook/roles/k8s/tasks/main.yml b/aether-playbook/roles/k8s/tasks/main.yml
new file mode 100644
index 0000000..0a357a6
--- /dev/null
+++ b/aether-playbook/roles/k8s/tasks/main.yml
@@ -0,0 +1,26 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- include_tasks: pre-install.yml
+ tags:
+ - etc-hosts
+ - ntp
+ - swapoff
+
+- include_tasks: docker.yml
+ tags: docker
+
+- include_tasks: k8s.yml
+ tags: k8s
diff --git a/aether-playbook/roles/k8s/tasks/pre-install.yml b/aether-playbook/roles/k8s/tasks/pre-install.yml
new file mode 100644
index 0000000..b0b2bfa
--- /dev/null
+++ b/aether-playbook/roles/k8s/tasks/pre-install.yml
@@ -0,0 +1,70 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Run "apt update"
+ apt:
+ update_cache: yes
+
+- name: Install ntp
+ apt: name=ntp state=present
+ when: ntp_enabled | bool
+ tags: ntp
+
+- name: Configure ntp file
+ template: src=etc/ntp.conf.j2 dest=/etc/ntp.conf
+ when: ntp_enabled | bool
+ notify: restart ntp
+ tags: ntp
+
+- name: Start the ntp service
+ service: name=ntp state=started enabled=yes
+ when: ntp_enabled | bool
+ tags: ntp
+
+- name: Set timezone to {{ ntp_timezone }}
+ timezone:
+ name: "{{ ntp_timezone }}"
+ when: ntp_enabled | bool
+ tags: ntp
+
+- name: Remove swapfile from /etc/fstab
+ mount:
+ name: "{{ item }}"
+ fstype: swap
+ state: absent
+ with_items:
+ - swap
+ - none
+ tags: swapoff
+
+- name: Check if swap is enabled
+ command: /sbin/swapon -s
+ register: swapon
+ changed_when: no
+ tags: swapoff
+
+- name: Disable swap
+ command: /sbin/swapoff -a
+ when: swapon.stdout
+ tags: swapoff
+
+- name: Add internal service domains to /etc/hosts
+ become: yes
+ lineinfile:
+ path: /etc/hosts
+ line: "{{ item['ip'] }}\t\t{{ item['name'] }}"
+ state: present
+ with_items: "{{ etc_hosts_entries }}"
+ tags: etc-hosts
diff --git a/aether-playbook/roles/k8s/templates/etc/ntp.conf.j2 b/aether-playbook/roles/k8s/templates/etc/ntp.conf.j2
new file mode 100644
index 0000000..0c46ff1
--- /dev/null
+++ b/aether-playbook/roles/k8s/templates/etc/ntp.conf.j2
@@ -0,0 +1,27 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# {{ ansible_managed }}
+
+# For more information about this file, see the man pages
+# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).
+
+driftfile /var/lib/ntp/drift
+
+restrict 127.0.0.1
+restrict -6 ::1
+
+{% for i in ntp_servers %}
+server {{ i }}
+{% endfor %}
diff --git a/aether-playbook/roles/reset/tasks/main.yml b/aether-playbook/roles/reset/tasks/main.yml
new file mode 100644
index 0000000..ffb8922
--- /dev/null
+++ b/aether-playbook/roles/reset/tasks/main.yml
@@ -0,0 +1,48 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Remove containers
+ shell: |
+ docker stop $(docker ps -aq)
+ docker system prune -f
+ docker volume rm $(docker volume ls -q)
+ docker image rm $(docker image ls -q)
+ register: remove_all_containers
+ retries: 4
+ delay: 5
+ until: remove_all_containers.rc == 0
+ tags: k8s
+
+- name: Remove directories
+ file:
+ path: "{{ item }}"
+ state: absent
+ ignore_errors: yes
+ with_items:
+ - /etc/kubernetes
+ - /etc/cni
+ - /opt/cni
+ - /opt/rke
+ - /run/secrets/kubernetes.io
+ - /run/calico
+ - /var/lib/calico
+ - /var/lib/etcd
+ - /var/lib/cni
+ - /var/lib/kubelet
+ - /var/lib/rancher/rke/log
+ - /var/log/containers
+ - /var/log/pods
+ - /var/run/calico
+ tags: k8s
diff --git a/aether-playbook/roles/router/defaults/main.yml b/aether-playbook/roles/router/defaults/main.yml
new file mode 100644
index 0000000..6af6545
--- /dev/null
+++ b/aether-playbook/roles/router/defaults/main.yml
@@ -0,0 +1,22 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+router_type: linux
+netplan_config_file: /etc/netplan/sgi-s1u-gateway.yaml
+
+# Provide below to run the playbook
+#sgi_gateway_ip:
+#s1u_gateway_ip:
+#sgi_s1u_gateway_iface:
diff --git a/aether-playbook/roles/router/linux/handlers/main.yml b/aether-playbook/roles/router/linux/handlers/main.yml
new file mode 100644
index 0000000..6408d79
--- /dev/null
+++ b/aether-playbook/roles/router/linux/handlers/main.yml
@@ -0,0 +1,20 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: netplan generate
+ command: netplan generate
+
+- name: netplan apply
+ command: netplan apply
diff --git a/aether-playbook/roles/router/linux/tasks/main.yml b/aether-playbook/roles/router/linux/tasks/main.yml
new file mode 100644
index 0000000..c37bca1
--- /dev/null
+++ b/aether-playbook/roles/router/linux/tasks/main.yml
@@ -0,0 +1,56 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Create netplan config file for SGI network gateway
+ template:
+ src: etc/netplan/sgi-s1u-gateway.yaml.j2
+ dest: "{{ netplan_config_file }}"
+ notify:
+ - netplan generate
+ - netplan apply
+ tags: router
+
+- name: Install iptables-persistent
+ apt:
+ name: iptables-persistent
+ state: present
+ update_cache: yes
+ tags: router
+
+- name: Ensure ip_forward enabled
+ sysctl:
+ name: net.ipv4.ip_forward
+ value: '1'
+ sysctl_set: yes
+ state: present
+ tags: router
+
+- name: Set default forwarding policy to ACCEPT
+ iptables:
+ chain: FORWARD
+ policy: ACCEPT
+ tags: router
+
+- name: Add SNAT
+ iptables:
+ table: nat
+ chain: POSTROUTING
+ out_interface: "{{ ansible_default_ipv4.interface }}"
+ jump: MASQUERADE
+ tags: router
+
+- name: Save iptables v4 rules
+ shell: iptables-save > /etc/iptables/rules.v4
+ tags: router
diff --git a/aether-playbook/roles/router/linux/templates/etc/netplan/sgi-s1u-gateway.yaml.j2 b/aether-playbook/roles/router/linux/templates/etc/netplan/sgi-s1u-gateway.yaml.j2
new file mode 100644
index 0000000..7c6cef7
--- /dev/null
+++ b/aether-playbook/roles/router/linux/templates/etc/netplan/sgi-s1u-gateway.yaml.j2
@@ -0,0 +1,26 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+network:
+ version: 2
+ renderer: networkd
+ ethernets:
+ {{ sgi_s1u_gateway_iface }}:
+ addresses:
+ - {{ sgi_gateway_ip }}
+ - {{ s1u_gateway_ip }}
+ routes:
+ - to: {{ ue_pool }}
+ via: {{ spgwu_sgi_ip }}
diff --git a/aether-playbook/roles/router/meta/main.yml b/aether-playbook/roles/router/meta/main.yml
new file mode 100644
index 0000000..3c4fe58
--- /dev/null
+++ b/aether-playbook/roles/router/meta/main.yml
@@ -0,0 +1,23 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+dependencies:
+ - role: router/linux
+ when:
+ - router_type == 'linux'
+
+# - role: router/vyos
+# when:
+# - router_type == 'vyos'
diff --git a/aether-playbook/roles/sriov-dpdk/defaults/main.yml b/aether-playbook/roles/sriov-dpdk/defaults/main.yml
new file mode 100644
index 0000000..c40b02c
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/defaults/main.yml
@@ -0,0 +1,17 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Provide SRIOV PF name for UPF to run the playbook
+#upf_sriov_pf:
diff --git a/aether-playbook/roles/sriov-dpdk/tasks/main.yml b/aether-playbook/roles/sriov-dpdk/tasks/main.yml
new file mode 100644
index 0000000..7b70b0f
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/tasks/main.yml
@@ -0,0 +1,108 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Fail if VT-d is not enabled
+- name: Ensure VT-d is enabled in BIOS
+ shell: "dmesg | grep DMAR-IR"
+ register: check_vt_d
+ changed_when: check_vt_d.rc != 0
+ failed_when: check_vt_d.rc != 0
+ tags: sriov-dpdk
+
+- name: Add kernel boot parameters to the grub for SRIOV
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^GRUB_CMDLINE_LINUX="(?!.* {{ item.regex }})(.*)"'
+ line: 'GRUB_CMDLINE_LINUX="\1 {{ item.context }}"'
+ state: present
+ backrefs: yes
+ loop:
+ - { regex: 'intel_iommu=on', context: 'intel_iommu=on' }
+ - { regex: 'hugepagesz=', context: 'hugepagesz=1G default_hugepagesz=1G hugepages=32' }
+ register: grub
+ tags: sriov-dpdk
+
+- name: Update grub
+ command: update-grub
+ when: grub.changed
+ register: update_grub
+ tags: sriov-dpdk
+
+- name: Load vfio_pci module to the kernel
+ modprobe:
+ name: vfio_pci
+ state: present
+ tags: sriov-dpdk
+
+- name: Set the vfio_pci module to load on boot
+ lineinfile:
+ dest: /etc/modules-load.d/vfio_pci.conf
+ create: yes
+ regexp: "^vfio_pci"
+ line: "vfio_pci"
+ tags: sriov-dpdk
+
+- name: Check VFIO bind devices for DPDK
+ shell: "ls -l /dev/vfio | wc -l"
+ register: check_dpdk_bind
+ changed_when: check_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
+
+- name: Create SRIOV-DPDK service
+ template:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ mode: "{{ item.mode }}"
+ loop:
+ - { src: 'usr/bin/sriov.sh.j2', dest: '/usr/bin/sriov.sh', mode: 'a+x' }
+ - { src: 'etc/systemd/system/sriov.service.j2', dest: '/etc/systemd/system/sriov.service', mode: 644 }
+ register: bind_dpdk
+ when: check_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
+
+- name: Enable SRIOV-DPDK service
+ systemd:
+ name: sriov
+ daemon_reload: true
+ enabled: yes
+ when: bind_dpdk.changed
+ tags: sriov-dpdk
+
+- name: Reboot machine
+ shell: sleep 2 && shutdown -r now "Ansible updates triggered"
+ async: 1
+ poll: 0
+ ignore_errors: true
+ when: update_grub.changed or bind_dpdk.changed
+ tags: sriov-dpdk
+
+- name: Wait for server to restart successfully
+ wait_for:
+ host: "{{ ansible_host }}"
+ search_regex: "OpenSSH"
+ port: 22
+ timeout: 300
+ connect_timeout: 50
+ delay: 5
+ delegate_to: localhost
+ become: false
+ tags: sriov-dpdk
+
+- name: Ensure enough VFIO bind devices
+ shell: "ls -l /dev/vfio | wc -l"
+ register: confirm_dpdk_bind
+ changed_when: confirm_dpdk_bind.stdout | int < 4
+ failed_when: confirm_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
diff --git a/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2 b/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2
new file mode 100644
index 0000000..7a73d88
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2
@@ -0,0 +1,25 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# {{ ansible_managed }}
+
+[Unit]
+Description=Create VFs on {{ upf_sriov_pf }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/sriov.sh {{ upf_sriov_pf }}
+
+[Install]
+WantedBy=default.target
diff --git a/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2 b/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2
new file mode 100644
index 0000000..6b2a8e6
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# copied from https://github.com/clearlinux/cloud-native-setup/blob/master/clr-k8s-examples/9-multi-network/systemd/sriov.sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+setup_pf() {
+ local pf=$1
+ local num_vfs
+
+ echo "Resetting PF $pf"
+ echo 0 | tee /sys/class/net/"$pf"/device/sriov_numvfs
+ num_vfs=$(cat /sys/class/net/"$pf"/device/sriov_totalvfs)
+ echo "Enabling $num_vfs VFs for $pf"
+ echo "$num_vfs" | tee /sys/class/net/"$pf"/device/sriov_numvfs
+ ip link set "$pf" up
+ sleep 1
+}
+
+vfio_bind() {
+ local pf=$1
+ local pfpci
+ local num_vfs
+
+ pfpci=$(readlink /sys/devices/pci*/*/*/net/"$pf"/device | awk '{print substr($1,10)}')
+ num_vfs=$(cat /sys/class/net/"$pf"/device/sriov_numvfs)
+
+ local vfpci
+ local mac
+ for ((idx = 0; idx < num_vfs; idx++)); do
+ #Some drivers does not support state change of VF
+ #ip link set dev $pf vf $idx state enable
+
+ local vfn="virtfn$idx"
+ # shellcheck disable=SC2012
+ vfpci=$(ls -l /sys/devices/pci*/*/"$pfpci" | awk -v vfn=$vfn 'vfn==$9 {print substr($11,4)}')
+ # Capture and set MAC of the VF before unbinding from linux, for later use in CNI
+ mac=$(cat /sys/bus/pci*/*/"$vfpci"/net/*/address)
+ ip link set dev "$pf" vf $idx mac "$mac"
+ # Bind VF to vfio-pci
+ echo "$vfpci" >/sys/bus/pci*/*/"$vfpci"/driver/unbind
+ echo "vfio-pci" >/sys/devices/pci*/*/"$vfpci"/driver_override
+ echo "$vfpci" >/sys/bus/pci/drivers/vfio-pci/bind
+ done
+}
+
+for pf in "$@"; do
+ setup_pf "$pf"
+ vfio_bind "$pf"
+done
diff --git a/aether-playbook/site.yml b/aether-playbook/site.yml
new file mode 100644
index 0000000..8edda32
--- /dev/null
+++ b/aether-playbook/site.yml
@@ -0,0 +1,58 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Check ansible version
+- hosts: localhost
+ become: no
+ gather_facts: False
+ tasks:
+ - name: "Check ansible version ==2.9.5"
+ assert:
+ msg: "Ansible must be v2.9.5"
+ that:
+ - ansible_version.string is version("2.9.5", "==")
+ vars:
+ ansible_connection: local
+
+# Prepare UPF nodes
+# 1) Check VT-d is enabled in BIOS
+# 2) Enable IOMMU and Hugepages
+# 3) Create VFs on SRIOV PF and bind to vfio
+# 4) Reboot to apply
+- hosts: upf
+ any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+ roles:
+ - { role: sriov-dpdk, when: upf_sriov_enabled }
+
+# Router setup
+# 1) Assign SGI and S1U gateway address
+# 2) Add routing to UE pool
+# 3) Add SNAT rule for UE pool
+- hosts: router
+ any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+ roles:
+ - { role: router, when: router_setup_enabled }
+
+# K8S node setup
+# 1) Time synch
+# 2) Add internal domains for Rancher and Docker registry to /etc/hosts
+# 3) Install Docker
+# 4) Install K8s
+- hosts: kube-master:kube-worker
+ any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+ roles:
+ - { role: k8s }
+
+#- hosts: accelleran