[EDGEPOD-186] Add Ansible playbook for Aether Edge
Change-Id: I71d73a5a4fdf7e5ccf12df924597af6ab90a68b4
diff --git a/aether-playbook/roles/sriov-dpdk/defaults/main.yml b/aether-playbook/roles/sriov-dpdk/defaults/main.yml
new file mode 100644
index 0000000..c40b02c
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/defaults/main.yml
@@ -0,0 +1,17 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Provide SRIOV PF name for UPF to run the playbook
+#upf_sriov_pf:
diff --git a/aether-playbook/roles/sriov-dpdk/tasks/main.yml b/aether-playbook/roles/sriov-dpdk/tasks/main.yml
new file mode 100644
index 0000000..7b70b0f
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/tasks/main.yml
@@ -0,0 +1,108 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Fail if VT-d is not enabled
+- name: Ensure VT-d is enabled in BIOS
+ shell: "dmesg | grep DMAR-IR"
+ register: check_vt_d
+ changed_when: check_vt_d.rc != 0
+ failed_when: check_vt_d.rc != 0
+ tags: sriov-dpdk
+
+- name: Add kernel boot parameters to the grub for SRIOV
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^GRUB_CMDLINE_LINUX="(?!.* {{ item.regex }})(.*)"'
+ line: 'GRUB_CMDLINE_LINUX="\1 {{ item.context }}"'
+ state: present
+ backrefs: yes
+ loop:
+ - { regex: 'intel_iommu=on', context: 'intel_iommu=on' }
+ - { regex: 'hugepagesz=', context: 'hugepagesz=1G default_hugepagesz=1G hugepages=32' }
+ register: grub
+ tags: sriov-dpdk
+
+- name: Update grub
+ command: update-grub
+ when: grub.changed
+ register: update_grub
+ tags: sriov-dpdk
+
+- name: Load vfio_pci module to the kernel
+ modprobe:
+ name: vfio_pci
+ state: present
+ tags: sriov-dpdk
+
+- name: Set the vfio_pci module to load on boot
+ lineinfile:
+ dest: /etc/modules-load.d/vfio_pci.conf
+ create: yes
+ regexp: "^vfio_pci"
+ line: "vfio_pci"
+ tags: sriov-dpdk
+
+- name: Check VFIO bind devices for DPDK
+ shell: "ls -l /dev/vfio | wc -l"
+ register: check_dpdk_bind
+ changed_when: check_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
+
+- name: Create SRIOV-DPDK service
+ template:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ mode: "{{ item.mode }}"
+ loop:
+ - { src: 'usr/bin/sriov.sh.j2', dest: '/usr/bin/sriov.sh', mode: 'a+x' }
+ - { src: 'etc/systemd/system/sriov.service.j2', dest: '/etc/systemd/system/sriov.service', mode: 644 }
+ register: bind_dpdk
+ when: check_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
+
+- name: Enable SRIOV-DPDK service
+ systemd:
+ name: sriov
+ daemon_reload: true
+ enabled: yes
+ when: bind_dpdk.changed
+ tags: sriov-dpdk
+
+- name: Reboot machine
+ shell: sleep 2 && shutdown -r now "Ansible updates triggered"
+ async: 1
+ poll: 0
+ ignore_errors: true
+ when: update_grub.changed or bind_dpdk.changed
+ tags: sriov-dpdk
+
+- name: Wait for server to restart successfully
+ wait_for:
+ host: "{{ ansible_host }}"
+ search_regex: "OpenSSH"
+ port: 22
+ timeout: 300
+ connect_timeout: 50
+ delay: 5
+ delegate_to: localhost
+ become: false
+ tags: sriov-dpdk
+
+- name: Ensure enough VFIO bind devices
+ shell: "ls -l /dev/vfio | wc -l"
+ register: confirm_dpdk_bind
+ changed_when: confirm_dpdk_bind.stdout | int < 4
+ failed_when: confirm_dpdk_bind.stdout | int < 4
+ tags: sriov-dpdk
diff --git a/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2 b/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2
new file mode 100644
index 0000000..7a73d88
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/templates/etc/systemd/system/sriov.service.j2
@@ -0,0 +1,25 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# {{ ansible_managed }}
+
+[Unit]
+Description=Create VFs on {{ upf_sriov_pf }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/sriov.sh {{ upf_sriov_pf }}
+
+[Install]
+WantedBy=default.target
diff --git a/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2 b/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2
new file mode 100644
index 0000000..6b2a8e6
--- /dev/null
+++ b/aether-playbook/roles/sriov-dpdk/templates/usr/bin/sriov.sh.j2
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# copied from https://github.com/clearlinux/cloud-native-setup/blob/master/clr-k8s-examples/9-multi-network/systemd/sriov.sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+setup_pf() {
+ local pf=$1
+ local num_vfs
+
+ echo "Resetting PF $pf"
+ echo 0 | tee /sys/class/net/"$pf"/device/sriov_numvfs
+ num_vfs=$(cat /sys/class/net/"$pf"/device/sriov_totalvfs)
+ echo "Enabling $num_vfs VFs for $pf"
+ echo "$num_vfs" | tee /sys/class/net/"$pf"/device/sriov_numvfs
+ ip link set "$pf" up
+ sleep 1
+}
+
+vfio_bind() {
+ local pf=$1
+ local pfpci
+ local num_vfs
+
+ pfpci=$(readlink /sys/devices/pci*/*/*/net/"$pf"/device | awk '{print substr($1,10)}')
+ num_vfs=$(cat /sys/class/net/"$pf"/device/sriov_numvfs)
+
+ local vfpci
+ local mac
+ for ((idx = 0; idx < num_vfs; idx++)); do
+ #Some drivers does not support state change of VF
+ #ip link set dev $pf vf $idx state enable
+
+ local vfn="virtfn$idx"
+ # shellcheck disable=SC2012
+ vfpci=$(ls -l /sys/devices/pci*/*/"$pfpci" | awk -v vfn=$vfn 'vfn==$9 {print substr($11,4)}')
+ # Capture and set MAC of the VF before unbinding from linux, for later use in CNI
+ mac=$(cat /sys/bus/pci*/*/"$vfpci"/net/*/address)
+ ip link set dev "$pf" vf $idx mac "$mac"
+ # Bind VF to vfio-pci
+ echo "$vfpci" >/sys/bus/pci*/*/"$vfpci"/driver/unbind
+ echo "vfio-pci" >/sys/devices/pci*/*/"$vfpci"/driver_override
+ echo "$vfpci" >/sys/bus/pci/drivers/vfio-pci/bind
+ done
+}
+
+for pf in "$@"; do
+ setup_pf "$pf"
+ vfio_bind "$pf"
+done