Adds QAT drivers installation and service configuration for QAT VFs
Fix lint, formatting, add galaxy metadata
Change-Id: I3173cefb356a07f9bead9e9ca01b38b16525e26f
diff --git a/.reuse/dep5 b/.reuse/dep5
index 087f2d5..18d8100 100644
--- a/.reuse/dep5
+++ b/.reuse/dep5
@@ -1,5 +1,5 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Files: VERSION .gitreview
+Files: VERSION README.md .gitreview
Copyright: 2021 Open Networking Foundation
License: Apache-2.0
diff --git a/README.md b/README.md
index 9e4079d..60c0533 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,3 @@
-<!--
-SPDX-FileCopyrightText: © 2021 Open Networking Foundation <support@opennetworking.org>
-SPDX-License-Identifier: Apache-2.0
---!>
# sriov
Create the maximum number of VFs a PF can support and bind half of them
diff --git a/defaults/main.yml b/defaults/main.yml
index 8bc4de2..4eccc44 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -9,3 +9,10 @@
# Provide number of 1Gi HugePages
sriov_1g_hugepages: 32
+
+# Provide QAT driver parameters
+sriov_qat_enabled: false
+sriov_qat_driver_dir: "/opt/qat"
+sriov_qat_latest_name: "QAT.L.4.16.0-00017.tar.gz"
+sriov_qat_latest_url: "https://downloadmirror.intel.com/710050/QAT.L.4.16.0-00017.tar.gz"
+sriov_qat_checksum: "sha256:315f48057253798fae386e514ce834d6b5939eb842491fe27bfc019c42c430f4"
diff --git a/handlers/main.yml b/handlers/main.yml
index 1e6905e..fd2805c 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -36,3 +36,18 @@
changed_when: confirm_vfs.stdout | int == 0
failed_when: confirm_vfs.stdout | int == 0
when: molecule_handler_notest is not defined
+
+- name: enable sriov_qat.service
+ systemd:
+ name: sriov_qat
+ daemon_reload: true
+ enabled: true
+
+- name: reboot qat
+ reboot:
+ msg: "Reboot initiated by Ansible - QAT drivers blacklist task"
+ connect_timeout: 5
+ reboot_timeout: 600
+ pre_reboot_delay: 0
+ post_reboot_delay: 30
+ test_command: whoami
diff --git a/meta/main.yml b/meta/main.yml
index 4bcc3e9..5781f24 100644
--- a/meta/main.yml
+++ b/meta/main.yml
@@ -5,8 +5,9 @@
# SPDX-License-Identifier: Apache-2.0
galaxy_info:
+ role_name: sriov
author: Open Networking Foundation
- description: Configure SRIOV
+ description: Configure SR-IOV
company: Open Networking Foundation
issue_tracker_url: https://jira.opennetworking.org/browse/AETHER-1839
@@ -18,7 +19,6 @@
platforms:
- name: Ubuntu
versions:
- - "16.04"
- "18.04"
galaxy_tags:
diff --git a/tasks/main.yml b/tasks/main.yml
index f7c365e..6e241b2 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -33,9 +33,13 @@
owner: root
group: root
loop:
- - {src: 'sriov', dest: '/etc/default/sriov', mode: 644}
- - {src: 'sriov.sh.j2', dest: '/usr/local/bin/sriov.sh', mode: 'a+x'}
- - {src: 'sriov.service.j2', dest: '/etc/systemd/system/sriov.service', mode: 644}
+ - {src: "sriov", dest: "/etc/default/sriov", mode: 644}
+ - {src: "sriov.sh.j2", dest: "/usr/local/bin/sriov.sh", mode: "a+x"}
+ - {
+ src: "sriov.service.j2",
+ dest: "/etc/systemd/system/sriov.service",
+ mode: 644,
+ }
notify: enable sriov.service
- name: Flush handlers
@@ -49,8 +53,15 @@
state: present
backrefs: true
loop:
- - {regex: 'intel_iommu=', context: 'intel_iommu=on'}
- - {regex: 'transparent_hugepage=', context: 'transparent_hugepage=never'}
- - {regex: 'default_hugepagesz=', context: 'default_hugepagesz=1G'}
- - {regex: 'hugepagesz=1G', context: 'hugepagesz=1G hugepages={{ sriov_1g_hugepages }}'}
+ - {regex: "intel_iommu=", context: "intel_iommu=on"}
+ - {regex: "transparent_hugepage=", context: "transparent_hugepage=never"}
+ - {regex: "default_hugepagesz=", context: "default_hugepagesz=1G"}
+ - {
+ regex: "hugepagesz=1G",
+ context: "hugepagesz=1G hugepages={{ sriov_1g_hugepages }}",
+ }
notify: update grub
+
+- name: Install qat driver and configure sriov vfs
+ when: sriov_qat_enabled
+ include_tasks: qat.yml
diff --git a/tasks/qat.yml b/tasks/qat.yml
new file mode 100644
index 0000000..1c113db
--- /dev/null
+++ b/tasks/qat.yml
@@ -0,0 +1,117 @@
+---
+# qat tasks/qat.yml
+#
+# SPDX-FileCopyrightText: © 2021 Open Networking Foundation <support@opennetworking.org>
+# SPDX-License-Identifier: Apache-2.0
+
+- name: Blacklist kernel QAT drivers
+ blockinfile:
+ create: true
+ dest: /etc/modprobe.d/blacklist-intel_qat.conf
+ mode: 0644
+ block: |
+ # Blacklist kernel QAT drivers
+ blacklist qat_c62x
+ blacklist qat_c62xvf
+ blacklist qat_c6xx
+ blacklist qat_c6xxvf
+ blacklist qat_c3xxx
+ blacklist qat_c3xxxvf
+ blacklist qat_d15xx
+ blacklist qat_d15xxvf
+ blacklist qat_c4xxx
+ blacklist qat_c4xxxvf
+ blacklist intel_qat
+ backup: true
+ notify: reboot qat
+
+- name: Flush handlers
+ meta: flush_handlers
+
+- name: Install QAT driver compile requirements
+ apt:
+ name:
+ - "libboost-all-dev"
+ - "pciutils-dev"
+ - "libssl-dev"
+ - "build-essential"
+ - "pkg-config"
+ - "yasm"
+ state: "present"
+ update_cache: true
+
+- name: Create root folder for QAT driver
+ file:
+ path: "{{ sriov_qat_driver_dir }}"
+ state: directory
+ owner: root
+ group: root
+ mode: "0644"
+
+- name: Download QAT driver .tgz file
+ get_url:
+ url: "{{ sriov_qat_latest_url }}"
+ dest: "{{ sriov_qat_driver_dir }}/{{ sriov_qat_latest_name }}"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ checksum: "{{ sriov_qat_checksum }}"
+
+- name: Unarchive QAT driver .tgz file
+ when: not sriov_qat_dkms_enabled
+ unarchive:
+ remote_src: true
+ list_files: true
+ src: "{{ sriov_qat_driver_dir }}/{{ sriov_qat_latest_name }}"
+ dest: "{{ sriov_qat_driver_dir }}"
+ owner: "root"
+ group: "root"
+ creates: "{{ sriov_qat_driver_dir }}/configure"
+
+- name: Configure QAT driver with SR-IOV host
+ when: not sriov_qat_dkms_enabled
+ command:
+ cmd: "./configure --enable-icp-sriov=host"
+ chdir: "{{ sriov_qat_driver_dir }}"
+ creates: "{{ sriov_qat_driver_dir }}/config.status"
+
+- name: Run the default target as root
+ when: not sriov_qat_dkms_enabled
+ command:
+ cmd: make
+ chdir: "{{ sriov_qat_driver_dir }}"
+ creates: "{{ sriov_qat_driver_dir }}/build/intel_qat.ko"
+
+- name: Run 'install' target as root
+ when: not sriov_qat_dkms_enabled
+ command:
+ chdir: "{{ sriov_qat_driver_dir }}"
+ cmd: "make install"
+ creates: "/usr/local/bin/adf_ctl"
+
+# - name: Run 'install-samples' target as root
+# # command:
+# chdir: "{{ sriov_qat_driver_dir }}"
+# cmd: "make install-samples"
+# creates: "{{ sriov_qat_driver_dir }}/build/cpa_sample_code"
+# become: yes
+
+- name: Create sriov_qat system servqat
+ template:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ mode: "{{ item.mode }}"
+ owner: root
+ group: root
+ loop:
+ - {
+ src: "sriov_qat.sh.j2",
+ dest: "/usr/local/bin/sriov_qat.sh",
+ mode: "a+x",
+ }
+ - {
+ src: "sriov_qat.service.j2",
+ dest: "/etc/systemd/system/sriov_qat.service",
+ mode: 644,
+ }
+ notify: enable sriov_qat.service
diff --git a/templates/sriov_qat.service.j2 b/templates/sriov_qat.service.j2
new file mode 100644
index 0000000..a600e82
--- /dev/null
+++ b/templates/sriov_qat.service.j2
@@ -0,0 +1,15 @@
+{#
+SPDX-FileCopyrightText: © 2022 Open Networking Foundation <support@opennetworking.org>
+SPDX-License-Identifier: Apache-2.0
+#}
+# {{ ansible_managed }}
+
+[Unit]
+Description=Create QAT VFs
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/sriov_qat.sh
+
+[Install]
+WantedBy=default.target
diff --git a/templates/sriov_qat.sh.j2 b/templates/sriov_qat.sh.j2
new file mode 100644
index 0000000..22dda99
--- /dev/null
+++ b/templates/sriov_qat.sh.j2
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+{#
+SPDX-FileCopyrightText: © 2022 Open Networking Foundation <support@opennetworking.org>
+SPDX-FileCopyrightText: © 2019 Intel Corporation
+SPDX-License-Identifier: Apache-2.0
+
+# adapted from https://github.com/clearlinux/cloud-native-setup/blob/master/clr-k8s-examples/9-multi-network/systemd/sriov.sh
+#}
+# {{ ansible_managed }}
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+QAT_PFs=($(lspci -Dmm | grep Quick | awk '{print $1}'))
+QAT_DIR="/sys/bus/pci/devices/"
+
+
+setup_qat_pf_vfs() {
+ echo "Restarting services qat_service and qat_service_vfs"
+ service qat_service restart
+ service qat_service_vfs restart
+}
+
+setup_pf() {
+ local pf=$1
+ local num_vfs
+
+ echo "Resetting PF $pf"
+ echo 0 | tee "$QAT_DIR"/"$pf"/sriov_numvfs
+ num_vfs=$(cat "$QAT_DIR"/"$pf"/sriov_totalvfs)
+ echo "Enabling $num_vfs VFs for $pf"
+ echo "$num_vfs" | tee "$QAT_DIR"/"$pf"/sriov_numvfs
+}
+
+vfio_bind() {
+ local pf=$1
+
+ vfs_dirs=($(ls "$QAT_DIR"/"$pf" | grep virtfn))
+
+ for vf_name in "${vfs_dirs[@]}"; do
+ local vf_dir="$QAT_DIR"/"$pf"/"$vf_name"
+
+ vfpci=$(ls -l "$vf_dir" | awk '{print substr($11,4)}')
+ vfpci_dir=$(readlink -f "$vf_dir")
+
+ echo "Binding qat VF $vfpci with vfio-pci"
+
+ echo "$vfpci" >"$vfpci_dir"/driver/unbind
+ echo "vfio-pci" >"$vfpci_dir"/driver_override
+ echo "$vfpci" >/sys/bus/pci/drivers/vfio-pci/bind
+ done
+}
+
+setup_qat_pf_vfs
+
+for pf in "${QAT_PFs[@]}"; do
+ # setup_pf "$pf"
+ vfio_bind "$pf"
+done