Adds dkms installation for QAT drivers and QAT SRIOV VFs
Change-Id: Id72e19bbfd6cbf4bf46c7a5fb0d8797eb184ab38
diff --git a/defaults/main.yml b/defaults/main.yml
index f6069c8..128860b 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -6,9 +6,17 @@
dkms_modules:
- ice
+ # - qat
# intel E810 network driver
# https://www.intel.com/content/www/us/en/download/19630/intel-network-adapter-driver-for-e810-series-devices-under-linux.html
dkms_ice_version: "1.6.7"
dkms_ice_dl_url: "https://downloadmirror.intel.com/29746/ice-1.6.7.tar.gz"
dkms_ice_checksum: "sha256:cf1124051e3d84dab2943a591735661b7323b72592779cd3e69854bd9ffaed33"
+
+# Provide QAT driver parameters
+sriov_qat_version: "4.16.0"
+sriov_qat_driver_dir: "/opt/qat"
+sriov_qat_latest_name: "QAT.L.4.16.0-00017.tar.gz"
+sriov_qat_latest_url: "https://downloadmirror.intel.com/710050/QAT.L.4.16.0-00017.tar.gz"
+sriov_qat_checksum: "sha256:315f48057253798fae386e514ce834d6b5939eb842491fe27bfc019c42c430f4"
diff --git a/tasks/qat.yml b/tasks/qat.yml
new file mode 100644
index 0000000..b5dabda
--- /dev/null
+++ b/tasks/qat.yml
@@ -0,0 +1,121 @@
+---
+# qat tasks/qat.yml
+#
+# SPDX-FileCopyrightText: © 2022 Open Networking Foundation <support@opennetworking.org>
+# SPDX-Lqatnse-Identifier: Apache-2.0
+
+- name: Blacklist kernel QAT drivers
+ blockinfile:
+ create: true
+ dest: /etc/modprobe.d/blacklist-intel_qat.conf
+ mode: 0644
+ block: |
+ # Blacklist kernel QAT drivers
+ blacklist qat_c62x
+ blacklist qat_c62xvf
+ blacklist qat_c6xx
+ blacklist qat_c6xxvf
+ blacklist qat_c3xxx
+ blacklist qat_c3xxxvf
+ blacklist qat_d15xx
+ blacklist qat_d15xxvf
+ blacklist qat_c4xxx
+ blacklist qat_c4xxxvf
+ blacklist intel_qat
+ backup: true
+ notify: reboot qat
+
+- name: Flush handlers
+ meta: flush_handlers
+
+- name: Install QAT driver compile requirements
+ apt:
+ name:
+ - "libboost-all-dev"
+ - "pciutils-dev"
+ - "libssl-dev"
+ - "build-essential"
+ - "pkg-config"
+ - "yasm"
+ state: "present"
+ update_cache: true
+
+- name: Create root folder for QAT driver
+ file:
+ path: "{{ sriov_qat_driver_dir }}"
+ state: directory
+ owner: root
+ group: root
+ mode: "0644"
+
+- name: Download QAT driver .tgz file
+ get_url:
+ url: "{{ sriov_qat_latest_url }}"
+ dest: "{{ sriov_qat_driver_dir }}/{{ sriov_qat_latest_name }}"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ checksum: "{{ sriov_qat_checksum }}"
+
+- name: Create qat directory in /usr/src
+ file:
+ path: "/usr/src/qat-{{ sriov_qat_version }}"
+ state: directory
+ owner: "root"
+ group: "root"
+ mode: "0755"
+
+- name: Decompress qat archive
+ unarchive:
+ remote_src: true
+ src: "{{ sriov_qat_driver_dir }}/{{ sriov_qat_latest_name }}"
+ dest: "/usr/src/qat-{{ sriov_qat_version }}"
+ owner: "root"
+ group: "root"
+ extra_opts:
+ - "--strip-components=1" # remove initial directory name
+ creates: "/usr/src/qat-{{ sriov_qat_version }}/README"
+
+- name: Copy over qat dkms.conf
+ template:
+ src: "qat_dkms.conf.j2"
+ dest: "/usr/src/qat-{{ sriov_qat_version }}/dkms.conf"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+
+- name: add qat to DKMS
+ command:
+ cmd: "dkms add -m qat -v {{ sriov_qat_version }}"
+ chdir: "/usr/src/qat-{{ sriov_qat_version }}"
+ creates: "/var/lib/dkms/qat/{{ sriov_qat_version }}/source/README"
+
+- name: build qat using DKMS
+ command:
+ cmd: "dkms build -m qat -v {{ sriov_qat_version }}"
+ chdir: "/usr/src/qat-{{ sriov_qat_version }}"
+
+- name: install qat modules with DKMS
+ command:
+ cmd: "dkms install -m qat -v {{ sriov_qat_version }}"
+ chdir: "/usr/src/qat-{{ sriov_qat_version }}"
+
+- name: Create sriov_qat system servqat
+ template:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ mode: "{{ item.mode }}"
+ owner: root
+ group: root
+ loop:
+ - {
+ src: "sriov_qat.sh.j2",
+ dest: "/usr/local/bin/sriov_qat.sh",
+ mode: "a+x",
+ }
+ - {
+ src: "sriov_qat.service.j2",
+ dest: "/etc/systemd/system/sriov_qat.service",
+ mode: 644,
+ }
+ notify: enable sriov_qat.service
diff --git a/templates/qat_dkms.conf.j2 b/templates/qat_dkms.conf.j2
new file mode 100644
index 0000000..6160d13
--- /dev/null
+++ b/templates/qat_dkms.conf.j2
@@ -0,0 +1,13 @@
+{#
+SPDX-FileCopyrightText: © 2020 Open Networking Foundation <support@opennetworking.org>
+SPDX-License-Identifier: Apache-2.0
+#}
+MAKE="echo; ./configure --enable-icp-sriov=host && make && make install"
+CLEAN="make clean && make distclean"
+BUILT_MODULE_NAME=qat_api
+BUILT_MODULE_LOCATION=build/
+PACKAGE_NAME=qat
+PACKAGE_VERSION={{ sriov_qat_version }}
+DEST_MODULE_LOCATION=/updates
+AUTOINSTALL=yes
+REMAKE_INITRD=yes
diff --git a/templates/sriov_qat.service.j2 b/templates/sriov_qat.service.j2
new file mode 100644
index 0000000..a600e82
--- /dev/null
+++ b/templates/sriov_qat.service.j2
@@ -0,0 +1,15 @@
+{#
+SPDX-FileCopyrightText: © 2022 Open Networking Foundation <support@opennetworking.org>
+SPDX-License-Identifier: Apache-2.0
+#}
+# {{ ansible_managed }}
+
+[Unit]
+Description=Create QAT VFs
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/sriov_qat.sh
+
+[Install]
+WantedBy=default.target
diff --git a/templates/sriov_qat.sh.j2 b/templates/sriov_qat.sh.j2
new file mode 100644
index 0000000..22dda99
--- /dev/null
+++ b/templates/sriov_qat.sh.j2
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+{#
+SPDX-FileCopyrightText: © 2022 Open Networking Foundation <support@opennetworking.org>
+SPDX-FileCopyrightText: © 2019 Intel Corporation
+SPDX-License-Identifier: Apache-2.0
+
+# adapted from https://github.com/clearlinux/cloud-native-setup/blob/master/clr-k8s-examples/9-multi-network/systemd/sriov.sh
+#}
+# {{ ansible_managed }}
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+QAT_PFs=($(lspci -Dmm | grep Quick | awk '{print $1}'))
+QAT_DIR="/sys/bus/pci/devices/"
+
+
+setup_qat_pf_vfs() {
+ echo "Restarting services qat_service and qat_service_vfs"
+ service qat_service restart
+ service qat_service_vfs restart
+}
+
+setup_pf() {
+ local pf=$1
+ local num_vfs
+
+ echo "Resetting PF $pf"
+ echo 0 | tee "$QAT_DIR"/"$pf"/sriov_numvfs
+ num_vfs=$(cat "$QAT_DIR"/"$pf"/sriov_totalvfs)
+ echo "Enabling $num_vfs VFs for $pf"
+ echo "$num_vfs" | tee "$QAT_DIR"/"$pf"/sriov_numvfs
+}
+
+vfio_bind() {
+ local pf=$1
+
+ vfs_dirs=($(ls "$QAT_DIR"/"$pf" | grep virtfn))
+
+ for vf_name in "${vfs_dirs[@]}"; do
+ local vf_dir="$QAT_DIR"/"$pf"/"$vf_name"
+
+ vfpci=$(ls -l "$vf_dir" | awk '{print substr($11,4)}')
+ vfpci_dir=$(readlink -f "$vf_dir")
+
+ echo "Binding qat VF $vfpci with vfio-pci"
+
+ echo "$vfpci" >"$vfpci_dir"/driver/unbind
+ echo "vfio-pci" >"$vfpci_dir"/driver_override
+ echo "$vfpci" >/sys/bus/pci/drivers/vfio-pci/bind
+ done
+}
+
+setup_qat_pf_vfs
+
+for pf in "${QAT_PFs[@]}"; do
+ # setup_pf "$pf"
+ vfio_bind "$pf"
+done