[CORD-2540]
Add items to diagnostic collection
Support MaaS deployed compute node for diagnostics
Change-Id: I4c9a35b17fe6bf14e092e02591795c81be0a5776
diff --git a/collect-diag-playbook.yml b/collect-diag-playbook.yml
index 4612eb1..ae0053b 100644
--- a/collect-diag-playbook.yml
+++ b/collect-diag-playbook.yml
@@ -1,4 +1,4 @@
-
+---
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
----
# collect-diag-playbook.yml
-# Collects diagnostic information for the currently running cord-pod config
+# Collects diagnostic information for the currently running POD
- name: Create diag_dir fact
hosts: head
@@ -36,13 +34,15 @@
- compute-diag
- name: Collect compute node diagnostics on head node
- hosts: compute
+ hosts: head
tasks:
- - name: rsync diag_dir from compute nodes
- synchronize:
- src: "/tmp/{{ hostvars[groups['head'][0]]['diag_dir'] }}/{{ inventory_hostname }}"
- dest: "~/{{ hostvars[groups['head'][0]]['diag_dir'] }}/"
- recursive: yes
- mode: pull
- delegate_to: "{{ groups['head'][0] }}"
+ - name: rsync diag_dir from compute nodes to head node
+ when: groups['compute'] | length > 0
+ command: >
+ rsync -a
+ "ubuntu@{{ item }}:/tmp/{{ hostvars[groups['head'][0]]['diag_dir'] }}/{{ item }}"
+ "~/{{ hostvars[groups['head'][0]]['diag_dir'] }}/"
+ with_items: "{{ groups['compute'] }}"
+ tags:
+ - skip_ansible_lint # the synchronize module can't set the remote username with delegate_to
diff --git a/pod-test-playbook.yml b/pod-test-playbook.yml
index 2561001..75e6fc6 100644
--- a/pod-test-playbook.yml
+++ b/pod-test-playbook.yml
@@ -1,4 +1,4 @@
-
+---
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
----
# pod-test-playbook.yml
-# Tests CiaB cord-pod XOS configuration
+# Tests rcord XOS configuration
- name: Create test client
hosts: head
diff --git a/roles/compute-diag/tasks/main.yml b/roles/compute-diag/tasks/main.yml
index bd16192..c2012e9 100644
--- a/roles/compute-diag/tasks/main.yml
+++ b/roles/compute-diag/tasks/main.yml
@@ -1,4 +1,4 @@
-
+---
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
----
# compute-diag/tasks/main.yml
- name: Create diag_dir subdir
@@ -29,14 +27,34 @@
args:
creates: "compute_diag_dir.stat.path/{{ item | regex_replace('[^\\w-]', '_')}}"
with_items:
- - "date"
- "arp -n"
- "brctl show"
- - "cat /etc/resolv.conf"
- "cat /etc/lsb-release"
+ - "cat /etc/resolv.conf"
+ - "date -u"
+ - "df -k"
+ - "free"
- "ifconfig -a"
+ - "ip addr list"
+ - "ip link list"
+ - "ip neighbor"
+ - "ip route"
+ - "ps aux"
- "route -n"
- - "sudo ovs-vsctl show"
+ - "sudo iptables -S"
+ - "sudo iptables -t nat -S"
+ - "sudo ovs-appctl bridge/dump-flows br-int"
+ - "sudo ovs-appctl fdb/show br-int"
+ - "sudo ovs-appctl vlog/list"
+ - "sudo ovs-dpctl dump-flows"
+ - "sudo ovs-ofctl dump-flows br-int"
+ - "sudo ovs-ofctl dump-ports br-int"
+ - "sudo ovs-ofctl dump-ports-desc br-int"
+ - "sudo ovs-ofctl dump-tables br-int"
+ - "sudo ovs-ofctl show br-int"
- "sudo ovs-vsctl get-controller br-int"
-
+ - "sudo ovs-vsctl list interface"
+ - "sudo ovs-vsctl list-br"
+ - "sudo ovs-vsctl list-ports br-int"
+ - "sudo ovs-vsctl show"
diff --git a/roles/head-diag/tasks/main.yml b/roles/head-diag/tasks/main.yml
index 182d1b6..7462112 100644
--- a/roles/head-diag/tasks/main.yml
+++ b/roles/head-diag/tasks/main.yml
@@ -1,4 +1,4 @@
-
+---
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,14 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
----
# head-diag/tasks/main.yml
- name: Create diag_dir subdirs
file:
- dest="~/{{ diag_dir }}/{{ item }}"
- state=directory
+ dest: "~/{{ diag_dir }}/{{ item }}"
+ state: directory
with_items:
- head
- juju
@@ -35,16 +33,25 @@
args:
creates: "~/{{ diag_dir }}/head/{{ item | regex_replace('[^\\w-]', '_')}}"
with_items:
- - "ifconfig -a"
- - "route -n"
- "arp -n"
- - "date"
- "brctl show"
- - "cat /etc/resolv.conf"
- "cat /etc/lsb-release"
- - "sudo virsh list"
- - "sudo lxc list"
+ - "cat /etc/resolv.conf"
+ - "date -u"
+ - "df -k"
+ - "free"
+ - "ifconfig -a"
+ - "ip addr list"
+ - "ip link list"
+ - "ip neighbor"
+ - "ip route"
+ - "ps aux"
+ - "route -n"
- "sudo docker ps -a"
+ - "sudo iptables -S"
+ - "sudo iptables -t nat -S"
+ - "sudo lxc list"
+ - "sudo virsh list"
- name: Juju diag collection
shell: "{{ item }} > ~/{{ diag_dir }}/juju/{{ item | regex_replace('[^\\w-]', '_')}}"
@@ -131,18 +138,35 @@
- "summary"
- name: Docker diag collection - Find names for all running Docker containers
- command: sudo docker ps --format '{{'{{'}} .Names {{'}}'}}'
+ command: sudo docker ps -a --format '{{'{{'}} .Names {{'}}'}}'
ignore_errors: yes
register: docker_containers
tags:
- skip_ansible_lint # collecting docker container names
- name: Docker diag collection - Collect logs from Docker containers
- shell: "sudo docker logs {{ item }} > ~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}} 2>&1"
+ shell: "sudo docker logs {{ item }} > ~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}}.logs 2>&1"
ignore_errors: yes
args:
- creates: "~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}}"
+ creates: "~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}}.logs"
with_items: "{{ docker_containers.stdout_lines }}"
tags:
- skip_ansible_lint # ansible-lint interprets the \n incorrectly
+- name: Docker diag collection - inspect Docker containers
+ shell: "sudo docker inspect {{ item }} > ~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}}.inspect 2>&1"
+ ignore_errors: yes
+ args:
+ creates: "~/{{ diag_dir }}/docker/{{ item | regex_replace('[^\\w-]', '_')}}.inspect"
+ with_items: "{{ docker_containers.stdout_lines }}"
+ tags:
+ - skip_ansible_lint # ansible-lint interprets the \n incorrectly
+
+# much cleaner to use !unsafe to avoid j2 parsing of golang {{ and }} templates
+# but ansible-lint doesn't support it: https://github.com/willthames/ansible-lint/issues/291
+- name: Docker diag collection - Collect memory and CPU stats from docker containers
+ shell: "sudo docker stats --no-stream --format \"table {{'{{'}}.Name{{'}}'}}\t{{'{{'}}.Container{{'}}'}}\t{{'{{'}}.CPUPerc{{'}}'}}\t{{'{{'}}.MemUsage{{'}}'}}\t{{'{{'}}.MemPerc{{'}}'}}\" > ~/{{ diag_dir }}/docker/stats 2>&1"
+ ignore_errors: yes
+ args:
+ creates: "~/{{ diag_dir }}/docker/stats"
+