[SEBA-308] Save container logs after each test case in ATT workflow test
Change-Id: I1feac976d752bf73b1bf7335fb0c19bb1dad4980
diff --git a/src/test/cord-api/Framework/Kubernetes.robot b/src/test/cord-api/Framework/Kubernetes.robot
index a76fd4f..3f46c52 100644
--- a/src/test/cord-api/Framework/Kubernetes.robot
+++ b/src/test/cord-api/Framework/Kubernetes.robot
@@ -21,6 +21,7 @@
Library RequestsLibrary
Library utils/utils.py
Library restApi.py
+Resource ../../Framework/utils/utils.robot
*** Keywords ***
Helm Chart is Removed
@@ -51,3 +52,25 @@
Run ${export_kubeconfig}; helm upgrade -f ${KUBERNETES_YAML} --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ${HELM_CHARTS_DIR}/voltha
Wait Until Keyword Succeeds 60s 10s Kubernetes PODs in Namespace are Running voltha ${VOLTHA_POD_NUM}
Sleep 10s
+
+Get Current Datetime On Kubernetes Node
+ [Arguments] ${ip} ${user} ${pass}
+ ${result}= Login And Run Command On Remote System date +"%Y-%m-%dT%H:%M:%S.%NZ" ${ip} ${user} ${pass}
+ ${result}= Get Line ${result} 0
+ [Return] ${result}
+
+Log Kubernetes Container Log Since Time
+ [Arguments] ${datetime} ${container_name}
+ ${rc} ${output}= Run And Return Rc And Output ${export_kubeconfig}; kubectl logs --timestamps --since-time=${datetime} ${container_name}
+ Log ${output}
+
+Log Kubernetes Containers Logs Since Time
+ [Arguments] ${datetime} ${container_list}
+ : FOR ${container_name} IN @{container_list}
+ \ Log Kubernetes Container Log Since Time ${datetime} ${container_name}
+
+Get Kubernetes POD Name By Prefix
+ [Arguments] ${prefix}
+ [Documentation] Return the first POD name that starts with the specified prefix
+ ${rc} ${output}= Run And Return Rc And Output ${export_kubeconfig}; kubectl get pods | grep '^${prefix}' | head -1 | awk '{print $1}'
+ [Return] ${output}
diff --git a/src/test/cord-api/Tests/WorkflowValidations/ATT_Test001.robot b/src/test/cord-api/Tests/WorkflowValidations/ATT_Test001.robot
index c4a572d..7383c8a 100644
--- a/src/test/cord-api/Tests/WorkflowValidations/ATT_Test001.robot
+++ b/src/test/cord-api/Tests/WorkflowValidations/ATT_Test001.robot
@@ -76,6 +76,7 @@
Validate DHCP and Ping True True ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
Clean Up Linux
Remove Whitelist
+ Wait Until Keyword Succeeds 300s 15s Validate ONU States UNKNOWN DISABLED ${onu_device}
Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI DISABLED AWAITING ${onu_device}
Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status awaiting-auth ${onu_device}
Validate Authentication False ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
@@ -158,7 +159,7 @@
Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
+ Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
Validate DHCP and Ping False False ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
Clean Up Linux
Create Subscriber
@@ -212,7 +213,7 @@
Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
+ Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
Validate DHCP and Ping False False ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
Clean Up Linux
Create Subscriber
@@ -257,7 +258,7 @@
Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
+ Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
Validate DHCP and Ping False False ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
Clean Up Linux
Create Subscriber
@@ -309,6 +310,9 @@
${olt_ip}= Evaluate ${olts}[0].get("ip")
${olt_user}= Evaluate ${olts}[0].get("user")
${olt_pass}= Evaluate ${olts}[0].get("pass")
+ ${k8s_node_ip}= Evaluate ${nodes}[0].get("ip")
+ ${k8s_node_user}= Evaluate ${nodes}[0].get("user")
+ ${k8s_node_pass}= Evaluate ${nodes}[0].get("pass")
Set Suite Variable ${src_ip}
Set Suite Variable ${src_user}
Set Suite Variable ${src_pass}
@@ -325,6 +329,17 @@
Set Suite Variable ${olt_ip}
Set Suite Variable ${olt_user}
Set Suite Variable ${olt_pass}
+ Set Suite Variable ${k8s_node_ip}
+ Set Suite Variable ${k8s_node_user}
+ Set Suite Variable ${k8s_node_pass}
+ @{container_list}= Create List
+ ${container_name}= Get Kubernetes POD Name By Prefix onos
+ Append To List ${container_list} ${container_name}
+ ${container_name}= Get Kubernetes POD Name By Prefix xos-core
+ Append To List ${container_list} ${container_name}
+ Set Suite Variable ${container_list}
+ ${datetime}= Get Current Datetime On Kubernetes Node ${k8s_node_ip} ${k8s_node_user} ${k8s_node_pass}
+ Set Suite Variable ${datetime}
Teardown Suite
[Documentation] Performs any additional cleanup required
@@ -334,6 +349,7 @@
Setup Test
[Documentation] Re-create Subscriber, whitelist, and olt-device models to test
Log Re-creating objects
+ ${datetime}= Get Current Datetime On Kubernetes Node ${k8s_node_ip} ${k8s_node_user} ${k8s_node_pass}
Create Whitelist
Create Subscriber
Create VOLT
@@ -342,6 +358,7 @@
[Documentation] Delete xos objects, kills processes and cleans up interfaces on src+dst servers
Clean Up Linux
Clean Up XOS
+ Log Kubernetes Containers Logs Since Time ${datetime} ${container_list}
Clean Up Linux
[Documentation] Kill processes and clean up interfaces on src+dst servers