[VOL-4359] Added ETCD data cleanup verification after device delete
Change-Id: Ic3e3eced67826c6c7277cc8c21f0e6241b7e9a0b
diff --git a/VERSION b/VERSION
index 56f5e1b..01b0f09 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.8.12
+2.8.13
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index 815cec4..fc5e0e7 100755
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -78,7 +78,8 @@
[Arguments] ${namespace} ${name} ${command} ${grep}=${EMPTY}
[Documentation] Uses kubectl to execute a command in the pod and return the output
${rc} ${exec_pod_name}= Run Keyword If '${grep}'=='${EMPTY}'
- ... Run and Return Rc and Output kubectl -n ${namespace} get pods -l app.kubernetes.io/name=${name} -o name
+ ... Run and Return Rc and Output
+ ... kubectl -n ${namespace} get pods -l app.kubernetes.io/name=${name} -o name | awk 'NR==1{print $1}'
... ELSE Run and Return Rc and Output
... kubectl -n ${namespace} get pods -l app.kubernetes.io/name=${name} -o name \| grep ${grep}
Log ${exec_pod_name}
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 9c24def..cb775bc 100755
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -700,12 +700,10 @@
Wait Until Keyword Succeeds ${timeout} 2s Test Empty Device List
FOR ${I} IN RANGE 0 ${num_olts}
${olt_serial_number}= Set Variable ${list_olts}[${I}][sn]
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Wait Until Keyword Succeeds ${timeout} 5s
... Validate Deleted Device Cleanup In ONOS ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${olt_serial_number}
END
- # Clear devices from ONOS
- #Remove All Devices From ONOS
- #... http://karaf:karaf@${ONOS_REST_IP}:${ONOS_REST_PORT}
+ Wait Until Keyword Succeeds ${timeout} 5s Validate Cleanup In ETCD ${INFRA_NAMESPACE}
Teardown
[Documentation] kills processes and cleans up interfaces on src+dst servers
@@ -1398,3 +1396,75 @@
END
${real_num_onus}= Get Length ${onu_list}
[Return] ${real_num_onus}
+
+Validate Cleanup In ETCD
+ [Documentation] The keyword verifies that device, ports, flows, meters are all cleared in ETCD
+ [Arguments] ${namespace}=default ${defaultkvstoreprefix}=voltha_voltha
+ ${podname}= Set Variable etcd
+ ${kvstoreprefix}= Get Kv Store Prefix ${defaultkvstoreprefix}
+ # Log Devices Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/devices --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Devices Data in Etcd!
+ # Log Flows Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/flows --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Flows Data in Etcd!
+ # Log LogicalDevices Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_devices --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Logical Devices Data in Etcd!
+ # Log LogicalFlows Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_flows --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Logical Flows Data in Etcd!
+ # Log LogicalMeters Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_meters --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Logical Meters Data in Etcd!
+ # Log LogicalPorts Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_ports --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Logical Ports Data in Etcd!
+ # Log Openolt Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openolt --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Openolt Data in Etcd!
+ # Log Openonu Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openonu --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Openonu Data in Etcd!
+ # Log Ports Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/ports --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Ports Data in Etcd!
+ # Log ResourceInstances Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_instances --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Resource Instances Data in Etcd!
+ # Log ResourceManager Output and Verify Output Should be Empty
+ ${commandget}= Catenate
+ ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_manager --keys-only'
+ ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
+ Log ${result}
+ Should Be Empty ${result} Stale Resource Manager Data in Etcd!
diff --git a/tests/bbsim/Voltha_BBSimTests.robot b/tests/bbsim/Voltha_BBSimTests.robot
index ef185f9..5cf1cbf 100755
--- a/tests/bbsim/Voltha_BBSimTests.robot
+++ b/tests/bbsim/Voltha_BBSimTests.robot
@@ -34,6 +34,7 @@
*** Variables ***
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/dt-workflow/Voltha_DT_FailureScenarios.robot b/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
index 7f729e8..0d4e985 100755
--- a/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
+++ b/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/dt-workflow/Voltha_DT_MultiOLT_Tests.robot b/tests/dt-workflow/Voltha_DT_MultiOLT_Tests.robot
index 98d4eaf..0f9b884 100755
--- a/tests/dt-workflow/Voltha_DT_MultiOLT_Tests.robot
+++ b/tests/dt-workflow/Voltha_DT_MultiOLT_Tests.robot
@@ -41,6 +41,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/dt-workflow/Voltha_DT_PODTests.robot b/tests/dt-workflow/Voltha_DT_PODTests.robot
index c44116e..b954c11 100755
--- a/tests/dt-workflow/Voltha_DT_PODTests.robot
+++ b/tests/dt-workflow/Voltha_DT_PODTests.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/functional/Voltha_AlarmTests.robot b/tests/functional/Voltha_AlarmTests.robot
index 9b7da61..57ebf31 100755
--- a/tests/functional/Voltha_AlarmTests.robot
+++ b/tests/functional/Voltha_AlarmTests.robot
@@ -40,6 +40,7 @@
${BBSIMCTL_NAMESPACE} voltha
${VOLTCTL_POD_NAME} voltctl
${BBSIMCTL_POD_NAME} bbsim
+${INFRA_NAMESPACE} default
*** Test Cases ***
Ensure required pods Running
diff --git a/tests/functional/Voltha_ErrorScenarios.robot b/tests/functional/Voltha_ErrorScenarios.robot
index bf00856..a9e53d6 100755
--- a/tests/functional/Voltha_ErrorScenarios.robot
+++ b/tests/functional/Voltha_ErrorScenarios.robot
@@ -41,6 +41,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/functional/Voltha_FailureScenarios2.robot b/tests/functional/Voltha_FailureScenarios2.robot
index 137fc9b..fb444c6 100755
--- a/tests/functional/Voltha_FailureScenarios2.robot
+++ b/tests/functional/Voltha_FailureScenarios2.robot
@@ -41,6 +41,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
${DEFAULTSPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
diff --git a/tests/functional/Voltha_MultiOLT_Tests.robot b/tests/functional/Voltha_MultiOLT_Tests.robot
index c9cc74c..c92ba70 100755
--- a/tests/functional/Voltha_MultiOLT_Tests.robot
+++ b/tests/functional/Voltha_MultiOLT_Tests.robot
@@ -41,6 +41,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
${DEFAULTSPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
diff --git a/tests/functional/Voltha_PODTests.robot b/tests/functional/Voltha_PODTests.robot
index cfd5f2f..936701e 100755
--- a/tests/functional/Voltha_PODTests.robot
+++ b/tests/functional/Voltha_PODTests.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
index 1a2ebf7..f17ec63 100755
--- a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
@@ -35,6 +35,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 300s
${of_id} 0
${logical_id} 0
diff --git a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
index 4c4953a..78e186e 100755
--- a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
@@ -35,6 +35,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
index eab6545..f235048 100755
--- a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
@@ -39,6 +39,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
index 938f27d..9881095 100755
--- a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
@@ -35,6 +35,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
index 3b86bbc..9449f5e 100755
--- a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
@@ -37,6 +37,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/pm-data/Voltha_ONUPMTests.robot b/tests/pm-data/Voltha_ONUPMTests.robot
index 4346925..7455f57 100755
--- a/tests/pm-data/Voltha_ONUPMTests.robot
+++ b/tests/pm-data/Voltha_ONUPMTests.robot
@@ -46,6 +46,7 @@
*** Variables ***
${namespace} voltha
+${INFRA_NAMESPACE} default
${timeout} 60s
${of_id} 0
${logical_id} 0
diff --git a/tests/scale/Voltha_Scale_Tests.robot b/tests/scale/Voltha_Scale_Tests.robot
index 7dcc4e8..8f6e667 100755
--- a/tests/scale/Voltha_Scale_Tests.robot
+++ b/tests/scale/Voltha_Scale_Tests.robot
@@ -64,6 +64,7 @@
${BBSIM_REST_PORT} 50071
${NAMESPACE} default
+${INFRA_NAMESPACE} default
# Scale pipeline values
${stackId} 1
diff --git a/tests/software-upgrades/ONOS_AppsUpgrade.robot b/tests/software-upgrades/ONOS_AppsUpgrade.robot
index 0b6418b..784287d 100755
--- a/tests/software-upgrades/ONOS_AppsUpgrade.robot
+++ b/tests/software-upgrades/ONOS_AppsUpgrade.robot
@@ -41,6 +41,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/software-upgrades/ONU_Upgrade.robot b/tests/software-upgrades/ONU_Upgrade.robot
index 978e2c5..5bc5961 100755
--- a/tests/software-upgrades/ONU_Upgrade.robot
+++ b/tests/software-upgrades/ONU_Upgrade.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/software-upgrades/Voltha_ComponentsUpgrade.robot b/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
index 820515a..6af3472 100755
--- a/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
+++ b/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/tt-workflow/Voltha_TT_FailureScenarios.robot b/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
index 5b54501..70a76e4 100755
--- a/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
+++ b/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot b/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
index e746223..5123814 100755
--- a/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
+++ b/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
diff --git a/tests/tt-workflow/Voltha_TT_PODTests.robot b/tests/tt-workflow/Voltha_TT_PODTests.robot
index 9cbb3c5..3c90ac7 100755
--- a/tests/tt-workflow/Voltha_TT_PODTests.robot
+++ b/tests/tt-workflow/Voltha_TT_PODTests.robot
@@ -42,6 +42,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
+${INFRA_NAMESPACE} default
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius