[VOL-4359] Added ETCD data cleanup verification after device delete

Change-Id: Ib69199c22608060b022ef1605ca7c3e76f4f2bf7
diff --git a/Makefile b/Makefile
index c73f4d8..ef68b1e 100755
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,7 @@
              --configure TooManyTestCases:50 -e TooManyTestCases \
              --configure TooFewTestSteps:1 \
              --configure TooFewKeywordSteps:1 \
-             --configure FileTooLong:1500 -e FileTooLong \
+             --configure FileTooLong:1600 -e FileTooLong \
              -e TrailingWhitespace
 
 PYTHON_FILES := $(wildcard libraries/*.py)
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 470c811..544d40c 100755
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -704,9 +704,7 @@
         Wait Until Keyword Succeeds    ${timeout}    5s
         ...    Validate Deleted Device Cleanup In ONOS    ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}    ${olt_serial_number}
     END
-    # Clear devices from ONOS
-    #Remove All Devices From ONOS
-    #...    http://karaf:karaf@${ONOS_REST_IP}:${ONOS_REST_PORT}
+    Wait Until Keyword Succeeds    ${timeout}    5s    Validate Cleanup In ETCD    ${INFRA_NAMESPACE}
 
 Teardown
     [Documentation]    kills processes and cleans up interfaces on src+dst servers
@@ -1430,3 +1428,75 @@
     END
     ${real_num_onus}=    Get Length    ${onu_list}
     [Return]    ${real_num_onus}
+
+Validate Cleanup In ETCD
+    [Documentation]    The keyword verifies that device, ports, flows, meters are all cleared in ETCD
+    [Arguments]    ${namespace}=default    ${defaultkvstoreprefix}=voltha_voltha
+    ${podname}=    Set Variable    etcd
+    ${kvstoreprefix}=    Get Kv Store Prefix    ${defaultkvstoreprefix}
+    # Log Devices Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/devices --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Devices Data in Etcd!
+    # Log Flows Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/flows --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Flows Data in Etcd!
+    # Log LogicalDevices Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_devices --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Logical Devices Data in Etcd!
+    # Log LogicalFlows Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_flows --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Logical Flows Data in Etcd!
+    # Log LogicalMeters Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_meters --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Logical Meters Data in Etcd!
+    # Log LogicalPorts Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_ports --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Logical Ports Data in Etcd!
+    # Log Openolt Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openolt --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Openolt Data in Etcd!
+    # Log Openonu Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openonu --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Openonu Data in Etcd!
+    # Log Ports Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/ports --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Ports Data in Etcd!
+    # Log ResourceInstances Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_instances --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Resource Instances Data in Etcd!
+    # Log ResourceManager Output and Verify Output Should be Empty
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_manager --keys-only'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    Log    ${result}
+    Should Be Empty    ${result}    Stale Resource Manager Data in Etcd!