fix etcd data check - part III
- delete onu etcd data in suite setup to ensure clean etcd data
- validate etcd in both teardown, test as well suite with Wait Until Keyword Succeeds
- add a flag to decide etcd check is executed in test or suite teardown
- some small further improvements

Change-Id: If323a0ce85e46fda46ed5aaa229788260df27c83
diff --git a/libraries/onu_utilities.robot b/libraries/onu_utilities.robot
index 2db750c..b19a9c4 100755
--- a/libraries/onu_utilities.robot
+++ b/libraries/onu_utilities.robot
@@ -234,6 +234,7 @@
         ${value}=    Get From List    ${jsondata}    ${INDEX}
         # TODO: The TP ID is hardcoded to 64 below. It is fine when testing single-tcont workflow.
         # When testing multi-tcont this may need some adjustment.
+        Exit For Loop If    not ('uni_config' in $value)
         ${tp_path}=    Get From Dictionary    ${value['uni_config'][0]['PersTpPathMap']}    64
         ${oltpononuuniid}=    Read Pon Onu Uni String    ${tp_path}
         ${list_id}=    Get Index From List    ${oltpononuuniidlist}   ${oltpononuuniid}
@@ -368,6 +369,20 @@
     Should Be Equal As Integers    ${uni}    ${uni_id}
     ...    msg=Uni-Id (${uni_id}) does not match onu (${uni}) from tp_path in etcd data!
 
+Delete ONU Go Adapter ETCD Data
+    [Documentation]    This keyword deletes openonu-go-adapter Data stored in etcd
+    [Arguments]    ${defaultkvstoreprefix}=voltha_voltha    ${validate}=False
+    ${namespace}=    Set Variable    default
+    ${podname}=    Set Variable    etcd
+    ${kvstoreprefix}=    Get Kv Store Prefix    ${defaultkvstoreprefix}
+    ${commandget}=    Catenate
+    ...    /bin/sh -c 'ETCDCTL_API=3 etcdctl del --prefix service/${kvstoreprefix}/openonu'
+    ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
+    log    ${result}
+    Run Keyword If    ${validate}    Wait Until Keyword Succeeds    ${timeout}    1s
+    ...    Validate Onu Data In Etcd    0    without_pm_data=False
+    [Return]    ${result}
+
 Wait for Ports in ONOS for all OLTs
     [Documentation]    Waits untill a certain number of ports are enabled in all OLTs
     [Arguments]    ${onos_ssh_connection}    ${count}    ${filter}    ${max_wait_time}=10m
diff --git a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
index 00860fc..8aa9ccb 100755
--- a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
@@ -68,6 +68,9 @@
 # if True (hard) kill will be used to restart onu adapter, else (soft) restart mechanism of k8s will be used
 # example: -v usekill2restart:True
 ${usekill2restart}    False
+# if True etcd check will be executed in test case teardown, if False etcd check will be executed in suite teardown
+# example: -v etcdcheckintestteardown:False
+${etcdcheckintestteardown}    True
 ${data_dir}    ../data
 ${suppressaddsubscriber}    True
 
@@ -169,6 +172,8 @@
     Set Suite Variable  ${onos_ssh_connection}
     # delete etcd MIB Template Data
     Delete MIB Template Data
+    # delete etcd onu data
+    Delete ONU Go Adapter ETCD Data    validate=True
 
 
 Teardown Suite
@@ -180,7 +185,8 @@
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     Run Keyword If    ${usekill2restart}    Restart Pod    ${namespace}    open-onu
-    Validate Onu Data In Etcd    0    without_pm_data=False
+    Run Keyword Unless    ${etcdcheckintestteardown}    Wait Until Keyword Succeeds    ${timeout}    1s
+    ...    Validate Onu Data In Etcd    0    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${onos_ssh_connection}  0   BBSM    ${timeout}
     Close All ONOS SSH Connections
 
@@ -217,11 +223,15 @@
 
 Teardown Test
     [Documentation]    Post-test Teardown
+    Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
+    Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
+    Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     # delete etcd MIB Template Data
     Delete MIB Template Data
     # check etcd data are empty
-    Validate Onu Data In Etcd    0    without_pm_data=False
+    Run Keyword If    ${etcdcheckintestteardown}    Wait Until Keyword Succeeds    ${timeout}    1s
+    ...    Validate Onu Data In Etcd    0    without_pm_data=False
     Sleep    5s
 
 Do Reconcile In Determined State
diff --git a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
index 755a0f9..ff7f136 100755
--- a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
@@ -214,6 +214,8 @@
     Set Suite Variable    ${onu_state}
     # delete etcd MIB Template Data
     Delete MIB Template Data
+    # delete etcd onu data
+    Delete ONU Go Adapter ETCD Data    validate=True
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
@@ -223,7 +225,7 @@
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
-    Validate Onu Data In Etcd    0    ${kvstoreprefix}    without_pm_data=False
+    Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    0    ${kvstoreprefix}    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${onos_ssh_connection}  0   BBSM    ${timeout}
     Close All ONOS SSH Connections
     Remove Tech Profile