VOL-3229 Fix etcd tests

Change-Id: Ieb6da1a7b180feee094a834017e5a8a38fec7198
diff --git a/tests/functional/K8S_SystemTest.robot b/tests/functional/K8S_SystemTest.robot
index 74f8ad5..6d74e19 100644
--- a/tests/functional/K8S_SystemTest.robot
+++ b/tests/functional/K8S_SystemTest.robot
@@ -29,47 +29,52 @@
 ${desired_ETCD_cluster_size}    3
 ${minimal_ETCD_cluster_size}    2
 ${namespace}      voltha
-${ETCD_resources}    etcdclusters.etcd.database.coreos.com
-${ETCD_name}      voltha-etcd-cluster
-${ETCD_pod_label_key}    etcd_cluster
+${ETCD_namespace}    default
+${ETCD_resources}    statefulsets
+${ETCD_name}      etcd
+${ETCD_pod_label_key}    app
 ${common_pod_label_key}    app
 ${rwcore_pod_label_value}    rw-core
 ${ofagent_pod_label_value}    ofagent
 ${adapter_openolt_pod_label_value}    adapter-open-olt
 
+# Per-test logging on failure is turned off by default; set this variable to enable
+${container_log_dir}    ${None}
+
 *** Test Cases ***
-ECTD Scale Test
+ETCD Scale Test
     [Documentation]    Perform the sanity test if some ETCD endpoints crash
     [Tags]    functional    bbsim
-    [Setup]    Run Keywords    Announce Message    START TEST SanityTest
-    ...        AND             Setup
+    [Setup]    Run Keywords    Start Logging    EtcdScaleTest
+    ...        AND    Setup
     [Teardown]    Run Keywords    Collect Logs
     ...           AND             Announce Message    END TEST SanityTest
     ...           AND    Teardown Suite
-    ${current_size}=    Get ETCD Running Size    voltha
+    ...           AND    Stop Logging    EtcdScaleTest
+    ${current_size}=    Get ETCD Replica Count    ${ETCD_namespace}
     Pass Execution If    '${current_size}' != '${desired_ETCD_cluster_size}'
     ...    'Skip the test if the cluster size smaller than minimal size 3'
     # The minimal cluster size after scale down
     # based on https://github.com/ETCD-io/ETCD/blob/master/Documentation/faq.md#what-is-failure-tolerance
-    Scale ETCD    ${namespace}    ${minimal_ETCD_cluster_size}
+    Scale ETCD    ${ETCD_namespace}    ${minimal_ETCD_cluster_size}
     Wait Until Keyword Succeeds    ${timeout}    2s
-    ...    Validate ETCD Size    ${namespace}    ${minimal_ETCD_cluster_size}
-    Wait Until Keyword Succeeds    ${timeout}    2s
-    ...    Check Expected Running Pods Number By Label    ${namespace}
-    ...    ${ETCD_pod_label_key}    ${ETCD_name}    2
+    ...    Validate ETCD Size    ${ETCD_namespace}    ${minimal_ETCD_cluster_size}
     # Perform the sanity-test
     Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
     # We scale up the size to 3, the recommended size of ETCD cluster.
-    Scale ETCD    ${namespace}    ${desired_ETCD_cluster_size}
+    Scale ETCD    ${ETCD_namespace}    ${desired_ETCD_cluster_size}
     Wait Until Keyword Succeeds    ${timeout}    2s
-    ...    Validate ETCD Size    ${namespace}    ${desired_ETCD_cluster_size}
+    ...    Validate ETCD Size    ${ETCD_namespace}    ${desired_ETCD_cluster_size}
 
 ETCD Failure Test
     [Documentation]    Failure Scenario Test: ETCD Crash
     [Tags]    FailureTest
-    Delete K8s Pods By Label    ${namespace}    ${ETCD_pod_label_key}    ${ETCD_name}
-    Wait Until Keyword Succeeds    ${timeout}    2s
-    ...    Pods Do Not Exist By Label    ${namespace}    ${ETCD_pod_label_key}    ${ETCD_name}
+    [Setup]    Start Logging    EtcdFailureTest
+    [Teardown]    Run Keywords    Collect Logs
+    ...              AND    Stop Logging    EtcdFailureTest
+    Delete K8s Pods By Label    ${ETCD_namespace}    ${ETCD_pod_label_key}    ${ETCD_name}
+    #Wait Until Keyword Succeeds    ${timeout}    2s
+    #...    Pods Do Not Exist By Label    ${ETCD_namespace}    ${ETCD_pod_label_key}    ${ETCD_name}
     Wait Until Keyword Succeeds    ${timeout}    2s
     ...    Pods Are Ready By Label    ${namespace}    ${common_pod_label_key}    ${rwcore_pod_label_value}
     Wait Until Keyword Succeeds    ${timeout}    2s
@@ -78,11 +83,11 @@
     ...    Pods Are Ready By Label    ${namespace}    ${common_pod_label_key}    ${adapter_openolt_pod_label_value}
 
 *** Keywords ***
-Get ETCD Running Size
+Get ETCD Replica Count
     [Arguments]    ${namespace}
-    [Documentation]    Get the number of running ETCD nodes
+    [Documentation]    Get the number of configured ETCD nodes
     ${rc}    ${size}=    Run and Return Rc and Output
-    ...    kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.size}'
+    ...    kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.replicas}'
     Should Be Equal As Integers    ${rc}    0
     [Return]    ${size}
 
@@ -90,12 +95,17 @@
     [Arguments]    ${namespace}    ${size}
     [Documentation]    Scale down the number of ETCD pod
     ${rc}=    Run and Return Rc
-    ...    kubectl -n ${namespace} patch ${ETCD_resources} ${ETCD_name} --type='merge' -p '{"spec":{"size":${size}}}'
+    ...    kubectl -n ${namespace} patch ${ETCD_resources} ${ETCD_name} -p '{"spec":{"replicas": ${size}}}'
     Should Be Equal As Integers    ${rc}    0
 
 Validate ETCD Size
     [Arguments]    ${namespace}    ${ETCD_cluster_size}
     [Documentation]    Scale down the number of ETCD pod
     ${rc}    ${size}=    Run and Return Rc and Output
-    ...    kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.size}'
-    Should Be Equal As Integers    ${size}    ${ETCD_cluster_size}
+    ...    kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.replicas}'
+    Should Be Equal As Integers    ${rc}    0
+    Should Be Equal As Integers    ${size}    ${ETCD_cluster_size}    Unexpected number of replicas
+    ${rc}    ${size}=    Run and Return Rc and Output
+    ...    kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.readyReplicas}'
+    Should Be Equal As Integers    ${rc}    0
+    Should Be Equal As Integers    ${size}    ${ETCD_cluster_size}    Unexpected number of ready replicas