[VOL-1961] Implement the new test: failure-test
- Check the rw-core status after ETCD crash
Change-Id: I64ac14ffcf547f735b8dea3c4237bedcc5cf3699
diff --git a/Makefile b/Makefile
index 6aee2ea..0026def 100644
--- a/Makefile
+++ b/Makefile
@@ -82,6 +82,11 @@
system-scale-test: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_SINGLE_PON_FILE)
system-scale-test: k8s-system-test
+failure-test: ROBOT_MISC_ARGS += -X -i FailureTest $(ROBOT_DEBUG_LOG_OPT)
+failure-test: ROBOT_FILE := $(ROBOT_SYSTEM_FILE)
+failure-test: ROBOT_CONFIG_FILE := $(ROBOT_FAIL_SINGLE_PON_FILE)
+failure-test: voltha-test
+
voltha-test: ROBOT_MISC_ARGS += -e notready
k8s-system-test: ROBOT_MISC_ARGS += -e notready
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index dfc0727..781d0b2 100644
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -162,6 +162,13 @@
... kubectl delete -n ${namespace} pod/${name}
Should Be Equal as Integers ${rc} 0
+Delete K8s Pods By Label
+ [Arguments] ${namespace} ${key} ${value}
+ [Documentation] Uses kubectl to delete a PODs, filtering by label
+ ${rc}= Run and Return Rc
+ ... kubectl -n ${namespace} delete pods -l${key}=${value}
+ Should Be Equal as Integers ${rc} 0
+
Scale K8s Deployment
[Arguments] ${namespace} ${name} ${count}
[Documentation] Uses kubectl to scale a named deployment
@@ -183,6 +190,14 @@
Should Be Equal As Integers ${count} 0
Should Be True ${count}==0
+Pods Does Not Exist By Label
+ [Arguments] ${namespace} ${key} ${value}
+ [Documentation] Succeeds if the named POD does not exist
+ ${rc} ${count} Run and Return Rc And Output
+ ... kubectl get -n ${namespace} pod -l${key}=${value} -o json | jq -r ".items[].metadata.name" | wc -l
+ Should Be Equal As Integers ${count} 0
+ Should Be True ${count}==0
+
Get Available Deployment Replicas
[Arguments] ${namespace} ${name}
[Documentation] Succeeds if the named POD exists and has a ready count > 0
@@ -216,3 +231,10 @@
${replicas}= Run Keyword If '${value}' == '' Set Variable 0
... ELSE Set Variable ${value}
Should be Equal as Integers ${replicas} ${expected_count}
+
+Pods Does Not Ready By Label
+ [Arguments] ${namespace} ${key} ${value}
+ [Documentation] Check PODs Ready Status
+ ${rc} ${count} Run and Return Rc and Output
+ ... kubectl -n ${namespace} get pods -l ${key}=${value} -o json | jq -r ".items[].status.containerStatuses[].ready" | grep -c false
+ Should Be Equal as Integers ${rc} 0
diff --git a/tests/functional/K8S_SystemTest.robot b/tests/functional/K8S_SystemTest.robot
index 03b5163..ca75ff1 100644
--- a/tests/functional/K8S_SystemTest.robot
+++ b/tests/functional/K8S_SystemTest.robot
@@ -13,58 +13,77 @@
# limitations under the License.
*** Settings ***
-Documentation Provide the function to scale up/down the etcd cluster
+Documentation Provide the function to perform system related test
Library OperatingSystem
+Resource ../../libraries/k8s.robot
*** Variables ***
${timeout} 60s
-${desired_etcd_cluster_size} 3
-${minimal_etcd_cluster_size} 2
+${desired_ETCD_cluster_size} 3
+${minimal_ETCD_cluster_size} 2
${namespace} voltha
-${etcd__resources} etcdclusters.etcd.database.coreos.com
-${etcd_name} voltha-etcd-cluster
+${ETCD_resources} ETCDclusters.ETCD.database.coreos.com
+${ETCD_name} voltha-ETCD-cluster
+${ETCD_pod_label_key} ETCD_cluster
+${common_pod_label_key} app
+${rwcore_pod_label_value} rw-core
+${ofagent_pod_label_value} ofagent
+${adapter_openolt_pod_label_value} adapter-open-olt
*** Test Cases ***
-Scale Down etcd Cluster
- [Documentation] Scale Down the etcd cluster to minimal size, skip test if current cluster size < 3
- [Tags] scaledown etcddown
- ${current_size}= Get etcd Running Size voltha
- Pass Execution If '${current_size}' != '${desired_etcd_cluster_size}'
+Scale Down ETCD Cluster
+ [Documentation] Scale Down the ETCD cluster to minimal size, skip test if current cluster size < 3
+ [Tags] scaledown ETCDdown
+ ${current_size}= Get ETCD Running Size voltha
+ Pass Execution If '${current_size}' != '${desired_ETCD_cluster_size}'
... 'Skip the test if the cluster size smaller than minimal size 3'
# The minimal cluster size after scale down
- # based on https://github.com/etcd-io/etcd/blob/master/Documentation/faq.md#what-is-failure-tolerance
- Scale etcd ${namespace} ${minimal_etcd_cluster_size}
- Wait Until Keyword Succeeds ${timeout} 2s Validate etcd Size ${namespace} ${minimal_etcd_cluster_size}
+ # based on https://github.com/ETCD-io/ETCD/blob/master/Documentation/faq.md#what-is-failure-tolerance
+ Scale ETCD ${namespace} ${minimal_ETCD_cluster_size}
+ Wait Until Keyword Succeeds ${timeout} 2s Validate ETCD Size ${namespace} ${minimal_ETCD_cluster_size}
-Scale Up etcd Cluster
- [Documentation] Recover the etcd cluster by scaling up its size
- [Tags] scaleup etcdup
- ${current_size}= Get etcd Running Size voltha
- Pass Execution If '${current_size}' != '${minimal_etcd_cluster_size}'
+Scale Up ETCD Cluster
+ [Documentation] Recover the ETCD cluster by scaling up its size
+ [Tags] scaleup ETCDup
+ ${current_size}= Get ETCD Running Size voltha
+ Pass Execution If '${current_size}' != '${minimal_ETCD_cluster_size}'
... 'Skip the test if the cluster size smaller than minimal size 3'
- Scale etcd ${namespace} ${desired_etcd_cluster_size}
- # We scale up the size to 3, the recommended size of etcd cluster.
- Wait Until Keyword Succeeds ${timeout} 2s Validate etcd Size ${namespace} ${desired_etcd_cluster_size}
+ Scale ETCD ${namespace} ${desired_ETCD_cluster_size}
+ # We scale up the size to 3, the recommended size of ETCD cluster.
+ Wait Until Keyword Succeeds ${timeout} 2s Validate ETCD Size ${namespace} ${desired_ETCD_cluster_size}
+
+ETCD Failure Test
+ [Documentation] Failure Scenario Test: ETCD Crash
+ [Tags] FailureTest
+ Delete K8s Pods By Label ${namespace} ${ETCD_pod_label_key} ${ETCD_name}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Pods Does Not Exist By Label ${namespace} ${ETCD_pod_label_key} ${ETCD_name}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Pods Does Not Ready By Label ${namespace} ${common_pod_label_key} ${rwcore_pod_label_value}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Pods Does Not Ready By Label ${namespace} ${common_pod_label_key} ${ofagent_pod_label_value}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Pods Does Not Ready By Label ${namespace} ${common_pod_label_key} ${adapter_openolt_pod_label_value}
*** Keywords ***
-Get etcd Running Size
+Get ETCD Running Size
[Arguments] ${namespace}
- [Documentation] Get the number of running etcd nodes
+ [Documentation] Get the number of running ETCD nodes
${rc} ${size}= Run and Return Rc and Output
- ... kubectl -n ${namespace} get ${etcd__resources} ${etcd_name} -o jsonpath='{.status.size}'
+ ... kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.size}'
Should Be Equal As Integers ${rc} 0
[Return] ${size}
-Scale etcd
+Scale ETCD
[Arguments] ${namespace} ${size}
- [Documentation] Scale down the number of etcd pod
+ [Documentation] Scale down the number of ETCD pod
${rc}= Run and Return Rc
- ... kubectl -n ${namespace} patch ${etcd__resources} ${etcd_name} --type='merge' -p '{"spec":{"size":${size}}}'
+ ... kubectl -n ${namespace} patch ${ETCD_resources} ${ETCD_name} --type='merge' -p '{"spec":{"size":${size}}}'
Should Be Equal As Integers ${rc} 0
-Validate etcd Size
- [Arguments] ${namespace} ${etcd_cluster_size}
- [Documentation] Scale down the number of etcd pod
+Validate ETCD Size
+ [Arguments] ${namespace} ${ETCD_cluster_size}
+ [Documentation] Scale down the number of ETCD pod
${rc} ${size}= Run and Return Rc and Output
- ... kubectl -n ${namespace} get ${etcd__resources} ${etcd_name} -o jsonpath='{.status.size}'
- Should Be Equal As Integers ${size} ${etcd_cluster_size}
+ ... kubectl -n ${namespace} get ${ETCD_resources} ${ETCD_name} -o jsonpath='{.status.size}'
+ Should Be Equal As Integers ${size} ${ETCD_cluster_size}