[VOL-2480]Failure Scenarios
Change-Id: I21648bcb23932e65cab58f1fe3a88dc46457c300
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 9e59780..6d0828b 100644
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -236,3 +236,38 @@
Should Be Equal As Integers ${rc} 0
Wait Until Keyword Succeeds ${timeout} 5s Validate Device Removed ${olt_device_id}
+Repeat Sanity Test
+ [Documentation] This keyword performs Sanity Test Procedure
+ ... Sanity test performs authentication, dhcp and pings for all the ONUs given for the POD
+ ... This keyword can be used to call in any other tests where sanity check is required
+ ... with wpa reassociation
+ ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${olt_serial_number}
+ Set Global Variable ${of_id}
+ FOR ${I} IN RANGE 0 ${num_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
+ ... Get ONU Port in ONOS ${src['onu']} ${of_id}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Verify Eapol Flows Added For ONU ${k8s_node_ip} ${ONOS_SSH_PORT} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
+ ... Validate Authentication After Reassociate
+ ... True ${src['dp_iface_name']} ${src['ip']} ${src['user']} ${src['pass']}
+ ... ${src['container_type']} ${src['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Verify ONU in AAA-Users ${k8s_node_ip} ${ONOS_SSH_PORT} ${onu_port}
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Execute ONOS CLI Command ${k8s_node_ip} ${ONOS_SSH_PORT}
+ ... volt-add-subscriber-access ${of_id} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
+ ... Validate DHCP and Ping True True
+ ... ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
+ ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
+ ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
+ ... ${dst['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s Run Keyword And Continue On Failure
+ ... Validate Subscriber DHCP Allocation ${k8s_node_ip} ${ONOS_SSH_PORT} ${onu_port}
+ Run Keyword and Ignore Error Get Device Output from Voltha ${onu_device_id}
+ Run Keyword and Ignore Error Collect Logs
+ END
diff --git a/tests/functional/Voltha_FailureScenarios.robot b/tests/functional/Voltha_FailureScenarios.robot
new file mode 100644
index 0000000..80d4af5
--- /dev/null
+++ b/tests/functional/Voltha_FailureScenarios.robot
@@ -0,0 +1,75 @@
+# Copyright 2017 - present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# FIXME Can we use the same test against BBSim and Hardware?
+
+*** Settings ***
+Documentation Test various end-to-end scenarios
+Suite Setup Common Test Suite Setup
+Test Setup Setup
+Test Teardown Teardown
+#Suite Teardown Teardown Suite
+Library Collections
+Library String
+Library OperatingSystem
+Library XML
+Library RequestsLibrary
+Library ../../libraries/DependencyLibrary.py
+Resource ../../libraries/onos.robot
+Resource ../../libraries/voltctl.robot
+Resource ../../libraries/voltha.robot
+Resource ../../libraries/utils.robot
+Resource ../../libraries/k8s.robot
+Resource ../../variables/variables.robot
+
+*** Variables ***
+${POD_NAME} flex-ocp-cord
+${KUBERNETES_CONF} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
+${KUBERNETES_CONFIGS_DIR} ~/pod-configs/kubernetes-configs
+#${KUBERNETES_CONFIGS_DIR} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
+${KUBERNETES_YAML} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.yml
+${HELM_CHARTS_DIR} ~/helm-charts
+${VOLTHA_POD_NUM} 8
+${NAMESPACE} voltha
+# For below variable value, using deployment name as using grep for
+# parsing radius pod name, we can also use full radius pod name
+${RESTART_POD_NAME} radius
+${timeout} 60s
+${of_id} 0
+${logical_id} 0
+${has_dataplane} False
+${external_libs} True
+${teardown_device} False
+${scripts} ../../scripts
+
+*** Test Cases ***
+Verify restart any container after VOLTHA is operational
+ [Documentation] Restart any container after VOLTHA is operational.
+ ... Prerequisite : ONUs are authenticated and pingable.
+ [Tags] functional VOL-1958 RestartPods notready
+ [Setup] NONE
+ [Teardown] NONE
+ ${waitforRestart} Set Variable 120s
+ ${podStatusOutput}= Run ${KUBECTL_CONFIG};kubectl get pods -n ${NAMESPACE}
+ Log ${podStatusOutput}
+ ${countBforRestart}= Run ${KUBECTL_CONFIG};kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
+ ${podName} Set Variable adapter-open-olt
+ Restart Pod ${NAMESPACE} ${podName}
+ Wait Until Keyword Succeeds ${waitforRestart} 2s Validate Pod Status ${podName} ${NAMESPACE}
+ ... Running
+ Repeat Sanity Test
+ ${podStatusOutput}= Run ${KUBECTL_CONFIG};kubectl get pods -n ${NAMESPACE}
+ Log ${podStatusOutput}
+ ${countAfterRestart}= Run ${KUBECTL_CONFIG};kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
+ Should Be Equal As Strings ${countAfterRestart} ${countBforRestart}
+ Log to console Pod ${podName} restarted and sanity checks passed successfully