Added OLT adapter restart test
Adds a test to restart the open olt adapter after the
ONU is authenticated and then completes to ONU DHCP
after the adapter is restarted.
Updates the core restart test to only loop over ONUs
for ONU specific steps as opposed to restart the core
for each ONU.
Addresses a bug in the voltha.robot library to ensure
the port forward process is seen by ps.
Change-Id: Ief16c9b6127267ea58fe4f4690b2c2868ad28664
diff --git a/Makefile b/Makefile
index eb72d07..45ec35f 100644
--- a/Makefile
+++ b/Makefile
@@ -62,6 +62,11 @@
rwcore-restart-single-kind: ROBOT_FILE := Voltha_PODTests.robot
rwcore-restart-single-kind: voltha-test
+single-kind: ROBOT_MISC_ARGS += -X -i $(TEST_TAGS) $(ROBOT_DEBUG_LOG_OPT)
+single-kind: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_SINGLE_PON_FILE)
+single-kind: ROBOT_FILE := Voltha_PODTests.robot
+single-kind: voltha-test
+
sanity-multi-kind: ROBOT_MISC_ARGS += -i sanity $(ROBOT_DEBUG_LOG_OPT)
sanity-multi-kind: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_MULT_PON_FILE)
sanity-multi-kind: bbsim-kind
diff --git a/libraries/voltha.robot b/libraries/voltha.robot
index f1f47aa..b8679b0 100644
--- a/libraries/voltha.robot
+++ b/libraries/voltha.robot
@@ -43,8 +43,13 @@
Restart VOLTHA Port Foward
[Arguments] ${name}
[Documentation] Uses a script to restart a kubectl port-forward
- ${rc} ${pid} Run And Return Rc And Output
- ... ps e -ww | grep _TAG=${name} | awk '{printf(\"%s %s\\n\",$1,$5)}' | grep kubectl | awk '{print $1}'
+ ${cmd} Catenate
+ ... ps e -ww -A |
+ ... grep _TAG=${name} |
+ ... grep -v grep |
+ ... awk '{printf(\"%s %s\\n\",$1,$5)}' |
+ ... grep -v bash | awk '{print $1}'
+ ${rc} ${pid} Run And Return Rc And Output ${cmd}
Should Be Equal as Integers ${rc} 0
Run Keyword If '${pid}' != '' Run And Return Rc kill -9 ${pid}
Should Be Equal as Integers ${rc} 0
diff --git a/tests/functional/Voltha_PODTests.robot b/tests/functional/Voltha_PODTests.robot
index b1d39e8..09e1434 100644
--- a/tests/functional/Voltha_PODTests.robot
+++ b/tests/functional/Voltha_PODTests.robot
@@ -441,26 +441,85 @@
... ${src['dp_iface_name']} wpa_supplicant.conf ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
Wait Until Keyword Succeeds ${timeout} 2s Verify ONU in AAA-Users ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} ${onu_port}
+ END
+
+ # Scale down the rw-core deployment to 0 PODs and once confirmed, scale it back to 1
+ Scale K8s Deployment voltha voltha-rw-core 0
+ Wait Until Keyword Succeeds ${timeout} 2s Pod Does Not Exist voltha voltha-rw-core
+ # Ensure the ofagent POD goes "not-ready" as expected
+ Wait Until keyword Succeeds ${timeout} 2s
+ ... Check Expected Available Deployment Replicas voltha voltha-ofagent 0
+ # Scale up the core deployment and make sure both it and the ofagent deployment are back
+ Scale K8s Deployment voltha voltha-rw-core 1
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Check Expected Available Deployment Replicas voltha voltha-rw-core 1
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Check Expected Available Deployment Replicas voltha voltha-ofagent 1
+ # For some reason scaling down and up the POD behind a service causes the port forward to stop working,
+ # so restart the port forwarding for the API service
+ Restart VOLTHA Port Foward voltha-api-minimal
+ # Ensure that the ofagent pod is up and ready and the device is available in ONOS, this
+ # represents system connectivity being restored
+ Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
+ ... http://karaf:karaf@${k8s_node_ip}:${ONOS_REST_PORT} ${of_id}
+
+ FOR ${I} IN RANGE 0 ${num_onus}
+ # Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
+ Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
+ ... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
+ ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
+ ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
+ ... ${dst['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s Run Keyword And Continue On Failure
+ ... Validate Subscriber DHCP Allocation ${k8s_node_ip} ${ONOS_SSH_PORT} ${onu_port}
+ END
+
+Sanity E2E Test for OLT/ONU on POD With OLT Adapters Fail and Restart
+ [Documentation] Deploys an device instance and waits for it to authenticate. After
+ ... authentication is successful the rw-core deployment is scaled to 0 instances to
+ ... simulate a POD crash. The test then scales the rw-core back to a single instance
+ ... and configures ONOS for access. The test succeeds if the device is able to
+ ... complete the DHCP sequence.
+ [Tags] bbsim olt-adapter-restart
+ [Setup] Clear All Devices Then Create New Device
+ ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${olt_serial_number}
+ Set Global Variable ${of_id}
+
+ FOR ${I} IN RANGE 0 ${num_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id}
+
+ # Bring up the device and verify it authenticates
+ Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ ... ${onu_device_id} onu=True onu_reason=omci-flows-pushed
+ Wait Until Keyword Succeeds ${timeout} 2s Verify Eapol Flows Added For ONU ${k8s_node_ip}
... ${ONOS_SSH_PORT} ${onu_port}
- # Scale down the rw-core deployment to 0 PODs and once confirmed, scale it back to 1
- Scale K8s Deployment voltha voltha-rw-core 0
- Wait Until Keyword Succeeds ${timeout} 2s Pod Does Not Exist voltha voltha-rw-core
- # Ensure the ofagent POD goes "not-ready" as expected
- Wait Until keyword Succeeds ${timeout} 2s
- ... Check Expected Available Deployment Replicas voltha voltha-ofagent 0
- # Scale up the core deployment and make sure both it and the ofagent deployment are back
- Scale K8s Deployment voltha voltha-rw-core 1
- Wait Until Keyword Succeeds ${timeout} 2s
- ... Check Expected Available Deployment Replicas voltha voltha-rw-core 1
- Wait Until Keyword Succeeds ${timeout} 2s
- ... Check Expected Available Deployment Replicas voltha voltha-ofagent 1
- # For some reason scaling down and up the POD behind a service causes the port forward to stop working,
- # so restart the port forwarding for the API service
- Restart VOLTHA Port Foward voltha-api-minimal
- # Ensure that the ofagent pod is up and ready and the device is available in ONOS, this
- # represents system connectivity being restored
- Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
- ... http://karaf:karaf@${k8s_node_ip}:${ONOS_REST_PORT} ${of_id}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate Authentication True
+ ... ${src['dp_iface_name']} wpa_supplicant.conf ${src['ip']} ${src['user']} ${src['pass']}
+ ... ${src['container_type']} ${src['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s Verify ONU in AAA-Users ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} ${onu_port}
+ END
+
+ # Scale down the open OLT adapter deployment to 0 PODs and once confirmed, scale it back to 1
+ Scale K8s Deployment voltha adapter-open-olt 0
+ Wait Until Keyword Succeeds ${timeout} 2s Pod Does Not Exist voltha adapter-open-olt
+ # Scale up the open OLT adapter deployment and make sure both it and the ofagent deployment are back
+ Scale K8s Deployment voltha adapter-open-olt 1
+ Wait Until Keyword Succeeds ${timeout} 2s
+ ... Check Expected Available Deployment Replicas voltha adapter-open-olt 1
+
+ # Ensure the device is available in ONOS, this represents system connectivity being restored
+ Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
+ ... http://karaf:karaf@${k8s_node_ip}:${ONOS_REST_PORT} ${of_id}
+
+ FOR ${I} IN RANGE 0 ${num_onus}
# Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command ${k8s_node_ip}
... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}