VOL-1670 - test failure or rw-core
Validates a a scale down/up test of the rw-core
to determine if the system can recover and continue
Change-Id: I6dd937724d2dd75b77c61505e0e3ee7cd36cb0db
diff --git a/.gitignore b/.gitignore
index 72fa7bc..124a8d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@
*.html
*.xml
*.pyc
+*.swp
diff --git a/Makefile b/Makefile
index 4343b64..48d9dd5 100644
--- a/Makefile
+++ b/Makefile
@@ -22,9 +22,11 @@
--configure TooFewTestSteps:1 --configure TooFewKeywordSteps:1
VERSION ?= $(shell cat ./VERSION)
ROBOT_SANITY_SINGLE_PON_FILE ?= $(ROOT_DIR)/tests/data/bbsim-kind.yaml
+ROBOT_FAIL_SINGLE_PON_FILE ?= $(ROOT_DIR)/tests/data/bbsim-kind.yaml
ROBOT_SANITY_MULT_PON_FILE ?= $(ROOT_DIR)/tests/data/bbsim-kind-2x2.yaml
ROBOT_SCALE_SINGLE_PON_FILE ?= $(ROOT_DIR)/tests/data/bbsim-kind-16.yaml
ROBOT_SCALE_MULT_PON_FILE ?= $(ROOT_DIR)/tests/data/bbsim-kind-8x2.yaml
+ROBOT_DEBUG_LOG_OPT ?=
ROBOT_SYSTEM_SETUP_MISC_ARGS ?= -i scaledown -l systemup_log.html -r systemup_report.html -o systemup_output.xml
ROBOT_SYSTEM_TEARDOWN_MISC_ARGS ?= -i scaleup -l systemdown_log.html -r systemdown_report.html -o sysstemdown_output.xml
@@ -46,11 +48,16 @@
# for backwards compatibility
sanity-kind: sanity-single-kind
-sanity-single-kind: ROBOT_MISC_ARGS += -i sanity
+sanity-single-kind: ROBOT_MISC_ARGS += -i sanity $(ROBOT_DEBUG_LOG_OPT)
sanity-single-kind: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_SINGLE_PON_FILE)
sanity-single-kind: bbsim-kind
-sanity-multi-kind: ROBOT_MISC_ARGS += -i sanity
+rwcore-restart-single-kind: ROBOT_MISC_ARGS += -X -i bbsimANDrwcore-restart $(ROBOT_DEBUG_LOG_OPT)
+rwcore-restart-single-kind: ROBOT_CONFIG_FILE := $(ROBOT_FAIL_SINGLE_PON_FILE)
+rwcore-restart-single-kind: ROBOT_FILE := Voltha_PODTests.robot
+rwcore-restart-single-kind: voltha-test
+
+sanity-multi-kind: ROBOT_MISC_ARGS += -i sanity $(ROBOT_DEBUG_LOG_OPT)
sanity-multi-kind: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_MULT_PON_FILE)
sanity-multi-kind: bbsim-kind
@@ -58,15 +65,15 @@
bbsim-kind: ROBOT_FILE := Voltha_PODTests.robot
bbsim-kind: voltha-test
-scale-single-kind: ROBOT_MISC_ARGS += -i active
+scale-single-kind: ROBOT_MISC_ARGS += -i active $(ROBOT_DEBUG_LOG_OPT)
scale-single-kind: ROBOT_CONFIG_FILE := $(ROBOT_SCALE_SINGLE_PON_FILE)
scale-single-kind: bbsim-scale-kind
-scale-multi-kind: ROBOT_MISC_ARGS += -i active
+scale-multi-kind: ROBOT_MISC_ARGS += -i active $(ROBOT_DEBUG_LOG_OPT)
scale-multi-kind: ROBOT_CONFIG_FILE := $(ROBOT_SCALE_MULT_PON_FILE)
scale-multi-kind: bbsim-scale-kind
-bbsim-scale-kind: ROBOT_MISC_ARGS += -X
+bbsim-scale-kind: ROBOT_MISC_ARGS += -X $(ROBOT_DEBUG_LOG_OPT)
bbsim-scale-kind: ROBOT_FILE := Voltha_ScaleFunctionalTests.robot
bbsim-scale-kind: voltha-test
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index 7362470..dfc0727 100644
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -28,7 +28,7 @@
*** Keywords ***
Lookup Service IP
[Arguments] ${namespace} ${name}
- [Documentation] Uses kubeclt to resolve a service name to an IP
+ [Documentation] Uses kubectl to resolve a service name to an IP
${rc} ${ip}= Run and Return Rc and Output
... kubectl get svc -n ${namespace} ${name} -o jsonpath={.spec.clusterIP}
Should Be Equal as Integers ${rc} 0
@@ -36,7 +36,7 @@
Lookup Service PORT
[Arguments] ${namespace} ${name}
- [Documentation] Uses kubeclt to resolve a service name to an PORT
+ [Documentation] Uses kubectl to resolve a service name to an PORT
${rc} ${port}= Run and Return Rc and Output
... kubectl get svc -n ${namespace} ${name} -o jsonpath={.spec.ports[0].port}
Should Be Equal as Integers ${rc} 0
@@ -155,3 +155,64 @@
END
[Return] ${errorPodList}
+Delete K8s Pod
+ [Arguments] ${namespace} ${name}
+ [Documentation] Uses kubectl to delete a named POD
+ ${rc} Run and Return Rc
+ ... kubectl delete -n ${namespace} pod/${name}
+ Should Be Equal as Integers ${rc} 0
+
+Scale K8s Deployment
+ [Arguments] ${namespace} ${name} ${count}
+ [Documentation] Uses kubectl to scale a named deployment
+ ${rc} Run and Return Rc
+ ... kubectl scale --replicas=${count} -n ${namespace} deploy/${name}
+ Should Be Equal as Integers ${rc} 0
+
+Pod Exists
+ [Arguments] ${namespace} ${name}
+ [Documentation] Succeeds it the named POD exists
+ ${rc} ${count} Run and Return Rc kubectl get -n ${namespace} pod -o json | jq -r ".items[].metadata.name" | grep ${name}
+ Should Be True ${count}>0
+
+Pod Does Not Exist
+ [Arguments] ${namespace} ${name}
+ [Documentation] Succeeds if the named POD does not exist
+ ${rc} ${count} Run and Return Rc And Output
+ ... kubectl get -n ${namespace} pod -o json | jq -r ".items[].metadata.name" | grep -c ${name}
+ Should Be Equal As Integers ${count} 0
+ Should Be True ${count}==0
+
+Get Available Deployment Replicas
+ [Arguments] ${namespace} ${name}
+ [Documentation] Succeeds if the named POD exists and has a ready count > 0
+ ${rc} ${count} Run and Return Rc and Output ${KUBECTL_CONFIG};kubectl get -n ${namespace} deploy/${name} -o jsonpath='{.status.availableReplicas}'
+ ${result}= Run Keyword If '${count}' == '' Set Variable 0
+ ... ELSE Set Variable ${count}
+ [Return] ${result}
+
+Check Expected Available Deployment Replicas
+ [Arguments] ${namespace} ${name} ${expected}
+ [Documentation] Succeeds if the named deployment has the expected number of available replicas
+ ${count}= Get Available Deployment Replicas ${namespace} ${name}
+ Should Be Equal As Integers ${expected} ${count}
+
+Get Deployment Replica Count
+ [Arguments] ${namespace} ${name}
+ [Documentation] Uses kubectl to fetch the number of configured replicas on a deployment
+ ${rc} ${value} Run and Return Rc and Output
+ ... kubectl -n ${namespace} get deploy/${name} -o 'jsonpath={.status.replicas}'
+ Should Be Equal as Integers ${rc} 0
+ ${replicas}= Run Keyword If '${value}' == '' Set Variable 0
+ ... ELSE Set Variable ${value}
+ [Return] ${replicas}
+
+Does Deployment Have Replicas
+ [Arguments] ${namespace} ${name} ${expected_count}
+ [Documentation] Uses kubectl to fetch the number of configured replicas on a deployment
+ ${rc} ${value} Run and Return Rc and Output
+ ... kubectl -n ${namespace} get deploy/${name} -o 'jsonpath={.status.replicas}'
+ Should Be Equal as Integers ${rc} 0
+ ${replicas}= Run Keyword If '${value}' == '' Set Variable 0
+ ... ELSE Set Variable ${value}
+ Should be Equal as Integers ${replicas} ${expected_count}
diff --git a/libraries/onos.robot b/libraries/onos.robot
index 67ae0af..e9dfa63 100644
--- a/libraries/onos.robot
+++ b/libraries/onos.robot
@@ -128,3 +128,22 @@
${allocations}= Execute ONOS CLI Command ${ip} ${port}
... dhcpl2relay-allocations | grep DHCPACK | grep ${onu_port}
Should Not Be Empty ${allocations} ONU port ${onu_port} not found in dhcpl2relay-allocations
+
+Device Is Available In ONOS
+ [Arguments] ${url} ${dpid}
+ [Documentation] Validates the device exists and it available in ONOS
+ ${rc} ${json} Run And Return Rc And Output curl --fail -sSL ${url}/onos/v1/devices/${dpid}
+ Should Be Equal As Integers 0 ${rc}
+ ${rc} ${value} Run And Return Rc And Output echo '${json}' | jq -r .available
+ Should Be Equal As Integers 0 ${rc}
+ Should Be Equal 'true' '${value}'
+
+Remove All Devices From ONOS
+ [Arguments] ${url}
+ [Documentation] Executes the device-remove command on each device in ONOS
+ ${rc} @{dpids} Run And Return Rc And Output curl --fail -sSL ${url}/onos/v1/devices | jq -r '.devices[].id'
+ Should Be Equal As Integers ${rc} 0
+ ${count}= Get length ${dpids}
+ :FOR ${dpid} IN @{dpids}
+ \ ${rc}= Run Keyword If '${dpid}' != '' Run And Return Rc curl -XDELETE --fail -sSL ${url}/onos/v1/devices/${dpid}
+ \ Run Keyword If '${dpid}' != '' Should Be Equal As Integers ${rc} 0
diff --git a/libraries/voltctl.robot b/libraries/voltctl.robot
index 71da9a1..2fbd900 100644
--- a/libraries/voltctl.robot
+++ b/libraries/voltctl.robot
@@ -33,7 +33,7 @@
${jsondata}= To Json ${output}
Log ${jsondata}
${length}= Get Length ${jsondata}
- [Return] ${length}
+ Should Be Equal As Integers ${length} 0
Create Device
[Arguments] ${ip} ${port}
@@ -56,6 +56,33 @@
${rc} ${output}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device disable ${device_id}
Should Be Equal As Integers ${rc} 0
+Disable Devices In Voltha
+ [Documentation] Disables all the known devices in voltha
+ [Arguments] ${filter}
+ ${arg}= Set Variable ${EMPTY}
+ ${arg}= Run Keyword If len('${filter}'.strip()) != 0 Set Variable --filter ${filter}
+ ${rc} ${devices}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device list ${arg} --orderby Root -q | xargs echo -n
+ Should Be Equal As Integers ${rc} 0
+ ${rc} ${output}= Run Keyword If len('${devices}') != 0 Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device disable ${devices}
+ Run Keyword If len('${devices}') != 0 Should Be Equal As Integers ${rc} 0
+
+Test Devices Disabled In Voltha
+ [Documentation] Tests to verify that all devices in VOLTHA are disabled
+ [Arguments] ${filter}
+ ${rc} ${count}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device list --filter '${filter},AdminState!=DISABLED' -q | wc -l
+ Should Be Equal As Integers ${rc} 0
+ Should Be Equal As Integers ${count} 0
+
+Delete Devices In Voltha
+ [Documentation] Disables all the known devices in voltha
+ [Arguments] ${filter}
+ ${arg}= Set Variable ${EMPTY}
+ ${arg}= Run Keyword If len('${filter}'.strip()) != 0 Set Variable --filter ${filter}
+ ${rc} ${devices}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device list ${arg} --orderby Root -q | xargs echo -n
+ Should Be Equal As Integers ${rc} 0
+ ${rc} ${output}= Run Keyword If len('${devices}') != 0 Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device delete ${devices}
+ Run Keyword If len('${devices}') != 0 Should Be Equal As Integers ${rc} 0
+
Get Device Flows from Voltha
[Arguments] ${device_id}
[Documentation] Gets device flows from VOLTHA
diff --git a/libraries/voltha.robot b/libraries/voltha.robot
new file mode 100644
index 0000000..1009eda
--- /dev/null
+++ b/libraries/voltha.robot
@@ -0,0 +1,52 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# voltha common functions
+
+*** Settings ***
+Documentation Library for various utilities
+Library SSHLibrary
+Library HttpLibrary.HTTP
+Library String
+Library DateTime
+Library Process
+Library Collections
+Library RequestsLibrary
+Library OperatingSystem
+
+*** Keywords ***
+Lookup Pod That Owns Device
+ [Arguments] ${device_id}
+ [Documentation] Uses a utility script to lookup which RW Core has current ownership of an OLT
+ ${rc} ${pod}= Run and Return Rc and Output
+ ... ../scripts/which_pod_owns_device.sh ${device_id}
+ Should Be Equal as Integers ${rc} 0
+ [Return] ${pod}
+
+Lookup Deployment That Owns Device
+ [Arguments] ${device_id}
+ [Documentation] Uses a utility script to lookup which RW Core has current ownership of an OLT
+ ${rc} ${deploy}= Run and Return Rc and Output
+ ... which_deployment_owns_device.sh ${device_id}
+ Should Be Equal as Integers ${rc} 0
+ [Return] ${deploy}
+
+Restart VOLTHA Port Foward
+ [Arguments] ${name}
+ [Documentation] Uses a script to restart a kubectl port-forward
+ ${rc} ${pid} Run And Return Rc And Output
+ ... ps e -ww | grep _TAG=${name} | grep -v grep | awk '{printf(\"%s %s\\n\",$1,$5)}' | grep -v bash | awk '{print $1}'
+ Should Be Equal as Integers ${rc} 0
+ Run Keyword If '${pid}' != '' Run And Return Rc kill -9 ${pid}
+ Should Be Equal as Integers ${rc} 0
diff --git a/scripts/which_deployment_owns_device.sh b/scripts/which_deployment_owns_device.sh
new file mode 100755
index 0000000..c23961d
--- /dev/null
+++ b/scripts/which_deployment_owns_device.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright 2019 Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DEVICE_ID=$1
+
+BEST_DATE=
+BEST_DEPLOY=
+for DEPLOY in $(kubectl -n voltha get deploy -o 'jsonpath={.items[*].metadata.name}'); do
+ if [[ "$DEPLOY" =~ voltha-rw-core-.* ]]; then
+ FOUND=$(kubectl -n voltha logs "deploy/$DEPLOY" 2>/dev/null | grep "$DEVICE_ID" | grep -i ownedbyme | tail -1)
+ if [ ! -z "$FOUND" ]; then
+ OWNED=$(echo "$FOUND" | grep '"owned":true')
+ if [ ! -z "$OWNED" ]; then
+ CUR_DATE=$(echo "$OWNED" | jq -r .ts)
+ CUR_DEPLOY=$DEPLOY
+ if [ -z "$BEST_DEPLOY" ]; then
+ BEST_DATE=$CUR_DATE
+ BEST_DEPLOY=$CUR_DEPLOY
+ elif [[ "$CUR_DATE" > "$BEST_DATE" ]]; then
+ BEST_DATE=$CUR_DATE
+ BEST_DEPLOY=$CUR_DEPLOY
+ fi
+ fi
+ fi
+ fi
+done
+if [ -z "$BEST_DEPLOY" ]; then
+ exit 1
+fi
+echo "$BEST_DEPLOY"
diff --git a/tests/functional/Voltha_PODTests.robot b/tests/functional/Voltha_PODTests.robot
index 775285e..668298b 100644
--- a/tests/functional/Voltha_PODTests.robot
+++ b/tests/functional/Voltha_PODTests.robot
@@ -1,4 +1,4 @@
-#Copyright 2017-present Open Networking Foundation
+# Copyright 2017 - present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@
Library ../../libraries/DependencyLibrary.py
Resource ../../libraries/onos.robot
Resource ../../libraries/voltctl.robot
+Resource ../../libraries/voltha.robot
Resource ../../libraries/utils.robot
Resource ../../libraries/k8s.robot
Resource ../../variables/variables.robot
@@ -50,6 +51,7 @@
${has_dataplane} True
${external_libs} True
${teardown_device} False
+${scripts} ../../scripts
*** Test Cases ***
Sanity E2E Test for OLT/ONU on POD
@@ -126,6 +128,66 @@
END
Run Keyword and Ignore Error Collect Logs
+Sanity E2E Test for OLT/ONU on POD With Core Fail and Restart
+ [Documentation] Deploys an device instance and waits for it to authenticate. After
+ ... authentication is successful the rw-core deployment is scaled to 0 instances to
+ ... simulate a POD crash. The test then scales the rw-core back to a single instance
+ ... and configures ONOS for access. The test succeeds if the device is able to
+ ... complete the DHCP sequence.
+ [Tags] bbsim rwcore-restart
+ [Setup] Clear All Devices Then Create New Device
+ ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${olt_serial_number}
+ Set Global Variable ${of_id}
+
+ FOR ${I} IN RANGE 0 ${num_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id}
+
+ # Bring up the device and verify it authenticates
+ Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ ... ${onu_device_id} onu=True onu_reason=omci-flows-pushed
+ Wait Until Keyword Succeeds ${timeout} 2s Verify Eapol Flows Added For ONU ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate Authentication True
+ ... ${src['dp_iface_name']} wpa_supplicant.conf ${src['ip']} ${src['user']} ${src['pass']}
+ ... ${src['container_type']} ${src['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s Verify ONU in AAA-Users ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} ${onu_port}
+
+ # Scale down the rw-core deployment to 0 PODs and once confirmed, scale it back to 1
+ Scale K8s Deployment voltha voltha-rw-core 0
+ Wait Until Keyword Succeeds ${timeout} 2s Pod Does Not Exist voltha voltha-rw-core
+ # Ensure the ofagent POD goes "not-ready" as expected
+ Wait Until keyword Succeeds ${timeout} 2s Check Expected Available Deployment Replicas voltha voltha-ofagent 0
+ # Scale up the core deployment and make sure both it and the ofagent deployment are back
+ Scale K8s Deployment voltha voltha-rw-core 1
+ Wait Until Keyword Succeeds ${timeout} 2s Check Expected Available Deployment Replicas voltha voltha-rw-core 1
+ Wait Until Keyword Succeeds ${timeout} 2s Check Expected Available Deployment Replicas voltha voltha-ofagent 1
+
+ # For some reason scaling down and up the POD behind a service causes the port forward to stop working,
+ # so restart the port forwarding for the API service
+ Restart VOLTHA Port Foward voltha-api-minimal
+
+ # Ensure that the ofagent pod is up and ready and the device is available in ONOS, this
+ # represents system connectivity being restored
+ Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
+ ... http://karaf:karaf@${k8s_node_ip}:${ONOS_REST_PORT} ${of_id}
+
+ # Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
+ Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command ${k8s_node_ip}
+ ... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
+ ... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
+ ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
+ ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
+ ... ${dst['container_name']}
+ Wait Until Keyword Succeeds ${timeout} 2s Run Keyword And Continue On Failure
+ ... Validate Subscriber DHCP Allocation ${k8s_node_ip} ${ONOS_SSH_PORT} ${onu_port}
+ END
+
*** Keywords ***
Setup Suite
[Documentation] Set up the test suite
@@ -134,8 +196,7 @@
Setup
[Documentation] Pre-test Setup
#test for empty device list
- ${length}= Test Empty Device List
- Should Be Equal As Integers ${length} 0
+ Test Empty Device List
#create/preprovision device
${olt_device_id}= Create Device ${olt_ip} ${OLT_PORT}
Set Suite Variable ${olt_device_id}
@@ -148,6 +209,28 @@
${logical_id}= Get Logical Device ID From SN ${olt_serial_number}
Set Suite Variable ${logical_id}
+Delete All Devices and Verify
+ [Documentation] Remove any devices from VOLTHA and ONOS
+
+ # Clear devices from VOLTHA
+ Disable Devices In Voltha Root=true
+ Wait Until Keyword Succeeds ${timeout} 2s Test Devices Disabled In Voltha Root=true
+ Delete Devices In Voltha Root=true
+ Wait Until Keyword Succeeds ${timeout} 2s Test Empty Device List
+
+ # Clear devices from ONOS
+ Remove All Devices From ONOS
+ ... http://karaf:karaf@${k8s_node_ip}:${ONOS_REST_PORT}
+
+Clear All Devices Then Create New Device
+ [Documentation] Remove any devices from VOLTHA and ONOS
+
+ # Remove all devices from voltha and nos
+ Delete All Devices and Verify
+
+ # Execute normal test Setup Keyword
+ Setup
+
Teardown
[Documentation] kills processes and cleans up interfaces on src+dst servers
Run Keyword If ${has_dataplane} Clean Up Linux
@@ -162,12 +245,7 @@
Teardown Suite
[Documentation] Clean up device if desired
- Run Keyword If ${teardown_device} Delete Device and Verify
- ${length}= Run Keyword If ${teardown_device} Test Empty Device List
- Run Keyword If ${teardown_device} Should Be Equal As Integers ${length} 0
- Run Keyword If ${teardown_device} Execute ONOS CLI Command ${k8s_node_ip} ${ONOS_SSH_PORT}
- ... device-remove ${of_id}
-
+ Run Keyword If ${teardown_device} Delete All Devices and Verify
Clean Up Linux
[Documentation] Kill processes and clean up interfaces on src+dst servers
diff --git a/tests/functional/Voltha_ScaleFunctionalTests.robot b/tests/functional/Voltha_ScaleFunctionalTests.robot
index 8ff1396..f60b4dd 100644
--- a/tests/functional/Voltha_ScaleFunctionalTests.robot
+++ b/tests/functional/Voltha_ScaleFunctionalTests.robot
@@ -45,8 +45,7 @@
... re-validate deployment -> Active OLT
[Tags] active
#test for empty device list
- ${length}= Test Empty Device List
- Should Be Equal As Integers ${length} 0
+ Test Empty Device List
#create/preprovision device
${olt_device_id}= Create Device ${olt_ip} ${OLT_PORT}
Set Global Variable ${olt_device_id}
@@ -121,8 +120,7 @@
Run Keyword If ${has_dataplane} Clean Up Linux
Run Keyword If ${external_libs} Log Kubernetes Containers Logs Since Time ${datetime} ${container_list}
Run Keyword If ${teardown_device} Delete Device and Verify
- ${length}= Run Keyword If ${teardown_device} Test Empty Device List
- Run Keyword If ${teardown_device} Should Be Equal As Integers ${length} 0
+ Run Keyword If ${teardown_device} Test Empty Device List
Run Keyword If ${teardown_device} Execute ONOS CLI Command ${k8s_node_ip} ${ONOS_SSH_PORT}
... device-remove ${of_id}
@@ -153,4 +151,4 @@
... ${olt_serial_number}
${rc} ${output}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl device delete ${olt_device_id}
Should Be Equal As Integers ${rc} 0
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device Removed ${olt_device_id}
\ No newline at end of file
+ Wait Until Keyword Succeeds ${timeout} 5s Validate Device Removed ${olt_device_id}