Added keyword Clean Up All Nodes to kill stale lxc processes in Suite Teardown

Change-Id: Iac5367cb5eb6158a1897ff05ff2fa1f77a6e3111
diff --git a/Makefile b/Makefile
index 460c005..424045d 100755
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,7 @@
              --configure TooManyTestCases:50 -e TooManyTestCases \
              --configure TooFewTestSteps:1 \
              --configure TooFewKeywordSteps:1 \
-             --configure FileTooLong:1500 -e FileTooLong \
+             --configure FileTooLong:1600 -e FileTooLong \
              -e TrailingWhitespace
 
 PYTHON_FILES := $(wildcard libraries/*.py)
diff --git a/VERSION b/VERSION
index 7ee4924..683d82a 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.8.25
+2.8.26
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 60e197b..43c0520 100755
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -712,9 +712,11 @@
 Teardown Suite
     [Documentation]    Clean up device if desired
     Start Logging Setup or Teardown  Teardown-${SUITE NAME}
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     Collect Logs
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
     Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Delete Device and Verify
@@ -1490,3 +1492,13 @@
     ${result}=    Exec Pod In Kube    ${namespace}    ${podname}    ${commandget}
     Log    ${result}
     Should Be Empty    ${result}    Stale Resource Manager Data in Etcd!
+
+Clean Up All Nodes
+    [Documentation]    Login to each node and kill all stale lxc prcoesses
+    FOR    ${node}    IN    @{nodes}
+        ${node_ip}=    Evaluate    ${node}.get("ip")
+        ${node_user}=    Evaluate    ${node}.get("user")
+        ${node_pass}=    Evaluate    ${node}.get("pass")
+        Run Keyword And Continue On Failure    Start Remote Command    kill -9 `pidof lxc`
+        ...    ${node_ip}    ${node_user}    ${node_pass}
+    END
diff --git a/tests/functional/Voltha_AlarmTests.robot b/tests/functional/Voltha_AlarmTests.robot
index 8973593..4437135 100755
--- a/tests/functional/Voltha_AlarmTests.robot
+++ b/tests/functional/Voltha_AlarmTests.robot
@@ -381,6 +381,7 @@
     Run Keyword If    ${teardown_device}    Execute ONOS CLI Command use single connection    ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}
     ...    device-remove ${of_id}
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
 
 # Onu Alarms
 
diff --git a/tests/functional/Voltha_ErrorScenarios.robot b/tests/functional/Voltha_ErrorScenarios.robot
index a9e53d6..7f51af7 100755
--- a/tests/functional/Voltha_ErrorScenarios.robot
+++ b/tests/functional/Voltha_ErrorScenarios.robot
@@ -332,3 +332,4 @@
     #Restore all ONUs
 #    Run Keyword If    ${has_dataplane}    RestoreONUs    ${num_onus}
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
diff --git a/tests/functional/Voltha_FailureScenarios.robot b/tests/functional/Voltha_FailureScenarios.robot
index e0e8eb7..add8f1b 100755
--- a/tests/functional/Voltha_FailureScenarios.robot
+++ b/tests/functional/Voltha_FailureScenarios.robot
@@ -805,3 +805,4 @@
 Teardown Suite
     [Documentation]    Clean up ONOS SSH connections
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
diff --git a/tests/functional/Voltha_MultiOLT_Tests.robot b/tests/functional/Voltha_MultiOLT_Tests.robot
index c92ba70..fbca7fe 100755
--- a/tests/functional/Voltha_MultiOLT_Tests.robot
+++ b/tests/functional/Voltha_MultiOLT_Tests.robot
@@ -191,3 +191,4 @@
 Teardown Suite
     [Documentation]    Clean up ONOS SSH connections
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
diff --git a/tests/functional/Voltha_ScaleFunctionalTests.robot b/tests/functional/Voltha_ScaleFunctionalTests.robot
index 58e6a8e..709336e 100755
--- a/tests/functional/Voltha_ScaleFunctionalTests.robot
+++ b/tests/functional/Voltha_ScaleFunctionalTests.robot
@@ -196,6 +196,7 @@
     Run Keyword If    ${teardown_device}    Execute ONOS CLI Command use single connection    ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}
     ...    device-remove ${of_id}
     Close All ONOS SSH Connections
+    Run Keyword If    ${has_dataplane}    Clean Up All Nodes
 
 Clean Up Linux
     [Documentation]    Kill processes and clean up interfaces on src+dst servers
diff --git a/tests/tt-workflow/Voltha_TT_FailureScenarios.robot b/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
index 70a76e4..52e26c0 100755
--- a/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
+++ b/tests/tt-workflow/Voltha_TT_FailureScenarios.robot
@@ -481,12 +481,6 @@
     ${switch_type}=    Get Variable Value    ${web_power_switch.type}
     Run Keyword If  "${switch_type}"!=""    Set Global Variable    ${powerswitch_type}    ${switch_type}
 
-Teardown Suite
-    [Documentation]    Tear down steps for the suite
-    Run Keyword If    ${has_dataplane}    Clean Up Linux
-    Run Keyword If    ${teardown_device}    Delete All Devices and Verify
-    Close All ONOS SSH Connections
-
 Clear All Devices Then Create New Device
     [Documentation]    Remove any devices from VOLTHA and ONOS & then Create new devices
     # Remove all devices from voltha and onos
diff --git a/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot b/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
index 5123814..f455726 100755
--- a/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
+++ b/tests/tt-workflow/Voltha_TT_MultiTcontTests.robot
@@ -578,10 +578,3 @@
     Delete All Devices and Verify
     # Execute normal test Setup Keyword
     Setup
-
-
-Teardown Suite
-    [Documentation]    Tear down steps for the suite
-    Run Keyword If    ${has_dataplane}    Clean Up Linux
-    Run Keyword If    ${teardown_device}    Delete All Devices And Verify
-    Close All ONOS SSH Connections