Adding logs to setup and teardown to better debug cleanup issues

Change-Id: I4972fd0057c125f63790a40aa7117db06912ed16
diff --git a/VERSION b/VERSION
index 63cb62f..92e6e77 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.8.14
+2.8.15
diff --git a/libraries/utils.robot b/libraries/utils.robot
index cb775bc..66dd136 100755
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -711,8 +711,11 @@
 
 Teardown Suite
     [Documentation]    Clean up device if desired
+    Start Logging Setup or Teardown  Teardown-${SUITE NAME}
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
+    Collect Logs
     Close All ONOS SSH Connections
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Delete Device and Verify
     [Arguments]    ${olt_serial_number}
@@ -1105,6 +1108,23 @@
     Set Test Variable    ${kail_process}
     Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     START ${label}
 
+Start Logging Setup or Teardown
+    [Arguments]    ${label}
+    [Documentation]    Start logging for suite ${label}
+    ${kail_process}=     Run Keyword If    "${container_log_dir}" != "${None}"   Start Process    kail    -n    ${NAMESPACE}
+    ...    -n    ${INFRA_NAMESPACE}    cwd=${container_log_dir}   stdout=${label}-combined.log
+    Set Suite Variable    ${kail_process}
+    Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     START ${label}
+
+Stop Logging Setup or Teardown
+    [Arguments]    ${label}
+    [Documentation]    End logging for suite;
+    Run    sync
+    Run Keyword If    ${kail_process}    Terminate Process    ${kail_process}
+    ${test_logfile}=    Run Keyword If    "${container_log_dir}" != "${None}"
+    ...    Join Path    ${container_log_dir}    ${label}-combined.log
+    Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     END ${label}
+
 Stop Logging
     [Arguments]    ${label}
     [Documentation]    End logging for test; remove logfile if test passed
diff --git a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
index f17ec63..50bb473 100755
--- a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
@@ -92,6 +92,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    techprofile:${techprofile},
@@ -109,16 +110,21 @@
     Delete MIB Template Data
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    0    ${kvstoreprefix}    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
     Close All ONOS SSH Connections
     Remove Tech Profile
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
diff --git a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
index 78e186e..35d8478 100755
--- a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
@@ -78,6 +78,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite inclusive enable device and sanity test of given workflow
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     Common Test Suite Setup
     ${switch_type}=    Get Variable Value    ${web_power_switch.type}
     Run Keyword If  "${switch_type}"!=""    Set Global Variable    ${powerswitch_type}    ${switch_type}
@@ -86,3 +87,5 @@
     Run Keyword If    "${workflow}"=="DT"    Perform Sanity Test DT
     ...    ELSE IF    "${workflow}"=="TT"    Perform Sanity Tests TT
     ...    ELSE       Perform Sanity Test
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
diff --git a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
index f235048..4de3058 100755
--- a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
@@ -166,6 +166,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    debugmode:${debugmode}, logging:${logging}, pausebeforecleanup:${pausebeforecleanup},
@@ -177,12 +178,15 @@
     Delete MIB Template Data
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -191,7 +195,9 @@
     Run Keyword Unless    ${etcdcheckintestteardown}    Wait Until Keyword Succeeds    ${timeout}    1s
     ...    Validate Onu Data In Etcd    0    ${kvstoreprefix}    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
     Close All ONOS SSH Connections
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
 
 Setup Test
     [Documentation]    Pre-test Setup
diff --git a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
index 9881095..3b24008 100755
--- a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
@@ -182,6 +182,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    state2test:${state2test}, testmode:${testmode}, techprofile:${techprofile},
@@ -215,19 +216,24 @@
     Delete MIB Template Data
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    0    ${kvstoreprefix}    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
     Close All ONOS SSH Connections
     Remove Tech Profile
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
 
 Setup Test
     [Documentation]    Pre-test Setup
diff --git a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
index 9449f5e..7905655 100755
--- a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
@@ -85,6 +85,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    debugmode:${debugmode}, logging:${logging}, pausebeforecleanup:${pausebeforecleanup},
@@ -92,11 +93,14 @@
     Common Test Suite Setup
     # delete etcd MIB Template Data
     Delete MIB Template Data
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -104,7 +108,9 @@
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM
     # delete etcd MIB Template Data (for repeating test)
     Delete MIB Template Data
+    Run Keyword If    ${logging}    Collect Logs
     Close All ONOS SSH Connections
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
 
 Perform ONU MIB Template Data Test
     [Documentation]    This keyword performs ONU MIB Template Data Test