Adding logs to setup and teardown to better debug cleanup issues

Change-Id: I4972fd0057c125f63790a40aa7117db06912ed16
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 544d40c..db2346f 100755
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -712,9 +712,11 @@
 
 Teardown Suite
     [Documentation]    Clean up device if desired
+    Start Logging Setup or Teardown  Teardown-${SUITE NAME}
     Run Keyword If    ${teardown_device}    Delete All Devices and Verify
     Run Keyword And Continue On Failure    Collect Logs
     Close All ONOS SSH Connections
+    Stop Logging Setup or Teardown    Teardown-${SUITE NAME}
 
 Delete Device and Verify
     [Arguments]    ${olt_serial_number}
@@ -1107,6 +1109,23 @@
     Set Test Variable    ${kail_process}
     Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     START ${label}
 
+Start Logging Setup or Teardown
+    [Arguments]    ${label}
+    [Documentation]    Start logging for suite ${label}
+    ${kail_process}=     Run Keyword If    "${container_log_dir}" != "${None}"   Start Process    kail    -n    ${NAMESPACE}
+    ...    -n    ${INFRA_NAMESPACE}    cwd=${container_log_dir}   stdout=${label}-combined.log
+    Set Suite Variable    ${kail_process}
+    Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     START ${label}
+
+Stop Logging Setup or Teardown
+    [Arguments]    ${label}
+    [Documentation]    End logging for suite;
+    Run    sync
+    Run Keyword If    ${kail_process}    Terminate Process    ${kail_process}
+    ${test_logfile}=    Run Keyword If    "${container_log_dir}" != "${None}"
+    ...    Join Path    ${container_log_dir}    ${label}-combined.log
+    Run Keyword If    ${has_dataplane}    Echo Message to OLT Logs     END ${label}
+
 Stop Logging
     [Arguments]    ${label}
     [Documentation]    End logging for test; remove logfile if test passed and ${logging} is set to False
diff --git a/tests/openonu-go-adapter/Voltha_ONUFlowChecks.robot b/tests/openonu-go-adapter/Voltha_ONUFlowChecks.robot
index ad700d2..3858c75 100755
--- a/tests/openonu-go-adapter/Voltha_ONUFlowChecks.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUFlowChecks.robot
@@ -104,6 +104,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite inclusive enable device and sanity test of given workflow
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     Common Test Suite Setup
     ${techprofile}=    Set Variable If    "${techprofile}"=="1T1GEM"    default    ${techprofile}
     Set Suite Variable    ${techprofile}
@@ -118,11 +119,14 @@
     Run Keyword If    "${workflow}"=="DT"    Perform Sanity Test DT
     ...    ELSE IF    "${workflow}"=="TT"    Perform Sanity Tests TT
     ...    ELSE       Perform Sanity Test
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -130,6 +134,8 @@
     Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    ${INFRA_NAMESPACE}    0    ${kvstoreprefix}
     ...    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
     Close All ONOS SSH Connections
     Remove Tech Profile    ${INFRA_NAMESPACE}
 
diff --git a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
index 07a44c9..512c498 100755
--- a/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUNegativeStateTests.robot
@@ -92,6 +92,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    techprofile:${techprofile},
@@ -109,11 +110,14 @@
     Delete MIB Template Data    ${INFRA_NAMESPACE}
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    namespace=${INFRA_NAMESPACE}    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -121,5 +125,7 @@
     Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    ${INFRA_NAMESPACE}    0    ${kvstoreprefix}
     ...    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
     Close All ONOS SSH Connections
     Remove Tech Profile    ${INFRA_NAMESPACE}
diff --git a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
index e401f11..64b6312 100755
--- a/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUOmciGetTest.robot
@@ -78,6 +78,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite inclusive enable device and sanity test of given workflow
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     Common Test Suite Setup
     ${switch_type}=    Get Variable Value    ${web_power_switch.type}
     Run Keyword If  "${switch_type}"!=""    Set Global Variable    ${powerswitch_type}    ${switch_type}
@@ -86,3 +87,5 @@
     Run Keyword If    "${workflow}"=="DT"    Perform Sanity Test DT
     ...    ELSE IF    "${workflow}"=="TT"    Perform Sanity Tests TT
     ...    ELSE       Perform Sanity Test
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
diff --git a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
index 11bfdb1..5a22947 100755
--- a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
@@ -170,6 +170,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    debugmode:${debugmode}, logging:${logging}, pausebeforecleanup:${pausebeforecleanup},
@@ -181,12 +182,15 @@
     Delete MIB Template Data    ${INFRA_NAMESPACE}
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    namespace=${INFRA_NAMESPACE}    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -195,6 +199,8 @@
     Run Keyword Unless    ${etcdcheckintestteardown}    Wait Until Keyword Succeeds    ${timeout}    1s
     ...    Validate Onu Data In Etcd    ${INFRA_NAMESPACE}    0    ${kvstoreprefix}    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
     Close All ONOS SSH Connections
 
 Setup Test
diff --git a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
index 8001c96..f9d4a21 100755
--- a/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUStateTests.robot
@@ -183,6 +183,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    state2test:${state2test}, testmode:${testmode}, techprofile:${techprofile},
@@ -216,11 +217,14 @@
     Delete MIB Template Data    ${INFRA_NAMESPACE}
     # delete etcd onu data
     Delete ONU Go Adapter ETCD Data    namespace=${INFRA_NAMESPACE}    validate=True
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -228,6 +232,8 @@
     Wait Until Keyword Succeeds    ${timeout}    1s    Validate Onu Data In Etcd    ${INFRA_NAMESPACE}    0    ${kvstoreprefix}
     ...    without_pm_data=False
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM    ${timeout}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
     Close All ONOS SSH Connections
     Remove Tech Profile    ${INFRA_NAMESPACE}
 
diff --git a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
index db6f8a4..ca476db 100755
--- a/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUTemplateTests.robot
@@ -85,6 +85,7 @@
 *** Keywords ***
 Setup Suite
     [Documentation]    Set up the test suite
+    Start Logging Setup or Teardown    Setup-${SUITE NAME}
     ${LogInfo}=    Catenate
     ...    \r\nPassed arguments:
     ...    debugmode:${debugmode}, logging:${logging}, pausebeforecleanup:${pausebeforecleanup},
@@ -92,11 +93,14 @@
     Common Test Suite Setup
     # delete etcd MIB Template Data
     Delete MIB Template Data    ${INFRA_NAMESPACE}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown    Setup-${SUITE NAME}
 
 Teardown Suite
     [Documentation]    Replaces the Suite Teardown in utils.robot.
     ...    Cleans up and checks all ONU ports disabled in ONOS.
     ...    Furthermore gives the possibility to pause the execution.
+    Start Logging Setup or Teardown   Teardown-${SUITE NAME}
     Run Keyword If    ${pausebeforecleanup}    Import Library    Dialogs
     Run Keyword If    ${pausebeforecleanup}    Pause Execution    Press OK to continue with clean up!
     Run Keyword If    ${pausebeforecleanup}    Log    Teardown will be continued...    console=yes
@@ -104,6 +108,8 @@
     Wait for Ports in ONOS for all OLTs      ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}  0   BBSM
     # delete etcd MIB Template Data (for repeating test)
     Delete MIB Template Data    ${INFRA_NAMESPACE}
+    Run Keyword If    ${logging}    Collect Logs
+    Stop Logging Setup or Teardown   Teardown-${SUITE NAME}
     Close All ONOS SSH Connections
 
 Perform ONU MIB Template Data Test