[VOL-2962] Restart ofagent before subscriber provision

Change-Id: Ib15f8197482b39407f8c2f220953de3aefdafcc1
diff --git a/tests/functional/Voltha_FailureScenarios2.robot b/tests/functional/Voltha_FailureScenarios2.robot
index 5c3c3d3..c08f4f8 100644
--- a/tests/functional/Voltha_FailureScenarios2.robot
+++ b/tests/functional/Voltha_FailureScenarios2.robot
@@ -92,6 +92,66 @@
     Run Keyword If    ${has_dataplane}    Clean Up Linux
     Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
 
+Verify restart ofagent container before subscriber is provisioned
+    [Documentation]    Restart ofagent container before subscriber is provisioned.
+    [Tags]    functional   VOL-2962   ofagentRestart
+    [Setup]    Start Logging    ofagentRestart
+    [Teardown]    Run Keywords    Collect Logs
+    ...           AND             Stop Logging    ofagentRestart
+    ...           AND             Scale K8s Deployment    ${NAMESPACE}    voltha-ofagent    1
+    setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    ${of_id}=    Wait Until Keyword Succeeds    ${timeout}    15s    Validate OLT Device in ONOS    ${olt_serial_number}
+    Set Global Variable    ${of_id}
+    FOR    ${I}    IN RANGE    0    ${num_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        ${onu_port}=    Wait Until Keyword Succeeds    ${timeout}    2s    Get ONU Port in ONOS    ${src['onu']}
+        ...    ${of_id}
+        # Bring up the device and verify it authenticates
+        Wait Until Keyword Succeeds    ${timeout}    5s    Validate Device        ENABLED    ACTIVE    REACHABLE
+        ...    ${onu_device_id}    onu=True    onu_reason=omci-flows-pushed
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify Eapol Flows Added For ONU    ${ONOS_SSH_IP}
+        ...    ${ONOS_SSH_PORT}    ${onu_port}
+        ${wpa_log}=    Run Keyword If    ${has_dataplane}    Catenate    SEPARATOR=.
+        ...    /tmp/wpa    ${src['dp_iface_name']}    log
+        Run Keyword If    ${has_dataplane}    Run Keyword And Continue On Failure    Validate Authentication    True
+        ...    ${src['dp_iface_name']}    wpa_supplicant.conf    ${src['ip']}    ${src['user']}    ${src['pass']}
+        ...    ${src['container_type']}    ${src['container_name']}    ${wpa_log}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU in AAA-Users    ${ONOS_SSH_IP}
+        ...    ${ONOS_SSH_PORT}     ${onu_port}
+    END
+    # Restart POD ofagent
+    ${waitforRestart}    Set Variable    120s
+    ${podStatusOutput}=    Run    kubectl get pods -n ${NAMESPACE}
+    Log    ${podStatusOutput}
+    ${countBforRestart}=    Run    kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
+    ${podName}    Set Variable     ofagent
+    Restart Pod    ${NAMESPACE}    ${podName}
+    Sleep    60s
+    Wait Until Keyword Succeeds    ${waitforRestart}    2s    Validate Pod Status    ofagent    ${NAMESPACE}
+    ...    Running
+    FOR    ${I}    IN RANGE    0    ${num_onus}
+        # Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
+        Wait Until Keyword Succeeds    ${timeout}    2s    Execute ONOS CLI Command    ${ONOS_SSH_IP}
+        ...    ${ONOS_SSH_PORT}    volt-add-subscriber-access ${of_id} ${onu_port}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Run Keyword and Ignore Error    Collect Logs
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        ${onu_port}=    Run Keyword And Continue On Failure    Wait Until Keyword Succeeds    ${timeout}    2s
+        ...    Get ONU Port in ONOS    ${src['onu']}    ${of_id}
+	# Verify DHCP-Allocations
+        Run Keyword And Continue On Failure    Wait Until Keyword Succeeds    ${timeout}    2s
+        ...    Validate Subscriber DHCP Allocation    ${ONOS_SSH_IP}    ${ONOS_SSH_PORT}    ${onu_port}
+        # Verify Ping
+        Run Keyword If    ${has_dataplane}    Run Keyword And Continue On Failure    Check Ping    True
+        ...    ${dst['dp_iface_ip_qinq']}    ${src['dp_iface_name']}    ${src['ip']}
+        ...    ${src['user']}    ${src['pass']}    ${src['container_type']}    ${src['container_name']}
+        Run Keyword and Ignore Error    Get Device Output from Voltha    ${onu_device_id}
+        Run Keyword and Ignore Error    Collect Logs
+    END
 
 *** Keywords ***
 Setup Suite