OpenOnuAdapter: Backport SW-Upgrade tests

Change-Id: Ice6c804f60722ae70cb09b358db22f43ea54a30c
diff --git a/Makefile b/Makefile
index ca96dc3..460c005 100755
--- a/Makefile
+++ b/Makefile
@@ -496,6 +496,13 @@
 onu-upgrade-test: ROBOT_CONFIG_FILE := $(ROBOT_SW_UPGRADE_FILE)
 onu-upgrade-test: software-upgrade-test
 
+# Requirement: Pass ONU image details in following parameters
+# image_version, image_url, image_vendor, image_activate_on_success, image_commit_on_success, image_crc
+onu-upgrade-test-multiolt-kind-att: ROBOT_MISC_ARGS +=  -e notready -i functionalMultipleONUs
+onu-upgrade-test-multiolt-kind-att: ROBOT_FILE := ONU_Upgrade.robot
+onu-upgrade-test-multiolt-kind-att: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_MULTIPLE_OLT_FILE)
+onu-upgrade-test-multiolt-kind-att: software-upgrade-test
+
 software-upgrade-test: vst_venv
 	source ./$</bin/activate ; set -u ;\
 	cd tests/software-upgrades ;\
diff --git a/VERSION b/VERSION
index 6c4bbe4..36b69a6 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.8.21
+2.8.22
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index fc5e0e7..5b0029a 100755
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -53,14 +53,17 @@
 
 Restart Pod By Label
     [Arguments]    ${namespace}    ${label_key}   ${label_value}
-    [Documentation]    Uses kubectl to force delete pod
+    [Documentation]    Uses kubectl to force delete pod(s)
     ${rc}    ${restart_pod_name}=    Run and Return Rc and Output
     ...    kubectl get pods -n ${namespace} -l ${label_key}=${label_value} --no-headers | awk '{print $1}'
     Log    ${restart_pod_name}
     Should Not Be Empty    ${restart_pod_name}    Unable to parse pod name
-    ${rc}    ${output}=    Run and Return Rc and Output
-    ...    kubectl delete pod ${restart_pod_name} -n ${namespace} --grace-period=0 --force
-    Log    ${output}
+    @{pods}=    Split String    ${restart_pod_name}    separator=${\n}
+    FOR    ${pod_name}    IN      @{pods}
+        ${rc}    ${output}=    Run and Return Rc and Output
+        ...    kubectl delete pod ${pod_name} -n ${namespace} --grace-period=0 --force
+        Log    ${output}
+    END
 
 Exec Pod
     [Arguments]    ${namespace}    ${name}    ${command}
diff --git a/libraries/voltctl.robot b/libraries/voltctl.robot
index e389641..f710aa9 100755
--- a/libraries/voltctl.robot
+++ b/libraries/voltctl.robot
@@ -702,6 +702,18 @@
     ...   Append To List    ${serial_numbers}    ${hosts.src[${INDEX}].onu}
     END
 
+Build ONU Device Id List
+    [Arguments]    ${device_ids}    ${olt_serial_number}=${EMPTY}    ${num_onus}=${num_all_onus}
+    [Documentation]    Appends all ONU Device IDs for the given OLT to the ${serial_numbers} list
+    FOR    ${INDEX}    IN RANGE    0    ${num_onus}
+        ${onu_device_id}=    Get Device ID From SN    ${hosts.src[${INDEX}]['onu']}
+        # skip if we have already append this ONU
+        ${onu_id}=    Get Index From List    ${device_ids}   ${onu_device_id}
+        Continue For Loop If    -1 != ${onu_id}
+        Run Keyword IF  "${olt_serial_number}"=="${hosts.src[${INDEX}].olt}" or "${olt_serial_number}"=="${EMPTY}"
+        ...   Append To List    ${device_ids}    ${onu_device_id}
+    END
+
 Get SN From Device ID
     [Arguments]    ${device_id}
     [Documentation]    Gets the device id by matching for ${device_id}
@@ -832,34 +844,36 @@
     [Arguments]    ${data}    ${image_version}    ${dev_id}    ${download_state}    ${expected_reason}    ${image_status}
     ${jsondata}=    To Json    ${data}
     ${length}=    Get Length    ${jsondata}
-    Should Be Equal As Integers    ${length}    1    No record found for ${dev_id} to validate device image
-    ${value}=    Get From List    ${jsondata}    0
-    Log    ${value}
-    ${deviceId}=    Get From Dictionary    ${value}    deviceId
-    Should Be Equal    '${deviceId}'    '${dev_id}'    No match found for ${dev_id} to validate device image
-    ...    values=False
-    ${imageState}=    Get From Dictionary    ${value}    imageState
-    ${version}=    Get From Dictionary    ${imageState}    version
-    ${downloadState}=    Get From Dictionary    ${imageState}    downloadState
-    ${reason}=    Get From Dictionary    ${imageState}    reason
-    ${imageStatus}=    Get From Dictionary    ${imageState}    imageState
-    Should Be Equal    '${version}'    '${image_version}'    Device ${dev_id}: '${version}' != '${image_version}'
-    ...    values=False
-    Should Be Equal    '${downloadState}'    '${download_state}'    Device ${dev_id}: '${downloadState}' != '${download_state}'
-    ...    values=False
-    Should Be Equal    '${reason}'    '${expected_reason}'    Device ${dev_id}: '${reason}' != '${expected_reason}'
-    ...    values=False
-    Should Be Equal    '${imageStatus}'    '${image_status}'    Device ${dev_id}: '${imageStatus}' != '${image_status}'
-    ...    values=False
+    Should Not Be Equal As Integers    ${length}    0    No record to validate device image
+    FOR    ${J}    IN RANGE    0    ${length}
+        ${value}=    Get From List    ${jsondata}    ${J}
+        Log    ${value}
+        ${deviceId}=    Get From Dictionary    ${value}    deviceId
+        Should Contain    ${dev_id}    ${deviceId}    No match found for ${deviceId} to validate device image!
+        ${imageState}=    Get From Dictionary    ${value}    imageState
+        ${version}=    Get From Dictionary    ${imageState}    version
+        ${dwlState}=    Get From Dictionary    ${imageState}    downloadState
+        ${reason}=    Get From Dictionary    ${imageState}    reason
+        ${imgStatus}=    Get From Dictionary    ${imageState}    imageState
+        Should Be Equal    '${version}'    '${image_version}'    Device ${deviceId}: '${version}' != '${image_version}'
+        ...    values=False
+        Should Be Equal    '${dwlState}'    '${download_state}'    Device ${deviceId}: '${dwlState}' != '${download_state}'
+        ...    values=False
+        Should Be Equal    '${reason}'    '${expected_reason}'    Device ${deviceId}: '${reason}' != '${expected_reason}'
+        ...    values=False
+        Should Be Equal    '${imgStatus}'    '${image_status}'    Device ${deviceId}: '${imgStatus}' != '${image_status}'
+        ...    values=False
+    END
 
 Download ONU Device Image
     [Documentation]    Downloads the given ONU software image
     [Arguments]    ${ver}    ${url}    ${vendor}    ${active}    ${commit}    ${crc}    ${id}
+    ...            ${download_state}=DOWNLOAD_STARTED    ${expected_reason}=NO_ERROR    ${image_state}=IMAGE_UNKNOWN
     ${rc}    ${output}=    Run and Return Rc and Output
     ...    voltctl -c ${VOLTCTL_CONFIG} device onuimage download ${ver} ${url} ${vendor} ${active} ${commit} ${crc} ${id} -o json
     Log    ${output}
     Should Be Equal As Integers    ${rc}    0
-    Validate ONU Device Image    ${output}    ${ver}    ${id}    DOWNLOAD_STARTED    NO_ERROR    IMAGE_INACTIVE
+    Validate ONU Device Image    ${output}    ${ver}    ${id}    ${download_state}    ${expected_reason}    ${image_state}
 
 Activate ONU Device Image
     [Documentation]    Activates the given ONU software image
@@ -879,6 +893,30 @@
     Should Be Equal As Integers    ${rc}    0
     Validate ONU Device Image    ${output}    ${ver}    ${id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_COMMITTING
 
+Abort ONU Device Image
+    [Documentation]    Aborts the given ONU software image
+    [Arguments]    ${ver}    ${id}    ${download_state}    ${expected_reason}    ${image_state}
+    ${rc}    ${output}=    Run and Return Rc and Output
+    ...    voltctl -c ${VOLTCTL_CONFIG} device onuimage abort ${ver} ${id} -o json
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+    Validate ONU Device Image    ${output}    ${ver}    ${id}    ${download_state}    ${expected_reason}    ${image_state}
+
+Remove Adapter Image
+    [Documentation]    Aborts the upgrade processing for a given ONU software image and thus removes the image from adapter
+    [Arguments]    ${ver}    ${id}
+    ${rc}    ${output}=    Run and Return Rc and Output
+    ...    voltctl -c ${VOLTCTL_CONFIG} device onuimage abort ${ver} ${id} -o json
+    Log    ${output}
+
+Remove Adapter Image from ONUs
+    [Documentation]    Aborts the upgrade processing for a given ONU software image and thus removes the image from adapter
+    ...                for the passed ONUs
+    [Arguments]    ${ver}    ${onu_id_list}
+    FOR  ${onu_device_id}  IN  @{onu_id_list}
+        Remove Adapter Image    ${ver}    ${onu_device_id}
+    END
+
 Verify ONU Device Image Status
     [Documentation]    Verfies the ONU device image state
     [Arguments]    ${image_version}    ${dev_id}    ${download_state}    ${expected_reason}    ${image_state}
@@ -891,7 +929,7 @@
 
 Verify ONU Device Image List
     [Documentation]    Verfies the ONU device image list
-    [Arguments]    ${dev_id}    ${image_version}    ${committed}    ${activated}    ${valid}
+    [Arguments]    ${dev_id}    ${image_version}    ${committed}    ${activated}    ${valid}    ${image_should_not_in_list}=False
     ${rc}    ${output}=    Run and Return Rc and Output
     ...    voltctl -c ${VOLTCTL_CONFIG} device onuimage list ${dev_id} -o json
     Should Be Equal As Integers    ${rc}    0
@@ -908,13 +946,15 @@
         ${matched}=    Set Variable If    '${version}' == '${image_version}'    True    False
         Exit For Loop If    ${matched}
     END
-    Should Be True    ${matched}     No ONU Image found with Version ${image_version}
-    Should Be Equal    '${isCommited}'    '${committed}'    Device ${dev_id}: '${isCommited}' != '${committed}'
-    ...    values=False
-    Should Be Equal    '${isActive}'    '${activated}'    Device ${dev_id}: '${isActive}' != '${activated}'
-    ...    values=False
-    Should Be Equal    '${isValid}'    '${valid}'    Device ${dev_id}: '${isValid}' != '${valid}'
-    ...    values=False
+    Run Keyword If    ${image_should_not_in_list}
+    ...            Should Not Be True    ${matched}    ONU Image found unexpectedly with Version ${image_version}
+    ...    ELSE    Should Be True    ${matched}    No ONU Image found with Version ${image_version}
+    Run Keyword If    ${matched}    Should Be Equal    '${isCommited}'    '${committed}'
+    ...    Device ${dev_id}: '${isCommited}' != '${committed}'    values=False
+    Run Keyword If    ${matched}    Should Be Equal    '${isActive}'    '${activated}'
+    ...    Device ${dev_id}: '${isActive}' != '${activated}'    values=False
+    Run Keyword If    ${matched}    Should Be Equal    '${isValid}'    '${valid}'
+    ...    Device ${dev_id}: '${isValid}' != '${valid}'    values=False
 
 # pm-data relevant keywords
 Read Default Interval From Pmconfig
diff --git a/tests/software-upgrades/ONU_Upgrade.robot b/tests/software-upgrades/ONU_Upgrade.robot
index 5bc5961..baefbb9 100755
--- a/tests/software-upgrades/ONU_Upgrade.robot
+++ b/tests/software-upgrades/ONU_Upgrade.robot
@@ -76,6 +76,20 @@
 ${image_crc}    ${EMPTY}
 # Example value: 0
 
+# when voltha is running in k8s port forwarding is needed
+# example: -v PORT_FORWARDING:False
+${PORT_FORWARDING}    True
+
+# Next values are default values for port forward, do not need to be passed, will be overwritten by values taken from image-url
+# bbsim webserver port
+# example: -v BBSIM_WEBSERVER_PORT:50074
+${BBSIM_WEBSERVER_PORT}    50074
+# bbsim instance
+# example: -v BBSIM_INSTANCE:bbsim0
+${BBSIM_INSTANCE}    bbsim0
+# port forward handle
+${portFwdHandle}     None
+
 *** Test Cases ***
 Test ONU Upgrade
     [Documentation]    Validates the ONU Upgrade doesn't affect the system functionality
@@ -87,8 +101,47 @@
     [Tags]    functional   ONUUpgrade
     [Setup]    Start Logging    ONUUpgrade
     [Teardown]    Run Keywords    Collect Logs
-    ...           AND             Stop Logging    ONUUpgrade
     ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgrade
+    Do ONU Upgrade All OLTs
+
+Test ONU Upgrade All Activate and Commit Combinations
+    [Documentation]    Validates the ONU Upgrade doesn't affect the system functionality by use all combinations of
+    ...    flags activate_on_success and commit_on_success
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Requirement: Pass image details in following parameters in the robot command
+    ...    onu_image_name, onu_image_url, onu_image_version, onu_image_crc, onu_image_local_dir
+    ...    Note: The TC expects the image url and other parameters to be common for all ONUs on all BBSim
+    ...    Check [VOL-4250] for more details
+    [Tags]    functional   ONUUpgradeAllCombies
+    [Setup]    Start Logging    ONUUpgradeAllCombies
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeAllCombies
+    ${false_false}=    Create Dictionary    activate    false    commit    false
+    ${true_false}=     Create Dictionary    activate    true     commit    false
+    ${false_true}=     Create Dictionary    activate    false    commit    true
+    ${true_true}=      Create Dictionary    activate    true     commit    true
+    ${flag_list}=    Create List    ${false_false}     ${true_false}    ${false_true}    ${true_true}
+    FOR    ${item}     IN      @{flag_list}
+        Do ONU Upgrade All OLTs    ${item['activate']}    ${item['commit']}
+        Delete All Devices and Verify
+        Restart And Check BBSIM    ${NAMESPACE}
+    END
+
+Test ONU Upgrade Correct Indication of Download Wrong Url
+    [Documentation]    Validates the ONU Upgrade download from wrong URL failure will be indicated correctly
+    ...    and doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4257] for more details
+    [Tags]    functional   ONUUpgradeDownloadWrongUrl
+    [Setup]    Start Logging    ONUUpgradeDownloadWrongUrl
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeDownloadWrongUrl
     # Add OLT device
     Setup
     # Performing Sanity Test to make sure subscribers are all DHCP and pingable
@@ -98,7 +151,10 @@
         ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
         ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
         ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
-        Test ONU Upgrade Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        Do ONU Upgrade Download Failure Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        ...    url=http://bbsim0:50074/images/wrong-image.img$
+        ...    dwl_dwlstate=DOWNLOAD_FAILED    dwl_reason=INVALID_URL    dwl_imgstate=IMAGE_UNKNOWN
+        ...        dwlstate=DOWNLOAD_UNKNOWN       reason=NO_ERROR           imgstate=IMAGE_UNKNOWN
         List ONUs    ${NAMESPACE}    ${bbsim_pod}
     END
     # Additional Verification
@@ -107,10 +163,293 @@
     Run Keyword If    ${has_dataplane}    Clean Up Linux
     Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
 
+Test ONU Upgrade Correct Indication of Download Failure
+    [Documentation]    Validates the ONU Upgrade download failure will be indicated correctly and
+    ...    doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-3935] for more details
+    [Tags]    functional   ONUUpgradeDownloadFailure
+    [Setup]    Start Logging    ONUUpgradeDownloadFailure
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeDownloadFailure
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Download Failure Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        ...    dwlstate=DOWNLOAD_FAILED       reason=CANCELLED_ON_ONU_STATE           imgstate=IMAGE_UNKNOWN
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication of Downloading Abort
+    [Documentation]    Validates the ONU Upgrade downloading abort will be indicated correctly and
+    ...    doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4318] for more details
+    [Tags]    functional   ONUUpgradeDownloadingAbort
+    [Setup]    Start Logging    ONUUpgradeDownloadingAbort
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeDownloadingAbort
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Downloading Abort Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication of Downloaded Abort
+    [Documentation]    Validates the ONU Upgrade downloaded abort will be indicated correctly and
+    ...    doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4320] for more details
+    [Tags]    functional   ONUUpgradeDownloadedAbort
+    [Setup]    Start Logging    ONUUpgradeDownloadedAbort
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeDownloadedAbort
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Downloaded Abort Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication of Activating Abort
+    [Documentation]    Validates the ONU Upgrade activating abort will be indicated correctly and
+    ...    doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4319] for more details
+    [Tags]    functional   ONUUpgradeActivatingAbort
+    [Setup]    Start Logging    ONUUpgradeActivatingAbort
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeActivatingAbort
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Activating Abort Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication of Active Abort
+    [Documentation]    Validates the ONU Upgrade active abort will be indicated correctly and
+    ...    doesn't affect the system functionality
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4320] for more details
+    [Tags]    functional   ONUUpgradeActiveAbort
+    [Setup]    Start Logging    ONUUpgradeActiveAbort
+    [Teardown]    Run Keywords    Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeActiveAbort
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Active Abort Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication of Committed Abort
+    [Documentation]    Validates the ONU Upgrade committed abort will be indicated correctly and
+    ...    doesn't affect the system functionality. Check BBSIM server transfer image counter after further download,
+    ...    counter should be incremented.
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4320] for more details
+    [Tags]    functional   ONUUpgradeCommittedAbort
+    [Setup]    Start Logging    ONUUpgradeCommittedAbort
+    [Teardown]    Run Keywords    Run Keyword If    ${portFwdHandle}!=None    Terminate Process    ${portFwdHandle}    kill=true
+    ...           AND             Set Suite Variable   ${portFwdHandle}    None
+    ...           AND             Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeCommittedAbort
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    # in case of bbsim is running in k8s port forwarding of WEBSERVER is needed
+    ${BBSIM_INSTANCE}   ${BBSIM_WEBSERVER_PORT}=   Run Keyword If   '${image_url}'!='${EMPTY}'   Get BBSIM Svc and Webserver Port
+    ${cmd}    Catenate
+    ...       kubectl port-forward --address 0.0.0.0 -n ${NAMESPACE} svc/${BBSIM_INSTANCE}
+    ...       ${BBSIM_WEBSERVER_PORT}:${BBSIM_WEBSERVER_PORT} &
+    ${portFwdHandle}=    Run Keyword If    ${PORT_FORWARDING}    Start Process   ${cmd}    shell=true
+    Set Suite Variable   ${portFwdHandle}
+    Sleep    5s
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Committed Abort Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Correct Indication Multiple Image Download
+    [Documentation]    Validates the ONU Upgrade multiple Image Download will be indicated correctly,
+    ...    In case of (re-) download the same image without aborting the cached one in openonu-adapter should taken,
+    ...    no further download from server has executed. Check BBSIM server transfer image counter after further download,
+    ...    counter should NOT be incremented.
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4320] for more details
+    [Tags]    functional   ONUUpgradeMultipleImageDownload
+    [Setup]    Start Logging    ONUUpgradeMultipleImageDownload
+    [Teardown]    Run Keywords    Run Keyword If    ${portFwdHandle}!=None    Terminate Process    ${portFwdHandle}    kill=true
+    ...           AND             Set Suite Variable   ${portFwdHandle}    None
+    ...           AND             Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeMultipleImageDownload
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    # in case of bbsim is running in k8s port forwarding of WEBSERVER is needed
+    ${BBSIM_INSTANCE}   ${BBSIM_WEBSERVER_PORT}=   Run Keyword If   '${image_url}'!='${EMPTY}'   Get BBSIM Svc and Webserver Port
+    ${cmd}    Catenate
+    ...       kubectl port-forward --address 0.0.0.0 -n ${NAMESPACE} svc/${BBSIM_INSTANCE}
+    ...       ${BBSIM_WEBSERVER_PORT}:${BBSIM_WEBSERVER_PORT} &
+    ${portFwdHandle}=    Run Keyword If    ${PORT_FORWARDING}    Start Process   ${cmd}    shell=true
+    Set Suite Variable   ${portFwdHandle}
+    Sleep    5s
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Multiple Image Download Per OLT    ${bbsim_pod}    ${olt_serial_number}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Test ONU Upgrade Image Download Simultaneously
+    [Documentation]    Validates the ONU Upgrade Image Download to all ONUs simultaneously.
+    ...    Test case should executed in multiple ONU (OLT) environment!
+    ...    Performs the sanity and verifies all the ONUs are authenticated/DHCP/pingable
+    ...    Check [VOL-4320] for more details
+    [Tags]    functionalMultipleONUs   ONUUpgradeImageDownloadSimultaneously
+    [Setup]    Start Logging    ONUUpgradeImageDownloadSimultaneously
+    [Teardown]    Run Keywords    Run Keyword If    ${portFwdHandle}!=None    Terminate Process    ${portFwdHandle}    kill=true
+    ...           AND             Set Suite Variable   ${portFwdHandle}    None
+    ...           AND             Run Keyword If    ${logging}    Collect Logs
+    ...           AND             Delete All Devices and Verify
+    ...           AND             Restart And Check BBSIM    ${NAMESPACE}
+    ...           AND             Stop Logging    ONUUpgradeImageDownloadSimultaneously
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    # in case of bbsim is running in k8s port forwarding of WEBSERVER is needed
+    ${BBSIM_INSTANCE}   ${BBSIM_WEBSERVER_PORT}=   Run Keyword If   '${image_url}'!='${EMPTY}'   Get BBSIM Svc and Webserver Port
+    ${cmd}    Catenate
+    ...       kubectl port-forward --address 0.0.0.0 -n ${NAMESPACE} svc/${BBSIM_INSTANCE}
+    ...       ${BBSIM_WEBSERVER_PORT}:${BBSIM_WEBSERVER_PORT} &
+    ${portFwdHandle}=    Run Keyword If    ${PORT_FORWARDING}    Start Process   ${cmd}    shell=true
+    Set Suite Variable   ${portFwdHandle}
+    Sleep    5s
+    Do ONU Upgrade Image Download Simultaneously
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
 *** Keywords ***
-Test ONU Upgrade Per OLT
+Do ONU Upgrade All OLTs
+    [Documentation]    This keyword performs the ONU Upgrade test on all OLTs
+    [Arguments]    ${activate_on_success}=${image_activate_on_success}    ${commit_on_success}=${image_commit_on_success}
+    # Add OLT device
+    Setup
+    # Performing Sanity Test to make sure subscribers are all DHCP and pingable
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Do ONU Upgrade Per OLT    ${bbsim_pod}    ${olt_serial_number}    ${activate_on_success}    ${commit_on_success}
+        List ONUs    ${NAMESPACE}    ${bbsim_pod}
+    END
+    # Additional Verification
+    Wait Until Keyword Succeeds    ${timeout}    2s    Delete All Devices and Verify
+    Setup
+    Run Keyword If    ${has_dataplane}    Clean Up Linux
+    Wait Until Keyword Succeeds    ${timeout}    2s    Perform Sanity Test
+
+Do ONU Upgrade Per OLT
     [Documentation]    This keyword performs the ONU Upgrade test on single OLT
-    [Arguments]    ${bbsim_pod}    ${olt_serial_number}
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}   ${activate_on_success}=${image_activate_on_success}
+    ...            ${commit_on_success}=${image_commit_on_success}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
     FOR    ${I}    IN RANGE    0    ${num_all_onus}
         ${src}=    Set Variable    ${hosts.src[${I}]}
         ${dst}=    Set Variable    ${hosts.dst[${I}]}
@@ -118,30 +457,31 @@
         ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
         # Download Image
         Download ONU Device Image    ${image_version}    ${image_url}    ${image_vendor}
-        ...    ${image_activate_on_success}    ${image_commit_on_success}
+        ...    ${activate_on_success}    ${commit_on_success}
         ...    ${image_crc}    ${onu_device_id}
-        ${imageState}=    Run Keyword If    '${image_activate_on_success}'=='true'    Set Variable    IMAGE_ACTIVE
-        ...    ELSE IF    '${image_activate_on_success}'=='true' and '${image_commit_on_success}'=='true'
+        ${imageState}=    Run Keyword If    '${activate_on_success}'=='true' and '${commit_on_success}'=='false'
+        ...    Set Variable    IMAGE_ACTIVE
+        ...    ELSE IF    '${activate_on_success}'=='true' and '${commit_on_success}'=='true'
         ...    Set Variable    IMAGE_COMMITTED
         ...    ELSE    Set Variable    IMAGE_INACTIVE
-        ${activated}=    Set Variable If    '${image_activate_on_success}'=='true'    True    False
-        ${committed}=    Set Variable If    '${image_activate_on_success}'=='true' and '${image_commit_on_success}'=='true'
+        ${activated}=    Set Variable If    '${activate_on_success}'=='true'    True    False
+        ${committed}=    Set Variable If    '${activate_on_success}'=='true' and '${commit_on_success}'=='true'
         ...    True    False
         Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
         ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    ${imageState}
         Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
         ...    ${image_version}    ${committed}    ${activated}    True
         # Activate Image
-        ${imageState}=    Set Variable If    '${image_commit_on_success}'=='true'    IMAGE_COMMITTED    IMAGE_ACTIVE
-        ${committed}=    Set Variable If    '${image_commit_on_success}'=='true'    True    False
-        Run Keyword If    '${image_activate_on_success}'=='false'    Run Keywords
-        ...    Activate ONU Device Image    ${image_version}    ${image_commit_on_success}    ${onu_device_id}
+        ${imageState}=    Set Variable If    '${commit_on_success}'=='true'    IMAGE_COMMITTED    IMAGE_ACTIVE
+        ${committed}=    Set Variable If    '${commit_on_success}'=='true'    True    False
+        Run Keyword If    '${activate_on_success}'=='false'    Run Keywords
+        ...    Activate ONU Device Image    ${image_version}    ${commit_on_success}    ${onu_device_id}
         ...    AND    Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
         ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    ${imageState}
         ...    AND    Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
         ...    ${image_version}    ${committed}    True    True
         # Commit Image
-        Run Keyword If    '${image_commit_on_success}'=='false'    Run Keywords
+        Run Keyword If    '${commit_on_success}'=='false'    Run Keywords
         ...    Commit ONU Device Image    ${image_version}    ${onu_device_id}
         ...    AND    Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
         ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_COMMITTED
@@ -150,8 +490,277 @@
         Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image On BBSim    ${NAMESPACE}    ${bbsim_pod}
         ...    ${src['onu']}    software_image_committed
         Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+        # to remove the image again from adapter, Multi-Onu tests could be restructured - not yet in focus
+        Remove Adapter Image    ${image_version}    ${onu_device_id}
     END
 
+Do ONU Upgrade Download Failure Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Dowload Failure test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    ...            ${dwl_dwlstate}=DOWNLOAD_STARTED    ${dwl_reason}=NO_ERROR    ${dwl_imgstate}=IMAGE_UNKNOWN
+    ...            ${dwlstate}=DOWNLOAD_SUCCEEDED    ${reason}=NO_ERROR    ${imgstate}=IMAGE_INACTIVE
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    INVALID_IMAGE    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        # Download Image
+        Download ONU Device Image    INVALID_IMAGE    ${url}    ${image_vendor}
+        ...    ${image_activate_on_success}    ${image_commit_on_success}    ${image_crc}    ${onu_device_id}
+        ...    download_state=${dwl_dwlstate}    expected_reason=${dwl_reason}    image_state=${dwl_imgstate}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    INVALID_IMAGE
+        ...    ${onu_device_id}    ${dwlstate}    ${reason}    ${imgstate}
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+        # to remove the image again from adapter, Multi-Onu tests could be restructured - not yet in focus
+        Remove Adapter Image    INVALID_IMAGE    ${onu_device_id}
+    END
+
+Do ONU Upgrade Downloading Abort Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Downloading Abort test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    ${image_activate_on_success}    ${image_commit_on_success}    ${image_crc}    ${onu_device_id}
+        Abort ONU Device Image    ${image_version}    ${onu_device_id}
+        ...    DOWNLOAD_STARTED    CANCELLED_ON_REQUEST    IMAGE_DOWNLOADING
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_CANCELLED    CANCELLED_ON_REQUEST    IMAGE_UNKNOWN
+        #   !!!    Expected is image is not visible in list   !!!
+#       Uncomment next lines in case of backporting of VOL-4302 (BBSIM)
+#        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+#        ...    ${image_version}    False    False    False    image_should_not_in_list=True
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+    END
+
+Do ONU Upgrade Downloaded Abort Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Downloaded Abort test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_INACTIVE
+        Abort ONU Device Image    ${image_version}    ${onu_device_id}
+        ...    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_INACTIVE
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_INACTIVE
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+        ...    ${image_version}    False    False    True
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+    END
+
+Do ONU Upgrade Activating Abort Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Activating Abort test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_INACTIVE
+        Activate ONU Device Image    ${image_version}    false    ${onu_device_id}
+        Abort ONU Device Image    ${image_version}    ${onu_device_id}
+        ...    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_ACTIVATING
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_ACTIVATION_ABORTED
+#       Uncomment next lines in case of backporting of VOL-4302 (BBSIM)
+#        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+#        ...    ${image_version}    False    True    True
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+    END
+
+Do ONU Upgrade Active Abort Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Active Abort test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_INACTIVE
+        Activate ONU Device Image    ${image_version}    false    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_ACTIVE
+        Abort ONU Device Image    ${image_version}    ${onu_device_id}
+        ...    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_ACTIVE
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    CANCELLED_ON_REQUEST    IMAGE_ACTIVE
+#       Uncomment next lines in case of backporting of VOL-4302 (BBSIM)
+#        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+#        ...    ${image_version}    False    True    True
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+    END
+
+Do ONU Upgrade Committed Abort Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Committed Abort test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+#       Uncomment next commented lines in case of backporting HTML request in BBSIM
+#        ${Images_Count_Start}=    Get Images Count
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+        # After download of image, check image counter of BBSIM, has to be incremented by 2, because bbsim increments counter
+        # whenever openonu adapter touch the image, so one increment for check image is available and one for downloading
+#        ${Images_Count_First}=    Get Images Count
+#        ${Images_Count_Start}=    Evaluate    ${Images_Count_Start}+2
+#        Should Be Equal as Integers    ${Images_Count_First}    ${Images_Count_Start}    Count of image download not correct!
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_INACTIVE
+        Activate ONU Device Image    ${image_version}    false    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_ACTIVE
+        Commit ONU Device Image    ${image_version}    ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_COMMITTED
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+        ...    ${image_version}    True    True    True
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image On BBSim    ${NAMESPACE}    ${bbsim_pod}
+        ...    ${src['onu']}    software_image_committed
+        Abort ONU Device Image    ${image_version}    ${onu_device_id}
+        ...    DOWNLOAD_UNKNOWN    NO_ERROR    IMAGE_UNKNOWN
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_UNKNOWN    NO_ERROR    IMAGE_UNKNOWN
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+        ...    ${image_version}    True    True    True
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+#        ${Images_Count_Intermediate}=    Get Images Count
+        # Repeat download of aborted image, check image counter of BBSIM, has to be incremented by 2, because bbsim increments
+        # whenever openonu adapter touch the image, so one increment for check image is available and one for downloading
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+#        ${Images_Count_End}=    Get Images Count
+#        ${Images_Count_Intermediate}=    Evaluate    ${Images_Count_Intermediate}+2
+#        Should Be Equal as Integers    ${Images_Count_End}    ${Images_Count_Intermediate}   Count of image download not correct!
+        Remove Adapter Image    ${image_version}    ${onu_device_id}
+    END
+
+Do ONU Upgrade Multiple Image Download Per OLT
+    [Documentation]    This keyword performs the ONU Upgrade Image Download test on single OLT
+    [Arguments]    ${bbsim_pod}    ${olt_serial_number}    ${url}=${image_url}
+    [Teardown]     Run Keyword If  '${KEYWORD STATUS}'=='FAIL'    Remove Adapter Image    ${image_version}    ${onu_device_id}
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${dst}=    Set Variable    ${hosts.dst[${I}]}
+        Continue For Loop If    "${olt_serial_number}"!="${src['olt']}"
+        ${onu_device_id}=    Get Device ID From SN    ${src['onu']}
+#       Uncomment next commented lines in case of backporting HTML request in BBSIM
+#        ${Images_Count_Start}=    Get Images Count
+        # Download Image
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    true    true    ${image_crc}    ${onu_device_id}
+        # After download of image, check image counter of BBSIM, has to be incremented by 2, because bbsim increments counter
+        # whenever openonu adapter touch the image, so one increment for check image is available and one for downloading
+#        ${Images_Count_First}=    Get Images Count
+#        ${Images_Count_Start}=    Evaluate    ${Images_Count_Start}+2
+#        Should Be Equal as Integers    ${Images_Count_First}    ${Images_Count_Start}    Count of image download not correct!
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_COMMITTED
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+        ...    ${image_version}    True    True    True
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image On BBSim    ${NAMESPACE}    ${bbsim_pod}
+        ...    ${src['onu']}    software_image_committed
+        Wait Until Keyword Succeeds    ${timeout}    5s    Perform Sanity Test     ${suppressaddsubscriber}
+#        ${Images_Count_Intermediate}=    Get Images Count
+        # Repeat download of same image, check image counter of BBSIM, has to be not incremented, because no download from
+        # server will be executed, the cached one from openonu-adapter will taken
+        Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+        ...    false    false    ${image_crc}    ${onu_device_id}
+#        ${Images_Count_End}=    Get Images Count
+#        Should Be Equal as Integers    ${Images_Count_End}    ${Images_Count_Intermediate}   Count of image download not correct!
+        Remove Adapter Image    ${image_version}    ${onu_device_id}
+    END
+
+Do ONU Upgrade Image Download Simultaneously
+    [Documentation]    This keyword performs the ONU Upgrade Image Download Simultaneously on all ONUs test
+    [Arguments]    ${url}=${image_url}
+    [Teardown]     Remove Adapter Image from ONUs    ${image_version}    ${list_onus}
+    # collect all ONU's device ids
+    ${list_onus}    Create List
+    Build ONU Device Id List    ${list_onus}
+    # prepare OLT-SN BBSIM-POD releation dictionary for later fast access
+    ${olt_bbsim_dict}=     Create Dictionary
+    FOR    ${J}    IN RANGE    0    ${num_olts}
+        ${olt_serial_number}=    Set Variable    ${list_olts}[${J}][sn]
+        ${bbsim_rel}=    Catenate    SEPARATOR=    bbsim    ${J}
+        ${bbsim_pod}=    Get Pod Name By Label    ${NAMESPACE}    release     ${bbsim_rel}
+        Set To Dictionary    ${olt_bbsim_dict}    ${olt_serial_number}    ${bbsim_pod}
+    END
+#   Uncomment next commented lines in case of backporting HTML request in BBSIM
+#    ${Images_Count_Start}=    Get Images Count
+    # Download Image to all ONUs simultaneously
+    ${onu_device_ids} =    Catenate    @{list_onus}
+    Download ONU Device Image    ${image_version}    ${url}    ${image_vendor}
+    ...    true    true    ${image_crc}    ${onu_device_ids}
+    # do all the check stuff
+    FOR  ${onu_device_id}  IN  @{list_onus}
+        Log  ${onu_device_id}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image Status    ${image_version}
+        ...    ${onu_device_id}    DOWNLOAD_SUCCEEDED    NO_ERROR    IMAGE_COMMITTED
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image List    ${onu_device_id}
+        ...    ${image_version}    True    True    True
+    END
+    # last but not least check bbsim
+    FOR    ${I}    IN RANGE    0    ${num_all_onus}
+        ${src}=    Set Variable    ${hosts.src[${I}]}
+        ${bbsim_pod}=    Get From Dictionary    ${olt_bbsim_dict}    ${src['olt']}
+        Wait Until Keyword Succeeds    ${timeout}    2s    Verify ONU Device Image On BBSim    ${NAMESPACE}    ${bbsim_pod}
+        ...    ${src['onu']}    software_image_committed
+    END
+#    ${Images_Count_End}=    Get Images Count
+#    ${Images_Count_Start}=    Evaluate    ${Images_Count_Start}+2
+#    Should Be Equal as Integers    ${Images_Count_End}    ${Images_Count_Start}   Count of image download not correct!
+
+Restart And Check BBSIM
+    [Documentation]    This keyword restarts bbsim and waits for it to come up again
+    ...    Following steps will be executed:
+    ...    - restart bbsim adaptor
+    ...    - check bbsim adaptor is ready again
+    [Arguments]    ${namespace}
+    ${bbsim_apps}   Create List    bbsim
+    ${label_key}    Set Variable   app
+    ${bbsim_label_value}    Set Variable   bbsim
+    Restart Pod By Label    ${namespace}    ${label_key}    ${bbsim_label_value}
+    Sleep    5s
+    Wait For Pods Ready    ${namespace}    ${bbsim_apps}
+
+Get BBSIM Svc and Webserver Port
+    [Documentation]    This keyword gets bbsim instance and bbsim webserver port from image url
+    @{words}=    Split String    ${image_url}    /
+    ${SvcAndPort}    Set Variable     @{words}[2]
+    ${bbsim_svc}    ${webserver_port}=    Split String    ${SvcAndPort}    :    1
+    ${svc_return}    Set Variable If    '${bbsim_svc}'!='${EMPTY}'    ${bbsim_svc}    ${BBSIM_INSTANCE}
+    ${port_return}   Set Variable If    '${webserver_port}'!='${EMPTY}'    ${webserver_port}    ${BBSIM_WEBSERVER_PORT}
+    [Return]    ${svc_return}    ${port_return}
+
 Setup Suite
     [Documentation]    Set up the test suite
     Common Test Suite Setup