[VOL-4534] Added test for Voltha Components Minor Version Rolling Upgrade
Change-Id: Ieebdb1d63ea7a1f7a760197e6f8644b87adc0dbd
diff --git a/tests/software-upgrades/Voltha_ComponentsUpgrade.robot b/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
index 4ae876c..6d5f150 100755
--- a/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
+++ b/tests/software-upgrades/Voltha_ComponentsUpgrade.robot
@@ -119,6 +119,92 @@
Run Keyword If ${has_dataplane} Clean Up Linux
Wait Until Keyword Succeeds ${timeout} 2s Perform Sanity Test
+
+Test Voltha Components Minor Version Rolling Upgrade
+ [Documentation] Validates that the system can handle operations during software upgrade for minor versions.
+ ... Requirement: Components to test needs to be passed in robot command variable 'voltha_comps_under_test' in the format:
+ ... <comp-label>,<comp-container>,<comp-image>*<comp-label>,<comp-container>,<comp-image>*
+ ... Check [VOL-4534] for more details
+ [Tags] functional VolthaCompMinorVeriRollingUpgrade
+ [Setup] Start Logging VolthaCompMinorVerRollingUpgrade
+ [Teardown] Run Keywords Collect Logs
+ ... AND Stop Logging VolthaCompMinorVerRollingUpgrade
+ ... AND Delete All Devices and Verify
+ # Add OLT device
+ Setup
+ Run Keyword If ${has_dataplane} Clean Up Linux
+ FOR ${I} IN RANGE 0 ${olt_count}
+ ${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
+ ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
+ ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS
+ ... ${olt_serial_number}
+ Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
+ ... http://karaf:karaf@${ONOS_REST_IP}:${ONOS_REST_PORT} ${of_id}
+ END
+ FOR ${I} IN RANGE 0 ${num_all_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${of_id}= Get ofID From OLT List ${src['olt']}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id} ${src['uni_id']}
+ # Check ONU port is Enabled in ONOS
+ Wait Until Keyword Succeeds 120s 2s
+ ... Verify UNI Port Is Enabled ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${src['onu']} ${src['uni_id']}
+ Wait Until Keyword Succeeds ${timeout} 5s
+ ... Verify Default Meter Present in ONOS ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
+ Wait Until Keyword Succeeds ${timeout} 2s Verify Eapol Flows Added For ONU ${ONOS_SSH_IP}
+ ... ${ONOS_SSH_PORT} ${of_id} ${onu_port}
+ # Bring up the device and verify it authenticates
+ Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ ... ${onu_device_id} onu=True onu_reason=omci-flows-pushed by_dev_id=True
+ ${wpa_log}= Run Keyword If ${has_dataplane} Catenate SEPARATOR=.
+ ... /tmp/wpa ${src['dp_iface_name']} log
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate Authentication True
+ ... ${src['dp_iface_name']} wpa_supplicant.conf ${src['ip']} ${src['user']} ${src['pass']}
+ ... ${src['container_type']} ${src['container_name']} ${wpa_log}
+ Wait Until Keyword Succeeds ${timeout} 2s Verify ONU in AAA-Users ${ONOS_SSH_IP}
+ ... ${ONOS_SSH_PORT} ${onu_port}
+ END
+ ${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
+ Log ${podStatusOutput}
+ ${countBeforeUpgrade}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
+ ${num_comps_under_test}= Get Length ${list_voltha_comps_under_test}
+ FOR ${I} IN RANGE 0 ${num_comps_under_test}
+ ${label}= Set Variable ${list_voltha_comps_under_test}[${I}][label]
+ ${container}= Set Variable ${list_voltha_comps_under_test}[${I}][container]
+ ${image}= Set Variable ${list_voltha_comps_under_test}[${I}][image]
+ ${pod_image} ${app_ver} ${helm_chart} Get Pod Image And App Version And Helm Chart By Label
+ ... ${NAMESPACE} app ${label}
+ Log ${label}: image, app ver & helm chart before upgrade: ${pod_image}, ${app_ver} & ${helm_chart}
+ ${deployment}= Wait Until Keyword Succeeds ${timeout} 15s
+ ... Get K8s Deployment by Pod Label ${NAMESPACE} app ${label}
+ Wait Until Keyword Succeeds ${timeout} 15s Deploy Pod New Image ${NAMESPACE} ${deployment}
+ ... ${container} ${image}
+ # Static sleep to let image-update progress before initiating subscriber-add system-operation
+ Sleep 2s
+ Provision Subscribers
+ Wait Until Keyword Succeeds ${timeout} 3s Validate Pods Status By Label ${NAMESPACE}
+ ... app ${label} Running
+ Wait Until Keyword Succeeds ${timeout} 3s Pods Are Ready By Label ${NAMESPACE} app ${label}
+ Wait Until Keyword Succeeds ${timeout} 3s Verify Pod Image ${NAMESPACE} app ${label} ${image}
+ ${pod_image_1} ${app_ver_1} ${helm_chart_1} Get Pod Image And App Version And Helm Chart By Label
+ ... ${NAMESPACE} app ${label}
+ Log ${label}: image, app ver & helm chart after upgrade: ${pod_image_1}, ${app_ver_1} & ${helm_chart_1}
+ Restart VOLTHA Port Forward voltha-api
+ Verify Provisioned Subscribers
+ Unprovision Subscribers
+ END
+ ${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
+ Log ${podStatusOutput}
+ ${countAfterUpgrade}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
+ Should Be Equal As Strings ${countAfterUpgrade} ${countBeforeUpgrade}
+ # Additional Verification
+ Wait Until Keyword Succeeds ${timeout} 2s Delete All Devices and Verify
+ Setup
+ Run Keyword If ${has_dataplane} Clean Up Linux
+ Wait Until Keyword Succeeds ${timeout} 2s Perform Sanity Test
+
*** Keywords ***
Setup Suite
[Documentation] Set up the test suite
@@ -147,3 +233,61 @@
END
Log ${list_voltha_comps_under_test}
Set Suite Variable ${list_voltha_comps_under_test}
+
+Provision Subscribers
+ [Documentation] This keyword provisions/adds all the subscribers on all the devices
+ FOR ${I} IN RANGE 0 ${num_all_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${of_id}= Get ofID From OLT List ${src['olt']}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id} ${src['uni_id']}
+ Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
+ ... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
+ END
+
+Verify Provisioned Subscribers
+ [Documentation] This keyword verifies all the subscribers on all the devices
+ FOR ${I} IN RANGE 0 ${num_all_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${of_id}= Get ofID From OLT List ${src['olt']}
+ ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id} ${src['uni_id']}
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
+ ... Verify No Pending Flows For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
+ ... Validate Subscriber DHCP Allocation ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
+ # Verify subscriber access flows are added for the ONU port
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ ... Verify Subscriber Access Flows Added For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
+ ... ${onu_port} ${nni_port} ${src['c_tag']} ${src['s_tag']}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
+ ... Wait Until Keyword Succeeds ${timeout} 2s
+ ... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
+ ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
+ END
+
+Unprovision Subscribers
+ [Documentation] This keyword unprovisions/deletes all the subscribers on all the devices
+ FOR ${I} IN RANGE 0 ${num_all_onus}
+ ${src}= Set Variable ${hosts.src[${I}]}
+ ${dst}= Set Variable ${hosts.dst[${I}]}
+ ${of_id}= Get ofID From OLT List ${src['olt']}
+ ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
+ ${onu_device_id}= Get Device ID From SN ${src['onu']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
+ ... ${of_id} ${src['uni_id']}
+ Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
+ ... ${ONOS_SSH_PORT} volt-remove-subscriber-access ${of_id} ${onu_port}
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
+ ... Verify No Pending Flows For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
+ Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
+ ... Wait Until Keyword Succeeds ${timeout} 2s
+ ... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
+ ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
+ ... ELSE Sleep 15s
+ END