[VOL-4493] OpenOnuAdapter flow deletion after adapter restart
Change-Id: I23f6123b296c113d25de4d3b6a7582ca3273be53
diff --git a/libraries/onos.robot b/libraries/onos.robot
index c988543..8ae58e7 100755
--- a/libraries/onos.robot
+++ b/libraries/onos.robot
@@ -818,9 +818,20 @@
Count flows
[Documentation] Count flows in a particular ${state} in ONOS
- [Arguments] ${ip} ${port} ${targetFlows} ${deviceId} ${state}
- ${flows}= Execute ONOS CLI Command use single connection ${ip} ${port}
- ... flows -s ${state} ${deviceId} | grep -v deviceId | wc -l
+ ... Optionally for a certain onu-port.
+ [Arguments] ${ip} ${port} ${deviceId} ${state} ${onu_port}=${EMPTY}
+ ${cmd}= Catenate SEPARATOR= flows -s ${state} ${deviceId} | grep -v deviceId
+ ${cmd}= Run Keyword If "${onu_port}"!="${EMPTY}" Catenate SEPARATOR= ${cmd} | grep ${onu_port}
+ ... ELSE Set Variable ${cmd}
+ ${cmd}= Catenate SEPARATOR= ${cmd} | wc -l
+ ${flows}= Execute ONOS CLI Command use single connection ${ip} ${port} ${cmd}
+ [return] ${flows}
+
+Validate number of flows
+ [Documentation] Validates number of flows in a particular ${state} in ONOS
+ ... Optionally for a certain onu-port.
+ [Arguments] ${ip} ${port} ${targetFlows} ${deviceId} ${state} ${onu_port}=${EMPTY}
+ ${flows}= Count flows ${ip} ${port} ${deviceId} ${state} ${onu_port}
Log Found ${state} ${flows} of ${targetFlows} expected flows on device ${deviceId}
Should Be Equal As Integers ${targetFlows} ${flows}
@@ -830,13 +841,13 @@
... ${provisioned} ${withEapol} ${withDhcp} ${withIgmp} ${withLldp}
${targetFlows}= Calculate flows by workflow ${workflow} ${uni_count} ${olt_count} ${provisioned}
... ${withEapol} ${withDhcp} ${withIgmp} ${withLldp}
- Wait Until Keyword Succeeds 10m 5s Count flows
+ Wait Until Keyword Succeeds 10m 5s Validate number of flows
... ${ip} ${port} ${targetFlows} ${deviceId} added
Wait for all flows to be removed
[Documentation] Wait for all flows to be removed from a particular device
[Arguments] ${ip} ${port} ${deviceId}
- Wait Until Keyword Succeeds 10m 5s Count flows
+ Wait Until Keyword Succeeds 10m 5s Validate number of flows
... ${ip} ${port} 0 ${deviceId} any
diff --git a/libraries/onu_utilities.robot b/libraries/onu_utilities.robot
index 83af853..d12a3b0 100755
--- a/libraries/onu_utilities.robot
+++ b/libraries/onu_utilities.robot
@@ -117,6 +117,7 @@
Reconcile Onu Adapter
[Documentation] Restarts the openonu adapter and waits for reconciling is finished and expected oper-state is reached
[Arguments] ${namespace} ${usekill2restart} ${oper_status} ${olt_to_be_deleted_sn}=${EMPTY}
+ ... ${flow_delete_params}=&{EMPTY}
# get time of restart of openonu adapter
${restart_ts}= Get Current Date
# restart OpenONU adapter
@@ -129,7 +130,11 @@
# delete the olt passed, if available (special feature)
${olt_to_be_deleted_device_id}= Run Keyword IF "${olt_to_be_deleted_sn}"!="${EMPTY}"
... Get OLTDeviceID From OLT List ${olt_to_be_deleted_sn}
- Run Keyword IF "${olt_to_be_deleted_sn}"!="${EMPTY}" Delete Device ${olt_to_be_deleted_device_id}
+ Run Keyword If "${olt_to_be_deleted_sn}"!="${EMPTY}" Delete Device ${olt_to_be_deleted_device_id}
+ # remove flows if params passed for it (special feature II)
+ Run Keyword If ${flow_delete_params}!=&{EMPTY} Wait Until Keyword Succeeds ${timeout} 2s
+ ... Execute ONOS CLI Command use single connection ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ ... volt-remove-subscriber-access ${flow_delete_params['of_id']} ${flow_delete_params['onu_port']}
# wait for the reconcile to complete
# - we check that communication to openonu-adapter is established again
# - we check that all ONUs leave reconcile state by validate a simple voltctl request will not responds with error
@@ -374,6 +379,67 @@
log ${result}
[Return] ${result}
+Validate Tech Profiles and Flows in ETCD Data Per Onu
+ [Documentation] This keyword validates tech profiles and flows data stored in etcd per onu.
+ ... It checks checks presence/absence of tech profiles and flows depending on must_exist.
+ ... The values/content of tech profiles and flows will be not validated!
+ [Arguments] ${onu_sn} ${namespace}=default ${defaultkvstoreprefix}=voltha_voltha ${must_exist}=True
+ ${kvstoreprefix}= Get Kv Store Prefix ${defaultkvstoreprefix}
+ ${etcddata}= Get ONU Go Adapter ETCD Data namespace=${namespace} defaultkvstoreprefix=${kvstoreprefix}
+ #prepare result for json convert
+ ${result}= Prepare ONU Go Adapter ETCD Data For Json ${etcddata}
+ ${jsondata}= To Json ${result}
+ ${length}= Get Length ${jsondata}
+ log ${jsondata}
+ ${matched}= Set Variable False
+ FOR ${INDEX} IN RANGE 0 ${length}
+ ${value}= Get From List ${jsondata} ${INDEX}
+ ${uni_config}= Get From Dictionary ${value} uni_config
+ ${uni_config}= Set Variable ${uni_config[0]}
+ ${sn}= Get From Dictionary ${value} serial_number
+ ${matched}= Set Variable If '${sn}'=='${onu_sn}' True False
+ Exit For Loop If ${matched}
+ END
+ Should Be True ${matched} No ETCD data found for ONU ${onu_sn}
+ Log ${uni_config}
+ ${tp_path}= Get From Dictionary ${uni_config} PersTpPathMap
+ Log ${tp_path}
+ ${flow_params}= Get From Dictionary ${uni_config} flow_params
+ Log ${flow_params}
+ # in case of ATT for ONU with removed flows there is default flow established
+ ${length}= Get Length ${flow_params}
+ ${cookieslice}= Run Keyword If ${length}>0 Run Keyword If 'cookie_slice' in ${flow_params[0]}
+ ... Get From Dictionary ${flow_params[0]} cookie_slice
+ ${setvid}= Run Keyword If ${length}>0 Run Keyword If 'set_vid' in ${flow_params[0]['vlan_rule_params']}
+ ... Get From Dictionary ${flow_params[0]['vlan_rule_params']} set_vid
+ ${cookie_slice_length}= Run Keyword If ${length}>0 Run Keyword If 'cookie_slice' in ${flow_params[0]}
+ ... Get Length ${cookieslice}
+ Run Keyword If "${workflow}"=="ATT" and not ${must_exist} Run Keywords
+ ... Should Be Equal As Numbers ${setvid} 4091 AND
+ ... Should Be Equal As Numbers ${cookie_slice_length} 1 AND
+ ... Should Not Be Empty ${tp_path} AND
+ ... Return From Keyword
+ ${tp_path_length}= Get Length ${tp_path}
+ # validate tp_path is not Empty for case must_exist, case not must_exist will be validated implicitly with FOR loop
+ Run Keyword If ${must_exist} Should Not Be Empty ${tp_path}
+ ${tp_path_keys}= Run Keyword If ${tp_path_length}==0 Create List
+ ... ELSE Get Dictionary Keys ${tp_path}
+ ${tp_path_values}= Run Keyword If ${tp_path_length}==0 Create List
+ ... ELSE Get Dictionary Values ${tp_path}
+ ${tp_path_empty}= Set Variable True
+ Log ${tp_path_values}
+ Log ${tp_path_keys}
+ ${expected_tp}= Set Variable 64
+ # In case of not empty tp_path each value will be checked depending on must_exist
+ FOR ${key} IN @{tp_path_keys}
+ Should Be Equal As Numbers ${expected_tp} ${key}
+ ${value}= Get From Dictionary ${tp_path} ${key}
+ Run Keyword If ${must_exist} Should Not Be Empty ${value}
+ ... ELSE Should Be Empty ${value}
+ ${expected_tp}= Evaluate ${expected_tp}+1
+ END
+ Run Keyword If ${must_exist} Should Not Be Empty ${flow_params}
+ ... ELSE Should Be Empty ${flow_params}
Validate Onu Data In Etcd
[Documentation] This keyword validates openonu-go-adapter Data stored in etcd.
@@ -560,6 +626,28 @@
... Validate Onu Data In Etcd namespace=${namespace} nbofetcddata=0 without_pm_data=False
[Return] ${result}
+Validate OLT Flows Per Onu
+ [Documentation] This keyword validates olt flows per onu
+ ... It checks checks presence/absence of olt flows depending on must_exist.
+ ... The values/content of olt flows will be not validated!
+ [Arguments] ${onu_device_id} ${must_exist}
+ ${rc} ${output}= Run and Return Rc and Output
+ ... voltctl -c ${VOLTCTL_CONFIG} device flows ${onu_device_id} -m 8MB -o json
+ Should Be Equal As Integers ${rc} 0
+ ${jsondata}= To Json ${output}
+ Log ${jsondata}
+ # in case of ATT for ONU with removed flows there is default flow established
+ ${length}= Get Length ${jsondata}
+ ${value}= Run Keyword If ${length}>0 Get From List ${jsondata} 0
+ ${setvid}= Run Keyword If ${length}>0 Run Keyword If 'setvlanid' in ${value}
+ ... Get From Dictionary ${value} setvlanid
+ Run Keyword If "${workflow}"=="ATT" and not ${must_exist} Run Keywords
+ ... Should Be Equal As Numbers ${setvid} 4091 AND
+ ... Should Be Equal As Numbers ${length} 1 AND
+ ... Return From Keyword
+ Run Keyword If ${must_exist} Should Not Be Empty ${jsondata}
+ ... ELSE Should Be Empty ${jsondata}
+
Wait for Ports in ONOS for all OLTs
[Documentation] Waits untill a certain number of ports are enabled in all OLTs
[Arguments] ${host} ${port} ${count} ${filter} ${max_wait_time}=10m ${determine_number}=False
diff --git a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
index c391631..de98195 100755
--- a/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
+++ b/tests/openonu-go-adapter/Voltha_ONUReconcileTests.robot
@@ -196,6 +196,24 @@
... AND Teardown Test
... AND Stop Logging OltDeletionAfterAdapterRestartOnuGo
+Flow Deletion After Adapter Restart
+ [Documentation] Validates the flow(s) deletion after adapter restart
+ ... - perform sanity test include add subscriber
+ ... - restart the ONU adapter preferred via "kubectl delete pod"
+ ... - remove flow(s) from one ONU immediately after the restart has been initiated
+ ... - wait until the restart of the ONU adapter and the reconcile processing are finished
+ ... - check removed flow(s) from ONU
+ ... - check for not removed flows still the same before restart (if available)
+ [Tags] functionalOnuGo FlowDeletionAfterAdapterRestartOnuGo
+ [Setup] Run Keywords Start Logging FlowDeletionAfterAdapterRestartOnuGo
+ ... AND Setup Test
+ Run Keyword If ${has_dataplane} Clean Up Linux
+ Do Flow Deletion After Adapter Restart
+ [Teardown] Run Keywords Run Keyword If ${logging} Get Logical Id of OLT
+ ... AND Run Keyword If ${logging} Collect Logs
+ ... AND Teardown Test
+ ... AND Stop Logging FlowDeletionAfterAdapterRestartOnuGo
+
*** Keywords ***
Setup Suite
[Documentation] Set up the test suite
@@ -394,7 +412,7 @@
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
Enable Device ${olt_device_id}
END
- # bring all onus to activ -> OMCI-Flows-Pushed
+ # bring all onus to active -> OMCI-Flows-Pushed
Run Keyword If "${workflow}"=="DT" Perform Sanity Test DT
... ELSE IF "${workflow}"=="TT" Perform Sanity Tests TT
... ELSE Perform Sanity Test
@@ -420,3 +438,90 @@
Continue For Loop If "${olt_to_be_deleted}"=="${src['olt']}"
Current State Test omci-flows-pushed ${src['onu']}
END
+
+Do Flow Deletion After Adapter Restart
+ [Documentation] This keyword removes flow(s) after adapter restart and checks removed flow(s)
+ ... - perform sanity test include add subscriber
+ ... - restart the ONU adapter preferred via "kubectl delete pod"
+ ... - remove flow(s) from one ONU immediately after the restart has been initiated
+ ... - wait until the restart of the ONU adapter and the reconcile processing are finished
+ ... - check removed flow(s) from ONU
+ ... - check for not removed flows still the same before restart (if available)
+ FOR ${I} IN RANGE 0 ${num_olts}
+ #get olt serial number
+ ${olt_serial_number}= Set Variable ${list_olts}[${I}][sn]
+ #validate olt states
+ ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
+ Enable Device ${olt_device_id}
+ END
+ # bring all onus to active -> OMCI-Flows-Pushed
+ Run Keyword If "${workflow}"=="DT" Perform Sanity Test DT
+ ... ELSE IF "${workflow}"=="TT" Perform Sanity Tests TT
+ ... ELSE Perform Sanity Test
+ # log ONOS flows before remove
+ ${flow}= Execute ONOS CLI Command use single connection
+ ... ${ONOS_SSH_IP} ${ONOS_SSH_PORT} flows -s any ${of_id}
+ Log ${flow}
+ # validate OLT flows before remove
+ ${onu_device_id_list} Create List
+ Build ONU Device Id List ${onu_device_id_list}
+ Log ${onu_device_id_list}
+ FOR ${onu_device_id} IN @{onu_device_id_list}
+ Log ${onu_device_id}
+ ${rc} ${output}= Run and Return Rc and Output
+ ... voltctl -c ${VOLTCTL_CONFIG} device flows ${onu_device_id} -m 8MB -o json
+ Should Be Equal As Integers ${rc} 0
+ ${jsondata}= To Json ${output}
+ Log ${jsondata}
+ END
+ # Collect data for remove flow(s)
+ ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${hosts.src[0]['olt']}
+ ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${hosts.src[0]['onu']}
+ ... ${of_id} ${hosts.src[0]['uni_id']}
+ ${params_for_remove_flow}= Create Dictionary of_id=${of_id} onu_port=${onu_port}
+ # Collect number of flows for comparing after Reconcile
+ ${olt_flows_list} Create List
+ FOR ${I} IN RANGE 0 ${num_olts}
+ ${olt_of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${list_olts}[${I}][sn]
+ ${flows}= Count flows ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${olt_of_id} added
+ ${olt_flows}= Create Dictionary olt=${olt_of_id} flows=${flows}
+ Append To List ${olt_flows_list} ${olt_flows}
+ END
+ ${flows_onu}= Count flows ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id} added ${onu_port}
+ # Restart onu adapter with deleting flows from first onu
+ Reconcile Onu Adapter ${NAMESPACE} ${usekill2restart} ACTIVE flow_delete_params=${params_for_remove_flow}
+ # validate flows in ONOS after remove
+ ${flow}= Execute ONOS CLI Command use single connection
+ ... ${ONOS_SSH_IP} ${ONOS_SSH_PORT} flows -s any ${of_id}
+ Log ${flow}
+ ${expected_flows_onu}= Set Variable If "${workflow}"=="ATT" 1 0
+ Wait Until Keyword Succeeds ${timeout} 2s Validate number of flows ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ ... ${expected_flows_onu} ${of_id} any ${onu_port}
+ # Beside onu port specific flows additional flows deleted depending on workflow
+ ${additional_flows_deleted}= Set Variable If
+ ... "${workflow}"=="DT" 1
+ ... "${workflow}"=="TT" 3
+ ... "${workflow}"=="ATT" 0
+ FOR ${I} IN RANGE 0 ${num_olts}
+ ${olt_of_id} Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS ${list_olts}[${I}][sn]
+ ${flows}= Count flows ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${olt_of_id} added
+ ${expected_flows}= Run Keyword If "${of_id}"=="${olt_flows_list}[${I}][olt]"
+ ... Evaluate ${olt_flows_list}[${I}][flows]-${flows_onu}-${additional_flows_deleted}
+ ... ELSE Set Variable ${olt_flows_list}[${I}][flows]
+ Log Found added ${flows} of ${expected_flows} expected flows on device ${list_olts}[${I}][sn]
+ Should Be Equal As Integers ${expected_flows} ${flows}
+ END
+ # validate etcd data
+ ${List_ONU_Serial} Create List
+ Build ONU SN List ${List_ONU_Serial}
+ ${onu_sn_no_flows}= Set Variable ${hosts.src[0]['onu']}
+ FOR ${onu_sn} IN @{List_ONU_Serial}
+ ${must_exist}= Set Variable If "${onu_sn}"=="${onu_sn_no_flows}" False True
+ Validate Tech Profiles and Flows in ETCD Data Per Onu ${onu_sn} ${INFRA_NAMESPACE} ${kvstoreprefix} ${must_exist}
+ END
+ ${onu_device_id_no_flows}= Get Device ID From SN ${hosts.src[0]['onu']}
+ FOR ${onu_device_id} IN @{onu_device_id_list}
+ Log ${onu_device_id}
+ ${must_exist}= Set Variable If "${onu_device_id}"=="${onu_device_id_no_flows}" False True
+ Validate OLT Flows Per Onu ${onu_device_id} ${must_exist}
+ END