blob: a9f6186a6aaa5c0ed3d87c114ca4080d0391209b [file] [log] [blame]
bharat raj58488b32023-05-11 22:46:18 +05301# Copyright 2017-present Open Networking Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# robot test functions
15
16*** Settings ***
17Documentation Library for various utilities
18Library SSHLibrary
19Library String
20Library DateTime
21Library Process
22Library Collections
23Library RequestsLibrary
24Library OperatingSystem
25Library CORDRobot
26Library ImportResource resources=CORDRobot
27Resource ./voltctl.robot
28Resource ./vgc.robot
29
30*** Keywords ***
31Check CLI Tools Configured
32 [Documentation] Tests that use 'voltctl' and 'kubectl' should execute this keyword in suite setup
33 # check voltctl and kubectl configured
34 ${voltctl_rc} ${voltctl_output}= Run And Return Rc And Output voltctl -c ${VOLTCTL_CONFIG} device list
35 Log ${voltctl_output}
36 ${kubectl_rc} ${kubectl_output}= Run And Return Rc And Output kubectl get pods
37 Log ${kubectl_output}
38 Run Keyword If ${voltctl_rc} != 0 or ${kubectl_rc} != 0 FATAL ERROR
39 ... VOLTCTL and KUBECTL not configured. Please configure before executing tests.
40
41Common Test Suite Setup
42 [Documentation] Setup the test suite
Cristina de Franciscoc4cfd7e2023-10-09 10:55:08 +020043 Set Global Variable ${KUBECTL_CONFIG} %{KUBECONFIG}
bharat raj58488b32023-05-11 22:46:18 +053044 Set Global Variable ${VOLTCTL_CONFIG} %{VOLTCONFIG}
45 ${k8s_node_ip}= Evaluate ${nodes}[0].get("ip")
46 ${VGC_REST_IP}= Get Environment Variable VGC_REST_IP ${k8s_node_ip}
47 ${VGC_SSH_IP}= Get Environment Variable VGC_SSH_IP ${k8s_node_ip}
48 Set Global Variable ${VGC_REST_IP}
49 Set Global Variable ${VGC_SSH_IP}
50 ${k8s_node_user}= Evaluate ${nodes}[0].get("user")
51 ${k8s_node_pass}= Evaluate ${nodes}[0].get("pass")
52 Check CLI Tools Configured
53 ${HEADERS} Create Dictionary Content-Type=application/json
54 Create VGC Session
55 ${num_olts} Get Length ${olts}
56 ${list_olts} Create List
57 # Create olt list from the configuration file
58 FOR ${I} IN RANGE 0 ${num_olts}
59 ${ip} Evaluate ${olts}[${I}].get("ip")
60 ${user} Evaluate ${olts}[${I}].get("user")
61 ${pass} Evaluate ${olts}[${I}].get("pass")
62 ${serial_number} Evaluate ${olts}[${I}].get("serial")
63 ${olt_ssh_ip} Evaluate ${olts}[${I}].get("sship")
64 ${type} Evaluate ${olts}[${I}].get("type")
65 ${power_switch_port} Evaluate ${olts}[${I}].get("power_switch_port")
66 ${orig_olt_port} Evaluate ${olts}[${I}].get("oltPort")
67 ${port}= Set Variable If "${orig_olt_port}" == "None" ${OLT_PORT} ${orig_olt_port}
68 ${onu_count}= Get ONU Count For OLT ${hosts.src} ${serial_number}
69 ${onu_list}= Get ONU List For OLT ${hosts.src} ${serial_number}
70 ${olt} Create Dictionary ip ${ip} user ${user} pass
71 ... ${pass} sn ${serial_number} onucount ${onu_count} type ${type}
72 ... sship ${olt_ssh_ip} oltport ${port} powerswitchport ${power_switch_port}
73 ... onus ${onu_list}
74 Append To List ${list_olts} ${olt}
75 END
76 ${num_all_onus}= Get Length ${hosts.src}
77 ${num_all_onus}= Convert to String ${num_all_onus}
78 #send sadis file to vgc
79 ${sadis_file}= Get Variable Value ${sadis.file}
80 Log To Console \nSadis File:${sadis_file}
Cristina de Franciscoc4cfd7e2023-10-09 10:55:08 +020081 Run Keyword Unless '${sadis_file}' == '${None}' Send File To VGC ${sadis_file} # apps/
bharat raj58488b32023-05-11 22:46:18 +053082 Set Suite Variable ${num_all_onus}
83 Set Suite Variable ${num_olts}
84 Set Suite Variable ${list_olts}
85 ${olt_count}= Get Length ${list_olts}
86 Set Suite Variable ${olt_count}
87 @{container_list}= Create List ${OLT_ADAPTER_APP_LABEL} adapter-open-onu voltha-api-server
88 ... voltha-ro-core voltha-rw-core-11 voltha-rw-core-12 voltha-ofagent
89 Set Suite Variable ${container_list}
90 ${datetime}= Get Current Date
91 Set Suite Variable ${datetime}
92
93Get ONU Count For OLT
94 [Arguments] ${src} ${serial_number}
95 [Documentation] Gets ONU Count for the specified OLT
96 ${src_length}= Get Length ${src}
97 ${count}= Set Variable 0
98 FOR ${I} IN RANGE 0 ${src_length}
99 ${sn} Evaluate ${src}[${I}].get("olt")
100 ${count}= Run Keyword If '${serial_number}' == '${sn}' Evaluate ${count} + 1
101 ... ELSE Set Variable ${count}
102 END
103 [Return] ${count}
104
105Get ONU List For OLT
106 [Arguments] ${src} ${serial_number}
107 [Documentation] Gets ONU List for the specified OLT
108 ${src_length}= Get Length ${src}
109 ${onu_list}= Create List
110 FOR ${I} IN RANGE 0 ${src_length}
111 ${sn} Evaluate ${src}[${I}].get("olt")
112 Run Keyword If '${serial_number}' == '${sn}' Append To List ${onu_list} ${src}[${I}][onu]
113 ... ELSE Set Variable ${onu_list}
114 END
115 [Return] ${onu_list}
116
117WPA Reassociate
118 [Documentation] Executes a particular wpa_cli reassociate, which performs force reassociation
119 [Arguments] ${iface} ${ip} ${user} ${pass}=${None}
120 ... ${container_type}=${None} ${container_name}=${None}
121 #Below for loops are used instead of sleep time, to execute reassociate command and check status
122 FOR ${i} IN RANGE 70
123 ${output}= Login And Run Command On Remote System
124 ... wpa_cli -i ${iface} reassociate ${ip} ${user}
125 ... ${pass} ${container_type} ${container_name}
126 ${passed}= Run Keyword And Return Status Should Contain ${output} OK
127 Exit For Loop If ${passed}
128 END
129 Should Be True ${passed} Status does not contain 'SUCCESS'
130 FOR ${i} IN RANGE 70
131 ${output}= Login And Run Command On Remote System
132 ... wpa_cli -i ${iface} status | grep SUCCESS ${ip} ${user}
133 ... ${pass} ${container_type} ${container_name}
134 ${passed}= Run Keyword And Return Status Should Contain ${output} SUCCESS
135 Exit For Loop If ${passed}
136 END
137 Should Be True ${passed} Status does not contain 'SUCCESS'
138
139Validate Authentication After Reassociate
140 [Arguments] ${auth_pass} ${iface} ${ip} ${user} ${pass}=${None}
141 ... ${container_type}=${None} ${container_name}=${None}
142 [Documentation]
143 ... Executes a particular reassociate request on the RG using wpa_cli.
144 ... auth_pass determines if authentication should pass
145 ${wpa_log}= Catenate SEPARATOR=. /tmp/wpa ${iface} log
146 ${output}= Login And Run Command On Remote System truncate -s 0 ${wpa_log}; cat ${wpa_log}
147 ... ${ip} ${user} ${pass} ${container_type} ${container_name}
148 Log ${output}
149 Should Not Contain ${output} authentication completed successfully
150 WPA Reassociate ${iface} ${ip} ${user} ${pass} ${container_type} ${container_name}
151 Run Keyword If '${auth_pass}' == 'True' Wait Until Keyword Succeeds ${timeout} 2s
152 ... Check Remote File Contents True ${wpa_log} ${iface}.*authentication completed successfully
153 ... ${ip} ${user} ${pass} ${container_type} ${container_name}
154 Run Keyword If '${auth_pass}' == 'False' Sleep 20s
155 Run Keyword If '${auth_pass}' == 'False' Check Remote File Contents False /tmp/wpa.log
156 ... ${iface}.*authentication completed successfully ${ip} ${user} ${pass}
157 ... ${container_type} ${container_name}
158
159Send Dhclient Request To Release Assigned IP
160 [Arguments] ${iface} ${ip} ${user} ${path_dhcpleasefile} ${pass}=${None}
161 ... ${container_type}=${None} ${container_name}=${None}
162 [Documentation] Executes a dhclient with option to release ip against a particular interface on the RG (src)
163 ${result}= Login And Run Command On Remote System
164 ... dhclient -nw -r ${iface} && rm ${path_dhcpleasefile}/dhclient.* ${ip} ${user}
165 ... ${pass} ${container_type} ${container_name}
166 Log ${result}
167 #Should Contain ${result} DHCPRELEASE
168 [Return] ${result}
169
170Check Remote File Contents For WPA Logs
171 [Arguments] ${file_should_exist} ${file} ${pattern} ${ip} ${user} ${pass}=${None}
172 ... ${container_type}=${None} ${container_name}=${None} ${prompt}=~$
173 [Documentation] Checks for particular pattern count in a file
174 ${result}= Login And Run Command On Remote System
175 ... cat ${file} | grep '${pattern}' | wc -l ${ip} ${user} ${pass}
176 ... ${container_type} ${container_name} ${prompt}
177 [Return] ${result}
178
179Perform Sanity Test DT
180 [Documentation] This keyword iterate all OLTs and performs Sanity Test Procedure for DT workflow
181 ... For repeating sanity test without subscriber changes set flag supress_add_subscriber=True.
182 ... In all other (common) cases flag has to be set False (default).
183 [Arguments] ${supress_add_subscriber}=False
184 FOR ${J} IN RANGE 0 ${num_olts}
185 ${olt_serial_number}= Set Variable ${list_olts}[${J}][sn]
186 ${num_onus}= Set Variable ${list_olts}[${J}][onucount]
187 ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
188 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC
189 ... ${olt_serial_number}
190 Set Global Variable ${of_id}
191 ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in VGC ${of_id}
192 Perform Sanity Test DT Per OLT ${of_id} ${nni_port} ${olt_serial_number} ${num_onus}
193 ... ${supress_add_subscriber}
194 # Verify VGC Flows
195 # Number of Access Flows on VGC equals 4 * the Number of Active ONUs (2 for each downstream and upstream)
196 ${vgc_flows_count}= Evaluate 4 * ${num_onus}
197 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
198 ... Verify Subscriber Access Flows Added Count DT ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id}
199 ... ${vgc_flows_count}
200 # Verify LLDP flow in VGC
201 #Wait Until Keyword Succeeds ${timeout} 5s
202 #... Verify LLDP Flow Added ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id} 1
203 # Verify VOLTHA Flows
204 # Number of per OLT Flows equals Twice the Number of Active ONUs (each for downstream and upstream) + 1 for LLDP
205 ${olt_flows}= Evaluate 2 * ${num_onus}
206 # Number of per ONU Flows equals 2 (one each for downstream and upstream)
207 ${onu_flows}= Set Variable 2
208 Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate OLT Flows
209 ... ${olt_flows} ${olt_device_id}
210 ${List_ONU_Serial} Create List
211 Set Suite Variable ${List_ONU_Serial}
212 Build ONU SN List ${List_ONU_Serial} ${olt_serial_number}
213 Log ${List_ONU_Serial}
214 Wait Until Keyword Succeeds ${timeout} 5s Validate ONU Flows
215 ... ${List_ONU_Serial} ${onu_flows}
216 END
217
218
219Perform Sanity Test DT Per OLT
220 [Arguments] ${of_id} ${nni_port} ${olt_serial_number} ${num_onus} ${supress_add_subscriber}=False
221 [Documentation] This keyword performs Sanity Test Procedure for DT Workflow
222 ... Sanity test performs dhcp and pings (without EAPOL and DHCP flows) for all the ONUs given for the POD
223 ... This keyword can be used to call in any other tests where sanity check is required
224 ... and avoids duplication of code.
225 ... For repeating sanity test without subscriber changes set flag supress_add_subscriber=True.
226 ... In all other (common) cases flag has to be set False (default).
227 FOR ${I} IN RANGE 0 ${num_all_onus}
228 ${src}= Set Variable ${hosts.src[${I}]}
229 ${dst}= Set Variable ${hosts.dst[${I}]}
230 Continue For Loop If "${olt_serial_number}"!="${src['olt']}"
231 ${onu_device_id}= Get Device ID From SN ${src['onu']}
232 ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
233 ... Get ONU Port in VGC ${src['onu']} ${of_id} ${src['uni_id']}
234 # Check ONU port is Enabled in VGC
235 Wait Until Keyword Succeeds 120s 2s
236 ... Verify UNI Port Is Enabled ${src['onu']} ${src['uni_id']}
237 Run Keyword Unless ${supress_add_subscriber}
238 ... Add Subscriber Details ${of_id} ${onu_port}
239 Wait Until Keyword Succeeds ${timeout} 5s
240 ... Verify Subscriber Access Flows Added For ONU DT in VGC ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id}
241 ... ${onu_port} ${nni_port} ${src['s_tag']}
242 # Verify ONU state in voltha
243 ${onu_reasons}= Create List omci-flows-pushed
244 Run Keyword If ${supress_add_subscriber} Append To List ${onu_reasons} onu-reenabled
245 Wait Until Keyword Succeeds ${timeout} 5s Validate Device
246 ... ENABLED ACTIVE REACHABLE
247 ... ${src['onu']} onu=True onu_reason=${onu_reasons}
248 # Verify Meters in VGC
249 Wait Until Keyword Succeeds ${timeout} 5s
250 ... Verify Meters in VGC Ietf ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id} ${onu_port}
251 # TODO: Yet to Verify on the GPON based Physical POD (VOL-2652)
252 Run Keyword If ${has_dataplane} Validate DHCP and Ping True
253 ... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
254 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
255 ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
256 ... ${dst['container_name']}
257 END
258
259Perform Sanity Test DT FTTB
260 [Documentation] This keyword iterate all OLTs and performs Sanity Test Procedure for DT-FTTB workflow
261 ... For repeating sanity test without subscriber changes set flag supress_add_subscriber=True.
262 ... In all other (common) cases flag has to be set False (default).
263 [Arguments] ${supress_add_subscriber}=False
264 FOR ${J} IN RANGE 0 ${num_olts}
265 ${olt_serial_number}= Set Variable ${list_olts}[${J}][sn]
266 ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
267 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC
268 ... ${olt_serial_number}
269 ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in VGC ${of_id}
270 Perform Sanity Test DT FTTB Per OLT ${of_id} ${nni_port} ${olt_serial_number}
271 ... ${supress_add_subscriber}
272 END
273
274Perform Sanity Test DT FTTB Per OLT
275 [Arguments] ${of_id} ${nni_port} ${olt_serial_number} ${supress_add_subscriber}=False
276 [Documentation] This keyword performs Sanity Test Procedure for DT-FTTB Workflow
277 ... Sanity test performs dhcp and pings (without EAPOL and DHCP flows) for all the ONUs given for the POD
278 ... This keyword can be used to call in any other tests where sanity check is required
279 ... and avoids duplication of code.
280 ... For repeating sanity test without subscriber changes set flag supress_add_subscriber=True.
281 ... In all other (common) cases flag has to be set False (default).
282 FOR ${I} IN RANGE 0 ${num_all_onus}
283 ${src}= Set Variable ${hosts.src[${I}]}
284 ${dst}= Set Variable ${hosts.dst[${I}]}
285 Continue For Loop If "${olt_serial_number}"!="${src['olt']}"
286 ${onu_device_id}= Get Device ID From SN ${src['onu']}
287 ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
288 ... Get ONU Port in VGC ${src['onu']} ${of_id} ${src['uni_id']}
289 # Check ONU port is Enabled in VGC
290 Wait Until Keyword Succeeds 120s 2s
291 ... Verify UNI Port Is Enabled ${src['onu']} ${src['uni_id']}
292 Run Keyword Unless ${supress_add_subscriber}
293 ... Add Subscriber Details ${of_id} ${onu_port}
294 Wait Until Keyword Succeeds ${timeout} 5s
295 ... Verify VGC Flows Added For DT FTTB ${of_id}
296 ... ${onu_port} ${nni_port} ${src['service']}
297 # Verify that the Subscriber is present at the given location
298 Wait Until Keyword Succeeds ${timeout} 5s
299 ... Verify Programmed Subscribers DT FTTB ${of_id}
300 ... ${onu_port} ${src['service']}
301 # Verify ONU state in voltha
302 ${onu_reasons}= Create List omci-flows-pushed
303 Run Keyword If ${supress_add_subscriber} Append To List ${onu_reasons} onu-reenabled
304 Wait Until Keyword Succeeds ${timeout} 5s Validate Device
305 ... ENABLED ACTIVE REACHABLE
306 ... ${src['onu']} onu=True onu_reason=${onu_reasons}
307 # Verify Meters in VGC
308 Wait Until Keyword Succeeds ${timeout} 5s
309 ... Verify Meters in VGC Ietf ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id} ${onu_port}
310 ... FTTB_SUBSCRIBER_TRAFFIC
311 Run Keyword If ${has_dataplane} Validate DHCP and Ping True
312 ... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
313 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
314 ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
315 ... ${dst['container_name']}
316 END
317
318Validate All OLT Flows
319 [Documentation] This keyword iterate all OLTs and performs Sanity Test Procedure for DT workflow
320 FOR ${J} IN RANGE 0 ${num_olts}
321 ${olt_serial_number}= Set Variable ${list_olts}[${J}][sn]
322 ${num_onus}= Set Variable ${list_olts}[${J}][onucount]
323 ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
324 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC
325 ... ${olt_serial_number}
326 Set Global Variable ${of_id}
327 # Verify VGC Flows
328 # Number of Access Flows on VGC equals 4 * the Number of Active ONUs (2 for each downstream and upstream)
329 ${vgc_flows_count}= Evaluate 4 * ${num_onus}
330 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
331 ... Verify Subscriber Access Flows Added Count DT ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id}
332 ... ${vgc_flows_count}
333 # Verify VOLTHA Flows
334 # Number of per OLT Flows equals Twice the Number of Active ONUs (each for downstream and upstream) + 1 for LLDP
335 ${olt_flows}= Evaluate 2 * ${num_onus}
336 # Number of per ONU Flows equals 2 (one each for downstream and upstream)
337 ${onu_flows}= Set Variable 2
338 Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate OLT Flows ${olt_flows}
339 ... ${olt_device_id}
340 ${List_ONU_Serial} Create List
341 Set Suite Variable ${List_ONU_Serial}
342 Build ONU SN List ${List_ONU_Serial} ${olt_serial_number}
343 Log ${List_ONU_Serial}
344 Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate ONU Flows
345 ... ${List_ONU_Serial} ${onu_flows}
346 END
347
348Setup Soak
349 [Documentation] Pre-test Setup for Soak Job
350 ${olt_ids} Create List
351 FOR ${I} IN RANGE 0 ${num_olts}
352 ${olt_serial_number}= Set Variable ${list_olts}[${I}][sn]
353 ${olt_device_id}= Get Device ID From SN ${olt_serial_number}
354 ${logical_id}= Get Logical Device ID From SN ${olt_serial_number}
355 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC
356 ... ${olt_serial_number}
357 ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in VGC ${of_id}
358 ${olt} Create Dictionary device_id ${olt_device_id} logical_id ${logical_id}
359 ... of_id ${of_id} sn ${olt_serial_number}
360 Append To List ${olt_ids} ${olt}
361 END
362 Set Global Variable ${olt_ids}
363
364Setup
365 [Documentation] Pre-test Setup
366 [Arguments] ${skip_empty_device_list_test}=False
367 #test for empty device list
368 Run Keyword If '${skip_empty_device_list_test}'=='False' Test Empty Device List
369 # TBD: Need for this Sleep
370 Run Keyword If ${has_dataplane} Sleep 180s
371 # Create a list of olt ids (logical and device_id)
372 ${olt_ids} Create List
373 FOR ${I} IN RANGE 0 ${num_olts}
374 #create/preprovision device
375 ${olt_device_id}= Run Keyword If "${list_olts}[${I}][type]" == "${None}"
376 ... Create Device ${list_olts}[${I}][ip] ${list_olts}[${I}][oltport]
377 ... ELSE Create Device ${list_olts}[${I}][ip] ${list_olts}[${I}][oltport] ${list_olts}[${I}][type]
378 ${olt_serial_number}= Set Variable ${list_olts}[${I}][sn]
379 #Set Suite Variable ${olt_device_id}
380 #validate olt states
381 Wait Until Keyword Succeeds ${timeout} 5s
382 ... Validate OLT Device PREPROVISIONED UNKNOWN UNKNOWN ${olt_device_id} by_dev_id=True
383 Sleep 5s
384 Enable Device ${olt_device_id}
385 # Increasing the timer to incorporate wait time for in-band
386 Wait Until Keyword Succeeds 540s 5s
387 ... Validate OLT Device ENABLED ACTIVE REACHABLE ${olt_serial_number}
388 ${logical_id}= Get Logical Device ID From SN ${olt_serial_number}
389 # Set Suite Variable ${logical_id}
390 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC
391 ... ${olt_serial_number}
392 ${olt} Create Dictionary device_id ${olt_device_id} logical_id ${logical_id}
393 ... of_id ${of_id} sn ${olt_serial_number}
394 Append To List ${olt_ids} ${olt}
395 END
396 Set Global Variable ${olt_ids}
397
398Get ofID From OLT List
399 [Documentation] Retrieves the corresponding of_id for the OLT serial number specified
400 [Arguments] ${serial_number}
401 FOR ${I} IN RANGE 0 ${olt_count}
402 ${sn}= Get From Dictionary ${olt_ids}[${I}] sn
403 ${of_id}= Run Keyword IF "${serial_number}"=="${sn}"
404 ... Get From Dictionary ${olt_ids}[${I}] of_id ELSE Set Variable ${of_id}
405 END
406 [Return] ${of_id}
407
408Get OLTDeviceID From OLT List
409 [Documentation] Retrieves the corresponding olt_device_id for the OLT serial number specified
410 [Arguments] ${serial_number}
411 ${olt_device_id}= Set Variable 0
412 FOR ${I} IN RANGE 0 ${olt_count}
413 ${sn}= Get From Dictionary ${olt_ids}[${I}] sn
414 ${olt_device_id}= Run Keyword IF "${serial_number}"=="${sn}"
415 ... Get From Dictionary ${olt_ids}[${I}] device_id ELSE Set Variable ${olt_device_id}
416 END
417 [Return] ${olt_device_id}
418
419Get Num of Onus From OLT SN
420 [Documentation] Retrieves the corresponding number of ONUs for a given OLT based on serial number specified
421 [Arguments] ${serial_number}
422 ${num_of_olt_onus}= Set Variable 0
423 FOR ${I} IN RANGE 0 ${olt_count}
424 ${sn}= Get From Dictionary ${olt_ids}[${I}] sn
425 ${num_of_olt_onus}= Run Keyword IF "${serial_number}"=="${sn}"
426 ... Get From Dictionary ${list_olts}[${I}] onucount ELSE Set Variable ${num_of_olt_onus}
427 END
428 [Return] ${num_of_olt_onus}
429
430Validate ONUs After OLT Disable
431 [Documentation] Validates the ONUs state in Voltha, ONUs port state in VGC
432 ... and that pings do not succeed After corresponding OLT is Disabled
433 [Arguments] ${num_onus} ${olt_serial_number}
434 FOR ${I} IN RANGE 0 ${num_onus}
435 ${src}= Set Variable ${hosts.src[${I}]}
436 ${dst}= Set Variable ${hosts.dst[${I}]}
437 Continue For Loop If "${olt_serial_number}"!="${src['olt']}"
438 ${of_id}= Get ofID From OLT List ${src['olt']}
439 ${onu_device_id}= Get Device ID From SN ${src['onu']}
440 ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in VGC ${src['onu']}
441 ... ${of_id}
442 ${valid_onu_states}= Create List stopping-openomci omci-flows-deleted
443 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
444 ... Validate Device ENABLED DISCOVERED
445 ... UNREACHABLE ${src['onu']} onu=True onu_reason=${valid_onu_states}
446 Wait Until Keyword Succeeds ${timeout} 2s
447 ... Verify UNI Port Is Disabled ${VGC_SSH_IP} ${VGC_SSH_PORT} ${src['onu']} ${src['uni_id']}
448 Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
449 ... Wait Until Keyword Succeeds ${timeout} 2s
450 ... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
451 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
452 END
453
454Delete All Devices and Verify
455 [Documentation] Remove any devices from VOLTHA and VGC
456 [Arguments] ${maclearning_enabled}=False
457 # Clear devices from VOLTHA
458 ${resp}= Get Request VGC devices
459 ${jsondata}= To Json ${resp.content}
460 ${length}= Get Length ${jsondata['devices']}
461 ${matched}= Set Variable False
462 ${matched}= Set Variable If '${length}' == '${num_olts}' True False
463 Run Keyword If ${matched} Disable Devices In Voltha Root=true
464 Wait Until Keyword Succeeds ${timeout} 2s Test Devices Disabled In Voltha Root=true
465 Delete Devices In Voltha Root=true
466 Wait Until Keyword Succeeds ${timeout} 2s Test Empty Device List
467 FOR ${I} IN RANGE 0 ${length}
468 ${value}= Get From List ${jsondata['devices']} ${I}
469 ${device_id}= Get From Dictionary ${value} id
470 ${olt_serial_number}= Get From Dictionary ${value} serial
471 #${olt_serial_number}= Set Variable ${list_olts}[${I}][sn]
472 Wait Until Keyword Succeeds ${timeout} 5s
473 ... Validate Deleted Device Cleanup In VGC ${VGC_SSH_IP} ${VGC_SSH_PORT} ${olt_serial_number} ${device_id}
474 ... ${maclearning_enabled}
475 END
476 Wait Until Keyword Succeeds ${timeout} 5s Validate Cleanup In ETCD ${INFRA_NAMESPACE}
477
478Teardown
479 [Documentation] kills processes and cleans up interfaces on src+dst servers
480 Run Keyword If ${has_dataplane} Clean Up Linux
481
482Teardown Suite
483 [Documentation] Clean up device if desired
484 Start Logging Setup or Teardown Teardown-${SUITE NAME}
485 Run Keyword If ${has_dataplane} Clean Up Linux
486 Run Keyword If ${teardown_device} Deactivate Subscribers In VGC
487 Run Keyword If ${teardown_device} Delete All Devices and Verify
488 Run Keyword And Continue On Failure Collect Logs
489 #Close All VGC SSH Connections
490 Run Keyword If ${has_dataplane} Clean Up All Nodes
491 Stop Logging Setup or Teardown Teardown-${SUITE NAME}
492
493Delete Device and Verify
494 [Arguments] ${olt_serial_number}
495 [Documentation] Disable -> Delete devices via voltctl and verify its removed
496 ${olt_device_id}= Get Device ID From SN ${olt_serial_number}
497 ${rc} ${output}= Run and Return Rc and Output
498 ... voltctl -c ${VOLTCTL_CONFIG} device disable ${olt_device_id}
499 Should Be Equal As Integers ${rc} 0
500 Sleep 5s
501 Wait Until Keyword Succeeds ${timeout} 5s
502 ... Validate OLT Device DISABLED UNKNOWN REACHABLE ${olt_serial_number}
503 ${rc} ${output}= Run and Return Rc and Output
504 ... voltctl -c ${VOLTCTL_CONFIG} device delete ${olt_device_id}
505 Sleep 50s
506 Should Be Equal As Integers ${rc} 0
507 Wait Until Keyword Succeeds ${timeout} 5s Validate Device Removed ${olt_device_id}
508 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 15s
509 ... Validate Deleted Device Cleanup In VGC ${VGC_SSH_IP} ${VGC_SSH_PORT} ${olt_serial_number} ${olt_device_id}
510
511Disable Enable PON Port Per OLT DT
512 [Arguments] ${olt_serial_number}
513 [Documentation] This keyword disables and then enables OLT PON port and
514 ... also validate ONUs for each corresponding case
515 ${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
516 ${olt_pon_port_list}= Retrieve OLT PON Ports ${olt_device_id}
517 ${olt_pon_port_list_len}= Get Length ${olt_pon_port_list}
518 FOR ${INDEX0} IN RANGE 0 ${olt_pon_port_list_len}
519 ${olt_pon_port}= Get From List ${olt_pon_port_list} ${INDEX0}
520 ${olt_peer_list}= Retrieve Peer List From OLT PON Port ${olt_device_id} ${olt_pon_port}
521 ${olt_peer_list_len}= Get Length ${olt_peer_list}
522 # Disable the OLT PON Port and Validate OLT Device
523 DisableOrEnable OLT PON Port disable ${olt_device_id} ${olt_pon_port}
524 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
525 ... Validate OLT PON Port Status ${olt_device_id} ${olt_pon_port}
526 ... DISABLED DISCOVERED
527 Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
528 ... Validate OLT Device ENABLED ACTIVE REACHABLE
529 ... ${olt_serial_number}
530 Wait Until Keyword Succeeds ${timeout} 5s
531 ... Validate ONUs for PON OLT Disable DT ${olt_serial_number} ${olt_peer_list}
532 # Enable the OLT PON Port back, and check ONU status are back to "ACTIVE"
533 DisableOrEnable OLT PON Port enable ${olt_device_id} ${olt_pon_port}
534 Wait Until Keyword Succeeds ${timeout} 5s
535 ... Validate OLT PON Port Status ${olt_device_id} ${olt_pon_port}
536 ... ENABLED ACTIVE
537 ${olt_peer_list_new}= Wait Until Keyword Succeeds ${timeout} 5s
538 ... Retrieve Peer List From OLT PON Port ${olt_device_id} ${olt_pon_port} ${olt_peer_list_len}
539 Wait Until Keyword Succeeds ${timeout} 5s
540 ... Validate ONUs for PON OLT Enable DT ${olt_serial_number} ${olt_peer_list_new}
541 END
542
543Validate ONUs for PON OLT Disable DT
544 [Arguments] ${olt_sn} ${olt_peer_list}
545 [Documentation] This keyword validates that Ping fails for ONUs connected to Disabled OLT PON port
546 ... And Pings succeed for other Active OLT PON port ONUs
547 ... Also it removes subscriber and deletes ONUs for Disabled OLT PON port to replicate DT workflow
548 FOR ${I} IN RANGE 0 ${num_all_onus}
549 ${src}= Set Variable ${hosts.src[${I}]}
550 ${dst}= Set Variable ${hosts.dst[${I}]}
551 Continue For Loop If "${olt_sn}"!="${src['olt']}"
552 ${onu_device_id}= Get Device ID From SN ${src['onu']}
553 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC ${src['olt']}
554 ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in VGC ${src['onu']}
555 ... ${of_id} ${src['uni_id']}
556 ${matched}= Match ONU in PON OLT Peer List ${olt_peer_list} ${onu_device_id}
557 ${valid_onu_states}= Create List stopping-openomci omci-flows-deleted
558 Run Keyword If ${matched}
559 ... Run Keywords
560 ... Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
561 ... Validate Device ENABLED DISCOVERED
562 ... UNREACHABLE ${src['onu']} onu=True onu_reason=${valid_onu_states}
563 ... AND Wait Until Keyword Succeeds ${timeout} 2s
564 ... Verify UNI Port Is Disabled ${VGC_SSH_IP} ${VGC_SSH_PORT} ${src['onu']} ${src['uni_id']}
565 ... AND Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
566 ... Wait Until Keyword Succeeds ${timeout} 2s
567 ... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
568 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']}
569 ... ${src['container_name']}
570 # Remove Subscriber Access (To replicate DT workflow)
571 ... AND Wait Until Keyword Succeeds ${timeout} 2s Remove Subscriber Access ${of_id} ${onu_port}
572 #Execute VGC CLI Command use single connection
573 #... ${VGC_SSH_IP} ${VGC_SSH_PORT} volt-remove-subscriber-access ${of_id} ${onu_port}
574 # Delete ONU Device (To replicate DT workflow)
575 ... AND Delete Device ${onu_device_id}
576 # Additional Sleep to let subscriber and ONU delete process
577 ... AND Sleep 10s
578 ... ELSE
579 ... Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
580 ... Wait Until Keyword Succeeds ${timeout} 2s
581 ... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
582 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']}
583 ... ${src['container_name']}
584 END
585
586Validate ONUs for PON OLT Enable DT
587 [Arguments] ${olt_sn} ${olt_peer_list}
588 [Documentation] This keyword validates Ping succeeds for all Enabled/Acitve OLT PON ports
589 ... Also performs subscriberAdd/DHCP/Ping for the ONUs on Re-Enabled OLT PON port
590 FOR ${I} IN RANGE 0 ${num_all_onus}
591 ${src}= Set Variable ${hosts.src[${I}]}
592 ${dst}= Set Variable ${hosts.dst[${I}]}
593 Continue For Loop If "${olt_sn}"!="${src['olt']}"
594 ${onu_device_id}= Get Device ID From SN ${src['onu']}
595 ${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in VGC ${src['olt']}
596 ${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in VGC ${of_id}
597 ${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in VGC ${src['onu']}
598 ... ${of_id} ${src['uni_id']}
599 ${matched}= Match ONU in PON OLT Peer List ${olt_peer_list} ${onu_device_id}
600 Run Keyword If ${matched}
601 ... Run Keywords
602 # Perform Cleanup
603 ... Run Keyword If ${has_dataplane} Clean Up Linux ${onu_device_id}
604 # Verify ONU port status
605 ... AND Wait Until Keyword Succeeds 120s 2s
606 ... Verify UNI Port Is Enabled ${src['onu']} ${src['uni_id']}
607 ... AND Wait Until Keyword Succeeds ${timeout} 2
608 # ... Execute VGC CLI Command use single connection ${VGC_SSH_IP} ${VGC_SSH_PORT}
609 # ... volt-add-subscriber-access ${of_id} ${onu_port}
610 ... Add Subscriber Details ${of_id} ${onu_port}
611 # Verify ONU state in voltha
612 ... AND Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
613 ... Validate Device ENABLED ACTIVE REACHABLE
614 ... ${src['onu']} onu=True onu_reason=omci-flows-pushed
615 # Verify subscriber access flows are added for the ONU port
616 ... AND Wait Until Keyword Succeeds ${timeout} 5s
617 ... Verify Subscriber Access Flows Added For ONU DT In VGC ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id}
618 ... ${onu_port} ${nni_port} ${src['s_tag']}
619 ... AND Run Keyword If ${has_dataplane}
620 ... Run Keyword And Continue On Failure Validate DHCP and Ping True
621 ... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
622 ... ${src['ip']} ${src['user']} ${src['pass']}
623 ... ${src['container_type']} ${src['container_name']}
624 ... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']}
625 ... ${dst['container_type']} ${dst['container_name']}
626 ... ELSE
627 ... Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
628 ... Wait Until Keyword Succeeds ${timeout} 2s
629 ... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
630 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']}
631 ... ${src['container_name']}
632 END
633
634Match ONU in PON OLT Peer List
635 [Arguments] ${olt_peer_list} ${onu_device_id}
636 [Documentation] This keyword matches if ONU device is present in OLT PON port peer list
637 ${matched}= Set Variable False
638 FOR ${olt_peer} IN @{olt_peer_list}
639 ${matched}= Set Variable If '${onu_device_id}' == '${olt_peer}' True False
640 Exit For Loop If ${matched}
641 END
642 [Return] ${matched}
643
644Collect Logs
645 [Documentation] Collect Logs from voltha for various commands
646 Run Keyword and Ignore Error Get Device List from Voltha
647 FOR ${I} IN RANGE 0 ${num_olts}
648 Run Keyword and Ignore Error Get Device Output from Voltha ${olt_ids}[${I}][device_id]
649 Run Keyword and Ignore Error Get Logical Device Output from Voltha ${olt_ids}[${I}][logical_id]
650 END
651
652Verify ping is successful except for given device
653 [Arguments] ${num_onus} ${exceptional_onu}
654 [Documentation] Checks that ping for all the devices are successful except the given ONU.
655 ${pingStatus} Set Variable True
656 FOR ${I} IN RANGE 0 ${num_all_onus}
657 ${src}= Set Variable ${hosts.src[${I}]}
658 ${dst}= Set Variable ${hosts.dst[${I}]}
659 ${pingStatus} Run Keyword If '${src['onu']}' == '${exceptional_onu}' Set Variable False
660 Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
661 ... Wait Until Keyword Succeeds 60s 2s
662 ... Check Ping ${pingStatus} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
663 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
664 END
665
666Verify ping is successful for ONUs not on this OLT
667 [Arguments] ${num_all_onus} ${exceptional_olt_id}
668 [Documentation] Checks that pings work for all the ONUs except for the ONUs on the given OLT.
669 #${pingStatus} Set Variable True
670 FOR ${I} IN RANGE 0 ${num_all_onus}
671 ${src}= Set Variable ${hosts.src[${I}]}
672 ${dst}= Set Variable ${hosts.dst[${I}]}
673 ${olt_device_id}= Get Device ID From SN ${src['olt']}
674 Continue For Loop If "${olt_device_id}"=="${exceptional_olt_id}"
675 #${pingStatus} Run Keyword If '${olt_device_id}' == '${exceptional_olt_id}' Set Variable False
676 Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
677 ... Wait Until Keyword Succeeds 60s 2s
678 ... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
679 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
680 END
681
682Echo Message to OLT Logs
683 [Arguments] ${message}
684 [Documentation] Echoes ${message} into the OLT logs
685 FOR ${I} IN RANGE 0 ${num_olts}
686 ${olt_user} Evaluate ${olts}[${I}].get("user")
687 ${olt_pass} Evaluate ${olts}[${I}].get("pass")
688 ${olt_ssh_ip} Evaluate ${olts}[${I}].get("sship")
689 ${olt_type} Evaluate ${olts}[${I}].get("type")
690 Continue For Loop If "${olt_user}" == "${None}"
691 Continue For Loop If "${olt_pass}" == "${None}"
692 ${command_timeout}= Set Variable If "${olt_type}"=="adtranolt" 300s 180s
693 Wait Until Keyword Succeeds ${command_timeout} 10s Execute Remote Command
694 ... printf '%s\n' '' '' '${message}' '' >> /var/log/openolt.log
695 ... ${olt_ssh_ip} ${olt_user} ${olt_pass}
696 Wait Until Keyword Succeeds ${command_timeout} 10s Execute Remote Command
697 ... printf '%s\n' '' '' '${message}' '' >> /var/log/dev_mgmt_daemon.log
698 ... ${olt_ssh_ip} ${olt_user} ${olt_pass}
699 Wait Until Keyword Succeeds ${command_timeout} 10s Execute Remote Command
700 ... printf '%s\n' '' '' '${message}' '' >> /var/log/openolt_process_watchdog.log
701 ... ${olt_ssh_ip} ${olt_user} ${olt_pass}
702 END
703
704Start Logging
705 [Arguments] ${label}
706 [Documentation] Start logging for test ${label}
707 ${kail_process}= Run Keyword If "${container_log_dir}" != "${None}" Start Process kail -n ${NAMESPACE}
708 ... -n ${INFRA_NAMESPACE} cwd=${container_log_dir} stdout=${label}-combined.log
709 Set Test Variable ${kail_process}
710 Run Keyword If ${has_dataplane} Echo Message to OLT Logs START ${label}
711
712Start Logging Setup or Teardown
713 [Arguments] ${label}
714 [Documentation] Start logging for suite ${label}
715 ${file}= Replace String ${label} ${SPACE} -
716 ${kail_process}= Run Keyword If "${container_log_dir}" != "${None}" Start Process kail -n ${NAMESPACE}
717 ... -n ${INFRA_NAMESPACE} cwd=${container_log_dir} stdout=${file}-combined.log
718 Set Suite Variable ${kail_process}
719 Run Keyword If ${has_dataplane} Echo Message to OLT Logs START ${label}
720
721Stop Logging Setup or Teardown
722 [Arguments] ${label}
723 [Documentation] End logging for suite;
724 Run sync
725 Run Keyword If ${kail_process} Terminate Process ${kail_process}
726 ${test_logfile}= Run Keyword If "${container_log_dir}" != "${None}"
727 ... Join Path ${container_log_dir} ${label}-combined.log
728 Run Keyword If ${has_dataplane} Echo Message to OLT Logs END ${label}
729
730Stop Logging
731 [Arguments] ${label}
732 [Documentation] End logging for test; remove logfile if test passed and ${logging} is set to False
733 Run sync
734 Run Keyword If ${kail_process} Terminate Process ${kail_process}
735 ${test_logfile}= Run Keyword If "${container_log_dir}" != "${None}"
736 ... Join Path ${container_log_dir} ${label}-combined.log
737 Run Keyword If Test Passed
738 ... Run Keyword If "${logging}" == "False"
739 ... Run Keyword If "${test_logfile}" != "${None}"
740 ... Remove File ${test_logfile}
741 Run Keyword If ${has_dataplane} Echo Message to OLT Logs END ${label}
742
743Clean Up Linux
744 [Documentation] Kill processes and clean up interfaces on src+dst servers
745 [Arguments] ${onu_id}=${EMPTY}
746 FOR ${I} IN RANGE 0 ${num_all_onus}
747 ${src}= Set Variable ${hosts.src[${I}]}
748 ${dst}= Set Variable ${hosts.dst[${I}]}
749 ${onu_device_id}= Get Device ID From SN ${src['onu']}
750 Continue For Loop If '${onu_id}' != '${EMPTY}' and '${onu_id}' != '${onu_device_id}'
751 Execute Remote Command sudo pkill wpa_supplicant ${src['ip']}
752 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
753 Execute Remote Command sudo pkill dhclient ${src['ip']}
754 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
755 Execute Remote Command sudo pkill mausezahn ${src['ip']}
756 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
757 Run Keyword If '${dst['ip']}' != '${None}' Execute Remote Command pkill dhcpd
758 ... ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']} ${dst['container_name']}
759 Delete IP Addresses from Interface on Remote Host ${src['dp_iface_name']} ${src['ip']}
760 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
761 Run Keyword If '${dst['ip']}' != '${None}' Delete Interface on Remote Host
762 ... ${dst['dp_iface_name']}.${src['s_tag']} ${dst['ip']} ${dst['user']} ${dst['pass']}
763 ... ${dst['container_type']} ${dst['container_name']}
764 ${bng_ip}= Get Variable Value ${dst['noroot_ip']}
765 ${bng_user}= Get Variable Value ${dst['noroot_user']}
766 ${bng_pass}= Get Variable Value ${dst['noroot_pass']}
767 Run Keyword If "${bng_ip}" != "${NONE}" and "${bng_user}" != "${NONE}" and "${bng_pass}" != "${NONE}"
768 ... Execute Remote Command sudo pkill mausezahn ${bng_ip} ${bng_user} ${bng_pass}
769 ... ${dst['container_type']} ${dst['container_name']}
770 END
771
772Clean Up Linux Per OLT
773 [Documentation] Kill processes and clean up interfaces on src+dst servers
774 [Arguments] ${olt_serial_number}
775 FOR ${I} IN RANGE 0 ${num_all_onus}
776 ${src}= Set Variable ${hosts.src[${I}]}
777 ${dst}= Set Variable ${hosts.dst[${I}]}
778 ${onu_device_id}= Get Device ID From SN ${src['onu']}
779 ${sn}= Get Device ID From SN ${src['olt']}
780 #Continue For Loop If '${onu_id}' != '${EMPTY}' and '${onu_id}' != '${onu_device_id}'
781 Continue For Loop If '${olt_serial_number}' == '${sn}'
782 Execute Remote Command sudo pkill wpa_supplicant ${src['ip']}
783 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
784 Execute Remote Command sudo pkill dhclient ${src['ip']}
785 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
786 Execute Remote Command sudo pkill mausezahn ${src['ip']}
787 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
788 Run Keyword If '${dst['ip']}' != '${None}' Execute Remote Command pkill dhcpd
789 ... ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']} ${dst['container_name']}
790 Delete IP Addresses from Interface on Remote Host ${src['dp_iface_name']} ${src['ip']}
791 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
792 Run Keyword If '${dst['ip']}' != '${None}' Delete Interface on Remote Host
793 ... ${dst['dp_iface_name']}.${src['s_tag']} ${dst['ip']} ${dst['user']} ${dst['pass']}
794 ... ${dst['container_type']} ${dst['container_name']}
795 ${bng_ip}= Get Variable Value ${dst['noroot_ip']}
796 ${bng_user}= Get Variable Value ${dst['noroot_user']}
797 ${bng_pass}= Get Variable Value ${dst['noroot_pass']}
798 Run Keyword If "${bng_ip}" != "${NONE}" and "${bng_user}" != "${NONE}" and "${bng_pass}" != "${NONE}"
799 ... Execute Remote Command sudo pkill mausezahn ${bng_ip} ${bng_user} ${bng_pass}
800 ... ${dst['container_type']} ${dst['container_name']}
801 END
802
803Clean dhclient
804 [Documentation] Kills dhclient processes only for all RGs
805 FOR ${I} IN RANGE 0 ${num_all_onus}
806 ${src}= Set Variable ${hosts.src[${I}]}
807 ${dst}= Set Variable ${hosts.dst[${I}]}
808 Execute Remote Command sudo pkill dhclient ${src['ip']}
809 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
810 END
811
812Clean WPA Process
813 [Documentation] Kills wpa_supplicant processes only for all RGs
814 FOR ${I} IN RANGE 0 ${num_all_onus}
815 ${src}= Set Variable ${hosts.src[${I}]}
816 ${dst}= Set Variable ${hosts.dst[${I}]}
817 Run Keyword And Ignore Error Kill Linux Process [w]pa_supplicant ${src['ip']}
818 ... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
819 END
820
821Should Be Larger Than
822 [Documentation] Verify that value_1 is > value_2
823 [Arguments] ${value_1} ${value_2}
824 Run Keyword If ${value_1} <= ${value_2}
825 ... Fail The value ${value_1} is not larger than ${value_2}
826
827Should Be Larger Than Or Equal To
828 [Documentation] Verify that value_1 is >= value_2
829 [Arguments] ${value_1} ${value_2}
830 Run Keyword If ${value_1} < ${value_2}
831 ... Fail The value ${value_1} is not larger than or equal to ${value_2}
832
833Should Be Lower Than
834 [Documentation] Verify that value_1 is < value_2
835 [Arguments] ${value_1} ${value_2}
836 Run Keyword If ${value_1} >= ${value_2}
837 ... Fail The value ${value_1} is not lower than ${value_2}
838
839Should Be Float
840 [Documentation] Verify that value is a floating point number type
841 [Arguments] ${value}
842 ${type} Evaluate type(${value}).__name__
843 Should Be Equal ${type} float
844
845Should Be Newer Than Or Equal To
846 [Documentation] Compare two RFC3339 dates
847 [Arguments] ${value_1} ${value_2}
848 ${unix_v1} Parse RFC3339 ${value_1}
849 ${unix_v2} Parse RFC3339 ${value_2}
850 Run Keyword If ${unix_v1} < ${unix_v2}
851 ... Fail The value ${value_1} is not newer than or equal to ${value_2}
852
853Get Current Time
854 [Documentation] Return the current time in RFC3339 format
855 ${output}= Run date -u +"%FT%T%:z"
856 [return] ${output}
857
858Parse RFC3339
859 [Documentation] Parse an RFC3339 timestamp
860 [Arguments] ${dateStr}
861 ${rc} ${output}= Run and Return Rc and Output date --date="${dateStr}" "+%s"
862 Should Be Equal As Numbers ${rc} 0
863 [return] ${output}
864
865Execute Remote Command
866 [Documentation] SSH into a remote host and execute a command on the bare host or in a container.
867 ... This replaces and simplifies the Login And Run Command On Remote System keyword in CORDRobot.
868 [Arguments] ${cmd} ${ip} ${user} ${pass}=${None}
869 ... ${container_type}=${None} ${container_name}=${None} ${timeout}=${None}
870 ${conn_id}= SSHLibrary.Open Connection ${ip}
871 Run Keyword If '${pass}' != '${None}'
872 ... SSHLibrary.Login ${user} ${pass}
873 ... ELSE
874 ... SSHLibrary.Login With Public Key ${user} %{HOME}/.ssh/id_rsa
875 ${namespace}= Run Keyword If '${container_type}' == 'K8S' SSHLibrary.Execute Command
876 ... kubectl get pods --all-namespaces | grep ${container_name} | awk '{print $1}'
877 ${stdout} ${stderr} ${rc}= Run Keyword If '${container_type}' == 'LXC'
878 ... SSHLibrary.Execute Command lxc exec ${container_name} -- ${cmd}
879 ... return_stderr=True return_rc=True timeout=${timeout}
880 ... ELSE IF '${container_type}' == 'K8S'
881 ... SSHLibrary.Execute Command kubectl -n ${namespace} exec ${container_name} -- ${cmd}
882 ... return_stderr=True return_rc=True timeout=${timeout}
883 ... ELSE
884 ... SSHLibrary.Execute Command ${cmd} return_stderr=True return_rc=True timeout=${timeout}
885
886 Log ${stdout}
887 Log ${stderr}
888 Log ${rc}
889 SSHLibrary.Close Connection
890 [Return] ${stdout} ${stderr} ${rc}
891
892Start Remote Command
893 [Documentation] SSH into a remote host and execute a command on the bare host or in a container.
894 ... This replaces and simplifies the Login And Run Command On Remote System keyword in CORDRobot.
895 [Arguments] ${cmd} ${ip} ${user} ${pass}=${None}
896 ... ${container_type}=${None} ${container_name}=${None}
897 ${conn_id}= SSHLibrary.Open Connection ${ip}
898 Run Keyword If '${pass}' != '${None}'
899 ... SSHLibrary.Login ${user} ${pass}
900 ... ELSE
901 ... SSHLibrary.Login With Public Key ${user} %{HOME}/.ssh/id_rsa
902 ${namespace}= Run Keyword If '${container_type}' == 'K8S' SSHLibrary.Execute Command
903 ... kubectl get pods --all-namespaces | grep ${container_name} | awk '{print $1}'
904 Run Keyword If '${container_type}' == 'LXC'
905 ... SSHLibrary.Start Command lxc exec ${container_name} -- ${cmd}
906 ... ELSE IF '${container_type}' == 'K8S'
907 ... SSHLibrary.Start Command kubectl -n ${namespace} exec ${container_name} -- ${cmd}
908 ... ELSE
909 ... SSHLibrary.Start Command ${cmd}
910 # It seems that closing the connection immediately will sometimes kill the command
911 Sleep 1s
912 SSHLibrary.Close Connection
913
914Run Iperf3 Test Client
915 [Arguments] ${src} ${server} ${args}
916 [Documentation] Login to ${src} and run the iperf3 client against ${server} using ${args}.
917 ... Return a Dictionary containing the results of the test.
918 ${output} ${stderr} ${rc}= Execute Remote Command iperf3 -J -c ${server} ${args} -l 1024 -M 1350 | jq -M -c '.'
919 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
920 Should Be Equal As Integers ${rc} 0
921 ${object}= Evaluate json.loads(r'''${output}''') json
922 [Return] ${object}
923
924Run Iperf Test Client for MCAST
925 [Arguments] ${src} ${server} ${args}
926 [Documentation] Login to ${src} and run the iperf client against ${server} using ${args}.
927 ... Return a Dictionary containing the results of the test.
928 ${output} ${stderr} ${rc}= Execute Remote Command sudo iperf -c ${server} ${args} | jq -M -c '.'
929 ... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
930 Should Be Equal As Integers ${rc} 0
931 ${object}= Evaluate json.loads(r'''${output}''') json
932 [Return] ${object}
933
934Run Ping In Background
935 [Arguments] ${output_file} ${dst_ip} ${iface} ${ip} ${user} ${pass}=${None}
936 ... ${container_type}=${None} ${container_name}=${None}
937 [Documentation] Runs the 'ping' on remote system in background and stores the result in a file
938 ${result}= Login And Run Command On Remote System
939 ... echo "ping -I ${iface} ${dst_ip} > ${output_file} &" > ping.sh; chmod +x ping.sh; ./ping.sh
940 ... ${ip} ${user} ${pass} ${container_type} ${container_name}
941 Log ${result}
942
943Stop Ping Running In Background
944 [Arguments] ${ip} ${user} ${pass}=${None}
945 ... ${container_type}=${None} ${container_name}=${None}
946 [Documentation] Stops the 'ping' running in background on remote system
947 ${cmd}= Run Keyword If '${container_type}' == 'LXC' or '${container_type}' == 'K8S'
948 ... Set Variable kill -SIGINT `pgrep ping`
949 ... ELSE
950 ... Set Variable sudo kill -SIGINT `pgrep ping`
951 ${result}= Login And Run Command On Remote System
952 ... ${cmd} ${ip} ${user} ${pass} ${container_type} ${container_name}
953 Log ${result}
954
955Retrieve Remote File Contents
956 [Documentation] Retrieves the contents of the file on remote system
957 [Arguments] ${file} ${ip} ${user} ${pass}=${None}
958 ... ${container_type}=${None} ${container_name}=${None} ${prompt}=~$
959 ${output}= Login And Run Command On Remote System
960 ... cat ${file}
961 ... ${ip} ${user} ${pass} ${container_type} ${container_name} ${prompt}
962 [Return] ${output}
963
964RestoreONUs
965 [Documentation] Restore all connected ONUs
966 [Arguments] ${num_all_onus}
967 FOR ${I} IN RANGE 0 ${num_all_onus}
968 ${src}= Set Variable ${hosts.src[${I}]}
969 ${container_type}= Get Variable Value ${src['container_type']} "null"
970 ${container_name}= Get Variable Value ${src['container_name']} "null"
971 ${onu_type}= Get Variable Value ${src['onu_type']} "null"
972 #Get ens6f0 from ens6f0.22
973 ${if_name}= Replace String Using Regexp ${src['dp_iface_name']} \\..* \
974 Run Keyword IF '${onu_type}' == 'alpha' AlphaONURestoreDefault ${src['ip']} ${src['user']}
975 ... ${src['pass']} ${if_name} admin admin ${container_type} ${container_name}
976 END
977
978AlphaONURestoreDefault
979 [Documentation] Restore the Alpha ONU to factory setting
980 [Arguments] ${rg_ip} ${rg_user} ${rg_pass} ${onu_ifname}
981 ... ${onu_user} ${onu_pass} ${container_type}=${None} ${container_name}=${None}
982 ${output}= Login And Run Command On Remote System sudo ifconfig ${onu_ifname} 192.168.1.3/24
983 ... ${rg_ip} ${rg_user} ${rg_pass} ${container_type} ${container_name}
984 ${cmd} Catenate
985 ... (echo open "192.168.1.1"; sleep 1;
986 ... echo "${onu_user}"; sleep 1;
987 ... echo "${onu_pass}"; sleep 1;
988 ... echo "restoredefault"; sleep 1) | telnet
989 ${output}= Login And Run Command On Remote System ${cmd}
990 ... ${rg_ip} ${rg_user} ${rg_pass} ${container_type} ${container_name}
991 Log To Console ${output}
992 ${output}= Login And Run Command On Remote System sudo ifconfig ${onu_ifname} 0
993 ... ${rg_ip} ${rg_user} ${rg_pass} ${container_type} ${container_name}
994
995Create traffic with each pbit and capture at other end
996 [Documentation] Generates upstream traffic using Mausezahn tool
997 ... with each pbit and validates reception at other end using tcpdump
998 [Arguments] ${target_ip} ${target_iface} ${src_iface}
999 ... ${packet_count} ${packet_type} ${c_vlan} ${s_vlan} ${direction} ${tcpdump_filter}
1000 ... ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
1001 ... ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
1002 FOR ${pbit} IN RANGE 8
1003 Execute Remote Command sudo pkill mausezahn
1004 ... ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
1005 ${var1}= Set Variable sudo mausezahn ${src_iface} -B ${target_ip} -c ${packet_count}
1006 ${var2}= Run Keyword If "${direction}"=="downstream"
1007 ... Set Variable -t ${packet_type} "dp=80, flags=rst, p=aa:aa:aa" -Q ${pbit}:${s_vlan},${pbit}:${c_vlan}
1008 ... ELSE
1009 ... Set Variable -t ${packet_type} "dp=80, flags=rst, p=aa:aa:aa" -Q ${pbit}:${c_vlan}
1010 ${cmd}= Set Variable ${var1} ${var2}
1011 Start Remote Command ${cmd} ${src_ip} ${src_user} ${src_pass}
1012 ... ${src_container_type} ${src_container_name}
1013 ${output} ${stderr} ${rc}= Execute Remote Command
1014 ... sudo tcpdump -l -U -c 30 -i ${target_iface} -e ${tcpdump_filter}
1015 ... ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
1016 ... timeout=30 seconds
1017 Execute Remote Command sudo pkill mausezahn
1018 ... ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
1019 Run Keyword If "${tcpdump_filter}"=="tcp"
1020 ... Should Match Regexp ${output} , p ${pbit},
1021 END
1022
1023Determine Number Of ONU
1024 [Arguments] ${olt_serial_number}=${EMPTY} ${num_onus}=${num_all_onus}
1025 [Documentation] Determine the number of different ONUs for the given OLT taken from host.src
1026 ${onu_list} Create List
1027 FOR ${INDEX} IN RANGE 0 ${num_onus}
1028 Continue For Loop If "${olt_serial_number}"!="${hosts.src[${INDEX}].olt}" and "${olt_serial_number}"!="${EMPTY}"
1029 ${onu_id}= Get Index From List ${onu_list} ${hosts.src[${INDEX}].onu}
1030 Run Keyword If -1 == ${onu_id} Append To List ${onu_list} ${hosts.src[${INDEX}].onu}
1031 END
1032 ${real_num_onus}= Get Length ${onu_list}
1033 [Return] ${real_num_onus}
1034
1035Validate Cleanup In ETCD
1036 [Documentation] The keyword verifies that device, ports, flows, meters are all cleared in ETCD
1037 [Arguments] ${namespace}=default ${defaultkvstoreprefix}=voltha/voltha_voltha
1038 ${podname}= Set Variable etcd
1039 ${kvstoreprefix}= Get Kv Store Prefix ${defaultkvstoreprefix}
1040 # Log Devices Output and Verify Output Should be Empty
1041 ${commandget}= Catenate
1042 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/devices --keys-only'
1043 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1044 Log ${result}
1045 Should Be Empty ${result} Stale Devices Data in Etcd!
1046 # Log Flows Output and Verify Output Should be Empty
1047 ${commandget}= Catenate
1048 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/flows --keys-only'
1049 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1050 Log ${result}
1051 Should Be Empty ${result} Stale Flows Data in Etcd!
1052 # Log LogicalDevices Output and Verify Output Should be Empty
1053 ${commandget}= Catenate
1054 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_devices --keys-only'
1055 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1056 Log ${result}
1057 Should Be Empty ${result} Stale Logical Devices Data in Etcd!
1058 # Log LogicalFlows Output and Verify Output Should be Empty
1059 ${commandget}= Catenate
1060 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_flows --keys-only'
1061 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1062 Log ${result}
1063 Should Be Empty ${result} Stale Logical Flows Data in Etcd!
1064 # Log LogicalMeters Output and Verify Output Should be Empty
1065 ${commandget}= Catenate
1066 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_meters --keys-only'
1067 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1068 Log ${result}
1069 Should Be Empty ${result} Stale Logical Meters Data in Etcd!
1070 # Log LogicalPorts Output and Verify Output Should be Empty
1071 ${commandget}= Catenate
1072 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/logical_ports --keys-only'
1073 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1074 Log ${result}
1075 Should Be Empty ${result} Stale Logical Ports Data in Etcd!
1076 # Log Openolt Output and Verify Output Should be Empty
1077 ${commandget}= Catenate
1078 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openolt --keys-only'
1079 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1080 Log ${result}
1081 Should Be Empty ${result} Stale Openolt Data in Etcd!
1082 # Log Openonu Output and Verify Output Should be Empty
1083 ${commandget}= Catenate
1084 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/openonu --keys-only'
1085 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1086 Log ${result}
1087 Should Be Empty ${result} Stale Openonu Data in Etcd!
1088 # Log Ports Output and Verify Output Should be Empty
1089 ${commandget}= Catenate
1090 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/ports --keys-only'
1091 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1092 Log ${result}
1093 Should Be Empty ${result} Stale Ports Data in Etcd!
1094 # Log ResourceInstances Output and Verify Output Should be Empty
1095 ${commandget}= Catenate
1096 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_instances --keys-only'
1097 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1098 Log ${result}
1099 Should Be Empty ${result} Stale Resource Instances Data in Etcd!
1100 # Log ResourceManager Output and Verify Output Should be Empty
1101 ${commandget}= Catenate
1102 ... /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/${kvstoreprefix}/resource_manager --keys-only'
1103 ${result}= Exec Pod In Kube ${namespace} ${podname} ${commandget}
1104 Log ${result}
1105 Should Be Empty ${result} Stale Resource Manager Data in Etcd!
1106
1107Clean Up All Nodes
1108 [Documentation] Login to each node and kill all stale lxc prcoesses
1109 ${num_nodes}= Get Length ${nodes}
1110 FOR ${I} IN RANGE 0 ${num_nodes}
1111 ${node_ip}= Evaluate ${nodes}[${I}].get("ip")
1112 ${node_user}= Evaluate ${nodes}[${I}].get("user")
1113 ${node_pass}= Evaluate ${nodes}[${I}].get("pass")
1114 Run Keyword And Continue On Failure Start Remote Command kill -9 `pidof lxc`
1115 ... ${node_ip} ${node_user} ${node_pass}
1116 END
1117
1118Reboot XGSPON ONU
1119 [Documentation] Reboots the XGSPON ONU and verifies the ONU state after the reboot
1120 [Arguments] ${olt_sn} ${onu_sn} ${reason}
1121 FOR ${I} IN RANGE 0 ${num_olts}
1122 ${serial_number} Evaluate ${olts}[${I}].get("serial")
1123 Continue For Loop If "${serial_number}"!="${olt_sn}"
1124 ${board_tech} Evaluate ${olts}[${I}].get("board_technology")
1125 ${onu_device_id}= Get Device ID From SN ${onu_sn}
1126 Run Keyword If "${board_tech}"=="XGS-PON" Run Keywords
1127 ... Reboot Device ${onu_device_id}
1128 ... AND Wait Until Keyword Succeeds 120s 5s
1129 ... Validate Device ENABLED ACTIVE
1130 ... REACHABLE ${onu_sn} onu=True onu_reason=${reason}
1131 END
1132
1133Set Non-Critical Tag for XGSPON Tech
1134 [Documentation] Dynamically sets the test tag for xgs-pon based to non-critical
1135 FOR ${I} IN RANGE 0 ${num_olts}
1136 ${board_tech} Evaluate ${olts}[${I}].get("board_technology")
1137 Run Keyword If "${board_tech}"=="XGS-PON" Run Keywords
1138 ... Set Tags non-critical
1139 ... AND Exit For Loop
1140 END
1141
1142Perform Reboot ONUs and OLTs Physically
1143 [Documentation] This keyword reboots ONUs and OLTs physically
1144 ... It runs only on the PODs that are configured with PowerSwitch that
1145 ... controls the power off/on ONUs/OLT remotely (simulating a physical reboot)
1146 [Arguments] ${power_cycle_olt}=False
1147 Power Switch Connection Suite ${web_power_switch.ip} ${web_power_switch.user} ${web_power_switch.password}
1148 @{onu_list}= Create List
1149 FOR ${I} IN RANGE 0 ${num_all_onus}
1150 ${src}= Set Variable ${hosts.src[${I}]}
1151 ${dst}= Set Variable ${hosts.dst[${I}]}
1152 # If the power switch port is not specified, continue
1153 Continue For Loop If '${src["power_switch_port"]}' == '${None}'
1154 # Skip if we have already handled this ONU
1155 ${sn}= Set Variable ${src['onu']}
1156 ${onu_id}= Get Index From List ${onu_list} ${sn}
1157 Continue For Loop If -1 != ${onu_id}
1158 Append To List ${onu_list} ${sn}
1159 Disable Switch Outlet ${src['power_switch_port']}
1160 Sleep 10s
1161 Enable Switch Outlet ${src['power_switch_port']}
1162 END
1163 Pass Execution If '${power_cycle_olt}'=='False' Skipping OLT(s) Power Switch Reboot
1164 # Waiting extra time for the ONUs to come up
1165 Sleep 30s
1166 FOR ${I} IN RANGE 0 ${olt_count}
1167 ${olt_serial_number}= Get From Dictionary ${list_olts}[${I}] sn
1168 ${power_switch_port}= Get From Dictionary ${list_olts}[${I}] powerswitchport
1169 ${olt_ssh_ip}= Get From Dictionary ${list_olts}[${I}] sship
1170 # If the power switch port is not specified, continue
1171 Continue For Loop If '${power_switch_port}' == '${None}'
1172 Disable Switch Outlet ${power_switch_port}
1173 Sleep 10s
1174 Enable Switch Outlet ${power_switch_port}
1175 Run Keyword If ${has_dataplane} Wait Until Keyword Succeeds 120s 10s
1176 ... Check Remote System Reachability True ${olt_ssh_ip}
1177 END
1178 # Waiting extra time for the ONUs to come up
1179 Sleep 60s
1180
1181Count Number of UNI ports for OLT
1182 [Documentation] Count Provisioned UNI ports, for ONUs connected with specified OLT
1183 [Arguments] ${olt_serial_number} ${type_of_service}
1184 ${num_of_provisioned_onus_ports}= Evaluate 0
1185 FOR ${I} IN RANGE 0 ${num_all_onus}
1186 ${src}= Set Variable ${hosts.src[${I}]}
1187 Continue For Loop If "${olt_serial_number}"!="${src['olt']}"
1188 Continue For Loop If "${type_of_service}"!="${src['service_type']}"
1189 ${num_of_provisioned_onus_ports}= Evaluate ${num_of_provisioned_onus_ports} + 1
1190 END
1191 [Return] ${num_of_provisioned_onus_ports}