blob: 6ae3c3ac8c410e590ef0983ffab900e932a66e12 [file] [log] [blame]
# Copyright 2017-2024 Open Networking Foundation (ONF) and the ONF Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# FIXME Can we use the same test against BBSim and Hardware?
*** Settings ***
Documentation Test various end-to-end scenarios
Suite Setup Setup Suite
Test Setup Setup
Test Teardown Teardown
Suite Teardown Teardown Suite
Library Collections
Library String
Library OperatingSystem
Library XML
Library RequestsLibrary
Library ../../libraries/DependencyLibrary.py
Resource ../../libraries/onos.robot
Resource ../../libraries/voltctl.robot
Resource ../../libraries/voltha.robot
Resource ../../libraries/utils.robot
Resource ../../libraries/k8s.robot
Resource ../../variables/variables.robot
Resource ../../libraries/power_switch.robot
*** Variables ***
${POD_NAME} flex-ocp-cord
${KUBERNETES_CONF} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
${KUBERNETES_CONFIGS_DIR} ~/pod-configs/kubernetes-configs
#${KUBERNETES_CONFIGS_DIR} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
${KUBERNETES_YAML} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.yml
${HELM_CHARTS_DIR} ~/helm-charts
${VOLTHA_POD_NUM} 8
${NAMESPACE} voltha
${INFRA_NAMESPACE} default
${STACK_NAME} voltha
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
${timeout} 60s
${of_id} 0
${logical_id} 0
${has_dataplane} True
${teardown_device} False
${scripts} ../../scripts
# Per-test logging on failure is turned off by default; set this variable to enable
${container_log_dir} ${None}
# logging flag to enable Collect Logs, can be passed via the command line too
# example: -v logging:False
${logging} True
# Flag specific to Soak Jobs
${SOAK_TEST} False
*** Test Cases ***
Verify ONU after Rebooting Physically for DT
[Documentation] Test the ONU functionality by physically turning on/off ONU.
... Assuming that all the ONUs are DHCP/pingable (i.e. assuming sanityDt test was executed)
... Test case runs only on the PODs that are configured with PowerSwitch that
... controls the power off/on ONUs/OLT remotely (simulating a physical reboot)
[Tags] functionalDt PowerSwitchOnuRebootDt VOL-2819 PowerSwitch
[Setup] Start Logging RebootOnu_PowerSwitch_Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Delete All Devices and Verify
... AND Run Keyword If ${logging} Collect Logs
... AND Stop Logging RebootOnu_PowerSwitch_Dt
# Add OLT device
Setup
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
Power Switch Connection Suite ${web_power_switch.ip} ${web_power_switch.user} ${web_power_switch.password}
Run Keyword If ${has_dataplane} Clean Up Linux
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
... Get ONU Port in ONOS ${src['onu']} ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
# If the power switch port is not specified, continue
Continue For Loop If '${src["power_switch_port"]}' == '${None}'
# Disable Power Switch
Disable Switch Outlet ${src['power_switch_port']}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
# Remove Subscriber Access (To replicate DT workflow)
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-remove-subscriber-access ${of_id} ${onu_port}
# Allow the remove subscriber command to clean up all the flows, schedulers and queues before deleting the device
Sleep 30s
# Delete ONU Device (To replicate DT workflow)
Delete Device ${onu_device_id}
Sleep 5s
# Enable Power Switch
Enable Switch Outlet ${src['power_switch_port']}
# Check ONU port is Enabled in ONOS
Wait Until Keyword Succeeds 120s 5s
... Verify UNI Port Is Enabled ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${src['onu']} ${src['uni_id']}
Wait Until Keyword Succeeds ${timeout} 2
... Execute ONOS CLI Command use single connection ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
... volt-add-subscriber-access ${of_id} ${onu_port}
# Verify ONU state in voltha
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Verify subscriber access flows are added for the ONU port
Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
#Run Keyword If ${has_dataplane} Clean Up Linux
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
... ${dst['container_name']}
END
# Verify flows for all OLTs
Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate All OLT Flows
Verify OLT after Rebooting Physically for DT
[Documentation] Test the physical reboot of the OLT
... Assuming that all the ONUs are DHCP/pingable (i.e. assuming sanityDt test was executed)
... Test performs a physical reboot, performs "reboot" from the OLT CLI
[Tags] functionalDt PhysicalOltRebootDt VOL-2817
[Setup] Start Logging RebootOlt_Physical_Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Delete All Devices and Verify
... AND Run Keyword If ${logging} Collect Logs
... AND Stop Logging RebootOlt_Physical_Dt
# Add OLT device
Setup
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
# Reboot the OLT from the OLT CLI
FOR ${I} IN RANGE 0 ${olt_count}
${olt_user}= Get From Dictionary ${list_olts}[${I}] user
${olt_pass}= Get From Dictionary ${list_olts}[${I}] pass
${olt_ssh_ip}= Get From Dictionary ${list_olts}[${I}] sship
${olt_serial_number}= Get From Dictionary ${list_olts}[${I}] sn
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
Run Keyword If ${has_dataplane} Login And Run Command On Remote System
... reboot ${olt_ssh_ip} ${olt_user} ${olt_pass} prompt=#
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
# Wait for the OLT to come back up
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
${olt_user}= Get From Dictionary ${list_olts}[${I}] user
${olt_pass}= Get From Dictionary ${list_olts}[${I}] pass
${olt_ssh_ip}= Get From Dictionary ${list_olts}[${I}] sship
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
Run Keyword If ${has_dataplane} Wait Until Keyword Succeeds 120s 10s
... Check Remote System Reachability True ${olt_ssh_ip}
Wait Until Keyword Succeeds 360s 5s
... Validate OLT Device ENABLED ACTIVE
... REACHABLE ${olt_serial_number}
END
# Waiting extra time for the ONUs to come up
Sleep 60s
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
Verify restart openonu-adapter container after subscriber provisioning for DT
[Documentation] Restart openonu-adapter container after VOLTHA is operational.
... Prerequisite : ONUs are authenticated and pingable.
[Tags] Restart-OpenOnu-Dt soak
[Setup] Start Logging Restart-OpenOnu-Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging Restart-OpenOnu-Dt
# Add OLT device
Run Keyword If '${SOAK_TEST}'=='False' Setup
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable adapter-open-onu
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
# Wait for 1 min after openonu adapter is restarted
Sleep 60s
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}
Log to console Pod ${podName} restarted and sanity checks passed successfully
Run Keyword If '${SOAK_TEST}'=='False' Delete All Devices and Verify
Verify restart openolt-adapter container after subscriber provisioning for DT
[Documentation] Restart openolt-adapter container after VOLTHA is operational.
... Prerequisite : ONUs are authenticated and pingable.
[Tags] Restart-OpenOlt-Dt soak
[Setup] Start Logging Restart-OpenOlt-Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging Restart-OpenOlt-Dt
# Add OLT device
Run Keyword If '${SOAK_TEST}'=='False' setup
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBforRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable ${OLT_ADAPTER_APP_LABEL}
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
# Wait for 1 min after openolt adapter is restarted
Sleep 60s
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBforRestart}
Log to console Pod ${podName} restarted and sanity checks passed successfully
Verify openolt adapter restart before subscriber provisioning for DT
[Documentation] Restart openolt-adapter container before adding the subscriber.
[Tags] functionalDt olt-adapter-restart-Dt
[Setup] Start Logging OltAdapterRestart-Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging OltAdapterRestart-Dt
# Add OLT device
Clear All Devices Then Create New Device
Run Keyword If ${has_dataplane} Clean Up Linux
Set Global Variable ${of_id}
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
Wait Until Keyword Succeeds ${timeout} 5s
... Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=initial-mib-downloaded by_dev_id=True
END
# Scale down the open OLT adapter deployment to 0 PODs and once confirmed, scale it back to 1
Scale K8s Deployment by Pod Label ${NAMESPACE} app ${OLT_ADAPTER_APP_LABEL} 0
Wait Until Keyword Succeeds ${timeout} 2s Pods Do Not Exist By Label ${NAMESPACE} app
... ${OLT_ADAPTER_APP_LABEL}
Sleep 10s
# Scale up the open OLT adapter deployment and make sure both it and the ofagent deployment are back
Scale K8s Deployment by Pod Label ${NAMESPACE} app ${OLT_ADAPTER_APP_LABEL} 1
Wait Until Keyword Succeeds ${timeout} 2s
... Check Expected Available Deployment Replicas By Pod Label ${NAMESPACE} app ${OLT_ADAPTER_APP_LABEL} 1
Wait Until Keyword Succeeds ${timeout} 3s Pods Are Ready By Label ${NAMESPACE} app ${OLT_ADAPTER_APP_LABEL}
# Ensure the device is available in ONOS, this represents system connectivity being restored
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS
... ${olt_serial_number}
Wait Until Keyword Succeeds ${timeout} 2s Device Is Available In ONOS
... http://karaf:karaf@${ONOS_REST_IP}:${ONOS_REST_PORT} ${of_id}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id} ${src['uni_id']}
# Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
# Verify Meters in ONOS
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify Meters in ONOS Ietf ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id} ${onu_port}
# Verify subscriber access flows are added for the ONU port
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
Wait Until Keyword Succeeds ${timeout} 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
... ${dst['container_name']}
END
Verify restart ofagent container after subscriber is provisioned for DT
[Documentation] Restart ofagent container after VOLTHA is operational.
... Prerequisite : ONUs are authenticated and pingable.
[Tags] functionalDt ofagentRestart-Dt soak
[Setup] Start Logging ofagentRestart-Dt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging ofagentRestart-Dt
... AND Scale K8s Deployment ${NAMESPACE} ${STACK_NAME}-voltha-ofagent 1
... AND Wait Until Keyword Succeeds ${timeout} 2s
... Validate Pods status By Label ${NAMESPACE} app ofagent Running
... AND Wait Until Keyword Succeeds ${timeout} 3s
... Pods Are Ready By Label ${NAMESPACE} app ofagent
# set timeout value
${waitforRestart} Set Variable 120s
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBforRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable ofagent
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
Wait Until Keyword Succeeds ${timeout} 3s Pods Are Ready By Label ${NAMESPACE} app ${podName}
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBforRestart}
# Scale Down the Of-Agent Deployment
Scale K8s Deployment ${NAMESPACE} ${STACK_NAME}-voltha-ofagent 0
Sleep 30s
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
... Get ONU Port in ONOS ${src['onu']} ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
# Verify ONU state in voltha
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Check ONU port is Disabled in ONOS
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify UNI Port Is Disabled ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${src['onu']} ${src['uni_id']}
# Verify subscriber access flows are added for the ONU port
Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
# Verify Ping
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Check Ping True
... ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']} ${src['ip']}
... ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
# Scale Up the Of-Agent Deployment
Scale K8s Deployment ${NAMESPACE} ${STACK_NAME}-voltha-ofagent 1
Wait Until Keyword Succeeds ${waitforRestart} 2s Validate Pod Status ofagent ${NAMESPACE}
... Running
Wait Until Keyword Succeeds ${timeout} 3s Pods Are Ready By Label ${NAMESPACE} app ${podName}
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
Log to console Pod ${podName} restarted and sanity checks passed successfully
Sanity E2E Test for OLT/ONU on POD With Core Fail and Restart for DT
[Documentation] Deploys an device instance and waits for it to authenticate. After
... authentication is successful the rw-core deployment is scaled to 0 instances to
... simulate a POD crash. The test then scales the rw-core back to a single instance
... and configures ONOS for access. The test succeeds if the device is able to
... complete the DHCP sequence.
[Tags] functionalDt rwcore-restart-Dt
[Setup] Run Keywords Start Logging RwCoreFailAndRestart-Dt
... AND Clear All Devices Then Create New Device
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging RwCoreFailAndRestart-Dt
#... AND Delete Device and Verify
Run Keyword If ${has_dataplane} Clean Up Linux
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS
... ${olt_serial_number}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
# Bring up the device and verify it authenticates
Wait Until Keyword Succeeds 360s 5s Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=initial-mib-downloaded by_dev_id=True
END
# Scale down the rw-core deployment to 0 PODs and once confirmed, scale it back to 1
Scale K8s Deployment voltha voltha-voltha-rw-core 0
Wait Until Keyword Succeeds ${timeout} 2s Pod Does Not Exist voltha voltha-voltha-rw-core
# Ensure the ofagent POD goes "not-ready" as expected
Wait Until keyword Succeeds ${timeout} 2s
... Check Expected Available Deployment Replicas voltha voltha-voltha-ofagent 0
# Scale up the core deployment and make sure both it and the ofagent deployment are back
Scale K8s Deployment voltha voltha-voltha-rw-core 1
Wait Until Keyword Succeeds ${timeout} 2s
... Check Expected Available Deployment Replicas voltha voltha-voltha-rw-core 1
Wait Until Keyword Succeeds ${timeout} 2s
... Check Expected Available Deployment Replicas voltha voltha-voltha-ofagent 1
# For some reason scaling down and up the POD behind a service causes the port forward to stop working,
# so restart the port forwarding for the API service
Restart VOLTHA Port Forward voltha-api
# Ensure that the ofagent pod is up and ready and the device is available in ONOS, this
# represents system connectivity being restored
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
${of_id}= Wait Until Keyword Succeeds ${timeout} 15s Validate OLT Device in ONOS
... ${olt_serial_number}
Wait Until Keyword Succeeds 120s 2s Device Is Available In ONOS
... http://karaf:karaf@${ONOS_REST_IP}:${ONOS_REST_PORT} ${of_id}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id} ${src['uni_id']}
# Add subscriber access and verify that DHCP completes to ensure system is still functioning properly
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
# Verify subscriber access flows are added for the ONU port
Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
... ${dst['container_name']}
END
Verify OLT Soft Reboot for DT
[Documentation] Test soft reboot of the OLT using voltctl command
[Tags] VOL-2818 OLTSoftRebootDt functionalDt
[Setup] Start Logging OLTSoftRebootDt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging OLTSoftRebootDt
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate OLT Device ENABLED ACTIVE
... REACHABLE ${olt_serial_number}
# Reboot the OLT using "voltctl device reboot" command
Reboot Device ${olt_device_id}
# Wait for the OLT to actually go down
Wait Until Keyword Succeeds 360s 5s Validate OLT Device ENABLED UNKNOWN UNREACHABLE
... ${olt_serial_number}
END
#Verify that ping fails
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
# Check OLT states
FOR ${I} IN RANGE 0 ${olt_count}
${olt_serial_number}= Get From Dictionary ${list_olts}[${I}] sn
${olt_ssh_ip}= Get From Dictionary ${list_olts}[${I}] sship
${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}
# Wait for the OLT to come back up
Run Keyword If ${has_dataplane} Wait Until Keyword Succeeds 120s 10s
... Check Remote System Reachability True ${olt_ssh_ip}
# Check OLT states
Wait Until Keyword Succeeds 360s 5s
... Validate OLT Device ENABLED ACTIVE
... REACHABLE ${olt_serial_number}
END
# Waiting extra time for the ONUs to come up
Sleep 60s
#Check after reboot that ONUs are active, DHCP/pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
Verify ONU Soft Reboot for DT
[Documentation] Test soft reboot of the ONU using voltctl command
[Tags] VOL-2820 ONUSoftRebootDt notready
[Setup] Start Logging ONUSoftRebootDt
#... AND Setup
[Teardown] Run Keywords Collect Logs
... AND Stop Logging ONUSoftRebootDt
#... AND Delete Device and Verify
#Reboot the ONU and verify that ping fails
FOR ${I} IN RANGE 0 ${num_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s
... Get ONU Port in ONOS ${src['onu']} ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
Reboot Device ${onu_device_id}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
Verify ping is successful except for given device ${num_onus} ${src['onu']}
# Remove Subscriber Access (To replicate DT workflow)
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-remove-subscriber-access ${of_id} ${onu_port}
# Delete ONU Device (To replicate DT workflow)
Delete Device ${onu_device_id}
Sleep 40s
# Check ONU port is Enabled in ONOS
Wait Until Keyword Succeeds ${timeout} 2s
... Verify UNI Port Is Enabled ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${src['onu']} ${src['uni_id']}
Wait Until Keyword Succeeds ${timeout} 2
... Execute ONOS CLI Command use single connection ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
... volt-add-subscriber-access ${of_id} ${onu_port}
# Verify ONU state in voltha
Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Verify subscriber access flows are added for the ONU port
Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
#Run Keyword If ${has_dataplane} Clean Up Linux
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure Validate DHCP and Ping True
... True ${src['dp_iface_name']} ${src['s_tag']} ${src['c_tag']} ${dst['dp_iface_ip_qinq']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
... ${dst['dp_iface_name']} ${dst['ip']} ${dst['user']} ${dst['pass']} ${dst['container_type']}
... ${dst['container_name']}
END
# Verify ONOS Flows
# Number of Access Flows on ONOS equals 4 * the Number of Active ONUs (2 for each downstream and upstream)
${onos_flows_count}= Evaluate 4 * ${num_onus}
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added Count DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
... ${of_id} ${onos_flows_count}
# Verify VOLTHA Flows
# Number of per OLT Flows equals Twice the Number of Active ONUs (each for downstream and upstream) + 1 for LLDP
${olt_flows}= Evaluate 2 * ${num_onus} + 1
# Number of per ONU Flows equals 2 (one each for downstream and upstream)
${onu_flows}= Set Variable 2
Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate OLT Flows ${olt_flows}
${List_ONU_Serial} Create List
Set Suite Variable ${List_ONU_Serial}
Build ONU SN List ${List_ONU_Serial}
Log ${List_ONU_Serial}
Run Keyword Wait Until Keyword Succeeds ${timeout} 5s Validate ONU Flows
... ${List_ONU_Serial} ${onu_flows}
Verify restart openonu-adapter container for DT
[Documentation] Restart openonu-adapter container after VOLTHA is operational.
... Run the ping continuously in background during container restart,
... and verify that there should be no affect on the dataplane.
... Also, verify that the voltha control plane functionality is not affected.
[Tags] functionalDt RestartOpenOnuPingDt
[Setup] Start Logging RestartOpenOnuPingDt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging RestartOpenOnuPingDt
Clear All Devices Then Create New Device
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable adapter-open-onu
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
# Wait for 1 min after openonu adapter is restarted
Sleep 60s
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}
END
# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber
Verify Control Plane After Pod Restart DT
Verify restart openolt-adapter container for DT
[Documentation] Restart openolt-adapter container after VOLTHA is operational.
... Run the ping continuously in background during container restart,
... and verify that there should be no affect on the dataplane.
... Also, verify that the voltha control plane functionality is not affected.
[Tags] functionalDt RestartOpenOltPingDt
[Setup] Start Logging RestartOpenOltPingDt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging RestartOpenOltPingDt
Clear All Devices Then Create New Device
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable ${OLT_ADAPTER_APP_LABEL}
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
# Wait for 1 min after openolt adapter is restarted
Sleep 60s
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}
END
# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber
Verify Control Plane After Pod Restart DT
Verify restart rw-core container for DT
[Documentation] Restart rw-core container after VOLTHA is operational.
... Run the ping continuously in background during container restart,
... and verify that there should be no affect on the dataplane.
... Also, verify that the voltha control plane functionality is not affected.
[Tags] functionalDt RestartRwCorePingDt
[Setup] Start Logging RestartRwCorePingDt
[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs
... AND Stop Logging RestartRwCorePingDt
Clear All Devices Then Create New Device
# Performing Sanity Test to make sure subscribers are all DHCP and pingable
Run Keyword If ${has_dataplane} Clean Up Linux
Perform Sanity Test DT
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
${podName} Set Variable rw-core
Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}
Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}
... app ${podName} Running
# Wait for 1 min after rw-core is restarted
Sleep 60s
# For some reason scaling down and up the POD behind a service causes the port forward to stop working,
# so restart the port forwarding for the API service
Restart VOLTHA Port Forward voltha-api
${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}
Log ${podStatusOutput}
${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l
Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
END
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${ping_output_file}= Set Variable /tmp/${src['onu']}_ping
${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}
... ${src['container_type']} ${src['container_name']}
Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}
END
# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber
Verify Control Plane After Pod Restart DT
*** Keywords ***
Setup Suite
[Documentation] Set up the test suite
Common Test Suite Setup
#power_switch.robot needs it to support different vendor's power switch
${switch_type}= Get Variable Value ${web_power_switch.type}
Run Keyword If "${switch_type}"!="" Set Global Variable ${powerswitch_type} ${switch_type}
# Run Pre-test Setup for Soak Job
# Note: As soak requirement, it expects that the devices under test are already created and enabled
Run Keyword If '${SOAK_TEST}'=='True' Setup Soak
Clear All Devices Then Create New Device
[Documentation] Remove any devices from VOLTHA and ONOS
# Remove all devices from voltha and nos
Delete All Devices and Verify
# Execute normal test Setup Keyword
Setup
Verify Control Plane After Pod Restart DT
[Documentation] Verifies the control plane functionality after the voltha pod restart
... by deleting and re-adding the subscriber
FOR ${I} IN RANGE 0 ${num_all_onus}
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${of_id}= Get ofID From OLT List ${src['olt']}
${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in ONOS ${of_id}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id} ${src['uni_id']}
${onu_device_id}= Get Device ID From SN ${src['onu']}
# Remove Subscriber Access
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-remove-subscriber-access ${of_id} ${onu_port}
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
# Disable and Re-Enable the ONU (To replicate DT current workflow)
# TODO: Delete and Auto-Discovery Add of ONU (not yet supported)
Disable Device ${onu_device_id}
Wait Until Keyword Succeeds ${timeout} 5s
... Validate Device DISABLED UNKNOWN
... REACHABLE ${src['onu']}
Enable Device ${onu_device_id}
Wait Until Keyword Succeeds ${timeout} 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']}
# Add Subscriber Access
Wait Until Keyword Succeeds ${timeout} 2s Execute ONOS CLI Command use single connection ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} volt-add-subscriber-access ${of_id} ${onu_port}
# Verify subscriber access flows are added for the ONU port
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
Wait Until Keyword Succeeds ${timeout} 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Workaround for issue seen in VOL-4489. Keep this workaround until VOL-4489 is fixed.
Run Keyword If ${has_dataplane} Reboot XGSPON ONU ${src['olt']} ${src['onu']} omci-flows-pushed
# Workaround ends here for issue seen in VOL-4489.
Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure
... Wait Until Keyword Succeeds ${timeout} 2s
... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
END