system upgrade test improvements
Change-Id: I47efcf09a8eec16c127db6d46a16a6f877b4889a
diff --git a/src/test/cord-api/Tests/WorkflowValidations/ISSU.robot b/src/test/cord-api/Tests/WorkflowValidations/ISSU.robot
deleted file mode 100644
index abf61ed..0000000
--- a/src/test/cord-api/Tests/WorkflowValidations/ISSU.robot
+++ /dev/null
@@ -1,281 +0,0 @@
-# Copyright 2017-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation In-Service-Software-Upgrade test suite
-Suite Setup Setup Suite
-Suite Teardown Teardown Suite
-Test Setup Setup Test
-Test Teardown Teardown Test
-Library Collections
-Library String
-Library OperatingSystem
-Library XML
-Library RequestsLibrary
-Library ../../Framework/utils/utils.py
-Resource ../../Framework/utils/utils.robot
-Library ../../Framework/restApi.py
-Resource ../../Framework/Subscriber.robot
-Resource ../../Framework/ATTWorkFlowDriver.robot
-Resource ../../Framework/Kubernetes.robot
-Resource ../../Framework/ONU.robot
-Resource ../../Framework/OLT.robot
-Resource ../../Framework/DHCP.robot
-Variables ../../Properties/RestApiProperties.py
-
-*** Variables ***
-${POD_NAME} flex-pod1-olt
-${KUBERNETES_CONFIGS_DIR} ~/pod-configs/kubernetes-configs
-${HELM_CHARTS_DIR} ~/helm-charts
-${WHITELIST_PATHFILE} ${CURDIR}/data/${POD_NAME}/ATTWhiteList.json
-${SUBSCRIBER_PATHFILE} ${CURDIR}/data/${POD_NAME}/ATTSubscriber.json
-${VOLT_DEVICE_PATHFILE} ${CURDIR}/data/${POD_NAME}/RealOLTDevice.json
-${KUBERNETES_CONF} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
-${KUBERNETES_YAML} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.yml
-${VOLTHA_POD_NUM} 8
-
-*** Test Cases ***
-
-OLT Reboot
- [Documentation] Validates E2E Ping Connectivity and object states for the given scenario:
- ... Configure whitelist with correct ONU location
- ... Validate successful authentication/DHCP/E2E ping
- ... Reboots OLT
- ... Validate that pings fail
- ... Validate successful authentication/DHCP/E2E ping after OLT comes back up
- [Tags] olt1
- Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status awaiting-auth ${onu_device}
- Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Validate DHCP and Ping True True ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
- # Reboot OLT
- Login And Run Command On Remote System sudo reboot ${olt_ip} ${olt_user} ${olt_pass} prompt=#
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability False ${olt_ip}
- Wait Until Keyword Succeeds 60s 2s Check Ping False ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability True ${olt_ip}
- Wait Until Keyword Succeeds 120s 10s Openolt is Up ${olt_ip} ${olt_user} ${olt_pass}
- # Validate successful pings since the OLT is Up
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Check Ping True ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
-
-Fabric Switch Reboot
- [Documentation] Validates E2E Ping Connectivity and object states for the given scenario:
- ... Configure whitelist with correct ONU location
- ... Validate successful authentication/DHCP/E2E ping
- ... Reboots Fabric Switch
- ... Validate that pings fail
- ... Validate successful authentication/DHCP/E2E ping after OLT comes back up
- #[Setup] None
- #[Teardown] None
- [Tags] fabric1
- Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status awaiting-auth ${onu_device}
- Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Validate DHCP and Ping True True ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
- # Reboot Fabric Switch
- Login And Run Command On Remote System sudo reboot ${fabric_ip} ${fabric_user} ${fabric_pass} prompt=#
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability False ${fabric_ip}
- Wait Until Keyword Succeeds 60s 2s Check Ping False ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
- #Validate DHCP and Ping False False ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability True ${fabric_ip}
- # Validate successful pings since Fabric Switch is Up
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Check Ping True ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
-
-Subscriber(RG) Reboot
- [Documentation] Validates E2E Ping Connectivity and object states for the given scenario:
- ... Configure whitelist with correct ONU location
- ... Validate successful authentication/DHCP/E2E ping
- ... Reboots RG
- ... Validate that pings fail
- ... Validate successful authentication/DHCP/E2E ping after OLT comes back up
- [Tags] RG-reboot
- Wait Until Keyword Succeeds 300s 15s Validate ONU States ACTIVE ENABLED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status awaiting-auth ${onu_device}
- Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Validate DHCP and Ping True True ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
- # Reboot RG
- Login And Run Command On Remote System sudo reboot ${src_ip} ${src_user} ${src_pass} prompt=$
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability False ${fabric_ip}
- Wait Until Keyword Succeeds 60s 2s Check Ping False ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
- Wait Until Keyword Succeeds 150s 10s Check Remote System Reachability True ${src_ip}
- Wait Until Keyword Succeeds 60s 2s Check Ping False ${dst_dp_ip} ${src_iface} ${src_ip} ${src_user} ${src_pass}
- # Perform Reauthentication/DHCP and Ping
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED AWAITING ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status awaiting-auth ${onu_device}
- Validate Authentication True ${src_iface} wpa_supplicant.conf ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Wait Until Keyword Succeeds 60s 2s Validate ATT Workflow Driver SI ENABLED APPROVED ${onu_device}
- Wait Until Keyword Succeeds 60s 2s Validate Subscriber Status enabled ${onu_device}
- Validate DHCP and Ping True True ${src_iface} ${s_tag} ${c_tag} ${dst_dp_ip} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name} ${dst_dp_iface} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
-
-*** Keywords ***
-Setup Suite
- ${auth} = Create List ${XOS_USER} ${XOS_PASSWD}
- ${HEADERS} Create Dictionary Content-Type=application/json
- Create Session ${server_ip} http://${server_ip}:${server_port} auth=${AUTH} headers=${HEADERS}
- ${att_workflow_service_id}= Get Service Owner Id ${ATT_SERVICE}
- ${volt_service_id}= Get Service Owner Id ${VOLT_SERVICE}
- ${AttWhiteListList}= utils.jsonToList ${WHITELIST_PATHFILE} AttWhiteListInfo
- Set Suite Variable ${AttWhiteListList}
- ${AttWhiteListDict}= utils.listToDict ${AttWhiteListList} 0
- ${AttWhiteListDict}= utils.setFieldValueInDict ${AttWhiteListDict} owner_id ${att_workflow_service_id}
- ${onu_device}= Get From Dictionary ${AttWhiteListDict} serial_number
- Set Global Variable ${onu_device}
- ${onu_location}= Get From Dictionary ${AttWhiteListDict} pon_port_id
- Set Global Variable ${onu_location}
- ${SubscriberList}= utils.jsonToList ${SUBSCRIBER_PATHFILE} SubscriberInfo
- Set Global Variable ${SubscriberList}
- ${SubscriberDict}= utils.listToDict ${SubscriberList} 0
- ${s_tag}= utils.getFieldValueFromDict ${SubscriberDict} s_tag
- ${c_tag}= utils.getFieldValueFromDict ${SubscriberDict} c_tag
- ${VoltDeviceList}= utils.jsonToList ${VOLT_DEVICE_PATHFILE} VOLTDeviceInfo
- Set Global Variable ${VoltDeviceList}
- Set Suite Variable ${s_tag}
- Set Suite Variable ${c_tag}
- Set Global Variable ${export_kubeconfig} export KUBECONFIG=${KUBERNETES_CONF}
- # Read variables from yaml file
- ${src_ip}= Evaluate ${hosts}.get("src").get("ip")
- ${src_user}= Evaluate ${hosts}.get("src").get("user")
- ${src_pass}= Evaluate ${hosts}.get("src").get("pass")
- ${src_container_type}= Evaluate ${hosts}.get("src").get("container_type")
- ${src_container_name}= Evaluate ${hosts}.get("src").get("container_name")
- ${src_iface}= Evaluate ${hosts}.get("src").get("dp_iface_name")
- ${dst_ip}= Evaluate ${hosts}.get("dst").get("ip")
- ${dst_user} = Evaluate ${hosts}.get("dst").get("user")
- ${dst_pass}= Evaluate ${hosts}.get("dst").get("pass")
- ${dst_container_type}= Evaluate ${hosts}.get("dst").get("container_type")
- ${dst_container_name}= Evaluate ${hosts}.get("dst").get("container_name")
- ${dst_dp_iface}= Evaluate ${hosts}.get("dst").get("dp_iface_name")
- ${dst_dp_ip}= Evaluate ${hosts}.get("dst").get("dp_iface_ip_qinq")
- ${olt_ip}= Evaluate ${olts}[0].get("ip")
- ${olt_user}= Evaluate ${olts}[0].get("user")
- ${olt_pass}= Evaluate ${olts}[0].get("pass")
- ${fabric_ip}= Evaluate ${fabric_switches}[0].get("ip")
- ${fabric_user}= Evaluate ${fabric_switches}[0].get("user")
- ${fabric_pass}= Evaluate ${fabric_switches}[0].get("pass")
- ${k8s_node_ip}= Evaluate ${nodes}[0].get("ip")
- ${k8s_node_user}= Evaluate ${nodes}[0].get("user")
- ${k8s_node_pass}= Evaluate ${nodes}[0].get("pass")
- Set Suite Variable ${src_ip}
- Set Suite Variable ${src_user}
- Set Suite Variable ${src_pass}
- Set Suite Variable ${src_container_type}
- Set Suite Variable ${src_container_name}
- Set Suite Variable ${src_iface}
- Set Suite Variable ${dst_ip}
- Set Suite Variable ${dst_user}
- Set Suite Variable ${dst_pass}
- Set Suite Variable ${dst_container_type}
- Set Suite Variable ${dst_container_name}
- Set Suite Variable ${dst_dp_iface}
- Set Suite Variable ${dst_dp_ip}
- Set Suite Variable ${olt_ip}
- Set Suite Variable ${olt_user}
- Set Suite Variable ${olt_pass}
- Set Suite Variable ${fabric_ip}
- Set Suite Variable ${fabric_user}
- Set Suite Variable ${fabric_pass}
- Set Suite Variable ${k8s_node_ip}
- Set Suite Variable ${k8s_node_user}
- Set Suite Variable ${k8s_node_pass}
- @{container_list}= Create List
- Append To List ${container_list} att-workflow-att-workflow-driver
- Append To List ${container_list} att-workflow-volt
- Append To List ${container_list} onos
- Append To List ${container_list} xos-core
- Append To List ${container_list} vcore
- Set Suite Variable ${container_list}
- ${datetime}= Get Current Datetime On Kubernetes Node ${k8s_node_ip} ${k8s_node_user} ${k8s_node_pass}
- Set Suite Variable ${datetime}
-
-Teardown Suite
- [Documentation] Performs any additional cleanup required
- Log Suite Teardown cleanup
- Delete All Sessions
-
-Setup Test
- [Documentation] Re-create Subscriber, whitelist, and olt-device models to test
- Log Re-creating objects
- ${datetime}= Get Current Datetime On Kubernetes Node ${k8s_node_ip} ${k8s_node_user} ${k8s_node_pass}
- Create Whitelist
- Create Subscriber
- Create VOLT
-
-Teardown Test
- [Documentation] Delete xos objects, kills processes and cleans up interfaces on src+dst servers
- Clean Up Linux
- Clean Up XOS
- Log Kubernetes Containers Logs Since Time ${datetime} ${container_list}
-
-Clean Up Linux
- [Documentation] Kill processes and clean up interfaces on src+dst servers
- Run Keyword And Ignore Error Kill Linux Process [w]pa_supplicant ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Run Keyword And Ignore Error Kill Linux Process [d]hclient ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Run Keyword If '${dst_ip}' != '${None}' Run Keyword And Ignore Error Kill Linux Process [d]hcpd ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
- Delete IP Addresses from Interface on Remote Host ${src_iface} ${src_ip} ${src_user} ${src_pass} ${src_container_type} ${src_container_name}
- Run Keyword If '${dst_ip}' != '${None}' Delete Interface on Remote Host ${dst_dp_iface}.${s_tag} ${dst_ip} ${dst_user} ${dst_pass} ${dst_container_type} ${dst_container_name}
-
-Clean Up XOS
- [Documentation] Clean up all XOS objects and reinstall voltha after OLT reboots
- Wait Until Keyword Succeeds 60s 2s Clean Up Objects ${VOLT_SUBSCRIBER}
- Wait Until Keyword Succeeds 60s 2s Clean Up Objects ${ATT_WHITELIST}
- Sleep 20s
- Wait Until Keyword Succeeds 30s 2s Validate Subscriber Count 0
- Sleep 10s
- Wait Until Keyword Succeeds 60s 2s Clean Up Objects ${VOLT_DEVICE}
- Sleep 20s
- Wait Until Keyword Succeeds 60s 2s Clean Up Objects ${ATT_SERVICEINSTANCES}
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability False ${olt_ip}
- Wait Until Keyword Succeeds 120s 10s Check Remote System Reachability True ${olt_ip}
- Wait Until Keyword Succeeds 120s 10s Openolt is Up ${olt_ip} ${olt_user} ${olt_pass}
- Sleep 15s
-
-Create Whitelist
- ${AttWhiteListDict}= utils.listToDict ${AttWhiteListList} 0
- CORD Post ${ATT_WHITELIST} ${AttWhiteListDict}
-
-Remove Whitelist
- ${whitelist_id}= Retrieve Whitelist Entry ${onu_device}
- CORD Delete ${ATT_WHITELIST} ${whitelist_id}
-
-Update Whitelist with Wrong Location
- ${whitelist_id}= Retrieve Whitelist Entry ${onu_device}
- CORD Put ${ATT_WHITELIST} {"pon_port_id": 55 } ${whitelist_id}
-
-Update Whitelist with Correct Location
- ${whitelist_id}= Retrieve Whitelist Entry ${onu_device}
- CORD Put ${ATT_WHITELIST} {"pon_port_id": ${onu_location} } ${whitelist_id}
-
-Create Subscriber
- ${SubscriberDict}= utils.listToDict ${SubscriberList} 0
- Wait Until Keyword Succeeds 120s 15s CORD Post ${VOLT_SUBSCRIBER} ${SubscriberDict}
-
-Remove Subscriber
- ${subscriber_id}= Retrieve Subscriber ${c_tag}
- CORD Delete ${VOLT_SUBSCRIBER} ${subscriber_id}
-
-Create VOLT
- ${VoltDeviceDict}= utils.listToDict ${VoltDeviceList} 0
- CORD Post ${VOLT_DEVICE} ${VoltDeviceDict}
diff --git a/src/test/cord-api/Tests/WorkflowValidations/SystemUpgrade.robot b/src/test/cord-api/Tests/WorkflowValidations/SystemUpgrade.robot
index c678f50..c674140 100644
--- a/src/test/cord-api/Tests/WorkflowValidations/SystemUpgrade.robot
+++ b/src/test/cord-api/Tests/WorkflowValidations/SystemUpgrade.robot
@@ -22,7 +22,6 @@
Library OperatingSystem
Library XML
Library RequestsLibrary
-Library /home/ubuntu/voltha/tests/atests/common/testCaseUtils.py
Library ../../Framework/utils/utils.py
Resource ../../Framework/utils/utils.robot
Library ../../Framework/restApi.py
@@ -40,7 +39,7 @@
${HELM_CHARTS_DIR} ~/helm-charts
${KUBERNETES_CONF} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.conf
${KUBERNETES_YAML} ${KUBERNETES_CONFIGS_DIR}/${POD_NAME}.yml
-${onos_tag} 1.13.9-rc4
+${onos_tag} 1.13.9
${voltha_tag} voltha-1.7
*** Test Cases ***
@@ -124,14 +123,14 @@
Upgrade Service
[Arguments] ${service}
${rc}= Run ${export_kubeconfig}; kubectl delete pod $(kubectl get pods | grep seba-services-tosca | head -1 | awk '{print $1}')
- ${rc}= Run ${export_kubeconfig}; kubectl delete pod $(kubectl get jobs | grep seba-services-tosca | head -1 | awk '{print $1}')
+ ${rc}= Run ${export_kubeconfig}; kubectl delete job $(kubectl get jobs | grep seba-services-tosca | head -1 | awk '{print $1}')
${rc}= Run And Return RC helm dep update ${HELM_CHARTS_DIR}/xos-profiles/seba-services
Should Be Equal As Integers ${rc} 0
- ${rc}= Run And Return RC helm upgrade --set ${service}.image.tag=master seba-services ${HELM_CHARTS_DIR}/xos-profiles/seba-services
+ ${rc}= Run And Return RC helm upgrade --recreate-pods --set ${service}.image.tag=master seba-services ${HELM_CHARTS_DIR}/xos-profiles/seba-services
Wait Until Keyword Succeeds 60s 5s Validate Service Running ${service} 1/1
Upgrade ONOS
- ${rc}= Run And Return RC helm upgrade --set images.onos.tag=${onos_tag} onos ${HELM_CHARTS_DIR}/onos
+ ${rc}= Run And Return RC helm upgrade --recreate-pods --set images.onos.tag=${onos_tag} onos ${HELM_CHARTS_DIR}/onos
Should Be Equal As Integers ${rc} 0
Wait Until Keyword Succeeds 60s 5s Validate Service Running onos 2/2
@@ -170,3 +169,4 @@
Run Keyword If '${dst0['ip']}' != '${None}' Run Keyword And Ignore Error Kill Linux Process [d]hcpd ${dst0['ip']} ${dst0['user']} ${dst0['pass']} ${dst0['container_type']} ${dst0['container_name']}
Delete IP Addresses from Interface on Remote Host ${src0['dp_iface_name']} ${src0['ip']} ${src0['user']} ${src0['pass']} ${src0['container_type']} ${src0['container_name']}
Run Keyword If '${dst0['ip']}' != '${None}' Delete Interface on Remote Host ${dst0['dp_iface_name']}.${src0['s_tag']} ${dst0['ip']} ${dst0['user']} ${dst0['pass']} ${dst0['container_type']} ${dst0['container_name']}
+