[VOL-3352] fix timer issues and build script timeouts
Change-Id: I19b2a0624931921bc0a03f811624ad3571137726
diff --git a/Jenkinsfile-voltha-build b/Jenkinsfile-voltha-build
index d4fd9bf..58c44a7 100644
--- a/Jenkinsfile-voltha-build
+++ b/Jenkinsfile-voltha-build
@@ -300,7 +300,7 @@
}
xconnect_out = sh returnStatus: true, script: """
sleep 30
- version=\$(sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "summary" | grep version)
+ version=\$(sshpass -p karaf ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "summary" | grep version)
sleep 10
if [[ \$version == *"version=2.2"* ]]; then
curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "endpoints": [${deployment_config.fabric_switches[0].bngPort},${aggPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect'
@@ -315,11 +315,11 @@
if ( params.inBandManagement ) {
stage('Reboot OLT') {
for(int i=0; i < deployment_config.olts.size(); i++) {
- timeout(5) {
+ timeout(10) {
sh returnStdout: true, script: """
ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; reboot' || true
- sleep 300
+ sleep 360
"""
}
timeout(15) {
diff --git a/libraries/utils.robot b/libraries/utils.robot
index 5a6ecdf..4b0fa9f 100644
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -312,7 +312,7 @@
... Validate OLT Device PREPROVISIONED UNKNOWN UNKNOWN ${olt_device_id}
Sleep 5s
Enable Device ${olt_device_id}
- Wait Until Keyword Succeeds 180s 5s
+ Wait Until Keyword Succeeds 380s 5s
... Validate OLT Device ENABLED ACTIVE REACHABLE ${olt_serial_number}
${logical_id}= Get Logical Device ID From SN ${olt_serial_number}
Set Suite Variable ${logical_id}
diff --git a/tests/dt-workflow/Voltha_DT_FailureScenarios.robot b/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
index 60c7fa7..8e496be 100644
--- a/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
+++ b/tests/dt-workflow/Voltha_DT_FailureScenarios.robot
@@ -103,7 +103,7 @@
... Execute ONOS CLI Command ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
... volt-add-subscriber-access ${of_id} ${onu_port}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Verify subscriber access flows are added for the ONU port
@@ -228,7 +228,7 @@
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id}
# Bring up the device and verify it authenticates
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ Wait Until Keyword Succeeds 360s 5s Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=omci-flows-pushed
END
# Scale down the open OLT adapter deployment to 0 PODs and once confirmed, scale it back to 1
@@ -294,7 +294,7 @@
${onu_port}= Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
... Get ONU Port in ONOS ${src['onu']} ${of_id}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Check ONU port is Disabled in ONOS
@@ -346,7 +346,7 @@
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id}
# Bring up the device and verify it authenticates
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ Wait Until Keyword Succeeds 360s 5s Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=initial-mib-downloaded
END
@@ -418,7 +418,7 @@
# Waiting extra time for the ONUs to come up
Sleep 60s
# Check OLT states
- Wait Until Keyword Succeeds ${timeout} 5s Validate OLT Device ENABLED ACTIVE REACHABLE
+ Wait Until Keyword Succeeds 360s 5s Validate OLT Device ENABLED ACTIVE REACHABLE
... ${olt_serial_number}
Run Keyword And Ignore Error Collect Logs
#Check after reboot that ONUs are active, DHCP and pingable
@@ -462,7 +462,7 @@
... Execute ONOS CLI Command ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
... volt-add-subscriber-access ${of_id} ${onu_port}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Verify subscriber access flows are added for the ONU port
diff --git a/tests/dt-workflow/Voltha_DT_PODTests.robot b/tests/dt-workflow/Voltha_DT_PODTests.robot
index a8e3765..41112a6 100644
--- a/tests/dt-workflow/Voltha_DT_PODTests.robot
+++ b/tests/dt-workflow/Voltha_DT_PODTests.robot
@@ -137,7 +137,7 @@
... Validate Device DISABLED UNKNOWN
... REACHABLE ${src['onu']}
Enable Device ${onu_device_id}
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']}
# Add Subscriber Access
@@ -148,7 +148,7 @@
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Verify Subscriber Access Flows Added For ONU DT ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${of_id}
... ${onu_port} ${nni_port} ${src['s_tag']}
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']} onu=True onu_reason=omci-flows-pushed
# TODO: Yet to Verify on the GPON based Physical POD (VOL-2652)
@@ -206,7 +206,7 @@
... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}
Sleep 5s
Enable Device ${onu_device_id}
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']} onu=True onu_reason=onu-reenabled
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
diff --git a/tests/functional/Voltha_ErrorScenarios.robot b/tests/functional/Voltha_ErrorScenarios.robot
index 8154b8d..ac18520 100644
--- a/tests/functional/Voltha_ErrorScenarios.robot
+++ b/tests/functional/Voltha_ErrorScenarios.robot
@@ -44,7 +44,7 @@
# For below variable value, using deployment name as using grep for
# parsing radius pod name, we can also use full radius pod name
${RESTART_POD_NAME} radius
-${timeout} 120s
+${timeout} 360s
${of_id} 0
${logical_id} 0
${has_dataplane} True
diff --git a/tests/functional/Voltha_FailureScenarios.robot b/tests/functional/Voltha_FailureScenarios.robot
index b0f2f58..4fa9600 100644
--- a/tests/functional/Voltha_FailureScenarios.robot
+++ b/tests/functional/Voltha_FailureScenarios.robot
@@ -96,7 +96,7 @@
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
... Verify Eapol Flows Added For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
Run Keyword If ${has_dataplane} Clean Up Linux
@@ -268,7 +268,7 @@
... ${of_id}
# Bring up the device and verify it authenticates
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ Wait Until Keyword Succeeds 360s 5s Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=omci-flows-pushed
Wait Until Keyword Succeeds ${timeout} 2s Verify Eapol Flows Added For ONU ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} ${onu_port}
@@ -345,7 +345,7 @@
${onu_port}= Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
... Get ONU Port in ONOS ${src['onu']} ${of_id}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Check ONU port is Disabled in ONOS
@@ -429,7 +429,7 @@
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
... Verify Eapol Flows Added For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
# Perform Authentication
@@ -486,7 +486,7 @@
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id}
# Bring up the device and verify it authenticates
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE REACHABLE
+ Wait Until Keyword Succeeds 360s 5s Validate Device ENABLED ACTIVE REACHABLE
... ${onu_device_id} onu=True onu_reason=omci-flows-pushed
Wait Until Keyword Succeeds ${timeout} 2s Verify Eapol Flows Added For ONU ${ONOS_SSH_IP}
... ${ONOS_SSH_PORT} ${onu_port}
@@ -575,7 +575,7 @@
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 2s
... Verify Eapol Flows Added For ONU ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${onu_port}
# Verify ONU state in voltha
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s Validate Device
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s Validate Device
... ENABLED ACTIVE REACHABLE
... ${src['onu']} onu=True onu_reason=omci-flows-pushed
Run Keyword If ${has_dataplane} Clean Up Linux
diff --git a/tests/functional/Voltha_PODTests.robot b/tests/functional/Voltha_PODTests.robot
index c5a099b..eb9e8e2 100644
--- a/tests/functional/Voltha_PODTests.robot
+++ b/tests/functional/Voltha_PODTests.robot
@@ -372,7 +372,7 @@
${onu_device_id}= Get Device ID From SN ${src['onu']}
${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s Get ONU Port in ONOS ${src['onu']}
... ${of_id}
- Wait Until Keyword Succeeds ${timeout} 5s Validate Device ENABLED ACTIVE
+ Wait Until Keyword Succeeds 180s 5s Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']} onu=True onu_reason=omci-flows-pushed
END
@@ -390,7 +390,7 @@
${src}= Set Variable ${hosts.src[${I}]}
${dst}= Set Variable ${hosts.dst[${I}]}
${onu_device_id}= Get Device ID From SN ${src['onu']}
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate Device ENABLED ACTIVE
... REACHABLE ${src['onu']} onu=True onu_reason=omci-flows-pushed
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
@@ -402,7 +402,7 @@
Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
... Validate Device DISABLED UNKNOWN
... REACHABLE ${src['onu']} onu=false
- Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s
+ Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s
... Validate OLT Device ENABLED ACTIVE
... REACHABLE ${olt_serial_number}
END