[VOL-3007] Moving the scale pipeline in robot
Change-Id: I729685325f6b05e9f9f17f14e96d597c885ebe68
diff --git a/.gitignore b/.gitignore
index a767f1c..7baae54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,4 @@
*.swp
# IDEs
-.idea
\ No newline at end of file
+.idea
diff --git a/Makefile b/Makefile
index f8b58df..fd803b5 100644
--- a/Makefile
+++ b/Makefile
@@ -46,6 +46,10 @@
# for backwards compatibility
sanity-kind: sanity-single-kind
+# for scale pipeline
+voltha-scale: ROBOT_MISC_ARGS += -i activation $(ROBOT_DEBUG_LOG_OPT)
+voltha-scale: voltha-scale-test
+
# target to invoke DT Workflow Sanity
sanity-kind-dt: ROBOT_MISC_ARGS += -i sanityDt $(ROBOT_DEBUG_LOG_OPT)
sanity-kind-dt: ROBOT_CONFIG_FILE := $(ROBOT_SANITY_DT_SINGLE_PON_FILE)
@@ -148,6 +152,11 @@
cd tests/dt-workflow ;\
robot -V $(ROBOT_CONFIG_FILE) $(ROBOT_MISC_ARGS) $(ROBOT_FILE)
+voltha-scale-test: vst_venv
+ source ./$</bin/activate ; set -u ;\
+ cd tests/scale ;\
+ robot $(ROBOT_MISC_ARGS) Voltha_Scale_Tests.robot
+
# self-test, lint, and setup targets
# virtualenv for the robot tools
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index d1ef61c..6dbf6b1 100644
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -138,7 +138,7 @@
[Arguments] ${datetime}
[Documentation] This keyword checks for the error occurence in the voltha pods
&{errorPodDict} Create Dictionary
- &{containerDict} Get Container Dictionary
+ &{containerDict} Get Container Dictionary voltha
FOR ${podName} IN @{PODLIST1}
${containerName} Get From Dictionary ${containerDict} ${podName}
${rc} ${logOutput} Run And Return Rc And Output
@@ -218,16 +218,17 @@
[Return] ${errorDict}
Get Container Dictionary
+ [Arguments] ${namespace}
[Documentation] Creates a mapping for pod name and container name and returns the same
&{containerDict} Create Dictionary
${containerName} Set Variable ${EMPTY}
- ${podName} Run kubectl get deployment -n voltha | awk 'NR>1 {print $1}'
+ ${podName} Run kubectl get deployment -n ${namespace} | awk 'NR>1 {print $1}'
@{podNameList}= Split To Lines ${podName}
Append To List ${podNameList} voltha-etcd-cluster voltha-kafka voltha-ro-core voltha-zookeeper
Log ${podNameList}
#Creatiing dictionary to correspond pod name and container name
FOR ${pod} IN @{podNameList}
- ${containerName} Run kubectl get pod -n voltha | grep ${pod} | awk '{print $1}'
+ ${containerName} Run kubectl get pod -n ${namespace} | grep ${pod} | awk '{print $1}'
&{containerDict} Set To Dictionary ${containerDict} ${pod} ${containerName}
END
Log ${containerDict}
diff --git a/libraries/onos.robot b/libraries/onos.robot
index 3d61332..4d10af5 100644
--- a/libraries/onos.robot
+++ b/libraries/onos.robot
@@ -234,19 +234,17 @@
${aaa_users}= Execute ONOS CLI Command ${ip} ${port} aaa-users | grep AUTHORIZED | grep ${onu_port}
Should Not Be Empty ${aaa_users} ONU port ${onu_port} not found in aaa-users
-Verify Number of AAA-Users
+Assert Number of AAA-Users
[Arguments] ${ip} ${port} ${expected_onus}
[Documentation] Matches for number of aaa-users authorized based on number of onus
- ##TODO: filter by onu serial number instead of count
${aaa_users}= Execute ONOS CLI Command ${ip} ${port} aaa-users | grep AUTHORIZED | wc -l
- Should Contain ${aaa_users} ${expected_onus}
+ Should Be Equal As Integers ${aaa_users} ${expected_onus}
Validate DHCP Allocations
[Arguments] ${ip} ${port} ${expected_onus}
[Documentation] Matches for number of dhcpacks based on number of onus
- ##TODO: filter by onu serial number instead of count
${allocations}= Execute ONOS CLI Command ${ip} ${port} dhcpl2relay-allocations | grep DHCPACK | wc -l
- Should Contain ${allocations} ${expected_onus}
+ Should Be Equal As Integers ${allocations} ${expected_onus}
Validate Subscriber DHCP Allocation
[Arguments] ${ip} ${port} ${onu_port}
@@ -278,3 +276,88 @@
Run Keyword If '${dpid}' != ''
... Should Be Equal As Integers ${rc} 0
END
+
+Assert Ports in ONOS
+ [Arguments] ${ip} ${port} ${count} ${filter}
+ [Documentation] Check that a certain number of ports are enabled in ONOS
+ ${ports}= Execute ONOS CLI Command ${ip} ${port}
+ ... ports -e | grep ${filter} | wc -l
+ Should Be Equal As Integers ${ports} ${count}
+
+Wait for Ports in ONOS
+ [Arguments] ${ip} ${port} ${count} ${filter}
+ [Documentation] Waits untill a certain number of ports are enabled in ONOS
+ Wait Until Keyword Succeeds 10m 5s Assert Ports in ONOS ${ip} ${port} ${count} ${filter}
+
+Wait for AAA Authentication
+ [Arguments] ${ip} ${port} ${count}
+ [Documentation] Waits untill a certain number of subscribers are authenticated in ONOS
+ Wait Until Keyword Succeeds 10m 5s Assert Number of AAA-Users ${ip} ${port} ${count}
+
+Wait for DHCP Ack
+ [Arguments] ${ip} ${port} ${count}
+ [Documentation] Waits untill a certain number of subscribers have received a DHCP_ACK
+ Wait Until Keyword Succeeds 10m 5s Validate DHCP Allocations ${ip} ${port} ${count}
+
+Provision subscriber
+ [Documentation] Calls volt-add-subscriber-access in ONOS
+ [Arguments] ${onos_ip} ${onos_port} ${of_id} ${onu_port}
+ Execute ONOS CLI Command ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ ... volt-add-subscriber-access ${of_id} ${onu_port}
+
+List Enabled UNI Ports
+ [Documentation] List all the UNI Ports, the only way we have is to filter out the one called NNI
+ ... Creates a list of dictionaries
+ [Arguments] ${onos_ip} ${onos_port} ${of_id}
+ [Return] [{'port': '16', 'of_id': 'of:00000a0a0a0a0a00'}, {'port': '32', 'of_id': 'of:00000a0a0a0a0a00'}]
+ ${result}= Create List
+ ${out}= Execute ONOS CLI Command ${onos_ip} ${onos_port}
+ ... ports -e ${of_id} | grep -v SWITCH | grep -v nni
+ @{unis}= Split To Lines ${out}
+ FOR ${uni} IN @{unis}
+ ${matches} = Get Regexp Matches ${uni} .*port=([0-9]+),.* 1
+ &{portDict} Create Dictionary of_id=${of_id} port=${matches[0]}
+ Append To List ${result} ${portDict}
+ END
+ Log ${result}
+ Return From Keyword ${result}
+
+Provision all subscribers on device
+ [Documentation] Calls volt-add-subscriber-access in ONOS for all the enabled UNI ports on a partivular device
+ [Arguments] ${onos_ip} ${onos_port} ${of_id}
+ ${unis}= List Enabled UNI Ports ${onos_ip} ${onos_port} ${of_id}
+ FOR ${uni} IN @{unis}
+ Provision subscriber ${onos_ip} ${onos_port} ${uni['of_id']} ${uni['port']}
+ END
+
+List OLTs
+ [Documentation] Returns a list of OLTs known to ONOS
+ [Arguments] ${onos_ip} ${onos_port}
+ [Return] ['of:00000a0a0a0a0a00', 'of:00000a0a0a0a0a01']
+ ${result}= Create List
+ ${out}= Execute ONOS CLI Command ${onos_ip} ${onos_port}
+ ... volt-olts
+ @{olts}= Split To Lines ${out}
+ FOR ${olt} IN @{olts}
+ Log ${olt}
+ ${matches} = Get Regexp Matches ${olt} ^OLT (.+)$ 1
+ # there may be some logs mixed with the output so only append if we have a match
+ ${matches_length}= Get Length ${matches}
+ Run Keyword If ${matches_length}==1
+ ... Append To List ${result} ${matches[0]}
+ END
+ Return From Keyword ${result}
+
+Count ADDED flows
+ [Documentation] Count the flows in ADDED state in ONOS
+ [Arguments] ${onos_ip} ${onos_port} ${targetFlows}
+ ${flows}= Execute ONOS CLI Command ${onos_ip} ${onos_port}
+ ... flows -s | grep ADDED | wc -l
+ Should Be Equal As Integers ${targetFlows} ${flows}
+
+Wait for all flows to in ADDED state
+ [Documentation] Waits until the flows have been provisioned
+ [Arguments] ${onos_ip} ${onos_port} ${workflow} ${uni_count} ${olt_count} ${provisioned}
+ ${targetFlows}= Calculate flows by workflow ${workflow} ${uni_count} ${olt_count} ${provisioned}
+ Wait Until Keyword Succeeds 10m 5s Count ADDED flows
+ ... ${onos_ip} ${onos_port} ${targetFlows}
diff --git a/libraries/utils.robot b/libraries/utils.robot
index b0e5548..258b465 100644
--- a/libraries/utils.robot
+++ b/libraries/utils.robot
@@ -483,3 +483,50 @@
${rc} ${output}= Run and Return Rc and Output date --date="${dateStr}" "+%s"
Should Be Equal As Numbers ${rc} 0
[return] ${output}
+
+Calculate flows by workflow
+ [Documentation] Calculate how many flows should be created based on the workflow, the number of UNIs
+ ... and whether the subscribers have been provisioned
+ [Arguments] ${workflow} ${uni_count} ${olt_count} ${provisioned}
+ ${expectedFlows}= Run Keyword If $workflow=="att" Calculate Att flows
+ ... ${uni_count} ${olt_count} ${provisioned}
+ ... ELSE IF $workflow=="dt" Calculate Dt Flows
+ ... ${uni_count} ${olt_count} ${provisioned}
+ ... ELSE IF $workflow=="tt" Calculate Tt Flows
+ ... ${uni_count} ${olt_count} ${provisioned}
+ ... ELSE Fail Workflow ${workflow} should be one of 'att', 'dt', 'tt'
+ Return From Keyword ${expectedFlows}
+
+Calculate Att flows
+ [Documentation] Calculate the flow for the ATT workflow
+ ... NOTE we may need to add support for IGMP enabled/disabled
+ [Arguments] ${uni_count} ${olt_count} ${provisioned}
+ # (1 EAPOL * ONUs) * (1 LLDP + 1 DHCP * OLTS) before provisioning
+ # (1 EAPOL, 1 DHCP, 1 IGMP, 4 DP * ONUs) * (1 LLDP + 1 DHCP * OLTS) before provisioning
+ ${flow_count}= Run Keyword If $provisioned=='false'
+ ... Evaluate (${uni_count} * 1) + (${olt_count} * 2)
+ ... ELSE
+ ... Evaluate (${uni_count} * 7) + (${olt_count} * 2)
+ Return From Keyword ${flow_count}
+
+Calculate Dt flows
+ [Documentation] Calculate the flow for the DT workflow
+ [Arguments] ${uni_count} ${olt_count} ${provisioned}
+ # (1 LLDP * OLTS) before provisioning
+ # (4 DP * ONUs) * (1 LLDP * OLTS) before provisioning
+ ${flow_count}= Run Keyword If $provisioned=='false'
+ ... Evaluate (${olt_count} * 1)
+ ... ELSE
+ ... Evaluate (${uni_count} * 4) + (${olt_count} * 1)
+ Return From Keyword ${flow_count}
+
+Calculate Tt flows
+ [Documentation] Calculate the flow for the TT workflow
+ [Arguments] ${uni_count} ${olt_count} ${provisioned}
+ # (1 LLDP + 1 DHCP * OLTS) before provisioning
+ # (1 DHCP, 1 IGMP, 4 DP * ONUs) * (1 LLDP + 1 DHCP * OLTS) before provisioning
+ ${flow_count}= Run Keyword If $provisioned=='false'
+ ... Evaluate (${olt_count} * 2)
+ ... ELSE
+ ... Evaluate (${uni_count} * 6) + (${olt_count} * 1)
+ Return From Keyword ${flow_count}
\ No newline at end of file
diff --git a/libraries/voltctl.robot b/libraries/voltctl.robot
index 0051e28..d315ff6 100644
--- a/libraries/voltctl.robot
+++ b/libraries/voltctl.robot
@@ -22,6 +22,7 @@
Library Collections
Library RequestsLibrary
Library OperatingSystem
+Resource ./utils.robot
*** Keywords ***
Test Empty Device List
@@ -34,11 +35,11 @@
Should Be Equal As Integers ${length} 0
Create Device
- [Arguments] ${ip} ${port}
+ [Arguments] ${ip} ${port} ${type}=openolt
[Documentation] Creates a device in VOLTHA
#create/preprovision device
${rc} ${device_id}= Run and Return Rc and Output
- ... ${VOLTCTL_CONFIG}; voltctl device create -t openolt -H ${ip}:${port}
+ ... ${VOLTCTL_CONFIG}; voltctl device create -t ${type} -H ${ip}:${port}
Should Be Equal As Integers ${rc} 0
[Return] ${device_id}
@@ -140,6 +141,13 @@
Log ${devices}
Should Be Equal As Integers ${rc1} 0
+Get Logical Device List from Voltha
+ [Documentation] Gets Logical Device List Output from Voltha (in json format)
+ ${rc1} ${devices}= Run and Return Rc and Output ${VOLTCTL_CONFIG}; voltctl logicaldevice list -o json
+ Log ${devices}
+ Should Be Equal As Integers ${rc1} 0
+ Return From Keyword ${devices}
+
Validate Device
[Documentation]
... Parses the output of "voltctl device list" and inspects a device ${id}, specified as either
@@ -447,3 +455,39 @@
Should Be Equal As Integers ${rc} 0
Run Keyword and Ignore Error Wait Until Keyword Succeeds 60s 1s Validate Device
... ENABLED DISCOVERED UNREACHABLE ${onu_id} onu=True
+
+Assert ONUs in Voltha
+ [Arguments] ${count}
+ [Documentation] Check that a certain number of devices reached the ACTIVE/ENABLE state
+ ${rc1} ${devices}= Run and Return Rc and Output
+ ... ${VOLTCTL_CONFIG}; voltctl device list | grep -v OLT | grep ACTIVE | wc -l
+ Should Be Equal As Integers ${rc1} 0
+ Should Be Equal As Integers ${devices} ${count}
+
+Wait for ONUs in VOLTHA
+ [Arguments] ${count}
+ [Documentation] Waits until a certain number of devices reached the ACTIVE/ENABLE state
+ Wait Until Keyword Succeeds 10m 5s Assert ONUs In Voltha ${count}
+
+Count Logical Devices flows
+ [Documentation] Count the flows across logical devices in VOLTHA
+ [Arguments] ${targetFlows}
+ ${output}= Get Logical Device List From Voltha
+ ${logical_devices}= To Json ${output}
+ ${total_flows}= Set Variable 0
+ FOR ${device} IN @{logical_devices}
+ ${rc} ${flows}= Run and Return Rc and Output
+ ... ${VOLTCTL_CONFIG}; voltctl logicaldevice flows ${device['id']} | grep -v ID | wc -l
+ Should Be Equal As Integers ${rc} 0
+ ${total_flows}= Evaluate ${total_flows} + ${flows}
+ END
+ Should Be Equal As Integers ${targetFlows} ${total_flows}
+
+Wait for Logical Devices flows
+ [Documentation] Waits until the flows have been provisioned in the logical device
+ [Arguments] ${workflow} ${uni_count} ${olt_count} ${provisioned}
+ ${targetFlows}= Calculate flows by workflow ${workflow} ${uni_count} ${olt_count} ${provisioned}
+ Log ${targetFlows}
+ # TODO extend Validate Logical Device Flows to check the correct number of flows
+ Wait Until Keyword Succeeds 10m 5s Count Logical Devices flows ${targetFlows}
+
diff --git a/requirements.txt b/requirements.txt
index 5d054b9..f7a9a66 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,7 @@
robotframework-lint
robotframework-requests
robotframework-sshlibrary
+robotframework-timer==0.0.5
yamllint
# replace when we can use upstream (needs python 3.6)
git+https://github.com/zdw/robotframework-importresource@b81b87aabaee0594e966687b41e3674b866f28ee
diff --git a/tests/scale/Voltha_Scale_Tests.robot b/tests/scale/Voltha_Scale_Tests.robot
new file mode 100644
index 0000000..c2c1527
--- /dev/null
+++ b/tests/scale/Voltha_Scale_Tests.robot
@@ -0,0 +1,174 @@
+# Copyright 2017 - present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tests can be enabled by passing the following tags:
+# - [setup] Creates and enable the OLT devices
+# - [activation] Checks that ONUs are active in VOLTHA and ports discevered in ONOS
+# - [flow-before] Checks that flows are pushed (before subscriber provisioning)
+# - [authentication] Checks that subscribers are correctly authenticated
+# - [provision] Provision the data-plane flows for all the subscribers
+# - [flow-after] Checks that flows are pushed (after subscriber provisioning)
+# - [dhcp] Checks that subscribers have received an IP
+#
+# To run the full test:
+# robot Voltha_Scale_Tests.robot
+#
+# To run only ceratain tests:
+# robot -i activation -i flow-before Voltha_Scale_Tests.robot
+#
+# To exclude only ceratain tests:
+# robot -e -i flow-before Voltha_Scale_Tests.robot
+#
+# Once te test complete you can extrapolate the results by using
+# python extract-times.py
+
+*** Settings ***
+Documentation Collect measurements on VOLTHA performances
+Suite Setup Setup Suite
+#Test Setup Setup
+#Test Teardown Teardown
+Suite Teardown Teardown Suite
+Library Collections
+Library String
+Library OperatingSystem
+Library XML
+Library Timer
+Library RequestsLibrary
+Library ../../libraries/DependencyLibrary.py
+Resource ../../libraries/onos.robot
+Resource ../../libraries/voltctl.robot
+Resource ../../libraries/voltha.robot
+Resource ../../libraries/utils.robot
+Resource ../../libraries/k8s.robot
+Resource ../../variables/variables.robot
+
+*** Variables ***
+${ONOS_SSH_IP} 127.0.0.1
+${ONOS_SSH_PORT} 8101
+
+# Scale pipeline values
+${olt} 1
+${pon} 1
+${onu} 1
+
+${enableLLDP} false
+${enableFlowProvisioning} true
+${enableSubscriberProvisioning} true
+
+${workflow} att
+#${flowsBeforeProvisioning} 1
+#${flowsAfterProvisioning} 1
+
+# Per-test logging on failure is turned off by default; set this variable to enable
+${container_log_dir} ${None}
+
+*** Test Cases ***
+
+Create and Enable devices
+ [Documentation] Create and enable the OLTs in VOLTHA
+ [Tags] setup
+ ${olt_device_ids}= Create List
+ FOR ${INDEX} IN RANGE 0 ${olt}
+ ${olt_device_id}= Create Device bbsim${INDEX} 50060 openolt 0f:f1:ce:c${INDEX}:ff:ee
+ Enable Device ${olt_device_id}
+ Append To List ${olt_device_ids} ${olt_device_id}
+ END
+
+ Set Suite Variable ${olt_device_ids}
+
+Onu Activation in VOLTHA
+ [Documentation] Check that all ONUs reach the ACTIVE/ENABLED state in VOLTHA
+ [Tags] non-critical activation plot-voltha-onus
+ Wait For ONUs In VOLTHA ${total_onus}
+
+Port Discovery in ONOS
+ [Documentation] Check that all the UNI ports show up in ONOS
+ [Tags] non-critical activation plot-onos-ports
+ Wait for Ports in ONOS ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${total_onus} BBSM
+
+Flows validation in VOLTHA before subscriber provisioning
+ [Documentation] Check that all the flows has been acknowledged
+ [Tags] non-critical flow-before plot-voltha-flows-before
+ # NOTE fail the test immediately if we're trying to check flows without provisioning them
+ Should Be Equal ${enableFlowProvisioning} true
+ Wait for Logical Devices flows ${workflow} ${total_onus} ${olt} false
+
+Flows validation in ONOS before subscriber provisioning
+ [Documentation] Check that all the flows has been stored in the logical device
+ [Tags] non-critical flow-before plot-onos-flows-before
+ # NOTE fail the test immediately if we're trying to check flows without provisioning them
+ Should Be Equal ${enableFlowProvisioning} true
+ Wait for all flows to in ADDED state ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ ... ${workflow} ${total_onus} ${olt} false
+
+Wait for subscribers to be Authenticated
+ [Documentation] Check that all subscribers have successfully authenticated
+ [Tags] non-critical authentication plot-onos-auth
+ Wait for AAA Authentication ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${total_onus}
+
+Provision subscribers
+ [Documentation] Provision data plane flows for all the subscribers
+ [Tags] non-critical provision
+ Should Be Equal ${enableSubscriberProvisioning} true
+ ${olts}= List OLTs ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ FOR ${olt} IN @{olts}
+ Provision all subscribers on device ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${olt}
+ END
+
+Flows validation in VOLTHA after subscriber provisioning
+ [Documentation] Check that all the flows has been stored in the logical device
+ [Tags] non-critical flow-after plot-voltha-flows-after
+ # NOTE fail the test immediately if we're trying to check flows without provisioning them
+ Should Be Equal ${enableFlowProvisioning} true
+ Wait for Logical Devices flows ${workflow} ${total_onus} ${olt} true
+
+Flows validation in ONOS after subscriber provisioning
+ [Documentation] Check that all the flows has been acknowledged
+ [Tags] non-critical flow-after plot-onos-flows-after
+ # NOTE fail the test immediately if we're trying to check flows without provisioning them
+ Should Be Equal ${enableFlowProvisioning} true
+ Wait for all flows to in ADDED state ${ONOS_SSH_IP} ${ONOS_SSH_PORT}
+ ... ${workflow} ${total_onus} ${olt} true
+
+Wait for subscribers to have an IP
+ [Documentation] Check that all subscribers have received a DHCP_ACK
+ [Tags] non-critical dhcp plot-onos-dhcp
+ Wait for DHCP Ack ${ONOS_SSH_IP} ${ONOS_SSH_PORT} ${total_onus}
+
+Disable and Delete devices
+ [Documentation] Disable and delete the OLTs in VOLTHA
+ [Tags] non-critical teardown
+ FOR ${olt_device_id} IN @{olt_device_ids}
+ Disable Device ${olt_device_id}
+ Delete Device ${olt_device_id}
+ END
+
+ Set Suite Variable ${olt_device_ids}
+
+*** Keywords ***
+Setup Suite
+ [Documentation] Setup test global variables and starts a timer
+ Set Suite Variable ${KUBECTL_CONFIG} export KUBECONFIG=%{KUBECONFIG}
+ Set Suite Variable ${VOLTCTL_CONFIG} export VOLTCONFIG=%{VOLTCONFIG}
+
+ ${total_onus}= Evaluate ${olt} * ${pon} * ${onu}
+ Set Suite Variable ${total_onus}
+
+ Configure Timer 10 minutes 0 seconds SuiteTimer
+ Start Timer SuiteTimer
+
+Teardown Suite
+ [Documentation] Verify the timer
+ Stop Timer SuiteTimer
+ Verify Single Timer 10 minutes 0 seconds SuiteTimer
diff --git a/tests/scale/collect-result.py b/tests/scale/collect-result.py
new file mode 100644
index 0000000..f65ae31
--- /dev/null
+++ b/tests/scale/collect-result.py
@@ -0,0 +1,76 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+dash = "-" * 75
+double_dash = "=" * 75
+
+
+def cut_string(str):
+ return (str[:48] + "..") if len(str) > 50 else str
+
+def format_key(str):
+ return str.replace("plot-", "").replace("-", " ")
+
+def read_file(file, plot_folder):
+ # create element tree object
+ tree = ET.parse(file)
+
+ # get root element
+ root = tree.getroot()
+
+ results = {}
+
+ print(double_dash)
+ print("{:<50}{:>10}{:>15}".format("Test Name", "Status", "Duration (s)"))
+ print(double_dash)
+ for test in root.findall("./suite/test"):
+ status = test.find(".status")
+ name = test.attrib["name"]
+ start = status.attrib["starttime"]
+ end = status.attrib["endtime"]
+ s = datetime.strptime(start[:-4], "%Y%m%d %H:%M:%S")
+ e = datetime.strptime(end[:-4], "%Y%m%d %H:%M:%S")
+ diff = e - s
+ print("{:<50}{:>10}{:>15}".format(cut_string(name), status.attrib["status"], diff.seconds))
+ print(dash)
+
+ # check if the test has a tag that starts with "plot-",
+ # if so store the result to create plot files for Jenkins
+ for tag in test.findall("./tags/tag"):
+ if "plot-" in tag.text:
+ results[tag.text] = diff.seconds
+
+ if not os.path.isdir(plot_folder):
+ os.mkdir(plot_folder)
+
+ for k, v in results.items():
+ f = open("%s/%s.txt" % (plot_folder, k), "a")
+ f.write("%s\n" % format_key(k))
+ f.write(str(v))
+ f.close()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(prog="collect-result")
+ parser.add_argument("-r", "--robot-output", help="he robot output.xml file to process", default="output.xml"),
+ parser.add_argument("-p", "--plot-folder", help="here to output the files needed for the Jenkins plots", default="plots")
+
+ args = parser.parse_args()
+
+ read_file(args.robot_output, args.plot_folder)