[VOL-4700] memory leak openolt-adapter and memory consumtions supervision
Change-Id: I3f1d087475545c86e953fcb06580ad973a0b0040
diff --git a/libraries/utility.py b/libraries/utility.py
index 9da6a98..e464e90 100755
--- a/libraries/utility.py
+++ b/libraries/utility.py
@@ -18,6 +18,7 @@
import inspect
import os
import operator
+import requests
# global definition of keys (find in given 'inventory_data')
_NAME = 'name'
@@ -136,3 +137,24 @@
if op == "range":
return ((compare(value1, ">=", value2[0])) and (compare(value1, "<=", value2[1])))
return compare(value1, op, value2)
+
+
+def get_memory_consumptions(address, container, namespace="default"):
+ """
+ Query Prometheus and generate instantaneous memory consumptions for given pods under test
+ :param address: string The address of the Prometheus instance to query
+ :container: string The pod name
+ :param namespace: string The pod namespace
+ :return: memory consumtion value
+ """
+ container_mem_query = ('container_memory_working_set_bytes{namespace="%s",container="%s"}' %
+ (namespace, container))
+ mem_params = {
+ "query": container_mem_query,
+ }
+ r = requests.get("http://%s/api/v1/query" % address, mem_params)
+ container_cpu = r.json()["data"]["result"]
+ if len(container_cpu) > 0:
+ return container_cpu[0]["value"][1]
+ else:
+ return -1
diff --git a/tests/memory-leak/VOLTHA_Memory_Leak_Tests.robot b/tests/memory-leak/VOLTHA_Memory_Leak_Tests.robot
index 043f653..6177ce3 100755
--- a/tests/memory-leak/VOLTHA_Memory_Leak_Tests.robot
+++ b/tests/memory-leak/VOLTHA_Memory_Leak_Tests.robot
@@ -24,6 +24,7 @@
Library XML
Library RequestsLibrary
Library ../../libraries/DependencyLibrary.py
+Library ../../libraries/utility.py WITH NAME utility
Resource ../../libraries/onos.robot
Resource ../../libraries/voltctl.robot
Resource ../../libraries/voltha.robot
@@ -68,7 +69,13 @@
${data_dir} ../data
# number of iterations
# example: -v iterations:10
-${iterations} 200
+${iterations} 50
+# address of Prometheus
+# example: -v prometheusaddr:0.0.0.0
+${prometheusaddr} 0.0.0.0
+# port of Prometheus
+# example: -v prometheusport:31301
+${prometheusport} 31301
# flag to choose the subscriber provisioning command type in ONOS
# TT often provision a single services for a subscriber (eg: hsia, voip, ...) one after the other.
@@ -84,46 +91,120 @@
... - remove flows
... - delete ONU devices
... - wait for onu auto detect
- ... Attention: Due VOL-4703 is not corrected errors in Sanity Test as well as Check Flows Removed will be ignored!
+ ... Attention: Due VOL-4703 is not corrected memory leak tests will run in pipeline for DT workflow only!
... This is a temporaly workaround only! Has to be checked after introduction of voltha-go-controller.
[Tags] functionalMemoryLeak MemoryLeakTestOnuGo
[Setup] Run Keywords Start Logging MemoryLeakTestOnuGo
... AND Setup
Run Keyword If ${has_dataplane} Clean Up Linux
+ ${output_file}= Catenate SEPARATOR=/ ${OUTPUT DIR} MemoryConsumptionsOpenOnuAdapter.txt
+ Create File ${output_file} This file contains the memory consumptions of openonu adapter.
Run Keyword If ${print2console} Log \r\nStart ${iterations} iterations. console=yes
+ ${start_mem_consumption}= utility.get_memory_consumptions ${prometheusaddr}:${prometheusport} adapter-open-onu voltha
+ Run Keyword If ${print2console} Log \r\nMemory consumptions Start ${start_mem_consumption} console=yes
+ ${time}= Get Time
+ Append To File ${output_file} \r\nMemory consumptions Start: ${start_mem_consumption} at ${time}.
FOR ${I} IN RANGE 1 ${iterations} + 1
Run Keyword If ${print2console} Log \r\nStart iteration ${I} of ${iterations}. console=yes
- Run Keyword If "${workflow}"=="DT" Run Keyword And Ignore Error Perform Sanity Test DT
- ... ELSE IF "${workflow}"=="TT" Run Keyword And Ignore Error Perform Sanity Tests TT
- ... ELSE Run Keyword And Ignore Error Perform Sanity Test
+ Run Keyword If "${workflow}"=="DT" Perform Sanity Test DT
+ ... ELSE IF "${workflow}"=="TT" Perform Sanity Tests TT
+ ... ELSE Perform Sanity Test
Sleep 5s
Run Keyword If ${print2console} Log Remove Flows. console=yes
Remove Flows all ONUs
Run Keyword If ${print2console} Log Check Flows removed. console=yes
- Run Keyword And Ignore Error Check All Flows Removed
+ Check All Flows Removed
Run Keyword If ${print2console} Log Get ONU Device IDs. console=yes
${onu_device_id_list}= Get ONUs Device IDs from Voltha
Run Keyword If ${print2console} Log Delete ONUs. console=yes
- Delete Devices In Voltha Type=brcm_openomci_onu
+ Wait Until Keyword Succeeds ${timeout} 1s Delete Devices In Voltha Type=brcm_openomci_onu
Run Keyword If ${print2console} Log Wait for ONUs come back. console=yes
- Wait Until Keyword Succeeds ${timeout} 1s Check for new ONU Device IDs ${onu_device_id_list}
+ Wait Until Keyword Succeeds ${timeout} 1s Check for new ONU Device IDs ${onu_device_id_list}
${list_onus} Create List
Build ONU SN List ${list_onus}
- Wait Until Keyword Succeeds ${timeout} 1s Check all ONU OperStatus ${list_onus} ACTIVE
+ Wait Until Keyword Succeeds ${timeout} 1s Check all ONU OperStatus ${list_onus} ACTIVE
Build ONU SN List ${list_onus}
${onu_reason}= Set Variable If "${workflow}"=="DT" initial-mib-downloaded
... "${workflow}"=="TT" initial-mib-downloaded
... "${workflow}"=="ATT" omci-flows-pushed
- Run Keyword And Ignore Error Wait Until Keyword Succeeds ${timeout} 1s
+ Wait Until Keyword Succeeds ${timeout} 1s
... Validate ONU Devices ENABLED ACTIVE REACHABLE ${list_onus} onu_reason=${onu_reason}
+ ${mem_consumption}= utility.get_memory_consumptions ${prometheusaddr}:${prometheusport} adapter-open-onu voltha
+ Run Keyword If ${print2console} Log \r\nMemory consumptions ${I} ${mem_consumption} console=yes
Run Keyword If ${print2console} Log End iteration ${I} of ${iterations}. console=yes
+ ${time}= Get Time
+ Append To File ${output_file} \r\nMemory consumptions ${I}: ${mem_consumption} at ${time}.
END
[Teardown] Run Keywords Printout ONU Serial Number and Device Id print2console=${print2console}
... AND Run Keyword If ${logging} Get Logical Id of OLT
... AND Run Keyword If ${logging} Collect Logs
... AND Teardown Test
+ ... AND Run Keyword And Ignore Error Wait Until Keyword Succeeds 300 10s
+ ... Validate Memory Consumptions adapter-open-onu voltha ${start_mem_consumption} ${output_file}
... AND Stop Logging MemoryLeakTestOnuGo
+Memory Leak Test Openolt Adapter
+ [Documentation] Test of try to catch memory leak in Openolt Adapter for all three workflows, ATT, DT and TT
+ ... Multiple run of OLT setup and teardown to try to catch memory leak.
+ ... - do workflow related sanity test (bring up onu to omci flows pushed and setup flows)
+ ... - delete OLT devices
+ ... - wait for OLT is removed
+ ... - add and enable OLT again
+ ... - wait for ONUs available again
+ ... Hint: default timeout in BBSim to mimic OLT reboot is 60 seconds!
+ ... This behaviour of BBSim can be modified by 'oltRebootDelay: 60' in BBSim section of helm chart or
+ ... used values.yaml during 'voltha up'.
+ ... Attention: Due VOL-4703 is not corrected memory leak tests will run in pipeline for DT workflow only!
+ ... This is a temporaly workaround only! Has to be checked after introduction of voltha-go-controller.
+ [Tags] functionalMemoryLeak MemoryLeakTestOlt
+ [Setup] Run Keywords Start Logging MemoryLeakTestOlt
+ ... AND Setup
+ Run Keyword If ${has_dataplane} Clean Up Linux
+ ${output_file}= Catenate SEPARATOR=/ ${OUTPUT DIR} MemoryConsumptionsOpenOltAdapter.txt
+ Create File ${output_file} This file contains the memory consumptions of openolt adapter.
+ Run Keyword If ${print2console} Log \r\nStart ${iterations} iterations. console=yes
+ ${start_mem_consumption}= utility.get_memory_consumptions ${prometheusaddr}:${prometheusport} adapter-open-olt voltha
+ Run Keyword If ${print2console} Log \r\nMemory consumptions Start ${start_mem_consumption} console=yes
+ ${time}= Get Time
+ Append To File ${output_file} \r\nMemory consumptions Start: ${start_mem_consumption} at ${time}.
+ FOR ${I} IN RANGE 1 ${iterations} + 1
+ Run Keyword If ${print2console} Log \r\nStart iteration ${I} of ${iterations}. console=yes
+ Run Keyword If "${workflow}"=="DT" Perform Sanity Test DT
+ ... ELSE IF "${workflow}"=="TT" Perform Sanity Tests TT
+ ... ELSE Perform Sanity Test
+ Sleep 5s
+ Run Keyword If ${print2console} Log Delete OLTs. console=yes
+ Delete Devices In Voltha Type=openolt
+ Run Keyword If ${print2console} Log Check OLTs removed. console=yes
+ Wait Until Keyword Succeeds ${timeout} 1s Test Empty Device List
+ Sleep 20s
+ Run Keyword If ${print2console} Log Add OLTs (calling Setup). console=yes
+ Setup
+ Run Keyword If ${print2console} Log Wait for ONUs come back. console=yes
+ ${list_onus} Create List
+ Build ONU SN List ${list_onus}
+ Wait Until Keyword Succeeds ${timeout} 1s Check all ONU OperStatus ${list_onus} ACTIVE
+ Build ONU SN List ${list_onus}
+ ${onu_reason}= Set Variable If "${workflow}"=="DT" initial-mib-downloaded
+ ... "${workflow}"=="TT" initial-mib-downloaded
+ ... "${workflow}"=="ATT" omci-flows-pushed
+ Wait Until Keyword Succeeds ${timeout} 1s
+ ... Validate ONU Devices ENABLED ACTIVE REACHABLE ${list_onus} onu_reason=${onu_reason}
+ ${mem_consumption}= utility.get_memory_consumptions ${prometheusaddr}:${prometheusport} adapter-open-olt voltha
+ Run Keyword If ${print2console} Log \r\nMemory consumptions ${I} ${mem_consumption} console=yes
+ Run Keyword If ${print2console} Log End iteration ${I} of ${iterations}. console=yes
+ ${time}= Get Time
+ Append To File ${output_file} \r\nMemory consumptions ${I}: ${mem_consumption} at ${time}.
+ END
+ [Teardown] Run Keywords Printout ONU Serial Number and Device Id print2console=${print2console}
+ ... AND Run Keyword If ${logging} Get Logical Id of OLT
+ ... AND Run Keyword If ${logging} Collect Logs
+ ... AND Teardown Test
+ ... AND Run Keyword And Ignore Error Wait Until Keyword Succeeds 300 10s
+ ... Validate Memory Consumptions adapter-open-olt voltha ${start_mem_consumption} ${output_file}
+ ... AND Stop Logging MemoryLeakTestOlt
+
+
*** Keywords ***
Setup Suite
[Documentation] Set up the test suite
@@ -194,3 +275,13 @@
FOR ${item} IN @{old_device_ids}
List Should Not Contain Value ${new_device_ids} ${item} Old device id ${item} still present.
END
+
+Validate Memory Consumptions
+ [Documentation] Validates meory consumptions of passed POD
+ [Arguments] ${container} ${namespace} ${start_value} ${output_file}
+ ${mem_consumption}= utility.get_memory_consumptions ${prometheusaddr}:${prometheusport} ${container} ${namespace}
+ Run Keyword If ${print2console} Log \r\nMemory consumptions Teardwown ${mem_consumption} console=yes
+ ${time}= Get Time
+ Append To File ${output_file} \r\nMemory consumptions Teardown: ${mem_consumption} at ${time}.
+ ${upper_bound}= Evaluate (${start_value} + (${start_value}*0.3))
+ Should Be True ${upper_bound} >= ${mem_consumption}