blob: 95fcd600ddb0c10a44bbb12259483f112d3f0f8a [file] [log] [blame]
Girish Gowdrae09a6202021-01-12 18:10:59 -08001/*
2 * Copyright 2021-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +000017//Package pmmgr provides the utilities to manage onu metrics
18package pmmgr
Girish Gowdrae09a6202021-01-12 18:10:59 -080019
20import (
21 "context"
Girish Gowdra0e533642021-03-02 22:02:51 -080022 "encoding/json"
Girish Gowdrae09a6202021-01-12 18:10:59 -080023 "fmt"
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +000024 "math"
25 "sync"
26 "time"
27
Girish Gowdrae0140f02021-02-02 16:55:09 -080028 "github.com/looplab/fsm"
Girish Gowdrae09a6202021-01-12 18:10:59 -080029 "github.com/opencord/omci-lib-go"
30 me "github.com/opencord/omci-lib-go/generated"
khenaidoo7d3c5582021-08-11 18:09:44 -040031 "github.com/opencord/voltha-lib-go/v7/pkg/db"
32 "github.com/opencord/voltha-lib-go/v7/pkg/db/kvstore"
33 "github.com/opencord/voltha-lib-go/v7/pkg/log"
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +000034 cmn "github.com/opencord/voltha-openonu-adapter-go/internal/pkg/common"
khenaidoo7d3c5582021-08-11 18:09:44 -040035 "github.com/opencord/voltha-protos/v5/go/extension"
36 "github.com/opencord/voltha-protos/v5/go/voltha"
Girish Gowdrae09a6202021-01-12 18:10:59 -080037)
38
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +000039// events of L2 PM FSM
Girish Gowdrae0140f02021-02-02 16:55:09 -080040const (
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +000041 L2PmEventInit = "L2PmEventInit"
42 L2PmEventTick = "L2PmEventTick"
43 L2PmEventSuccess = "L2PmEventSuccess"
44 L2PmEventFailure = "L2PmEventFailure"
45 L2PmEventAddMe = "L2PmEventAddMe"
46 L2PmEventDeleteMe = "L2PmEventDeleteMe"
47 L2PmEventStop = "L2PmEventStop"
Girish Gowdrae0140f02021-02-02 16:55:09 -080048)
49
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +000050// states of L2 PM FSM
51const (
52 L2PmStNull = "L2PmStNull"
53 L2PmStStarting = "L2PmStStarting"
54 L2PmStSyncTime = "L2PmStSyncTime"
55 L2PmStIdle = "L2PmStIdle"
56 L2PmStCreatePmMe = "L2PmStCreatePm"
57 L2PmStDeletePmMe = "L2PmStDeletePmMe"
58 L2PmStCollectData = "L2PmStCollectData"
59)
60
61// CL2PmFsmIdleState - TODO: add comment
62const CL2PmFsmIdleState = L2PmStIdle
Girish Gowdrae0140f02021-02-02 16:55:09 -080063
Girish Gowdra5a7c4922021-01-22 18:33:41 -080064// general constants used for overall Metric Collection management
65const (
66 DefaultMetricCollectionFrequency = 15 * 60 // unit in seconds. This setting can be changed from voltha NBI PmConfig configuration
67 GroupMetricEnabled = true // This is READONLY and cannot be changed from VOLTHA NBI
68 DefaultFrequencyOverrideEnabled = true // This is READONLY and cannot be changed from VOLTHA NBI
69 FrequencyGranularity = 5 // The frequency (in seconds) has to be multiple of 5. This setting cannot changed later.
70)
71
Himani Chawla43f95ff2021-06-03 00:24:12 +053072// constants for ethernet frame extended pm collection
73const (
74 ExtendedPmCreateAttempts = 3
75 UnsupportedCounterValue32bit uint64 = 4294967294
76 UnsupportedCounterValue64bit uint64 = 18446744073709551614
77 dropEvents = "DropEvents"
78 octets = "Octets"
79 frames = "Frames"
80 broadcastFrames = "BroadcastFrames"
81 multicastFrames = "MulticastFrames"
82 crcErroredFrames = "CrcErroredFrames"
83 undersizeFrames = "UndersizeFrames"
84 oversizeFrames = "OversizeFrames"
85 frames64Octets = "Frames64Octets"
86 frames65To127Octets = "Frames65To127Octets"
87 frames128To255Octets = "Frames128To255Octets"
88 frames256To511Octets = "Frames256To511Octets"
89 frames512To1023Octets = "Frames512To1023Octets"
90 frames1024To1518Octets = "Frames1024To1518Octets"
91)
92
Girish Gowdra5a7c4922021-01-22 18:33:41 -080093// OpticalPowerGroupMetrics are supported optical pm names
94var OpticalPowerGroupMetrics = map[string]voltha.PmConfig_PmType{
Girish Gowdrae20a4f62021-03-09 16:06:23 -080095 "ani_g_instance_id": voltha.PmConfig_CONTEXT,
96 "transmit_power_dBm": voltha.PmConfig_GAUGE,
97 "receive_power_dBm": voltha.PmConfig_GAUGE,
Girish Gowdra5a7c4922021-01-22 18:33:41 -080098}
99
100// OpticalPowerGroupMetrics specific constants
101const (
Girish Gowdrae0140f02021-02-02 16:55:09 -0800102 OpticalPowerGroupMetricName = "PON_Optical"
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800103 OpticalPowerGroupMetricEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
104 OpticalPowerMetricGroupCollectionFrequency = 5 * 60 // unit in seconds. This setting can be changed from voltha NBI PmConfig configuration
105)
106
107// UniStatusGroupMetrics are supported UNI status names
108var UniStatusGroupMetrics = map[string]voltha.PmConfig_PmType{
Girish Gowdrad3436802021-06-28 13:15:40 -0700109 "uni_port_no": voltha.PmConfig_CONTEXT,
110 "me_class_id": voltha.PmConfig_CONTEXT,
111 "entity_id": voltha.PmConfig_CONTEXT,
112 "configuration_ind": voltha.PmConfig_GAUGE,
113 "oper_status": voltha.PmConfig_GAUGE,
114 "uni_admin_state": voltha.PmConfig_GAUGE,
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800115}
116
117// UniStatusGroupMetrics specific constants
118const (
Girish Gowdrae0140f02021-02-02 16:55:09 -0800119 UniStatusGroupMetricName = "UNI_Status"
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800120 UniStatusGroupMetricEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
121 UniStatusMetricGroupCollectionFrequency = 5 * 60 // unit in seconds. This setting can be changed from voltha NBI PmConfig configuration
122)
123
Girish Gowdrae0140f02021-02-02 16:55:09 -0800124// *** Classical L2 PM Counters begin ***
125
126// EthernetBridgeHistory are supported ethernet bridge history counters fetched from
127// Ethernet Frame Performance Monitoring History Data Downstream and Ethernet Frame Performance Monitoring History Data Upstream MEs.
128var EthernetBridgeHistory = map[string]voltha.PmConfig_PmType{
129 "class_id": voltha.PmConfig_CONTEXT,
130 "entity_id": voltha.PmConfig_CONTEXT,
131 "interval_end_time": voltha.PmConfig_CONTEXT,
132 "parent_class_id": voltha.PmConfig_CONTEXT,
133 "parent_entity_id": voltha.PmConfig_CONTEXT,
134 "upstream": voltha.PmConfig_CONTEXT,
135
136 "drop_events": voltha.PmConfig_COUNTER,
137 "octets": voltha.PmConfig_COUNTER,
138 "packets": voltha.PmConfig_COUNTER,
139 "broadcast_packets": voltha.PmConfig_COUNTER,
140 "multicast_packets": voltha.PmConfig_COUNTER,
141 "crc_errored_packets": voltha.PmConfig_COUNTER,
142 "undersize_packets": voltha.PmConfig_COUNTER,
143 "oversize_packets": voltha.PmConfig_COUNTER,
144 "64_octets": voltha.PmConfig_COUNTER,
145 "65_to_127_octets": voltha.PmConfig_COUNTER,
146 "128_to_255_octets": voltha.PmConfig_COUNTER,
147 "256_to_511_octets": voltha.PmConfig_COUNTER,
148 "512_to_1023_octets": voltha.PmConfig_COUNTER,
149 "1024_to_1518_octets": voltha.PmConfig_COUNTER,
150}
151
152// EthernetUniHistory are supported ethernet uni history counters fetched from
153// Ethernet Performance Monitoring History Data ME.
154var EthernetUniHistory = map[string]voltha.PmConfig_PmType{
155 "class_id": voltha.PmConfig_CONTEXT,
156 "entity_id": voltha.PmConfig_CONTEXT,
157 "interval_end_time": voltha.PmConfig_CONTEXT,
158
159 "fcs_errors": voltha.PmConfig_COUNTER,
160 "excessive_collision_counter": voltha.PmConfig_COUNTER,
161 "late_collision_counter": voltha.PmConfig_COUNTER,
162 "frames_too_long": voltha.PmConfig_COUNTER,
163 "buffer_overflows_on_rx": voltha.PmConfig_COUNTER,
164 "buffer_overflows_on_tx": voltha.PmConfig_COUNTER,
165 "single_collision_frame_counter": voltha.PmConfig_COUNTER,
166 "multiple_collisions_frame_counter": voltha.PmConfig_COUNTER,
167 "sqe_counter": voltha.PmConfig_COUNTER,
168 "deferred_tx_counter": voltha.PmConfig_COUNTER,
169 "internal_mac_tx_error_counter": voltha.PmConfig_COUNTER,
170 "carrier_sense_error_counter": voltha.PmConfig_COUNTER,
171 "alignment_error_counter": voltha.PmConfig_COUNTER,
172 "internal_mac_rx_error_counter": voltha.PmConfig_COUNTER,
173}
174
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800175// FecHistory is supported FEC Performance Monitoring History Data related metrics
176var FecHistory = map[string]voltha.PmConfig_PmType{
177 "class_id": voltha.PmConfig_CONTEXT,
178 "entity_id": voltha.PmConfig_CONTEXT,
179 "interval_end_time": voltha.PmConfig_CONTEXT,
180
181 "corrected_bytes": voltha.PmConfig_COUNTER,
182 "corrected_code_words": voltha.PmConfig_COUNTER,
183 "uncorrectable_code_words": voltha.PmConfig_COUNTER,
184 "total_code_words": voltha.PmConfig_COUNTER,
185 "fec_seconds": voltha.PmConfig_COUNTER,
186}
187
188// GemPortHistory is supported GEM Port Network Ctp Performance Monitoring History Data
189// related metrics
190var GemPortHistory = map[string]voltha.PmConfig_PmType{
191 "class_id": voltha.PmConfig_CONTEXT,
192 "entity_id": voltha.PmConfig_CONTEXT,
193 "interval_end_time": voltha.PmConfig_CONTEXT,
194
195 "transmitted_gem_frames": voltha.PmConfig_COUNTER,
196 "received_gem_frames": voltha.PmConfig_COUNTER,
197 "received_payload_bytes": voltha.PmConfig_COUNTER,
198 "transmitted_payload_bytes": voltha.PmConfig_COUNTER,
199 "encryption_key_errors": voltha.PmConfig_COUNTER,
200}
201
Himani Chawla43f95ff2021-06-03 00:24:12 +0530202var maskToEthernetFrameExtendedPM32Bit = map[uint16][]string{
203 0x3F00: {"drop_events", "octets", "frames", "broadcast_frames", "multicast_frames", "crc_errored_frames"},
204 0x00FC: {"undersize_frames", "oversize_frames", "64_octets", "65_to_127_octets", "128_to_255_octets", "256_to_511_octets"},
205 0x0003: {"512_to_1023_octets", "1024_to_1518_octets"},
206}
207
208var maskToEthernetFrameExtendedPM64Bit = map[uint16][]string{
209 0x3800: {"drop_events", "octets", "frames"},
210 0x0700: {"broadcast_frames", "multicast_frames", "crc_errored_frames"},
211 0x00E0: {"undersize_frames", "oversize_frames", "64_octets"},
212 0x001C: {"65_to_127_octets", "128_to_255_octets", "256_to_511_octets"},
213 0x0003: {"512_to_1023_octets", "1024_to_1518_octets"},
214}
215
Girish Gowdrae0140f02021-02-02 16:55:09 -0800216// Constants specific for L2 PM collection
217const (
218 L2PmCollectionInterval = 15 * 60 // Unit in seconds. Do not change this as this fixed by OMCI specification for L2 PM counters
219 SyncTimeRetryInterval = 15 // Unit seconds
220 L2PmCreateAttempts = 3
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800221 L2PmDeleteAttempts = 3
Girish Gowdrae0140f02021-02-02 16:55:09 -0800222 L2PmCollectAttempts = 3
Girish Gowdra453750f2021-02-16 16:36:46 -0800223 // Per Table 11.2.9-1 – OMCI baseline message limitations in G.988 spec, the max GET Response
224 // payload size is 25. We define 24 (one less) to allow for dynamic insertion of IntervalEndTime
225 // attribute (1 byte) in L2 PM GET Requests.
Himani Chawla43f95ff2021-06-03 00:24:12 +0530226 MaxL2PMGetPayLoadSize = 24
227 MaxEthernetFrameExtPmPayloadSize = 25
Girish Gowdrae0140f02021-02-02 16:55:09 -0800228)
229
230// EthernetUniHistoryName specific constants
231const (
232 EthernetBridgeHistoryName = "Ethernet_Bridge_Port_History"
233 EthernetBridgeHistoryEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
234 EthernetBridgeHistoryFrequency = L2PmCollectionInterval
235)
236
237// EthernetBridgeHistory specific constants
238const (
239 EthernetUniHistoryName = "Ethernet_UNI_History"
240 EthernetUniHistoryEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
241 EthernetUniHistoryFrequency = L2PmCollectionInterval
242)
243
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800244// FecHistory specific constants
245const (
246 FecHistoryName = "FEC_History"
247 FecHistoryEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
248 FecHistoryFrequency = L2PmCollectionInterval
249)
250
251// GemPortHistory specific constants
252const (
253 GemPortHistoryName = "GEM_Port_History"
254 GemPortHistoryEnabled = true // This setting can be changed from voltha NBI PmConfig configuration
255 GemPortHistoryFrequency = L2PmCollectionInterval
256)
257
Girish Gowdra0e533642021-03-02 22:02:51 -0800258// KV Store related constants
259const (
Himani Chawla43f95ff2021-06-03 00:24:12 +0530260 cPmKvStorePrefix = "%s/openonu/pm-data/%s" // <some-base-path>/openonu/pm-data/<onu-device-id>
261 cPmAdd = "add"
262 cPmAdded = "added"
263 cPmRemove = "remove"
264 cPmRemoved = "removed"
265 cExtPmKvStorePrefix = "%s/omci_me" //<some-base-path>/omci_me/<onu_vendor>/<onu_equipment_id>/<onu_sw_version>
Girish Gowdra0e533642021-03-02 22:02:51 -0800266)
267
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800268// Defines the type for generic metric population function
269type groupMetricPopulateFunc func(context.Context, me.ClassID, uint16, me.AttributeValueMap, me.AttributeValueMap, map[string]float32, *int) error
270
Girish Gowdrae0140f02021-02-02 16:55:09 -0800271// *** Classical L2 PM Counters end ***
272
Girish Gowdra0e533642021-03-02 22:02:51 -0800273type pmMEData struct {
274 InstancesActive []uint16 `json:"instances_active"` // list of active ME instance IDs for the group
275 InstancesToDelete []uint16 `json:"instances_to_delete"` // list of ME instance IDs marked for deletion for the group
276 InstancesToAdd []uint16 `json:"instances_to_add"` // list of ME instance IDs marked for addition for the group
277}
278
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800279type groupMetric struct {
280 groupName string
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000281 Enabled bool
282 Frequency uint32 // valid only if FrequencyOverride is enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800283 metricMap map[string]voltha.PmConfig_PmType
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000284 NextCollectionInterval time.Time // valid only if FrequencyOverride is enabled.
285 IsL2PMCounter bool // true for only L2 PM counters
Girish Gowdrae0140f02021-02-02 16:55:09 -0800286 collectAttempts uint32 // number of attempts to collect L2 PM data
Girish Gowdra0e533642021-03-02 22:02:51 -0800287 pmMEData *pmMEData
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800288}
289
290type standaloneMetric struct {
291 metricName string
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000292 Enabled bool
293 Frequency uint32 // valid only if FrequencyOverride is enabled.
294 NextCollectionInterval time.Time // valid only if FrequencyOverride is enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800295}
296
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000297// OnuMetricsManager - TODO: add comment
298type OnuMetricsManager struct {
299 deviceID string
300 pDeviceHandler cmn.IdeviceHandler
301 pOnuDeviceEntry cmn.IonuDeviceEntry
302 PAdaptFsm *cmn.AdapterFsm
Girish Gowdrae09a6202021-01-12 18:10:59 -0800303
Himani Chawla43f95ff2021-06-03 00:24:12 +0530304 opticalMetricsChan chan me.AttributeValueMap
305 uniStatusMetricsChan chan me.AttributeValueMap
306 l2PmChan chan me.AttributeValueMap
307 extendedPmMeChan chan me.AttributeValueMap
308 syncTimeResponseChan chan bool // true is success, false is fail
309 l2PmCreateOrDeleteResponseChan chan bool // true is success, false is fail
310 extendedPMCreateOrDeleteResponseChan chan me.Results // true is sucesss, false is fail
Girish Gowdrae0140f02021-02-02 16:55:09 -0800311
312 activeL2Pms []string // list of active l2 pm MEs created on the ONU.
313 l2PmToDelete []string // list of L2 PMs to delete
314 l2PmToAdd []string // list of L2 PM to add
Girish Gowdrae09a6202021-01-12 18:10:59 -0800315
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000316 GroupMetricMap map[string]*groupMetric
317 StandaloneMetricMap map[string]*standaloneMetric
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800318
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000319 StopProcessingOmciResponses chan bool
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -0700320 omciProcessingActive bool
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800321
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000322 StopTicks chan bool
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -0700323 tickGenerationActive bool
Girish Gowdrae0140f02021-02-02 16:55:09 -0800324
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000325 NextGlobalMetricCollectionTime time.Time // valid only if pmConfig.FreqOverride is set to false.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800326
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000327 OnuMetricsManagerLock sync.RWMutex
Girish Gowdra0e533642021-03-02 22:02:51 -0800328
329 pmKvStore *db.Backend
Himani Chawla43f95ff2021-06-03 00:24:12 +0530330
331 supportedEthernetFrameExtendedPMClass me.ClassID
Himani Chawlac77d5372021-07-12 15:42:26 +0530332 ethernetFrameExtendedPmUpStreamMEByEntityID map[uint16]*me.ManagedEntity
333 ethernetFrameExtendedPmDownStreamMEByEntityID map[uint16]*me.ManagedEntity
Himani Chawla43f95ff2021-06-03 00:24:12 +0530334 extPmKvStore *db.Backend
335 onuEthernetFrameExtendedPmLock sync.RWMutex
336 isDeviceReadyToCollectExtendedPmStats bool
Girish Gowdrae09a6202021-01-12 18:10:59 -0800337}
338
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000339// NewOnuMetricsManager returns a new instance of the NewOnuMetricsManager
Girish Gowdra0e533642021-03-02 22:02:51 -0800340// The metrics manager module is responsible for configuration and management of individual and group metrics.
341// Currently all the metrics are managed as a group which fall into two categories - L2 PM and "all others"
342// The L2 PM counters have a fixed 15min interval for PM collection while all other group counters have
343// the collection interval configurable.
344// The global PM config is part of the voltha.Device struct and is backed up on KV store (by rw-core).
345// This module also implements resiliency for L2 PM ME instances that are active/pending-delete/pending-add.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000346func NewOnuMetricsManager(ctx context.Context, dh cmn.IdeviceHandler, onuDev cmn.IonuDeviceEntry) *OnuMetricsManager {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800347
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000348 var metricsManager OnuMetricsManager
349 metricsManager.deviceID = dh.GetDeviceID()
350 logger.Debugw(ctx, "init-OnuMetricsManager", log.Fields{"device-id": metricsManager.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800351 metricsManager.pDeviceHandler = dh
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000352 metricsManager.pOnuDeviceEntry = onuDev
Girish Gowdrae09a6202021-01-12 18:10:59 -0800353
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000354 commMetricsChan := make(chan cmn.Message)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800355 metricsManager.opticalMetricsChan = make(chan me.AttributeValueMap)
356 metricsManager.uniStatusMetricsChan = make(chan me.AttributeValueMap)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800357 metricsManager.l2PmChan = make(chan me.AttributeValueMap)
Himani Chawla43f95ff2021-06-03 00:24:12 +0530358 metricsManager.extendedPmMeChan = make(chan me.AttributeValueMap)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800359
360 metricsManager.syncTimeResponseChan = make(chan bool)
361 metricsManager.l2PmCreateOrDeleteResponseChan = make(chan bool)
Himani Chawla43f95ff2021-06-03 00:24:12 +0530362 metricsManager.extendedPMCreateOrDeleteResponseChan = make(chan me.Results)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800363
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000364 metricsManager.StopProcessingOmciResponses = make(chan bool)
365 metricsManager.StopTicks = make(chan bool)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800366
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000367 metricsManager.GroupMetricMap = make(map[string]*groupMetric)
368 metricsManager.StandaloneMetricMap = make(map[string]*standaloneMetric)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800369
Himani Chawlac77d5372021-07-12 15:42:26 +0530370 metricsManager.ethernetFrameExtendedPmUpStreamMEByEntityID = make(map[uint16]*me.ManagedEntity)
371 metricsManager.ethernetFrameExtendedPmDownStreamMEByEntityID = make(map[uint16]*me.ManagedEntity)
Himani Chawla43f95ff2021-06-03 00:24:12 +0530372
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000373 if dh.GetPmConfigs() == nil { // dh.GetPmConfigs() is NOT nil if adapter comes back from a restart. We should NOT go back to defaults in this case
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800374 metricsManager.initializeAllGroupMetrics()
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800375 }
376
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800377 metricsManager.populateLocalGroupMetricData(ctx)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800378
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800379 if err := metricsManager.initializeL2PmFsm(ctx, commMetricsChan); err != nil {
Girish Gowdrae0140f02021-02-02 16:55:09 -0800380 return nil
381 }
Girish Gowdrae0140f02021-02-02 16:55:09 -0800382
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800383 // initialize the next metric collection intervals.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000384 metricsManager.InitializeMetricCollectionTime(ctx)
Girish Gowdra0e533642021-03-02 22:02:51 -0800385
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000386 baseKvStorePath := fmt.Sprintf(cPmKvStorePrefix, dh.GetBackendPathPrefix(), metricsManager.deviceID)
387 metricsManager.pmKvStore = dh.SetBackend(ctx, baseKvStorePath)
Girish Gowdra0e533642021-03-02 22:02:51 -0800388 if metricsManager.pmKvStore == nil {
389 logger.Errorw(ctx, "Can't initialize pmKvStore - no backend connection to PM module",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000390 log.Fields{"device-id": metricsManager.deviceID, "service": baseKvStorePath})
Girish Gowdra0e533642021-03-02 22:02:51 -0800391 return nil
392 }
Girish Gowdra50e56422021-06-01 16:46:04 -0700393 // restore data from KV store
394 if err := metricsManager.restorePmData(ctx); err != nil {
395 logger.Errorw(ctx, "error restoring pm data", log.Fields{"err": err})
396 // we continue given that it does not effect the actual services for the ONU,
397 // but there may be some negative effect on PM collection (there may be some mismatch in
398 // the actual PM config and what is present on the device).
399 }
Girish Gowdra0e533642021-03-02 22:02:51 -0800400
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000401 baseExtPmKvStorePath := fmt.Sprintf(cExtPmKvStorePrefix, dh.GetBackendPathPrefix())
402 metricsManager.extPmKvStore = dh.SetBackend(ctx, baseExtPmKvStorePath)
Himani Chawla43f95ff2021-06-03 00:24:12 +0530403 if metricsManager.extPmKvStore == nil {
404 logger.Errorw(ctx, "Can't initialize extPmKvStore - no backend connection to PM module",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000405 log.Fields{"device-id": metricsManager.deviceID, "service": baseExtPmKvStorePath})
Himani Chawla43f95ff2021-06-03 00:24:12 +0530406 return nil
407 }
408
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000409 logger.Info(ctx, "init-OnuMetricsManager completed", log.Fields{"device-id": metricsManager.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800410 return &metricsManager
411}
412
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000413// InitializeMetricCollectionTime - TODO: add comment
414func (mm *OnuMetricsManager) InitializeMetricCollectionTime(ctx context.Context) {
415 if mm.pDeviceHandler.GetPmConfigs().FreqOverride {
416 // If mm.pDeviceHandler.GetPmConfigs().FreqOverride is set to true, then group/standalone metric specific interval applies
417 mm.OnuMetricsManagerLock.Lock()
418 defer mm.OnuMetricsManagerLock.Unlock()
419 for _, v := range mm.GroupMetricMap {
420 if v.Enabled && !v.IsL2PMCounter { // L2 PM counter collection is managed in a L2PmFsm
421 v.NextCollectionInterval = time.Now().Add(time.Duration(v.Frequency) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800422 }
423 }
424
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000425 for _, v := range mm.StandaloneMetricMap {
426 if v.Enabled {
427 v.NextCollectionInterval = time.Now().Add(time.Duration(v.Frequency) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800428 }
429 }
430 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000431 // If mm.pDeviceHandler.GetPmConfigs().FreqOverride is set to false, then overall metric specific interval applies
432 mm.NextGlobalMetricCollectionTime = time.Now().Add(time.Duration(mm.pDeviceHandler.GetPmConfigs().DefaultFreq) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800433 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000434 logger.Infow(ctx, "initialized standalone group/metric collection time", log.Fields{"device-id": mm.deviceID})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800435}
436
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000437// UpdateDefaultFrequency - TODO: add comment
438func (mm *OnuMetricsManager) UpdateDefaultFrequency(ctx context.Context, pmConfigs *voltha.PmConfigs) error {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800439 // Verify that the configured DefaultFrequency is > 0 and is a multiple of FrequencyGranularity
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800440 if pmConfigs.DefaultFreq == 0 || (pmConfigs.DefaultFreq > 0 && pmConfigs.DefaultFreq%FrequencyGranularity != 0) {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800441 logger.Errorf(ctx, "frequency-%u-should-be-a-multiple-of-%u", pmConfigs.DefaultFreq, FrequencyGranularity)
442 return fmt.Errorf("frequency-%d-should-be-a-multiple-of-%d", pmConfigs.DefaultFreq, FrequencyGranularity)
443 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000444 mm.pDeviceHandler.GetPmConfigs().DefaultFreq = pmConfigs.DefaultFreq
445 // re-set the NextGlobalMetricCollectionTime based on the new DefaultFreq
446 mm.NextGlobalMetricCollectionTime = time.Now().Add(time.Duration(mm.pDeviceHandler.GetPmConfigs().DefaultFreq) * time.Second)
447 logger.Debugw(ctx, "frequency-updated--new-frequency", log.Fields{"device-id": mm.deviceID, "frequency": mm.pDeviceHandler.GetPmConfigs().DefaultFreq})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800448 return nil
449}
450
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000451// UpdateGroupFreq - TODO: add comment
452func (mm *OnuMetricsManager) UpdateGroupFreq(ctx context.Context, aGroupName string, pmConfigs *voltha.PmConfigs) error {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800453 var newGroupFreq uint32
454 found := false
455 groupSliceIdx := 0
456 var group *voltha.PmGroupConfig
457 for groupSliceIdx, group = range pmConfigs.Groups {
458 if group.GroupName == aGroupName {
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800459 // freq 0 is not allowed and it should be multiple of FrequencyGranularity
460 if group.GroupFreq == 0 || (group.GroupFreq > 0 && group.GroupFreq%FrequencyGranularity != 0) {
461 logger.Errorf(ctx, "frequency-%u-should-be-a-multiple-of-%u", group.GroupFreq, FrequencyGranularity)
462 return fmt.Errorf("frequency-%d-should-be-a-multiple-of-%d", group.GroupFreq, FrequencyGranularity)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800463 }
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800464 newGroupFreq = group.GroupFreq
465 found = true
466 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800467 }
468 }
469 // if not found update group freq and next collection interval for the group
470 if !found {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000471 logger.Errorw(ctx, "group name not found", log.Fields{"device-id": mm.deviceID, "groupName": aGroupName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800472 return fmt.Errorf("group-name-not-found-%v", aGroupName)
473 }
474
475 updated := false
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000476 mm.OnuMetricsManagerLock.Lock()
477 defer mm.OnuMetricsManagerLock.Unlock()
478 for k, v := range mm.GroupMetricMap {
479 if k == aGroupName && !v.IsL2PMCounter { // We cannot allow the L2 PM counter frequency to be updated. It is 15min fixed by OMCI spec
480 v.Frequency = newGroupFreq
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800481 // update internal pm config
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000482 mm.pDeviceHandler.GetPmConfigs().Groups[groupSliceIdx].GroupFreq = newGroupFreq
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800483 // Also updated the next group metric collection time from now
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000484 v.NextCollectionInterval = time.Now().Add(time.Duration(newGroupFreq) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800485 updated = true
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000486 logger.Infow(ctx, "group frequency updated", log.Fields{"device-id": mm.deviceID, "newGroupFreq": newGroupFreq, "groupName": aGroupName})
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800487 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800488 }
489 }
490 if !updated {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000491 logger.Errorw(ctx, "group frequency not updated", log.Fields{"device-id": mm.deviceID, "newGroupFreq": newGroupFreq, "groupName": aGroupName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800492 return fmt.Errorf("internal-error-during-group-freq-update--groupname-%s-freq-%d", aGroupName, newGroupFreq)
493 }
494 return nil
495}
496
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000497// UpdateMetricFreq - TODO: add comment
498func (mm *OnuMetricsManager) UpdateMetricFreq(ctx context.Context, aMetricName string, pmConfigs *voltha.PmConfigs) error {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800499 var newMetricFreq uint32
500 found := false
501 metricSliceIdx := 0
502 var metric *voltha.PmConfig
503 for metricSliceIdx, metric = range pmConfigs.Metrics {
504 if metric.Name == aMetricName {
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800505 // freq 0 is not allowed and it should be multiple of FrequencyGranularity
506 if metric.SampleFreq == 0 || (metric.SampleFreq > 0 && metric.SampleFreq%FrequencyGranularity != 0) {
507 logger.Errorf(ctx, "frequency-%u-should-be-a-multiple-of-%u", metric.SampleFreq, FrequencyGranularity)
508 return fmt.Errorf("frequency-%d-should-be-a-multiple-of-%d", metric.SampleFreq, FrequencyGranularity)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800509 }
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800510 newMetricFreq = metric.SampleFreq
511 found = true
512 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800513 }
514 }
515 if !found {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000516 logger.Errorw(ctx, "metric name not found", log.Fields{"device-id": mm.deviceID, "metricName": aMetricName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800517 return fmt.Errorf("metric-name-not-found-%v", aMetricName)
518 }
519
520 updated := false
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000521 mm.OnuMetricsManagerLock.Lock()
522 defer mm.OnuMetricsManagerLock.Unlock()
523 for k, v := range mm.GroupMetricMap {
Girish Gowdraaf0ad632021-01-27 13:00:01 -0800524 if k == aMetricName {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000525 v.Frequency = newMetricFreq
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800526 // update internal pm config
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000527 mm.pDeviceHandler.GetPmConfigs().Metrics[metricSliceIdx].SampleFreq = newMetricFreq
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800528 // Also updated the next standalone metric collection time from now
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000529 v.NextCollectionInterval = time.Now().Add(time.Duration(newMetricFreq) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800530 updated = true
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000531 logger.Infow(ctx, "metric frequency updated", log.Fields{"device-id": mm.deviceID, "newMetricFreq": newMetricFreq, "aMetricName": aMetricName})
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800532 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800533 }
534 }
535 if !updated {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000536 logger.Errorw(ctx, "metric frequency not updated", log.Fields{"device-id": mm.deviceID, "newMetricFreq": newMetricFreq, "aMetricName": aMetricName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800537 return fmt.Errorf("internal-error-during-standalone-metric-update--matricnane-%s-freq-%d", aMetricName, newMetricFreq)
538 }
539 return nil
540}
541
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000542// UpdateGroupSupport - TODO: add comment
543func (mm *OnuMetricsManager) UpdateGroupSupport(ctx context.Context, aGroupName string, pmConfigs *voltha.PmConfigs) error {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800544 groupSliceIdx := 0
545 var group *voltha.PmGroupConfig
546
547 for groupSliceIdx, group = range pmConfigs.Groups {
548 if group.GroupName == aGroupName {
549 break
550 }
551 }
552 if group == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000553 logger.Errorw(ctx, "group metric not found", log.Fields{"device-id": mm.deviceID, "groupName": aGroupName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800554 return fmt.Errorf("group-not-found--groupName-%s", aGroupName)
555 }
556
557 updated := false
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000558 mm.OnuMetricsManagerLock.Lock()
559 defer mm.OnuMetricsManagerLock.Unlock()
560 for k, v := range mm.GroupMetricMap {
561 if k == aGroupName && v.Enabled != group.Enabled {
562 mm.pDeviceHandler.GetPmConfigs().Groups[groupSliceIdx].Enabled = group.Enabled
563 v.Enabled = group.Enabled
Girish Gowdrae0140f02021-02-02 16:55:09 -0800564 if group.Enabled {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000565 if v.IsL2PMCounter {
Girish Gowdrae0140f02021-02-02 16:55:09 -0800566 // If it is a L2 PM counter we need to mark the PM to be added
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800567 mm.l2PmToAdd = mm.appendIfMissingString(mm.l2PmToAdd, v.groupName)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800568 // If the group support flag toggles too soon, we need to delete the group name from l2PmToDelete slice
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800569 mm.l2PmToDelete = mm.removeIfFoundString(mm.l2PmToDelete, v.groupName)
570
571 // The GemPortHistory group requires some special handling as the instance IDs are not pre-defined
572 // unlike other L2 PM counters. We need to fetch the active gemport instance IDs in the system to
573 // take further action
574 if v.groupName == GemPortHistoryName {
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800575 mm.updateGemPortNTPInstanceToAddForPerfMonitoring(ctx)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800576 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000577 } else if mm.pDeviceHandler.GetPmConfigs().FreqOverride { // otherwise just update the next collection interval
578 v.NextCollectionInterval = time.Now().Add(time.Duration(v.Frequency) * time.Second)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800579 }
580 } else { // group counter is disabled
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000581 if v.IsL2PMCounter {
Girish Gowdrae0140f02021-02-02 16:55:09 -0800582 // If it is a L2 PM counter we need to mark the PM to be deleted
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800583 mm.l2PmToDelete = mm.appendIfMissingString(mm.l2PmToDelete, v.groupName)
Girish Gowdrae0140f02021-02-02 16:55:09 -0800584 // If the group support flag toggles too soon, we need to delete the group name from l2PmToAdd slice
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800585 mm.l2PmToAdd = mm.removeIfFoundString(mm.l2PmToAdd, v.groupName)
586
587 // The GemPortHistory group requires some special handling as the instance IDs are not pre-defined
588 // unlike other L2 PM counters. We need to fetch the active gemport instance IDs in the system to
589 // take further action
590 if v.groupName == GemPortHistoryName {
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800591 mm.updateGemPortNTPInstanceToDeleteForPerfMonitoring(ctx)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -0800592 }
Girish Gowdrae0140f02021-02-02 16:55:09 -0800593 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800594 }
595 updated = true
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000596 if v.IsL2PMCounter {
Girish Gowdrae0140f02021-02-02 16:55:09 -0800597 logger.Infow(ctx, "l2 pm group metric support updated",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000598 log.Fields{"device-id": mm.deviceID, "groupName": aGroupName, "enabled": group.Enabled, "l2PmToAdd": mm.l2PmToAdd, "l2PmToDelete": mm.l2PmToDelete})
Girish Gowdrae0140f02021-02-02 16:55:09 -0800599 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000600 logger.Infow(ctx, "group metric support updated", log.Fields{"device-id": mm.deviceID, "groupName": aGroupName, "enabled": group.Enabled})
Girish Gowdrae0140f02021-02-02 16:55:09 -0800601 }
602 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800603 }
604 }
605
606 if !updated {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000607 logger.Errorw(ctx, "group metric support not updated", log.Fields{"device-id": mm.deviceID, "groupName": aGroupName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800608 return fmt.Errorf("internal-error-during-group-support-update--groupName-%s", aGroupName)
609 }
610 return nil
611}
612
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000613// UpdateMetricSupport - TODO: add comment
614func (mm *OnuMetricsManager) UpdateMetricSupport(ctx context.Context, aMetricName string, pmConfigs *voltha.PmConfigs) error {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800615 metricSliceIdx := 0
616 var metric *voltha.PmConfig
617
618 for metricSliceIdx, metric = range pmConfigs.Metrics {
619 if metric.Name == aMetricName {
620 break
621 }
622 }
623
624 if metric == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000625 logger.Errorw(ctx, "standalone metric not found", log.Fields{"device-id": mm.deviceID, "metricName": aMetricName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800626 return fmt.Errorf("metric-not-found--metricname-%s", aMetricName)
627 }
628
629 updated := false
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000630 mm.OnuMetricsManagerLock.Lock()
631 defer mm.OnuMetricsManagerLock.Unlock()
632 for k, v := range mm.StandaloneMetricMap {
633 if k == aMetricName && v.Enabled != metric.Enabled {
634 mm.pDeviceHandler.GetPmConfigs().Metrics[metricSliceIdx].Enabled = metric.Enabled
635 v.Enabled = metric.Enabled
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800636 // If the standalone metric is now enabled and frequency override is enabled, set the next metric collection time
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000637 if metric.Enabled && mm.pDeviceHandler.GetPmConfigs().FreqOverride {
638 v.NextCollectionInterval = time.Now().Add(time.Duration(v.Frequency) * time.Second)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800639 }
640 updated = true
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000641 logger.Infow(ctx, "standalone metric support updated", log.Fields{"device-id": mm.deviceID, "metricName": aMetricName, "enabled": metric.Enabled})
Girish Gowdra36ccf7d2021-02-25 20:42:51 -0800642 break
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800643 }
644 }
645 if !updated {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000646 logger.Errorw(ctx, "standalone metric support not updated", log.Fields{"device-id": mm.deviceID, "metricName": aMetricName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800647 return fmt.Errorf("internal-error-during-standalone-support-update--metricname-%s", aMetricName)
648 }
649 return nil
650}
651
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000652// CollectAllGroupAndStandaloneMetrics - TODO: add comment
653func (mm *OnuMetricsManager) CollectAllGroupAndStandaloneMetrics(ctx context.Context) {
654 if mm.pDeviceHandler.GetPmConfigs().Grouped { // metrics are managed as a group.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800655 go mm.collectAllGroupMetrics(ctx)
656 } else {
657 go mm.collectAllStandaloneMetrics(ctx)
658 }
659}
660
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000661func (mm *OnuMetricsManager) collectAllGroupMetrics(ctx context.Context) {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800662 go func() {
663 logger.Debug(ctx, "startCollector before collecting optical metrics")
ozgecanetsiab36ed572021-04-01 10:38:48 +0300664 metricInfo, err := mm.collectOpticalMetrics(ctx)
665 if err != nil {
666 logger.Errorw(ctx, "collectOpticalMetrics failed",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000667 log.Fields{"device-id": mm.deviceID, "Error": err})
ozgecanetsiab36ed572021-04-01 10:38:48 +0300668 return
669 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800670 if metricInfo != nil {
671 mm.publishMetrics(ctx, metricInfo)
672 }
673 }()
674
675 go func() {
676 logger.Debug(ctx, "startCollector before collecting uni metrics")
ozgecanetsiab36ed572021-04-01 10:38:48 +0300677 metricInfo, err := mm.collectUniStatusMetrics(ctx)
678 if err != nil {
679 logger.Errorw(ctx, "collectOpticalMetrics failed",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000680 log.Fields{"device-id": mm.deviceID, "Error": err})
ozgecanetsiab36ed572021-04-01 10:38:48 +0300681 return
682 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800683 if metricInfo != nil {
684 mm.publishMetrics(ctx, metricInfo)
685 }
686 }()
687
688 // Add more here
689}
690
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000691func (mm *OnuMetricsManager) collectAllStandaloneMetrics(ctx context.Context) {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800692 // None exists as of now, add when available here
693}
694
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000695// CollectGroupMetric - TODO: add comment
696func (mm *OnuMetricsManager) CollectGroupMetric(ctx context.Context, groupName string) {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800697 switch groupName {
698 case OpticalPowerGroupMetricName:
699 go func() {
ozgecanetsiab36ed572021-04-01 10:38:48 +0300700 if mi, _ := mm.collectOpticalMetrics(ctx); mi != nil {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800701 mm.publishMetrics(ctx, mi)
702 }
703 }()
704 case UniStatusGroupMetricName:
705 go func() {
ozgecanetsiab36ed572021-04-01 10:38:48 +0300706 if mi, _ := mm.collectUniStatusMetrics(ctx); mi != nil {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800707 mm.publishMetrics(ctx, mi)
708 }
709 }()
710 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000711 logger.Errorw(ctx, "unhandled group metric name", log.Fields{"device-id": mm.deviceID, "groupName": groupName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800712 }
713}
714
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000715// CollectStandaloneMetric - TODO: add comment
716func (mm *OnuMetricsManager) CollectStandaloneMetric(ctx context.Context, metricName string) {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800717 switch metricName {
718 // None exist as of now, add when available
719 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000720 logger.Errorw(ctx, "unhandled standalone metric name", log.Fields{"device-id": mm.deviceID, "metricName": metricName})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800721 }
722}
723
724// collectOpticalMetrics collects groups metrics related to optical power from ani-g ME.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000725func (mm *OnuMetricsManager) collectOpticalMetrics(ctx context.Context) ([]*voltha.MetricInformation, error) {
726 logger.Debugw(ctx, "collectOpticalMetrics", log.Fields{"device-id": mm.deviceID})
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800727
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000728 mm.OnuMetricsManagerLock.RLock()
729 if !mm.GroupMetricMap[OpticalPowerGroupMetricName].Enabled {
730 mm.OnuMetricsManagerLock.RUnlock()
731 logger.Debugw(ctx, "optical power group metric is not enabled", log.Fields{"device-id": mm.deviceID})
ozgecanetsiab36ed572021-04-01 10:38:48 +0300732 return nil, nil
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800733 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000734 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800735
Girish Gowdrae09a6202021-01-12 18:10:59 -0800736 var metricInfoSlice []*voltha.MetricInformation
737 metricsContext := make(map[string]string)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000738 metricsContext["onuID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetProxyAddress().OnuId)
739 metricsContext["intfID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetProxyAddress().ChannelId)
740 metricsContext["devicetype"] = mm.pDeviceHandler.GetDeviceType()
Girish Gowdrae09a6202021-01-12 18:10:59 -0800741
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800742 raisedTs := time.Now().Unix()
Girish Gowdrae09a6202021-01-12 18:10:59 -0800743 mmd := voltha.MetricMetaData{
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800744 Title: OpticalPowerGroupMetricName,
Girish Gowdrae09a6202021-01-12 18:10:59 -0800745 Ts: float64(raisedTs),
746 Context: metricsContext,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000747 DeviceId: mm.deviceID,
748 LogicalDeviceId: mm.pDeviceHandler.GetLogicalDeviceID(),
749 SerialNo: mm.pDeviceHandler.GetDevice().SerialNumber,
Girish Gowdrae09a6202021-01-12 18:10:59 -0800750 }
751
Girish Gowdrae09a6202021-01-12 18:10:59 -0800752 // get the ANI-G instance IDs
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000753 anigInstKeys := mm.pOnuDeviceEntry.GetOnuDB().GetSortedInstKeys(ctx, me.AniGClassID)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800754loop:
755 for _, anigInstID := range anigInstKeys {
756 var meAttributes me.AttributeValueMap
757 opticalMetrics := make(map[string]float32)
758 // Get the ANI-G instance optical power attributes
759 requestedAttributes := me.AttributeValueMap{"OpticalSignalLevel": 0, "TransmitOpticalLevel": 0}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000760 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, me.AniGClassID, anigInstID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300761 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000762 logger.Errorw(ctx, "GetMe failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
763 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300764 return nil, err
765 }
766
767 if meInstance != nil {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800768 select {
769 case meAttributes = <-mm.opticalMetricsChan:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000770 logger.Debugw(ctx, "received optical metrics", log.Fields{"device-id": mm.deviceID})
771 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
772 logger.Errorw(ctx, "timeout waiting for omci-get response for optical metrics", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800773 // The metrics will be empty in this case
774 break loop
775 }
776 // Populate metric only if it was enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800777 for k := range OpticalPowerGroupMetrics {
778 switch k {
779 case "ani_g_instance_id":
780 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
781 opticalMetrics[k] = float32(val.(uint16))
782 }
Girish Gowdrae20a4f62021-03-09 16:06:23 -0800783 case "transmit_power_dBm":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800784 if val, ok := meAttributes["TransmitOpticalLevel"]; ok && val != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000785 opticalMetrics[k] = float32(math.Round((float64(cmn.TwosComplementToSignedInt16(val.(uint16)))/500.0)*10) / 10) // convert to dBm rounded of to single decimal place
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800786 }
Girish Gowdrae20a4f62021-03-09 16:06:23 -0800787 case "receive_power_dBm":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800788 if val, ok := meAttributes["OpticalSignalLevel"]; ok && val != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000789 opticalMetrics[k] = float32(math.Round((float64(cmn.TwosComplementToSignedInt16(val.(uint16)))/500.0)*10) / 10) // convert to dBm rounded of to single decimal place
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800790 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800791 default:
792 // do nothing
793 }
794 }
795 }
796 // create slice of metrics given that there could be more than one ANI-G instance and
797 // optical metrics are collected per ANI-G instance
798 metricInfo := voltha.MetricInformation{Metadata: &mmd, Metrics: opticalMetrics}
799 metricInfoSlice = append(metricInfoSlice, &metricInfo)
800 }
801
ozgecanetsiab36ed572021-04-01 10:38:48 +0300802 return metricInfoSlice, nil
Girish Gowdrae09a6202021-01-12 18:10:59 -0800803}
804
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800805// collectUniStatusMetrics collects UNI status group metric from various MEs (uni-g, pptp and veip).
Girish Gowdrae09a6202021-01-12 18:10:59 -0800806// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000807func (mm *OnuMetricsManager) collectUniStatusMetrics(ctx context.Context) ([]*voltha.MetricInformation, error) {
808 logger.Debugw(ctx, "collectUniStatusMetrics", log.Fields{"device-id": mm.deviceID})
809 mm.OnuMetricsManagerLock.RLock()
810 if !mm.GroupMetricMap[UniStatusGroupMetricName].Enabled {
811 mm.OnuMetricsManagerLock.RUnlock()
812 logger.Debugw(ctx, "uni status group metric is not enabled", log.Fields{"device-id": mm.deviceID})
ozgecanetsiab36ed572021-04-01 10:38:48 +0300813 return nil, nil
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800814 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000815 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800816
Girish Gowdrae09a6202021-01-12 18:10:59 -0800817 var metricInfoSlice []*voltha.MetricInformation
818 metricsContext := make(map[string]string)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000819 metricsContext["onuID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetDevice().ProxyAddress.OnuId)
820 metricsContext["intfID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetDevice().ProxyAddress.ChannelId)
821 metricsContext["devicetype"] = mm.pDeviceHandler.GetDeviceType()
Girish Gowdrae09a6202021-01-12 18:10:59 -0800822
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800823 raisedTs := time.Now().Unix()
Girish Gowdrae09a6202021-01-12 18:10:59 -0800824 mmd := voltha.MetricMetaData{
Girish Gowdra9b1577b2021-04-21 12:56:13 -0700825 Title: UniStatusGroupMetricName,
Girish Gowdrae09a6202021-01-12 18:10:59 -0800826 Ts: float64(raisedTs),
827 Context: metricsContext,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000828 DeviceId: mm.deviceID,
829 LogicalDeviceId: mm.pDeviceHandler.GetLogicalDeviceID(),
830 SerialNo: mm.pDeviceHandler.GetDevice().SerialNumber,
Girish Gowdrae09a6202021-01-12 18:10:59 -0800831 }
832
Girish Gowdrae09a6202021-01-12 18:10:59 -0800833 // get the UNI-G instance IDs
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000834 unigInstKeys := mm.pOnuDeviceEntry.GetOnuDB().GetSortedInstKeys(ctx, me.UniGClassID)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800835loop1:
836 for _, unigInstID := range unigInstKeys {
837 // TODO: Include additional information in the voltha.MetricMetaData - like portno, uni-id, instance-id
838 // to uniquely identify this ME instance and also to correlate the ME instance to physical instance
839 unigMetrics := make(map[string]float32)
840 var meAttributes me.AttributeValueMap
841 // Get the UNI-G instance optical power attributes
842 requestedAttributes := me.AttributeValueMap{"AdministrativeState": 0}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000843 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, me.UniGClassID, unigInstID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300844 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000845 logger.Errorw(ctx, "UNI-G failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
846 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300847 return nil, err
848 }
849 if meInstance != nil {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800850 // Wait for metrics or timeout
851 select {
852 case meAttributes = <-mm.uniStatusMetricsChan:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000853 logger.Debugw(ctx, "received uni-g metrics", log.Fields{"device-id": mm.deviceID})
854 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
855 logger.Errorw(ctx, "timeout waiting for omci-get response for uni status", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800856 // The metrics could be empty in this case
857 break loop1
858 }
859 // Populate metric only if it was enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800860 for k := range UniStatusGroupMetrics {
861 switch k {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800862 case "uni_admin_state":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800863 if val, ok := meAttributes["AdministrativeState"]; ok && val != nil {
864 unigMetrics[k] = float32(val.(byte))
865 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800866 default:
867 // do nothing
868 }
869 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800870 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
Girish Gowdra0e533642021-03-02 22:02:51 -0800871 entityID := val.(uint16)
872 unigMetrics["entity_id"] = float32(entityID)
873 // TODO: Rlock needed for reading uniEntityMap? May not be needed given uniEntityMap is populated setup at initial ONU bring up
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000874 for _, uni := range *mm.pDeviceHandler.GetUniEntityMap() {
875 if uni.EntityID == entityID {
876 unigMetrics["uni_port_no"] = float32(uni.PortNo)
Girish Gowdrada3a52f2021-03-17 11:24:11 -0700877 break
Girish Gowdra0e533642021-03-02 22:02:51 -0800878 }
879 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800880 }
Girish Gowdrada3a52f2021-03-17 11:24:11 -0700881 unigMetrics["me_class_id"] = float32(me.UniGClassID)
Girish Gowdra0e533642021-03-02 22:02:51 -0800882
Girish Gowdrae09a6202021-01-12 18:10:59 -0800883 // create slice of metrics given that there could be more than one UNI-G instance
884 metricInfo := voltha.MetricInformation{Metadata: &mmd, Metrics: unigMetrics}
885 metricInfoSlice = append(metricInfoSlice, &metricInfo)
886 }
887 }
888
889 // get the PPTP instance IDs
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000890 pptpInstKeys := mm.pOnuDeviceEntry.GetOnuDB().GetSortedInstKeys(ctx, me.PhysicalPathTerminationPointEthernetUniClassID)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800891loop2:
892 for _, pptpInstID := range pptpInstKeys {
893 // TODO: Include additional information in the voltha.MetricMetaData - like portno, uni-id, instance-id
894 // to uniquely identify this ME instance and also to correlate the ME instance to physical instance
895 var meAttributes me.AttributeValueMap
896 pptpMetrics := make(map[string]float32)
897
Girish Gowdrad3436802021-06-28 13:15:40 -0700898 requestedAttributes := me.AttributeValueMap{"ConfigurationInd": 0, "OperationalState": 0, "AdministrativeState": 0}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000899 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, me.PhysicalPathTerminationPointEthernetUniClassID, pptpInstID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300900 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000901 logger.Errorw(ctx, "GetMe failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
902 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300903 return nil, err
904 }
905 if meInstance != nil {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800906 // Wait for metrics or timeout
907 select {
908 case meAttributes = <-mm.uniStatusMetricsChan:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000909 logger.Debugw(ctx, "received pptp metrics", log.Fields{"device-id": mm.deviceID})
910 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
911 logger.Errorw(ctx, "timeout waiting for omci-get response for uni status", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800912 // The metrics could be empty in this case
913 break loop2
914 }
915
916 // Populate metric only if it was enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800917 for k := range UniStatusGroupMetrics {
918 switch k {
Girish Gowdrad3436802021-06-28 13:15:40 -0700919 case "configuration_ind":
920 if val, ok := meAttributes["ConfigurationInd"]; ok && val != nil {
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800921 pptpMetrics[k] = float32(val.(byte))
922 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800923 case "oper_status":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800924 if val, ok := meAttributes["OperationalState"]; ok && val != nil {
925 pptpMetrics[k] = float32(val.(byte))
926 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800927 case "uni_admin_state":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800928 if val, ok := meAttributes["AdministrativeState"]; ok && val != nil {
929 pptpMetrics[k] = float32(val.(byte))
930 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800931 default:
932 // do nothing
933 }
934 }
935 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800936 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
Girish Gowdra0e533642021-03-02 22:02:51 -0800937 entityID := val.(uint16)
938 pptpMetrics["entity_id"] = float32(entityID)
939 // TODO: Rlock needed for reading uniEntityMap? May not be needed given uniEntityMap is populated setup at initial ONU bring up
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000940 for _, uni := range *mm.pDeviceHandler.GetUniEntityMap() {
941 if uni.EntityID == entityID {
942 pptpMetrics["uni_port_no"] = float32(uni.PortNo)
Girish Gowdrada3a52f2021-03-17 11:24:11 -0700943 break
Girish Gowdra0e533642021-03-02 22:02:51 -0800944 }
945 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800946 }
Girish Gowdrada3a52f2021-03-17 11:24:11 -0700947 pptpMetrics["me_class_id"] = float32(me.PhysicalPathTerminationPointEthernetUniClassID)
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800948
Girish Gowdrae09a6202021-01-12 18:10:59 -0800949 // create slice of metrics given that there could be more than one PPTP instance and
950 metricInfo := voltha.MetricInformation{Metadata: &mmd, Metrics: pptpMetrics}
951 metricInfoSlice = append(metricInfoSlice, &metricInfo)
952 }
953
954 // get the VEIP instance IDs
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000955 veipInstKeys := mm.pOnuDeviceEntry.GetOnuDB().GetSortedInstKeys(ctx, me.VirtualEthernetInterfacePointClassID)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800956loop3:
957 for _, veipInstID := range veipInstKeys {
958 // TODO: Include additional information in the voltha.MetricMetaData - like portno, uni-id, instance-id
959 // to uniquely identify this ME instance and also to correlate the ME instance to physical instance
960 var meAttributes me.AttributeValueMap
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800961 veipMetrics := make(map[string]float32)
Girish Gowdrae09a6202021-01-12 18:10:59 -0800962
963 requestedAttributes := me.AttributeValueMap{"OperationalState": 0, "AdministrativeState": 0}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000964 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, me.VirtualEthernetInterfacePointClassID, veipInstID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300965 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000966 logger.Errorw(ctx, "GetMe failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
967 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +0300968 return nil, err
969 }
970 if meInstance != nil {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800971 // Wait for metrics or timeout
972 select {
973 case meAttributes = <-mm.uniStatusMetricsChan:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +0000974 logger.Debugw(ctx, "received veip metrics", log.Fields{"device-id": mm.deviceID})
975 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
976 logger.Errorw(ctx, "timeout waiting for omci-get response for uni status", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -0800977 // The metrics could be empty in this case
978 break loop3
979 }
980
981 // Populate metric only if it was enabled.
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800982 for k := range UniStatusGroupMetrics {
983 switch k {
Girish Gowdrae09a6202021-01-12 18:10:59 -0800984 case "oper_status":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800985 if val, ok := meAttributes["OperationalState"]; ok && val != nil {
986 veipMetrics[k] = float32(val.(byte))
987 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800988 case "uni_admin_state":
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800989 if val, ok := meAttributes["AdministrativeState"]; ok && val != nil {
990 veipMetrics[k] = float32(val.(byte))
991 }
Girish Gowdrae09a6202021-01-12 18:10:59 -0800992 default:
993 // do nothing
994 }
995 }
996 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800997
Girish Gowdra5a7c4922021-01-22 18:33:41 -0800998 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
Girish Gowdra0e533642021-03-02 22:02:51 -0800999 entityID := val.(uint16)
1000 veipMetrics["entity_id"] = float32(entityID)
1001 // TODO: Rlock needed for reading uniEntityMap? May not be needed given uniEntityMap is populated setup at initial ONU bring up
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001002 for _, uni := range *mm.pDeviceHandler.GetUniEntityMap() {
1003 if uni.EntityID == entityID {
1004 veipMetrics["uni_port_no"] = float32(uni.PortNo)
Girish Gowdrada3a52f2021-03-17 11:24:11 -07001005 break
Girish Gowdra0e533642021-03-02 22:02:51 -08001006 }
1007 }
Girish Gowdra5a7c4922021-01-22 18:33:41 -08001008 }
Girish Gowdrada3a52f2021-03-17 11:24:11 -07001009 veipMetrics["me_class_id"] = float32(me.VirtualEthernetInterfacePointClassID)
Girish Gowdra5a7c4922021-01-22 18:33:41 -08001010
Girish Gowdrae09a6202021-01-12 18:10:59 -08001011 // create slice of metrics given that there could be more than one VEIP instance
Girish Gowdra5a7c4922021-01-22 18:33:41 -08001012 metricInfo := voltha.MetricInformation{Metadata: &mmd, Metrics: veipMetrics}
Girish Gowdrae09a6202021-01-12 18:10:59 -08001013 metricInfoSlice = append(metricInfoSlice, &metricInfo)
1014 }
1015
ozgecanetsiab36ed572021-04-01 10:38:48 +03001016 return metricInfoSlice, nil
Girish Gowdrae09a6202021-01-12 18:10:59 -08001017}
1018
1019// publishMetrics publishes the metrics on kafka
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001020func (mm *OnuMetricsManager) publishMetrics(ctx context.Context, metricInfo []*voltha.MetricInformation) {
Girish Gowdrae09a6202021-01-12 18:10:59 -08001021 var ke voltha.KpiEvent2
Girish Gowdra5a7c4922021-01-22 18:33:41 -08001022 ts := time.Now().Unix()
Girish Gowdrae09a6202021-01-12 18:10:59 -08001023 ke.SliceData = metricInfo
1024 ke.Type = voltha.KpiEventType_slice
1025 ke.Ts = float64(ts)
1026
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001027 if err := mm.pDeviceHandler.GetEventProxy().SendKpiEvent(ctx, "STATS_EVENT", &ke, voltha.EventCategory_EQUIPMENT, voltha.EventSubCategory_ONU, ts); err != nil {
Girish Gowdrae09a6202021-01-12 18:10:59 -08001028 logger.Errorw(ctx, "failed-to-send-pon-stats", log.Fields{"err": err})
1029 }
1030}
1031
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001032// ProcessOmciMessages - TODO: add comment
1033func (mm *OnuMetricsManager) ProcessOmciMessages(ctx context.Context) {
1034 logger.Infow(ctx, "Start routine to process OMCI-GET messages for device-id", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001035 // Flush metric collection channels to be safe.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001036 // It is possible that there is stale data on this channel if the ProcessOmciMessages routine
Girish Gowdrae09a6202021-01-12 18:10:59 -08001037 // is stopped right after issuing a OMCI-GET request and started again.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001038 // The ProcessOmciMessages routine will get stopped if startCollector routine (in device_handler.go)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001039 // is stopped - as a result of ONU going down.
1040 mm.flushMetricCollectionChannels(ctx)
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07001041 mm.updateOmciProcessingStatus(true)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001042 for {
1043 select {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001044 case <-mm.StopProcessingOmciResponses: // stop this routine
1045 logger.Infow(ctx, "Stop routine to process OMCI-GET messages for device-id", log.Fields{"device-id": mm.deviceID})
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07001046 mm.updateOmciProcessingStatus(false)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001047 return
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001048 case message, ok := <-mm.PAdaptFsm.CommChan:
Girish Gowdrae09a6202021-01-12 18:10:59 -08001049 if !ok {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001050 logger.Errorw(ctx, "Message couldn't be read from channel", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001051 continue
1052 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001053 logger.Debugw(ctx, "Received message on ONU metrics channel", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001054
1055 switch message.Type {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001056 case cmn.OMCI:
1057 msg, _ := message.Data.(cmn.OmciMessage)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001058 mm.handleOmciMessage(ctx, msg)
1059 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001060 logger.Warn(ctx, "Unknown message type received", log.Fields{"device-id": mm.deviceID, "message.Type": message.Type})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001061 }
1062 }
1063 }
1064}
1065
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001066func (mm *OnuMetricsManager) handleOmciMessage(ctx context.Context, msg cmn.OmciMessage) {
1067 logger.Debugw(ctx, "omci Msg", log.Fields{"device-id": mm.deviceID,
Girish Gowdrae09a6202021-01-12 18:10:59 -08001068 "msgType": msg.OmciMsg.MessageType, "msg": msg})
1069 switch msg.OmciMsg.MessageType {
1070 case omci.GetResponseType:
1071 //TODO: error handling
1072 _ = mm.handleOmciGetResponseMessage(ctx, msg)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001073 case omci.SynchronizeTimeResponseType:
1074 _ = mm.handleOmciSynchronizeTimeResponseMessage(ctx, msg)
1075 case omci.CreateResponseType:
1076 _ = mm.handleOmciCreateResponseMessage(ctx, msg)
1077 case omci.DeleteResponseType:
1078 _ = mm.handleOmciDeleteResponseMessage(ctx, msg)
Himani Chawla43f95ff2021-06-03 00:24:12 +05301079 case omci.GetCurrentDataResponseType:
1080 _ = mm.handleOmciGetCurrentDataResponseMessage(ctx, msg)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001081 default:
1082 logger.Warnw(ctx, "Unknown Message Type", log.Fields{"msgType": msg.OmciMsg.MessageType})
1083
1084 }
1085}
1086
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001087func (mm *OnuMetricsManager) handleOmciGetResponseMessage(ctx context.Context, msg cmn.OmciMessage) error {
Girish Gowdrae09a6202021-01-12 18:10:59 -08001088 msgLayer := (*msg.OmciPacket).Layer(omci.LayerTypeGetResponse)
1089 if msgLayer == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001090 logger.Errorw(ctx, "omci Msg layer could not be detected for GetResponse - handling stopped", log.Fields{"device-id": mm.deviceID})
1091 return fmt.Errorf("omci Msg layer could not be detected for GetResponse - handling stopped: %s", mm.deviceID)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001092 }
1093 msgObj, msgOk := msgLayer.(*omci.GetResponse)
1094 if !msgOk {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001095 logger.Errorw(ctx, "omci Msg layer could not be assigned for GetResponse - handling stopped", log.Fields{"device-id": mm.deviceID})
1096 return fmt.Errorf("omci Msg layer could not be assigned for GetResponse - handling stopped: %s", mm.deviceID)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001097 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001098 logger.Debugw(ctx, "OMCI GetResponse Data", log.Fields{"device-id": mm.deviceID, "data-fields": msgObj, "result": msgObj.Result})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001099 if msgObj.Result == me.Success {
1100 meAttributes := msgObj.Attributes
1101 switch msgObj.EntityClass {
1102 case me.AniGClassID:
1103 mm.opticalMetricsChan <- meAttributes
1104 return nil
1105 case me.UniGClassID:
1106 mm.uniStatusMetricsChan <- meAttributes
1107 return nil
1108 case me.PhysicalPathTerminationPointEthernetUniClassID:
1109 mm.uniStatusMetricsChan <- meAttributes
1110 return nil
1111 case me.VirtualEthernetInterfacePointClassID:
1112 mm.uniStatusMetricsChan <- meAttributes
1113 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08001114 case me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID,
1115 me.EthernetFramePerformanceMonitoringHistoryDataDownstreamClassID,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001116 me.EthernetPerformanceMonitoringHistoryDataClassID,
1117 me.FecPerformanceMonitoringHistoryDataClassID,
1118 me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID:
Girish Gowdrae0140f02021-02-02 16:55:09 -08001119 mm.l2PmChan <- meAttributes
Himani Chawla43f95ff2021-06-03 00:24:12 +05301120 return nil
1121 case me.EthernetFrameExtendedPmClassID,
1122 me.EthernetFrameExtendedPm64BitClassID:
1123 mm.extendedPmMeChan <- meAttributes
1124 return nil
1125 default:
1126 logger.Errorw(ctx, "unhandled omci get response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001127 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Himani Chawla43f95ff2021-06-03 00:24:12 +05301128 }
1129 } else {
1130 meAttributes := msgObj.Attributes
1131 switch msgObj.EntityClass {
1132 case me.EthernetFrameExtendedPmClassID,
1133 me.EthernetFrameExtendedPm64BitClassID:
1134 // not all counters may be supported in which case we have seen some ONUs throwing
1135 // AttributeFailure error code, while correctly populating other counters it supports
1136 mm.extendedPmMeChan <- meAttributes
1137 return nil
Girish Gowdrae09a6202021-01-12 18:10:59 -08001138 default:
1139 logger.Errorw(ctx, "unhandled omci get response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001140 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae09a6202021-01-12 18:10:59 -08001141 }
1142 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001143 return fmt.Errorf("unhandled-omci-get-response-message")
1144}
1145
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001146func (mm *OnuMetricsManager) handleOmciGetCurrentDataResponseMessage(ctx context.Context, msg cmn.OmciMessage) error {
Himani Chawla43f95ff2021-06-03 00:24:12 +05301147 msgLayer := (*msg.OmciPacket).Layer(omci.LayerTypeGetCurrentDataResponse)
1148 if msgLayer == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001149 logger.Errorw(ctx, "omci Msg layer could not be detected for GetCurrentDataResponse - handling stopped", log.Fields{"device-id": mm.deviceID})
1150 return fmt.Errorf("omci Msg layer could not be detected for GetCurrentDataResponse - handling stopped: %s", mm.deviceID)
Himani Chawla43f95ff2021-06-03 00:24:12 +05301151 }
1152 msgObj, msgOk := msgLayer.(*omci.GetCurrentDataResponse)
1153 if !msgOk {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001154 logger.Errorw(ctx, "omci Msg layer could not be assigned for GetCurrentDataResponse - handling stopped", log.Fields{"device-id": mm.deviceID})
1155 return fmt.Errorf("omci Msg layer could not be assigned for GetCurrentDataResponse - handling stopped: %s", mm.deviceID)
Himani Chawla43f95ff2021-06-03 00:24:12 +05301156 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001157 logger.Debugw(ctx, "OMCI GetCurrentDataResponse Data", log.Fields{"device-id": mm.deviceID, "data-fields": msgObj, "result": msgObj.Result})
Himani Chawla43f95ff2021-06-03 00:24:12 +05301158 if msgObj.Result == me.Success {
1159 meAttributes := msgObj.Attributes
1160 switch msgObj.EntityClass {
1161 case me.EthernetFrameExtendedPmClassID,
1162 me.EthernetFrameExtendedPm64BitClassID:
1163 mm.extendedPmMeChan <- meAttributes
1164 return nil
1165 default:
1166 logger.Errorw(ctx, "unhandled omci get current data response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001167 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Himani Chawla43f95ff2021-06-03 00:24:12 +05301168 }
1169 } else {
1170 meAttributes := msgObj.Attributes
1171 switch msgObj.EntityClass {
1172 case me.EthernetFrameExtendedPmClassID,
1173 me.EthernetFrameExtendedPm64BitClassID:
1174 // not all counters may be supported in which case we have seen some ONUs throwing
1175 // AttributeFailure error code, while correctly populating other counters it supports
1176 mm.extendedPmMeChan <- meAttributes
1177 return nil
1178 default:
1179 logger.Errorw(ctx, "unhandled omci get current data response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001180 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Himani Chawla43f95ff2021-06-03 00:24:12 +05301181 }
1182 }
1183 return fmt.Errorf("unhandled-omci-get-current-data-response-message")
1184}
1185
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001186func (mm *OnuMetricsManager) handleOmciSynchronizeTimeResponseMessage(ctx context.Context, msg cmn.OmciMessage) error {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001187 msgLayer := (*msg.OmciPacket).Layer(omci.LayerTypeSynchronizeTimeResponse)
1188 if msgLayer == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001189 logger.Errorw(ctx, "omci Msg layer could not be detected for synchronize time response - handling stopped", log.Fields{"device-id": mm.deviceID})
1190 return fmt.Errorf("omci Msg layer could not be detected for synchronize time response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001191 }
1192 msgObj, msgOk := msgLayer.(*omci.SynchronizeTimeResponse)
1193 if !msgOk {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001194 logger.Errorw(ctx, "omci Msg layer could not be assigned for synchronize time response - handling stopped", log.Fields{"device-id": mm.deviceID})
1195 return fmt.Errorf("omci Msg layer could not be assigned for synchronize time response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001196 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001197 logger.Debugw(ctx, "OMCI synchronize time response Data", log.Fields{"device-id": mm.deviceID, "data-fields": msgObj})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001198 if msgObj.Result == me.Success {
1199 switch msgObj.EntityClass {
1200 case me.OnuGClassID:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001201 logger.Infow(ctx, "omci synchronize time success", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001202 mm.syncTimeResponseChan <- true
1203 return nil
1204 default:
1205 logger.Errorw(ctx, "unhandled omci message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001206 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001207 }
1208 }
1209 mm.syncTimeResponseChan <- false
1210 logger.Errorf(ctx, "unhandled-omci-synchronize-time-response-message--error-code-%v", msgObj.Result)
1211 return fmt.Errorf("unhandled-omci-synchronize-time-response-message--error-code-%v", msgObj.Result)
Girish Gowdrae09a6202021-01-12 18:10:59 -08001212}
1213
1214// flushMetricCollectionChannels flushes all metric collection channels for any stale OMCI responses
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001215func (mm *OnuMetricsManager) flushMetricCollectionChannels(ctx context.Context) {
Girish Gowdrae09a6202021-01-12 18:10:59 -08001216 // flush commMetricsChan
1217 select {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001218 case <-mm.PAdaptFsm.CommChan:
Girish Gowdrae09a6202021-01-12 18:10:59 -08001219 logger.Debug(ctx, "flushed common metrics channel")
1220 default:
1221 }
1222
1223 // flush opticalMetricsChan
1224 select {
1225 case <-mm.opticalMetricsChan:
1226 logger.Debug(ctx, "flushed optical metrics channel")
1227 default:
1228 }
1229
1230 // flush uniStatusMetricsChan
1231 select {
1232 case <-mm.uniStatusMetricsChan:
1233 logger.Debug(ctx, "flushed uni status metrics channel")
1234 default:
1235 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001236
1237 // flush syncTimeResponseChan
1238 select {
1239 case <-mm.syncTimeResponseChan:
1240 logger.Debug(ctx, "flushed sync time response channel")
1241 default:
1242 }
1243
1244 // flush l2PmChan
1245 select {
1246 case <-mm.l2PmChan:
1247 logger.Debug(ctx, "flushed L2 PM collection channel")
1248 default:
1249 }
1250
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001251 // flush StopTicks
Girish Gowdrae0140f02021-02-02 16:55:09 -08001252 select {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001253 case <-mm.StopTicks:
1254 logger.Debug(ctx, "flushed StopTicks channel")
Girish Gowdrae0140f02021-02-02 16:55:09 -08001255 default:
1256 }
1257
1258}
1259
1260// ** L2 PM FSM Handlers start **
1261
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001262func (mm *OnuMetricsManager) l2PMFsmStarting(ctx context.Context, e *fsm.Event) {
Girish Gowdra0e533642021-03-02 22:02:51 -08001263
Girish Gowdrae0140f02021-02-02 16:55:09 -08001264 // Loop through all the group metrics
1265 // If it is a L2 PM Interval metric and it is enabled, then if it is not in the
1266 // list of active L2 PM list then mark it for creation
1267 // It it is a L2 PM Interval metric and it is disabled, then if it is in the
1268 // list of active L2 PM list then mark it for deletion
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001269 mm.OnuMetricsManagerLock.Lock()
1270 for n, g := range mm.GroupMetricMap {
1271 if g.IsL2PMCounter { // it is a l2 pm counter
1272 if g.Enabled { // metric enabled.
Girish Gowdrae0140f02021-02-02 16:55:09 -08001273 found := false
1274 inner1:
1275 for _, v := range mm.activeL2Pms {
1276 if v == n {
1277 found = true // metric already present in active l2 pm list
1278 break inner1
1279 }
1280 }
1281 if !found { // metric not in active l2 pm list. Mark this to be added later
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001282 mm.l2PmToAdd = mm.appendIfMissingString(mm.l2PmToAdd, n)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001283 }
1284 } else { // metric not enabled.
1285 found := false
1286 inner2:
1287 for _, v := range mm.activeL2Pms {
1288 if v == n {
1289 found = true // metric is found in active l2 pm list
1290 break inner2
1291 }
1292 }
1293 if found { // metric is found in active l2 pm list. Mark this to be deleted later
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001294 mm.l2PmToDelete = mm.appendIfMissingString(mm.l2PmToDelete, n)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001295 }
1296 }
1297 }
1298 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001299 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001300 logger.Debugw(ctx, "pms to add and delete",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001301 log.Fields{"device-id": mm.deviceID, "pms-to-add": mm.l2PmToAdd, "pms-to-delete": mm.l2PmToDelete})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001302 go func() {
1303 // push a tick event to move to next state
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001304 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventTick); err != nil {
1305 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001306 }
1307 }()
1308}
1309
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001310func (mm *OnuMetricsManager) l2PMFsmSyncTime(ctx context.Context, e *fsm.Event) {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001311 // Sync time with the ONU to establish 15min boundary for PM collection.
1312 if err := mm.syncTime(ctx); err != nil {
1313 go func() {
1314 time.Sleep(SyncTimeRetryInterval * time.Second) // retry to sync time after this timeout
1315 // This will result in FSM attempting to sync time again
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001316 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventFailure); err != nil {
1317 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001318 }
1319 }()
1320 }
1321 // Initiate a tick generation routine every L2PmCollectionInterval
1322 go mm.generateTicks(ctx)
1323
1324 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001325 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventSuccess); err != nil {
1326 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001327 }
1328 }()
1329}
1330
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001331func (mm *OnuMetricsManager) l2PMFsmNull(ctx context.Context, e *fsm.Event) {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001332 // We need to reset the local data so that the L2 PM MEs are re-provisioned once the ONU is back up based on the latest PM CONFIG
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001333 mm.OnuMetricsManagerLock.Lock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001334 mm.activeL2Pms = nil
1335 mm.l2PmToAdd = nil
1336 mm.l2PmToDelete = nil
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001337 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001338 // If the FSM was stopped, then clear PM data from KV store
1339 // The FSM is stopped when ONU goes down. It is time to clear its data from store
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001340 if e.Event == L2PmEventStop {
Girish Gowdra0e533642021-03-02 22:02:51 -08001341 _ = mm.clearPmGroupData(ctx) // ignore error
1342 }
1343
Girish Gowdrae0140f02021-02-02 16:55:09 -08001344}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001345func (mm *OnuMetricsManager) l2PMFsmIdle(ctx context.Context, e *fsm.Event) {
1346 logger.Debugw(ctx, "Enter state idle", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001347
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001348 mm.OnuMetricsManagerLock.RLock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001349 numOfPmToDelete := len(mm.l2PmToDelete)
1350 numOfPmToAdd := len(mm.l2PmToAdd)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001351 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001352
1353 if numOfPmToDelete > 0 {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001354 logger.Debugw(ctx, "state idle - pms to delete", log.Fields{"device-id": mm.deviceID, "pms-to-delete": numOfPmToDelete})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001355 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001356 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventDeleteMe); err != nil {
1357 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001358 }
1359 }()
1360 } else if numOfPmToAdd > 0 {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001361 logger.Debugw(ctx, "state idle - pms to add", log.Fields{"device-id": mm.deviceID, "pms-to-add": numOfPmToAdd})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001362 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001363 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventAddMe); err != nil {
1364 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001365 }
1366 }()
1367 }
1368}
1369
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001370func (mm *OnuMetricsManager) l2PmFsmCollectData(ctx context.Context, e *fsm.Event) {
1371 logger.Debugw(ctx, "state collect data", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001372 // Copy the activeL2Pms for which we want to collect the metrics since activeL2Pms can change dynamically
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001373 mm.OnuMetricsManagerLock.RLock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001374 copyOfActiveL2Pms := make([]string, len(mm.activeL2Pms))
1375 _ = copy(copyOfActiveL2Pms, mm.activeL2Pms)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001376 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001377
1378 for _, n := range copyOfActiveL2Pms {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001379 var metricInfoSlice []*voltha.MetricInformation
Girish Gowdra0e533642021-03-02 22:02:51 -08001380
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001381 // mm.GroupMetricMap[n].pmMEData.InstancesActive could dynamically change, so make a copy
1382 mm.OnuMetricsManagerLock.RLock()
1383 copyOfEntityIDs := make([]uint16, len(mm.GroupMetricMap[n].pmMEData.InstancesActive))
1384 _ = copy(copyOfEntityIDs, mm.GroupMetricMap[n].pmMEData.InstancesActive)
1385 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001386
Girish Gowdrae0140f02021-02-02 16:55:09 -08001387 switch n {
1388 case EthernetBridgeHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001389 logger.Debugw(ctx, "state collect data - collecting data for EthernetFramePerformanceMonitoringHistoryData ME", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08001390 for _, entityID := range copyOfEntityIDs {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001391 if metricInfo := mm.collectEthernetFramePerformanceMonitoringHistoryData(ctx, true, entityID); metricInfo != nil { // upstream
1392 metricInfoSlice = append(metricInfoSlice, metricInfo)
1393 }
1394 if metricInfo := mm.collectEthernetFramePerformanceMonitoringHistoryData(ctx, false, entityID); metricInfo != nil { // downstream
1395 metricInfoSlice = append(metricInfoSlice, metricInfo)
1396 }
1397 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001398 case EthernetUniHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001399 logger.Debugw(ctx, "state collect data - collecting data for EthernetPerformanceMonitoringHistoryData ME", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08001400 for _, entityID := range copyOfEntityIDs {
1401 if metricInfo := mm.collectEthernetUniHistoryData(ctx, entityID); metricInfo != nil { // upstream
1402 metricInfoSlice = append(metricInfoSlice, metricInfo)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001403 }
1404 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001405
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001406 case FecHistoryName:
Girish Gowdra0e533642021-03-02 22:02:51 -08001407 for _, entityID := range copyOfEntityIDs {
1408 if metricInfo := mm.collectFecHistoryData(ctx, entityID); metricInfo != nil { // upstream
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001409 metricInfoSlice = append(metricInfoSlice, metricInfo)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001410 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001411 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001412 case GemPortHistoryName:
Girish Gowdra0e533642021-03-02 22:02:51 -08001413 for _, entityID := range copyOfEntityIDs {
1414 if metricInfo := mm.collectGemHistoryData(ctx, entityID); metricInfo != nil { // upstream
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001415 metricInfoSlice = append(metricInfoSlice, metricInfo)
1416 }
1417 }
1418
Girish Gowdrae0140f02021-02-02 16:55:09 -08001419 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001420 logger.Errorw(ctx, "unsupported l2 pm", log.Fields{"device-id": mm.deviceID, "name": n})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001421 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001422 mm.handleMetricsPublish(ctx, n, metricInfoSlice)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001423 }
1424 // Does not matter we send success or failure here.
1425 // Those PMs that we failed to collect data will be attempted to collect again in the next PM collection cycle (assuming
1426 // we have not exceed max attempts to collect the PM data)
1427 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001428 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventSuccess); err != nil {
1429 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001430 }
1431 }()
1432}
1433
Girish Gowdra0e533642021-03-02 22:02:51 -08001434// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001435func (mm *OnuMetricsManager) l2PmFsmCreatePM(ctx context.Context, e *fsm.Event) error {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001436 // Copy the l2PmToAdd for which we want to collect the metrics since l2PmToAdd can change dynamically
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001437 mm.OnuMetricsManagerLock.RLock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001438 copyOfL2PmToAdd := make([]string, len(mm.l2PmToAdd))
1439 _ = copy(copyOfL2PmToAdd, mm.l2PmToAdd)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001440 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001441
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001442 logger.Debugw(ctx, "state create pm - start", log.Fields{"device-id": mm.deviceID, "pms-to-add": copyOfL2PmToAdd})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001443 for _, n := range copyOfL2PmToAdd {
1444 resp := false
Girish Gowdra0e533642021-03-02 22:02:51 -08001445 atLeastOneSuccess := false // flag indicates if at least one ME instance of the PM was successfully created.
1446 cnt := 0
Girish Gowdrae0140f02021-02-02 16:55:09 -08001447 switch n {
1448 case EthernetBridgeHistoryName:
Girish Gowdrae0140f02021-02-02 16:55:09 -08001449 // Create ME twice, one for each direction. Boolean true is used to indicate upstream and false for downstream.
Himani Chawlac77d5372021-07-12 15:42:26 +05301450 for _, direction := range []bool{true, false} {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001451 for _, uniPort := range *mm.pDeviceHandler.GetUniEntityMap() {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001452 // Attach the EthernetFramePerformanceMonitoringHistoryData ME to MacBridgePortConfigData on the UNI port
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001453 entityID := cmn.MacBridgePortAniBaseEID + uniPort.EntityID
Girish Gowdra0e533642021-03-02 22:02:51 -08001454 _ = mm.updatePmData(ctx, n, entityID, cPmAdd) // TODO: ignore error for now
1455 inner1:
1456 // retry L2PmCreateAttempts times to create the instance of PM
1457 for cnt = 0; cnt < L2PmCreateAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001458 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteEthernetPerformanceMonitoringHistoryME(
1459 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, direction, true, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001460 if err != nil {
1461 logger.Errorw(ctx, "EthernetPerformanceMonitoringHistoryME create or delete failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001462 log.Fields{"device-id": mm.deviceID})
1463 pPMFsm := mm.PAdaptFsm
ozgecanetsiab36ed572021-04-01 10:38:48 +03001464 if pPMFsm != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001465 go func(p_pmFsm *cmn.AdapterFsm) {
1466 _ = p_pmFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001467 }(pPMFsm)
1468 }
1469 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteEthernetPerformanceMonitoringHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001470 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001471 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001472 if resp = mm.waitForResponseOrTimeout(ctx, true, entityID, "EthernetFramePerformanceMonitoringHistoryData"); resp {
1473 atLeastOneSuccess = true
1474 _ = mm.updatePmData(ctx, n, entityID, cPmAdded) // TODO: ignore error for now
1475 break inner1
1476 }
1477 }
1478 if cnt == L2PmCreateAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1479 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
Girish Gowdrae0140f02021-02-02 16:55:09 -08001480 }
1481 }
1482 }
1483 case EthernetUniHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001484 for _, uniPort := range *mm.pDeviceHandler.GetUniEntityMap() {
1485 if uniPort.PortType == cmn.UniPPTP { // This metric is only applicable for PPTP Uni Type
Girish Gowdra0e533642021-03-02 22:02:51 -08001486 // Attach the EthernetPerformanceMonitoringHistoryData ME to PPTP port instance
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001487 entityID := uniPort.EntityID
Girish Gowdra0e533642021-03-02 22:02:51 -08001488 _ = mm.updatePmData(ctx, n, entityID, cPmAdd) // TODO: ignore error for now
1489 inner2:
1490 // retry L2PmCreateAttempts times to create the instance of PM
1491 for cnt = 0; cnt < L2PmCreateAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001492 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteEthernetUniHistoryME(
1493 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, true, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001494 if err != nil {
1495 logger.Errorw(ctx, "CreateOrDeleteEthernetUNIHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001496 log.Fields{"device-id": mm.deviceID})
1497 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001498 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteEthernetUniHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001499 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001500 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001501 if resp = mm.waitForResponseOrTimeout(ctx, true, entityID, "EthernetPerformanceMonitoringHistoryData"); resp {
1502 atLeastOneSuccess = true
1503 _ = mm.updatePmData(ctx, n, entityID, cPmAdded) // TODO: ignore error for now
1504 break inner2
1505 }
1506 }
1507 if cnt == L2PmCreateAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1508 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
Girish Gowdrae0140f02021-02-02 16:55:09 -08001509 }
1510 }
1511 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001512 case FecHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001513 for _, anigInstID := range mm.pOnuDeviceEntry.GetOnuDB().GetSortedInstKeys(ctx, me.AniGClassID) {
Girish Gowdra0e533642021-03-02 22:02:51 -08001514 // Attach the FecPerformanceMonitoringHistoryData ME to the ANI-G ME instance
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001515 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteFecHistoryME(
1516 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, true, mm.PAdaptFsm.CommChan, anigInstID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001517 if err != nil {
1518 logger.Errorw(ctx, "CreateOrDeleteFecHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001519 log.Fields{"device-id": mm.deviceID})
1520 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001521 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteFecHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001522 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001523 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001524 _ = mm.updatePmData(ctx, n, anigInstID, cPmAdd) // TODO: ignore error for now
1525 inner3:
1526 // retry L2PmCreateAttempts times to create the instance of PM
1527 for cnt = 0; cnt < L2PmCreateAttempts; cnt++ {
1528 if resp = mm.waitForResponseOrTimeout(ctx, true, anigInstID, "FecPerformanceMonitoringHistoryData"); resp {
1529 atLeastOneSuccess = true
1530 _ = mm.updatePmData(ctx, n, anigInstID, cPmAdded) // TODO: ignore error for now
1531 break inner3
1532 }
1533 }
1534 if cnt == L2PmCreateAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1535 _ = mm.updatePmData(ctx, n, anigInstID, cPmRemoved) // TODO: ignore error for now
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001536 }
1537 }
1538 case GemPortHistoryName:
1539
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001540 mm.OnuMetricsManagerLock.RLock()
1541 copyOfGemPortInstIDsToAdd := make([]uint16, len(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd))
1542 _ = copy(copyOfGemPortInstIDsToAdd, mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd)
1543 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001544
1545 if len(copyOfGemPortInstIDsToAdd) == 0 {
1546 // If there are no gemport history MEs to be created, just skip further processing
1547 // Otherwise down below (after 'switch' case handling) we assume the ME creation failed because resp and atLeastOneSuccess flag are false.
1548 // Normally there are no GemPortHistory MEs to create at start up. They come in only after provisioning service on the ONU.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001549 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001550 mm.l2PmToAdd = mm.removeIfFoundString(mm.l2PmToAdd, n)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001551 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001552 continue
1553 }
1554
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001555 for _, v := range copyOfGemPortInstIDsToAdd {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001556 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteGemPortHistoryME(
1557 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, true, mm.PAdaptFsm.CommChan, v)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001558 if err != nil {
1559 logger.Errorw(ctx, "CreateOrDeleteGemPortHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001560 log.Fields{"device-id": mm.deviceID})
1561 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001562 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteGemPortHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001563 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001564 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001565 _ = mm.updatePmData(ctx, n, v, cPmAdd) // TODO: ignore error for now
1566 inner4:
1567 // retry L2PmCreateAttempts times to create the instance of PM
1568 for cnt = 0; cnt < L2PmCreateAttempts; cnt++ {
1569 if resp = mm.waitForResponseOrTimeout(ctx, true, v, "GemPortNetworkCtpPerformanceMonitoringHistoryData"); resp {
1570 atLeastOneSuccess = true
1571 _ = mm.updatePmData(ctx, n, v, cPmAdded) // TODO: ignore error for now
1572 break inner4
1573 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001574 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001575 if cnt == L2PmCreateAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1576 _ = mm.updatePmData(ctx, n, v, cPmRemoved) // TODO: ignore error for now
1577 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001578 }
1579
Girish Gowdrae0140f02021-02-02 16:55:09 -08001580 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001581 logger.Errorw(ctx, "unsupported l2 pm", log.Fields{"device-id": mm.deviceID, "name": n})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001582 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001583 // On success of at least one instance of the PM for a given ME, update the local list maintained for active PMs and PMs to add
1584 if atLeastOneSuccess {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001585 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001586 mm.activeL2Pms = mm.appendIfMissingString(mm.activeL2Pms, n)
Girish Gowdra69570d92021-04-22 18:26:20 -07001587 // gem ports can be added dynamically for perf monitoring. We want to clear the GemPortHistoryName from mm.l2PmToAdd
1588 // only if no more new gem port instances created.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001589 if n != GemPortHistoryName || (n == GemPortHistoryName && len(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd) == 0) {
Girish Gowdra69570d92021-04-22 18:26:20 -07001590 mm.l2PmToAdd = mm.removeIfFoundString(mm.l2PmToAdd, n)
1591 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001592 logger.Debugw(ctx, "success-resp", log.Fields{"pm-name": n, "active-l2-pms": mm.activeL2Pms, "pms-to-add": mm.l2PmToAdd})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001593 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001594 } else {
Girish Gowdra0e533642021-03-02 22:02:51 -08001595 // If we are here then no instance of the PM of the given ME were created successfully, so locally disable the PM
Girish Gowdrae0140f02021-02-02 16:55:09 -08001596 // and also remove it from l2PmToAdd slice so that we do not try to create the PM ME anymore
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001597 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001598 logger.Debugw(ctx, "exceeded-max-add-retry-attempts--disabling-group", log.Fields{"groupName": n})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001599 mm.GroupMetricMap[n].Enabled = false
Girish Gowdra0e533642021-03-02 22:02:51 -08001600 mm.l2PmToAdd = mm.removeIfFoundString(mm.l2PmToAdd, n)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001601
Girish Gowdrae0140f02021-02-02 16:55:09 -08001602 logger.Warnw(ctx, "state create pm - failed to create pm",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001603 log.Fields{"device-id": mm.deviceID, "metricName": n,
Girish Gowdra0e533642021-03-02 22:02:51 -08001604 "active-l2-pms": mm.activeL2Pms, "pms-to-add": mm.l2PmToAdd})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001605 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001606 }
1607 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001608 mm.OnuMetricsManagerLock.RLock()
1609 logger.Debugw(ctx, "state create pm - done", log.Fields{"device-id": mm.deviceID, "active-l2-pms": mm.activeL2Pms, "pms-to-add": mm.l2PmToAdd})
1610 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001611 // Does not matter we send success or failure here.
1612 // Those PMs that we failed to create will be attempted to create again in the next PM creation cycle (assuming
1613 // we have not exceed max attempts to create the PM ME)
1614 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001615 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventSuccess); err != nil {
1616 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001617 }
1618 }()
ozgecanetsiab36ed572021-04-01 10:38:48 +03001619 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08001620}
1621
Girish Gowdra0e533642021-03-02 22:02:51 -08001622// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001623func (mm *OnuMetricsManager) l2PmFsmDeletePM(ctx context.Context, e *fsm.Event) error {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001624 // Copy the l2PmToDelete for which we want to collect the metrics since l2PmToDelete can change dynamically
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001625 mm.OnuMetricsManagerLock.RLock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001626 copyOfL2PmToDelete := make([]string, len(mm.l2PmToDelete))
1627 _ = copy(copyOfL2PmToDelete, mm.l2PmToDelete)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001628 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001629
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001630 logger.Debugw(ctx, "state delete pm", log.Fields{"device-id": mm.deviceID, "pms-to-delete": copyOfL2PmToDelete})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001631 for _, n := range copyOfL2PmToDelete {
1632 resp := false
Girish Gowdra0e533642021-03-02 22:02:51 -08001633 cnt := 0
1634 atLeastOneDeleteFailure := false
1635
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001636 // mm.GroupMetricMap[n].pmMEData.InstancesActive could dynamically change, so make a copy
1637 mm.OnuMetricsManagerLock.RLock()
1638 copyOfEntityIDs := make([]uint16, len(mm.GroupMetricMap[n].pmMEData.InstancesActive))
1639 _ = copy(copyOfEntityIDs, mm.GroupMetricMap[n].pmMEData.InstancesActive)
1640 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001641
1642 if len(copyOfEntityIDs) == 0 {
1643 // if there are no enityIDs to remove for the PM ME just clear the PM name entry from cache and continue
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001644 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001645 mm.activeL2Pms = mm.removeIfFoundString(mm.activeL2Pms, n)
1646 mm.l2PmToDelete = mm.removeIfFoundString(mm.l2PmToDelete, n)
1647 logger.Debugw(ctx, "success-resp", log.Fields{"pm-name": n, "active-l2-pms": mm.activeL2Pms, "pms-to-delete": mm.l2PmToDelete})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001648 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001649 continue
1650 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001651 logger.Debugw(ctx, "entities to delete", log.Fields{"device-id": mm.deviceID, "metricName": n, "entityIDs": copyOfEntityIDs})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001652 switch n {
1653 case EthernetBridgeHistoryName:
Girish Gowdrae0140f02021-02-02 16:55:09 -08001654 // Create ME twice, one for each direction. Boolean true is used to indicate upstream and false for downstream.
Himani Chawlac77d5372021-07-12 15:42:26 +05301655 for _, direction := range []bool{true, false} {
Girish Gowdra0e533642021-03-02 22:02:51 -08001656 for _, entityID := range copyOfEntityIDs {
1657 inner1:
1658 // retry L2PmDeleteAttempts times to delete the instance of PM
1659 for cnt = 0; cnt < L2PmDeleteAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001660 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteEthernetPerformanceMonitoringHistoryME(
1661 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, direction, false, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001662 if err != nil {
1663 logger.Errorw(ctx, "CreateOrDeleteEthernetPerformanceMonitoringHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001664 log.Fields{"device-id": mm.deviceID})
1665 pPMFsm := mm.PAdaptFsm
ozgecanetsiab36ed572021-04-01 10:38:48 +03001666 if pPMFsm != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001667 go func(p_pmFsm *cmn.AdapterFsm) {
1668 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001669 }(pPMFsm)
1670 }
1671 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteEthernetPerformanceMonitoringHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001672 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001673 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001674 _ = mm.updatePmData(ctx, n, entityID, cPmRemove) // TODO: ignore error for now
1675 if resp = mm.waitForResponseOrTimeout(ctx, false, entityID, "EthernetFramePerformanceMonitoringHistoryData"); !resp {
1676 atLeastOneDeleteFailure = true
1677 } else {
1678 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
1679 break inner1
1680 }
1681 }
1682 if cnt == L2PmDeleteAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1683 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
Girish Gowdrae0140f02021-02-02 16:55:09 -08001684 }
1685 }
1686 }
1687 case EthernetUniHistoryName:
Girish Gowdra0e533642021-03-02 22:02:51 -08001688 for _, entityID := range copyOfEntityIDs {
1689 inner2:
1690 // retry L2PmDeleteAttempts times to delete the instance of PM
1691 for cnt = 0; cnt < L2PmDeleteAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001692 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteEthernetUniHistoryME(
1693 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, false, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001694 if err != nil {
1695 logger.Errorw(ctx, "CreateOrDeleteEthernetUniHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001696 log.Fields{"device-id": mm.deviceID})
1697 pmFsm := mm.PAdaptFsm
ozgecanetsiab36ed572021-04-01 10:38:48 +03001698 if pmFsm != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001699 go func(p_pmFsm *cmn.AdapterFsm) {
1700 _ = p_pmFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001701 }(pmFsm)
1702 return err
1703 }
1704 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteEthernetUniHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001705 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001706 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001707 if resp = mm.waitForResponseOrTimeout(ctx, false, entityID, "EthernetPerformanceMonitoringHistoryData"); !resp {
Girish Gowdra0e533642021-03-02 22:02:51 -08001708 atLeastOneDeleteFailure = true
1709 } else {
1710 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001711 break inner2
Girish Gowdrae0140f02021-02-02 16:55:09 -08001712 }
1713 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001714 if cnt == L2PmDeleteAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1715 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
1716 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001717 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001718 case FecHistoryName:
Girish Gowdra0e533642021-03-02 22:02:51 -08001719 for _, entityID := range copyOfEntityIDs {
1720 inner3:
1721 // retry L2PmDeleteAttempts times to delete the instance of PM
1722 for cnt = 0; cnt < L2PmDeleteAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001723 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteFecHistoryME(
1724 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, false, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001725 if err != nil {
1726 logger.Errorw(ctx, "CreateOrDeleteFecHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001727 log.Fields{"device-id": mm.deviceID})
1728 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001729 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteFecHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001730 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001731 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001732 if resp := mm.waitForResponseOrTimeout(ctx, false, entityID, "FecPerformanceMonitoringHistoryData"); !resp {
1733 atLeastOneDeleteFailure = true
1734 } else {
1735 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
1736 break inner3
1737 }
1738 }
1739 if cnt == L2PmDeleteAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1740 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001741 }
1742 }
1743 case GemPortHistoryName:
Girish Gowdra0e533642021-03-02 22:02:51 -08001744 for _, entityID := range copyOfEntityIDs {
1745 inner4:
1746 // retry L2PmDeleteAttempts times to delete the instance of PM
1747 for cnt = 0; cnt < L2PmDeleteAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001748 _, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteGemPortHistoryME(
1749 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, false, mm.PAdaptFsm.CommChan, entityID)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001750 if err != nil {
1751 logger.Errorw(ctx, "CreateOrDeleteGemPortHistoryME failed, failure PM FSM!",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001752 log.Fields{"device-id": mm.deviceID})
1753 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001754 return fmt.Errorf(fmt.Sprintf("CreateOrDeleteGemPortHistoryMe-failed-%s-%s",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001755 mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001756 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001757 if resp = mm.waitForResponseOrTimeout(ctx, false, entityID, "GemPortNetworkCtpPerformanceMonitoringHistoryData"); !resp {
1758 atLeastOneDeleteFailure = true
1759 } else {
1760 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
1761 break inner4
1762 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001763 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001764 if cnt == L2PmDeleteAttempts { // if we reached max attempts just give up hope on this given instance of the PM ME
1765 _ = mm.updatePmData(ctx, n, entityID, cPmRemoved) // TODO: ignore error for now
1766 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001767 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08001768 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001769 logger.Errorw(ctx, "unsupported l2 pm", log.Fields{"device-id": mm.deviceID, "name": n})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001770 }
Girish Gowdra0e533642021-03-02 22:02:51 -08001771 // If we could not completely clean up the PM ME then just give up.
1772 if atLeastOneDeleteFailure {
1773 logger.Warnw(ctx, "state delete pm - failed to delete at least one instance of the PM ME",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001774 log.Fields{"device-id": mm.deviceID, "metricName": n,
Girish Gowdra0e533642021-03-02 22:02:51 -08001775 "active-l2-pms": mm.activeL2Pms, "pms-to-delete": mm.l2PmToDelete})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001776 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001777 logger.Debugw(ctx, "exceeded-max-delete-retry-attempts--disabling-group", log.Fields{"groupName": n})
1778 mm.activeL2Pms = mm.removeIfFoundString(mm.activeL2Pms, n)
1779 mm.l2PmToDelete = mm.removeIfFoundString(mm.l2PmToDelete, n)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001780 mm.GroupMetricMap[n].Enabled = false
1781 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08001782 } else { // success case
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001783 mm.OnuMetricsManagerLock.Lock()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001784 mm.activeL2Pms = mm.removeIfFoundString(mm.activeL2Pms, n)
Girish Gowdra69570d92021-04-22 18:26:20 -07001785 // gem ports can be deleted dynamically from perf monitoring. We want to clear the GemPortHistoryName from mm.l2PmToDelete
1786 // only if no more new gem port instances removed.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001787 if n != GemPortHistoryName || (n == GemPortHistoryName && len(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete) == 0) {
Girish Gowdra69570d92021-04-22 18:26:20 -07001788 mm.l2PmToDelete = mm.removeIfFoundString(mm.l2PmToDelete, n)
1789 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001790 logger.Debugw(ctx, "success-resp", log.Fields{"pm-name": n, "active-l2-pms": mm.activeL2Pms, "pms-to-delete": mm.l2PmToDelete})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001791 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001792 }
1793 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001794 mm.OnuMetricsManagerLock.RLock()
1795 logger.Debugw(ctx, "state delete pm - done", log.Fields{"device-id": mm.deviceID, "active-l2-pms": mm.activeL2Pms, "pms-to-delete": mm.l2PmToDelete})
1796 mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdrae0140f02021-02-02 16:55:09 -08001797 // Does not matter we send success or failure here.
1798 // Those PMs that we failed to delete will be attempted to create again in the next PM collection cycle
1799 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001800 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventSuccess); err != nil {
1801 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001802 }
1803 }()
ozgecanetsiab36ed572021-04-01 10:38:48 +03001804 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08001805}
1806
1807// ** L2 PM FSM Handlers end **
1808
1809// syncTime synchronizes time with the ONU to establish a 15 min boundary for PM collection and reporting.
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001810func (mm *OnuMetricsManager) syncTime(ctx context.Context) error {
1811 if err := mm.pOnuDeviceEntry.GetDevOmciCC().SendSyncTime(ctx, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan); err != nil {
1812 logger.Errorw(ctx, "cannot send sync time request", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001813 return err
1814 }
1815
1816 select {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001817 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
1818 logger.Errorw(ctx, "timed out waiting for sync time response from onu", log.Fields{"device-id": mm.deviceID})
1819 return fmt.Errorf("timed-out-waiting-for-sync-time-response-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001820 case syncTimeRes := <-mm.syncTimeResponseChan:
1821 if !syncTimeRes {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001822 return fmt.Errorf("failed-to-sync-time-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001823 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001824 logger.Infow(ctx, "sync time success", log.Fields{"device-id": mm.deviceID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001825 return nil
1826 }
1827}
1828
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001829func (mm *OnuMetricsManager) collectEthernetFramePerformanceMonitoringHistoryData(ctx context.Context, upstream bool, entityID uint16) *voltha.MetricInformation {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001830 var mEnt *me.ManagedEntity
1831 var omciErr me.OmciErrors
1832 var classID me.ClassID
1833 var meAttributes me.AttributeValueMap
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001834 logger.Debugw(ctx, "collecting data for EthernetFramePerformanceMonitoringHistoryData", log.Fields{"device-id": mm.deviceID, "entityID": entityID, "upstream": upstream})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001835 meParam := me.ParamData{EntityID: entityID}
1836 if upstream {
1837 if mEnt, omciErr = me.NewEthernetFramePerformanceMonitoringHistoryDataUpstream(meParam); omciErr == nil || mEnt == nil || omciErr.GetError() != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001838 logger.Errorw(ctx, "error creating me", log.Fields{"device-id": mm.deviceID, "entityID": entityID, "upstream": upstream})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001839 return nil
1840 }
1841 classID = me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID
1842 } else {
1843 if mEnt, omciErr = me.NewEthernetFramePerformanceMonitoringHistoryDataDownstream(meParam); omciErr == nil || mEnt == nil || omciErr.GetError() != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001844 logger.Errorw(ctx, "error creating me", log.Fields{"device-id": mm.deviceID, "entityID": entityID, "upstream": upstream})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001845 return nil
1846 }
1847 classID = me.EthernetFramePerformanceMonitoringHistoryDataDownstreamClassID
1848 }
1849
Girish Gowdrae0140f02021-02-02 16:55:09 -08001850 intervalEndTime := -1
1851 ethPMHistData := make(map[string]float32)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001852 if err := mm.populateGroupSpecificMetrics(ctx, mEnt, classID, entityID, meAttributes, ethPMHistData, &intervalEndTime); err != nil {
1853 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08001854 }
1855
1856 // Populate some relevant context for the EthernetFramePerformanceMonitoringHistoryData PM
1857 ethPMHistData["class_id"] = float32(classID)
1858 ethPMHistData["interval_end_time"] = float32(intervalEndTime)
1859 ethPMHistData["parent_class_id"] = float32(me.MacBridgeConfigurationDataClassID) // EthernetFramePerformanceMonitoringHistoryData is attached to MBPCD ME
1860 ethPMHistData["parent_entity_id"] = float32(entityID)
1861 if upstream {
1862 ethPMHistData["upstream"] = float32(1)
1863 } else {
1864 ethPMHistData["upstream"] = float32(0)
1865 }
1866
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001867 metricInfo := mm.populateOnuMetricInfo(EthernetBridgeHistoryName, ethPMHistData)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001868
Girish Gowdrae0140f02021-02-02 16:55:09 -08001869 logger.Debugw(ctx, "collecting data for EthernetFramePerformanceMonitoringHistoryData successful",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001870 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "upstream": upstream, "metricInfo": metricInfo})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001871 return &metricInfo
1872}
1873
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001874func (mm *OnuMetricsManager) collectEthernetUniHistoryData(ctx context.Context, entityID uint16) *voltha.MetricInformation {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001875 var mEnt *me.ManagedEntity
1876 var omciErr me.OmciErrors
1877 var classID me.ClassID
1878 var meAttributes me.AttributeValueMap
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001879 logger.Debugw(ctx, "collecting data for EthernetFramePerformanceMonitoringHistoryData", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001880 meParam := me.ParamData{EntityID: entityID}
1881 if mEnt, omciErr = me.NewEthernetPerformanceMonitoringHistoryData(meParam); omciErr == nil || mEnt == nil || omciErr.GetError() != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001882 logger.Errorw(ctx, "error creating me", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001883 return nil
1884 }
1885 classID = me.EthernetPerformanceMonitoringHistoryDataClassID
1886
Girish Gowdrae0140f02021-02-02 16:55:09 -08001887 intervalEndTime := -1
1888 ethUniHistData := make(map[string]float32)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001889 if err := mm.populateGroupSpecificMetrics(ctx, mEnt, classID, entityID, meAttributes, ethUniHistData, &intervalEndTime); err != nil {
1890 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08001891 }
1892
1893 // Populate some relevant context for the EthernetPerformanceMonitoringHistoryData PM
1894 ethUniHistData["class_id"] = float32(classID)
1895 ethUniHistData["interval_end_time"] = float32(intervalEndTime)
1896
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001897 metricInfo := mm.populateOnuMetricInfo(EthernetUniHistoryName, ethUniHistData)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001898
Girish Gowdrae0140f02021-02-02 16:55:09 -08001899 logger.Debugw(ctx, "collecting data for EthernetPerformanceMonitoringHistoryData successful",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001900 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "metricInfo": metricInfo})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001901 return &metricInfo
1902}
1903
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001904func (mm *OnuMetricsManager) collectFecHistoryData(ctx context.Context, entityID uint16) *voltha.MetricInformation {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001905 var mEnt *me.ManagedEntity
1906 var omciErr me.OmciErrors
1907 var classID me.ClassID
1908 var meAttributes me.AttributeValueMap
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001909 logger.Debugw(ctx, "collecting data for FecPerformanceMonitoringHistoryData", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001910 meParam := me.ParamData{EntityID: entityID}
1911 if mEnt, omciErr = me.NewFecPerformanceMonitoringHistoryData(meParam); omciErr == nil || mEnt == nil || omciErr.GetError() != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001912 logger.Errorw(ctx, "error creating me", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001913 return nil
1914 }
1915 classID = me.FecPerformanceMonitoringHistoryDataClassID
1916
1917 intervalEndTime := -1
1918 fecHistData := make(map[string]float32)
1919 if err := mm.populateGroupSpecificMetrics(ctx, mEnt, classID, entityID, meAttributes, fecHistData, &intervalEndTime); err != nil {
1920 return nil
1921 }
1922
1923 // Populate some relevant context for the EthernetPerformanceMonitoringHistoryData PM
1924 fecHistData["class_id"] = float32(classID)
1925 fecHistData["interval_end_time"] = float32(intervalEndTime)
1926
1927 metricInfo := mm.populateOnuMetricInfo(FecHistoryName, fecHistData)
1928
1929 logger.Debugw(ctx, "collecting data for FecPerformanceMonitoringHistoryData successful",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001930 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "metricInfo": metricInfo})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001931 return &metricInfo
1932}
1933
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001934func (mm *OnuMetricsManager) collectGemHistoryData(ctx context.Context, entityID uint16) *voltha.MetricInformation {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001935 var mEnt *me.ManagedEntity
1936 var omciErr me.OmciErrors
1937 var classID me.ClassID
1938 var meAttributes me.AttributeValueMap
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001939 logger.Debugw(ctx, "collecting data for GemPortNetworkCtpPerformanceMonitoringHistoryData", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001940 meParam := me.ParamData{EntityID: entityID}
1941 if mEnt, omciErr = me.NewGemPortNetworkCtpPerformanceMonitoringHistoryData(meParam); omciErr == nil || mEnt == nil || omciErr.GetError() != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001942 logger.Errorw(ctx, "error creating me", log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001943 return nil
1944 }
1945 classID = me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID
1946
1947 intervalEndTime := -1
1948 gemHistData := make(map[string]float32)
1949 if err := mm.populateGroupSpecificMetrics(ctx, mEnt, classID, entityID, meAttributes, gemHistData, &intervalEndTime); err != nil {
1950 return nil
1951 }
1952
1953 // Populate some relevant context for the GemPortNetworkCtpPerformanceMonitoringHistoryData PM
1954 gemHistData["class_id"] = float32(classID)
1955 gemHistData["interval_end_time"] = float32(intervalEndTime)
1956
1957 metricInfo := mm.populateOnuMetricInfo(GemPortHistoryName, gemHistData)
1958
1959 logger.Debugw(ctx, "collecting data for GemPortNetworkCtpPerformanceMonitoringHistoryData successful",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001960 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "metricInfo": metricInfo})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001961 return &metricInfo
1962}
1963
Girish Gowdrae0140f02021-02-02 16:55:09 -08001964// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001965func (mm *OnuMetricsManager) populateEthernetBridgeHistoryMetrics(ctx context.Context, classID me.ClassID, entityID uint16,
Girish Gowdrae0140f02021-02-02 16:55:09 -08001966 meAttributes me.AttributeValueMap, requestedAttributes me.AttributeValueMap, ethPMHistData map[string]float32, intervalEndTime *int) error {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001967 upstream := false
1968 if classID == me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID {
1969 upstream = true
1970 }
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07001971 // Insert "IntervalEndTime" as part of the requested attributes as we need this to compare the get responses when get request is multipart
1972 requestedAttributes["IntervalEndTime"] = 0
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001973 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, classID, entityID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001974 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001975 logger.Errorw(ctx, "GetME failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
1976 pmFsm := mm.PAdaptFsm
ozgecanetsiab36ed572021-04-01 10:38:48 +03001977 if pmFsm != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001978 go func(p_pmFsm *cmn.AdapterFsm) {
1979 _ = p_pmFsm.PFsm.Event(L2PmEventFailure)
ozgecanetsiab36ed572021-04-01 10:38:48 +03001980 }(pmFsm)
1981 return err
1982 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001983 return fmt.Errorf(fmt.Sprintf("GetME-failed-%s-%s", mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03001984 }
1985 if meInstance != nil {
Girish Gowdrae0140f02021-02-02 16:55:09 -08001986 select {
1987 case meAttributes = <-mm.l2PmChan:
1988 logger.Debugw(ctx, "received ethernet pm history data metrics",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001989 log.Fields{"device-id": mm.deviceID, "upstream": upstream, "entityID": entityID})
1990 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Girish Gowdrae0140f02021-02-02 16:55:09 -08001991 logger.Errorw(ctx, "timeout waiting for omci-get response for ethernet pm history data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001992 log.Fields{"device-id": mm.deviceID, "upstream": upstream, "entityID": entityID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08001993 // The metrics will be empty in this case
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001994 return fmt.Errorf("timeout-during-l2-pm-collection-for-ethernet-bridge-history-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001995 }
1996 // verify that interval end time has not changed during metric collection. If it changed, we abort the procedure
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08001997 if valid := mm.updateAndValidateIntervalEndTime(ctx, entityID, meAttributes, intervalEndTime); !valid {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00001998 return fmt.Errorf("interval-end-time-changed-during-metric-collection-for-ethernet-bridge-history-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08001999 }
2000 }
2001 for k := range EthernetBridgeHistory {
2002 // populate ethPMHistData only if metric key not already present (or populated), since it is possible that we populate
2003 // the attributes in multiple iterations for a given L2 PM ME as there is a limit on the max OMCI GET payload size.
2004 if _, ok := ethPMHistData[k]; !ok {
2005 switch k {
Girish Gowdra0e533642021-03-02 22:02:51 -08002006 case "entity_id":
2007 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
2008 ethPMHistData[k] = float32(val.(uint16))
2009 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08002010 case "drop_events":
2011 if val, ok := meAttributes["DropEvents"]; ok && val != nil {
2012 ethPMHistData[k] = float32(val.(uint32))
2013 }
2014 case "octets":
2015 if val, ok := meAttributes["Octets"]; ok && val != nil {
2016 ethPMHistData[k] = float32(val.(uint32))
2017 }
2018 case "packets":
2019 if val, ok := meAttributes["Packets"]; ok && val != nil {
2020 ethPMHistData[k] = float32(val.(uint32))
2021 }
2022 case "broadcast_packets":
2023 if val, ok := meAttributes["BroadcastPackets"]; ok && val != nil {
2024 ethPMHistData[k] = float32(val.(uint32))
2025 }
2026 case "multicast_packets":
2027 if val, ok := meAttributes["MulticastPackets"]; ok && val != nil {
2028 ethPMHistData[k] = float32(val.(uint32))
2029 }
2030 case "crc_errored_packets":
2031 if val, ok := meAttributes["CrcErroredPackets"]; ok && val != nil {
2032 ethPMHistData[k] = float32(val.(uint32))
2033 }
2034 case "undersize_packets":
2035 if val, ok := meAttributes["UndersizePackets"]; ok && val != nil {
2036 ethPMHistData[k] = float32(val.(uint32))
2037 }
2038 case "oversize_packets":
2039 if val, ok := meAttributes["OversizePackets"]; ok && val != nil {
2040 ethPMHistData[k] = float32(val.(uint32))
2041 }
2042 case "64_octets":
2043 if val, ok := meAttributes["Packets64Octets"]; ok && val != nil {
2044 ethPMHistData[k] = float32(val.(uint32))
2045 }
2046 case "65_to_127_octets":
2047 if val, ok := meAttributes["Packets65To127Octets"]; ok && val != nil {
2048 ethPMHistData[k] = float32(val.(uint32))
2049 }
2050 case "128_to_255_octets":
2051 if val, ok := meAttributes["Packets128To255Octets"]; ok && val != nil {
2052 ethPMHistData[k] = float32(val.(uint32))
2053 }
2054 case "256_to_511_octets":
2055 if val, ok := meAttributes["Packets256To511Octets"]; ok && val != nil {
2056 ethPMHistData[k] = float32(val.(uint32))
2057 }
2058 case "512_to_1023_octets":
2059 if val, ok := meAttributes["Packets512To1023Octets"]; ok && val != nil {
2060 ethPMHistData[k] = float32(val.(uint32))
2061 }
2062 case "1024_to_1518_octets":
2063 if val, ok := meAttributes["Packets1024To1518Octets"]; ok && val != nil {
2064 ethPMHistData[k] = float32(val.(uint32))
2065 }
2066 default:
2067 // do nothing
2068 }
2069 }
2070 }
2071 return nil
2072}
2073
2074// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002075func (mm *OnuMetricsManager) populateEthernetUniHistoryMetrics(ctx context.Context, classID me.ClassID, entityID uint16,
Girish Gowdrae0140f02021-02-02 16:55:09 -08002076 meAttributes me.AttributeValueMap, requestedAttributes me.AttributeValueMap, ethPMUniHistData map[string]float32, intervalEndTime *int) error {
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002077 // Insert "IntervalEndTime" as part of the requested attributes as we need this to compare the get responses when get request is multipart
ozgecanetsiab36ed572021-04-01 10:38:48 +03002078 if _, ok := requestedAttributes["IntervalEndTime"]; !ok {
2079 requestedAttributes["IntervalEndTime"] = 0
2080 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002081 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, classID, entityID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +03002082 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002083 logger.Errorw(ctx, "GetMe failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
2084 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
2085 return fmt.Errorf(fmt.Sprintf("GetME-failed-%s-%s", mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03002086 }
2087 if meInstance != nil {
Girish Gowdrae0140f02021-02-02 16:55:09 -08002088 select {
2089 case meAttributes = <-mm.l2PmChan:
2090 logger.Debugw(ctx, "received ethernet uni history data metrics",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002091 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
2092 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Girish Gowdrae0140f02021-02-02 16:55:09 -08002093 logger.Errorw(ctx, "timeout waiting for omci-get response for ethernet uni history data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002094 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002095 // The metrics will be empty in this case
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002096 return fmt.Errorf("timeout-during-l2-pm-collection-for-ethernet-uni-history-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002097 }
2098 // verify that interval end time has not changed during metric collection. If it changed, we abort the procedure
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002099 if valid := mm.updateAndValidateIntervalEndTime(ctx, entityID, meAttributes, intervalEndTime); !valid {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002100 return fmt.Errorf("interval-end-time-changed-during-metric-collection-for-ethernet-uni-history-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002101 }
2102 }
2103 for k := range EthernetUniHistory {
2104 // populate ethPMUniHistData only if metric key not already present (or populated), since it is possible that we populate
2105 // the attributes in multiple iterations for a given L2 PM ME as there is a limit on the max OMCI GET payload size.
2106 if _, ok := ethPMUniHistData[k]; !ok {
2107 switch k {
Girish Gowdra0e533642021-03-02 22:02:51 -08002108 case "entity_id":
2109 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
2110 ethPMUniHistData[k] = float32(val.(uint16))
2111 }
Girish Gowdrae0140f02021-02-02 16:55:09 -08002112 case "fcs_errors":
2113 if val, ok := meAttributes["FcsErrors"]; ok && val != nil {
2114 ethPMUniHistData[k] = float32(val.(uint32))
2115 }
2116 case "excessive_collision_counter":
2117 if val, ok := meAttributes["ExcessiveCollisionCounter"]; ok && val != nil {
2118 ethPMUniHistData[k] = float32(val.(uint32))
2119 }
2120 case "late_collision_counter":
2121 if val, ok := meAttributes["LateCollisionCounter"]; ok && val != nil {
2122 ethPMUniHistData[k] = float32(val.(uint32))
2123 }
2124 case "frames_too_long":
2125 if val, ok := meAttributes["FramesTooLong"]; ok && val != nil {
2126 ethPMUniHistData[k] = float32(val.(uint32))
2127 }
2128 case "buffer_overflows_on_rx":
2129 if val, ok := meAttributes["BufferOverflowsOnReceive"]; ok && val != nil {
2130 ethPMUniHistData[k] = float32(val.(uint32))
2131 }
2132 case "buffer_overflows_on_tx":
2133 if val, ok := meAttributes["BufferOverflowsOnTransmit"]; ok && val != nil {
2134 ethPMUniHistData[k] = float32(val.(uint32))
2135 }
2136 case "single_collision_frame_counter":
2137 if val, ok := meAttributes["SingleCollisionFrameCounter"]; ok && val != nil {
2138 ethPMUniHistData[k] = float32(val.(uint32))
2139 }
2140 case "multiple_collisions_frame_counter":
2141 if val, ok := meAttributes["MultipleCollisionsFrameCounter"]; ok && val != nil {
2142 ethPMUniHistData[k] = float32(val.(uint32))
2143 }
2144 case "sqe_counter":
2145 if val, ok := meAttributes["SqeCounter"]; ok && val != nil {
2146 ethPMUniHistData[k] = float32(val.(uint32))
2147 }
2148 case "deferred_tx_counter":
2149 if val, ok := meAttributes["DeferredTransmissionCounter"]; ok && val != nil {
2150 ethPMUniHistData[k] = float32(val.(uint32))
2151 }
2152 case "internal_mac_tx_error_counter":
2153 if val, ok := meAttributes["InternalMacTransmitErrorCounter"]; ok && val != nil {
2154 ethPMUniHistData[k] = float32(val.(uint32))
2155 }
2156 case "carrier_sense_error_counter":
2157 if val, ok := meAttributes["CarrierSenseErrorCounter"]; ok && val != nil {
2158 ethPMUniHistData[k] = float32(val.(uint32))
2159 }
2160 case "alignment_error_counter":
2161 if val, ok := meAttributes["AlignmentErrorCounter"]; ok && val != nil {
2162 ethPMUniHistData[k] = float32(val.(uint32))
2163 }
2164 case "internal_mac_rx_error_counter":
2165 if val, ok := meAttributes["InternalMacReceiveErrorCounter"]; ok && val != nil {
2166 ethPMUniHistData[k] = float32(val.(uint32))
2167 }
2168 default:
2169 // do nothing
2170 }
2171 }
2172 }
2173 return nil
2174}
2175
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002176// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002177func (mm *OnuMetricsManager) populateFecHistoryMetrics(ctx context.Context, classID me.ClassID, entityID uint16,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002178 meAttributes me.AttributeValueMap, requestedAttributes me.AttributeValueMap, fecHistData map[string]float32, intervalEndTime *int) error {
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002179 // Insert "IntervalEndTime" as part of the requested attributes as we need this to compare the get responses when get request is multipart
ozgecanetsiab36ed572021-04-01 10:38:48 +03002180 if _, ok := requestedAttributes["IntervalEndTime"]; !ok {
2181 requestedAttributes["IntervalEndTime"] = 0
2182 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002183 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, classID, entityID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +03002184 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002185 logger.Errorw(ctx, "GetMe failed, failure PM FSM!", log.Fields{"device-id": mm.deviceID})
2186 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
2187 return fmt.Errorf(fmt.Sprintf("GetME-failed-%s-%s", mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03002188 }
2189 if meInstance != nil {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002190 select {
2191 case meAttributes = <-mm.l2PmChan:
2192 logger.Debugw(ctx, "received fec history data metrics",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002193 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
2194 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002195 logger.Errorw(ctx, "timeout waiting for omci-get response for fec history data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002196 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002197 // The metrics will be empty in this case
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002198 return fmt.Errorf("timeout-during-l2-pm-collection-for-fec-history-%v", mm.deviceID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002199 }
2200 // verify that interval end time has not changed during metric collection. If it changed, we abort the procedure
2201 if valid := mm.updateAndValidateIntervalEndTime(ctx, entityID, meAttributes, intervalEndTime); !valid {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002202 return fmt.Errorf("interval-end-time-changed-during-metric-collection-for-fec-history-%v", mm.deviceID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002203 }
2204 }
2205 for k := range FecHistory {
2206 // populate fecHistData only if metric key not already present (or populated), since it is possible that we populate
2207 // the attributes in multiple iterations for a given L2 PM ME as there is a limit on the max OMCI GET payload size.
2208 if _, ok := fecHistData[k]; !ok {
2209 switch k {
Girish Gowdra0e533642021-03-02 22:02:51 -08002210 case "entity_id":
2211 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
2212 fecHistData[k] = float32(val.(uint16))
2213 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002214 case "corrected_bytes":
2215 if val, ok := meAttributes["CorrectedBytes"]; ok && val != nil {
2216 fecHistData[k] = float32(val.(uint32))
2217 }
2218 case "corrected_code_words":
2219 if val, ok := meAttributes["CorrectedCodeWords"]; ok && val != nil {
2220 fecHistData[k] = float32(val.(uint32))
2221 }
2222 case "uncorrectable_code_words":
2223 if val, ok := meAttributes["UncorrectableCodeWords"]; ok && val != nil {
2224 fecHistData[k] = float32(val.(uint32))
2225 }
2226 case "total_code_words":
2227 if val, ok := meAttributes["TotalCodeWords"]; ok && val != nil {
2228 fecHistData[k] = float32(val.(uint32))
2229 }
2230 case "fec_seconds":
2231 if val, ok := meAttributes["FecSeconds"]; ok && val != nil {
2232 fecHistData[k] = float32(val.(uint16))
2233 }
2234 default:
2235 // do nothing
2236 }
2237 }
2238 }
2239 return nil
2240}
2241
2242// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002243func (mm *OnuMetricsManager) populateGemPortMetrics(ctx context.Context, classID me.ClassID, entityID uint16,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002244 meAttributes me.AttributeValueMap, requestedAttributes me.AttributeValueMap, gemPortHistData map[string]float32, intervalEndTime *int) error {
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002245 // Insert "IntervalEndTime" is part of the requested attributes as we need this to compare the get responses when get request is multipart
ozgecanetsiab36ed572021-04-01 10:38:48 +03002246 if _, ok := requestedAttributes["IntervalEndTime"]; !ok {
2247 requestedAttributes["IntervalEndTime"] = 0
2248 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002249 meInstance, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMe(ctx, classID, entityID, requestedAttributes, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
ozgecanetsiab36ed572021-04-01 10:38:48 +03002250 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002251 logger.Errorw(ctx, "GetMe failed", log.Fields{"device-id": mm.deviceID})
2252 _ = mm.PAdaptFsm.PFsm.Event(L2PmEventFailure)
2253 return fmt.Errorf(fmt.Sprintf("GetME-failed-%s-%s", mm.deviceID, err))
ozgecanetsiab36ed572021-04-01 10:38:48 +03002254 }
2255 if meInstance != nil {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002256 select {
2257 case meAttributes = <-mm.l2PmChan:
2258 logger.Debugw(ctx, "received gem port history data metrics",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002259 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
2260 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002261 logger.Errorw(ctx, "timeout waiting for omci-get response for gem port history data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002262 log.Fields{"device-id": mm.deviceID, "entityID": entityID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002263 // The metrics will be empty in this case
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002264 return fmt.Errorf("timeout-during-l2-pm-collection-for-gemport-history-%v", mm.deviceID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002265 }
2266 // verify that interval end time has not changed during metric collection. If it changed, we abort the procedure
2267 if valid := mm.updateAndValidateIntervalEndTime(ctx, entityID, meAttributes, intervalEndTime); !valid {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002268 return fmt.Errorf("interval-end-time-changed-during-metric-collection-for-gemport-history-%v", mm.deviceID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002269 }
2270 }
2271 for k := range GemPortHistory {
2272 // populate gemPortHistData only if metric key not already present (or populated), since it is possible that we populate
2273 // the attributes in multiple iterations for a given L2 PM ME as there is a limit on the max OMCI GET payload size.
2274 if _, ok := gemPortHistData[k]; !ok {
2275 switch k {
Girish Gowdra0e533642021-03-02 22:02:51 -08002276 case "entity_id":
2277 if val, ok := meAttributes["ManagedEntityId"]; ok && val != nil {
2278 gemPortHistData[k] = float32(val.(uint16))
2279 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002280 case "transmitted_gem_frames":
2281 if val, ok := meAttributes["TransmittedGemFrames"]; ok && val != nil {
2282 gemPortHistData[k] = float32(val.(uint32))
2283 }
2284 case "received_gem_frames":
2285 if val, ok := meAttributes["ReceivedGemFrames"]; ok && val != nil {
2286 gemPortHistData[k] = float32(val.(uint32))
2287 }
2288 case "received_payload_bytes":
2289 if val, ok := meAttributes["ReceivedPayloadBytes"]; ok && val != nil {
2290 gemPortHistData[k] = float32(val.(uint64))
2291 }
2292 case "transmitted_payload_bytes":
2293 if val, ok := meAttributes["TransmittedPayloadBytes"]; ok && val != nil {
2294 gemPortHistData[k] = float32(val.(uint64))
2295 }
2296 case "encryption_key_errors":
2297 if val, ok := meAttributes["EncryptionKeyErrors"]; ok && val != nil {
2298 gemPortHistData[k] = float32(val.(uint32))
2299 }
2300 default:
2301 // do nothing
2302 }
2303 }
2304 }
2305 return nil
2306}
2307
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002308func (mm *OnuMetricsManager) handleOmciCreateResponseMessage(ctx context.Context, msg cmn.OmciMessage) error {
Girish Gowdrae0140f02021-02-02 16:55:09 -08002309 msgLayer := (*msg.OmciPacket).Layer(omci.LayerTypeCreateResponse)
2310 if msgLayer == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002311 logger.Errorw(ctx, "omci Msg layer could not be detected for create response - handling stopped", log.Fields{"device-id": mm.deviceID})
2312 return fmt.Errorf("omci Msg layer could not be detected for create response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002313 }
2314 msgObj, msgOk := msgLayer.(*omci.CreateResponse)
2315 if !msgOk {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002316 logger.Errorw(ctx, "omci Msg layer could not be assigned for create response - handling stopped", log.Fields{"device-id": mm.deviceID})
2317 return fmt.Errorf("omci Msg layer could not be assigned for delete response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002318 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002319 logger.Debugw(ctx, "OMCI create response Data", log.Fields{"device-id": mm.deviceID, "data-fields": msgObj})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002320 switch msgObj.EntityClass {
2321 case me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID,
2322 me.EthernetFramePerformanceMonitoringHistoryDataDownstreamClassID,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002323 me.EthernetPerformanceMonitoringHistoryDataClassID,
2324 me.FecPerformanceMonitoringHistoryDataClassID,
2325 me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID:
Girish Gowdrae0140f02021-02-02 16:55:09 -08002326 // If the result is me.InstanceExists it means the entity was already created. It is ok handled that as success
2327 if msgObj.Result == me.Success || msgObj.Result == me.InstanceExists {
2328 mm.l2PmCreateOrDeleteResponseChan <- true
2329 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002330 logger.Warnw(ctx, "failed to create me", log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002331 mm.l2PmCreateOrDeleteResponseChan <- false
2332 }
2333 return nil
Himani Chawla43f95ff2021-06-03 00:24:12 +05302334 case me.EthernetFrameExtendedPmClassID,
2335 me.EthernetFrameExtendedPm64BitClassID:
2336 mm.extendedPMCreateOrDeleteResponseChan <- msgObj.Result
2337 return nil
Girish Gowdrae0140f02021-02-02 16:55:09 -08002338 default:
2339 logger.Errorw(ctx, "unhandled omci create response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002340 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002341 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002342 return fmt.Errorf("unhandled-omci-create-response-message-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002343}
2344
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002345func (mm *OnuMetricsManager) handleOmciDeleteResponseMessage(ctx context.Context, msg cmn.OmciMessage) error {
Girish Gowdrae0140f02021-02-02 16:55:09 -08002346 msgLayer := (*msg.OmciPacket).Layer(omci.LayerTypeDeleteResponse)
2347 if msgLayer == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002348 logger.Errorw(ctx, "omci Msg layer could not be detected for delete response - handling stopped", log.Fields{"device-id": mm.deviceID})
2349 return fmt.Errorf("omci Msg layer could not be detected for create response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002350 }
2351 msgObj, msgOk := msgLayer.(*omci.DeleteResponse)
2352 if !msgOk {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002353 logger.Errorw(ctx, "omci Msg layer could not be assigned for delete response - handling stopped", log.Fields{"device-id": mm.deviceID})
2354 return fmt.Errorf("omci Msg layer could not be assigned for delete response - handling stopped: %s", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002355 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002356 logger.Debugw(ctx, "OMCI delete response Data", log.Fields{"device-id": mm.deviceID, "data-fields": msgObj})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002357 switch msgObj.EntityClass {
2358 case me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID,
2359 me.EthernetFramePerformanceMonitoringHistoryDataDownstreamClassID,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002360 me.EthernetPerformanceMonitoringHistoryDataClassID,
2361 me.FecPerformanceMonitoringHistoryDataClassID,
2362 me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID:
Girish Gowdrae0140f02021-02-02 16:55:09 -08002363 // If the result is me.UnknownInstance it means the entity was already deleted. It is ok handled that as success
2364 if msgObj.Result == me.Success || msgObj.Result == me.UnknownInstance {
2365 mm.l2PmCreateOrDeleteResponseChan <- true
2366 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002367 logger.Warnw(ctx, "failed to delete me", log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002368 mm.l2PmCreateOrDeleteResponseChan <- false
2369 }
2370 return nil
2371 default:
2372 logger.Errorw(ctx, "unhandled omci delete response message",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002373 log.Fields{"device-id": mm.deviceID, "class-id": msgObj.EntityClass})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002374 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002375 return fmt.Errorf("unhandled-omci-delete-response-message-%v", mm.deviceID)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002376}
2377
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002378func (mm *OnuMetricsManager) generateTicks(ctx context.Context) {
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002379 mm.updateTickGenerationStatus(true)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002380 for {
2381 select {
2382 case <-time.After(L2PmCollectionInterval * time.Second):
2383 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002384 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventTick); err != nil {
2385 logger.Errorw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdrae0140f02021-02-02 16:55:09 -08002386 }
2387 }()
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002388 case <-mm.StopTicks:
2389 logger.Infow(ctx, "stopping ticks", log.Fields{"device-id": mm.deviceID})
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002390 mm.updateTickGenerationStatus(false)
Girish Gowdrae0140f02021-02-02 16:55:09 -08002391 return
2392 }
2393 }
2394}
2395
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002396func (mm *OnuMetricsManager) handleMetricsPublish(ctx context.Context, metricName string, metricInfoSlice []*voltha.MetricInformation) {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002397 // Publish metrics if it is valid
2398 if metricInfoSlice != nil {
2399 mm.publishMetrics(ctx, metricInfoSlice)
2400 } else {
2401 // If collectAttempts exceeds L2PmCollectAttempts then remove it from activeL2Pms
2402 // slice so that we do not collect data from that PM ME anymore
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002403 mm.OnuMetricsManagerLock.Lock()
2404 mm.GroupMetricMap[metricName].collectAttempts++
2405 if mm.GroupMetricMap[metricName].collectAttempts > L2PmCollectAttempts {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002406 mm.activeL2Pms = mm.removeIfFoundString(mm.activeL2Pms, metricName)
2407 }
2408 logger.Warnw(ctx, "state collect data - no metrics collected",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002409 log.Fields{"device-id": mm.deviceID, "metricName": metricName, "collectAttempts": mm.GroupMetricMap[metricName].collectAttempts})
2410 mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002411 }
2412}
2413
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002414func (mm *OnuMetricsManager) populateGroupSpecificMetrics(ctx context.Context, mEnt *me.ManagedEntity, classID me.ClassID, entityID uint16,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002415 meAttributes me.AttributeValueMap, data map[string]float32, intervalEndTime *int) error {
2416 var grpFunc groupMetricPopulateFunc
2417 switch classID {
2418 case me.EthernetFramePerformanceMonitoringHistoryDataUpstreamClassID, me.EthernetFramePerformanceMonitoringHistoryDataDownstreamClassID:
2419 grpFunc = mm.populateEthernetBridgeHistoryMetrics
2420 case me.EthernetPerformanceMonitoringHistoryDataClassID:
2421 grpFunc = mm.populateEthernetUniHistoryMetrics
2422 case me.FecPerformanceMonitoringHistoryDataClassID:
2423 grpFunc = mm.populateFecHistoryMetrics
2424 case me.GemPortNetworkCtpPerformanceMonitoringHistoryDataClassID:
2425 grpFunc = mm.populateGemPortMetrics
2426 default:
2427 return fmt.Errorf("unknown-classid-%v", classID)
2428 }
2429
2430 size := 0
2431 requestedAttributes := make(me.AttributeValueMap)
2432 for _, v := range mEnt.GetAttributeDefinitions() {
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002433 if v.Name == "ManagedEntityId" || v.Name == "IntervalEndTime" || v.Name == "ThresholdData12Id" {
2434 // Exclude the ManagedEntityId , it will be inserted by omci library based on 'entityID' information
2435 // Exclude IntervalEndTime. It will be inserted by the group PM populater function.
2436 // Exclude ThresholdData12Id as that is of no particular relevance for metrics collection.
2437 continue
2438 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002439 if (v.Size + size) <= MaxL2PMGetPayLoadSize {
2440 requestedAttributes[v.Name] = v.DefValue
2441 size = v.Size + size
2442 } else { // We exceeded the allow omci get size
2443 // Let's collect the attributes via get now and collect remaining in the next iteration
2444 if err := grpFunc(ctx, classID, entityID, meAttributes, requestedAttributes, data, intervalEndTime); err != nil {
2445 logger.Errorw(ctx, "error during metric collection",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002446 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "err": err})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002447 return err
2448 }
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002449 requestedAttributes = make(me.AttributeValueMap) // reset map
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002450 requestedAttributes[v.Name] = v.DefValue // populate the metric that was missed in the current iteration
2451 size = v.Size // reset size
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002452 }
2453 }
2454 // Collect the omci get attributes for the last bunch of attributes.
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002455 if err := grpFunc(ctx, classID, entityID, meAttributes, requestedAttributes, data, intervalEndTime); err != nil {
2456 logger.Errorw(ctx, "error during metric collection",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002457 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "err": err})
Girish Gowdra6c04fbc2021-04-22 15:34:49 -07002458 return err
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002459 }
2460 return nil
2461}
2462
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002463func (mm *OnuMetricsManager) populateOnuMetricInfo(title string, data map[string]float32) voltha.MetricInformation {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002464 metricsContext := make(map[string]string)
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002465 metricsContext["onuID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetDevice().ProxyAddress.OnuId)
2466 metricsContext["intfID"] = fmt.Sprintf("%d", mm.pDeviceHandler.GetDevice().ProxyAddress.ChannelId)
2467 metricsContext["devicetype"] = mm.pDeviceHandler.GetDeviceType()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002468
2469 raisedTs := time.Now().Unix()
2470 mmd := voltha.MetricMetaData{
2471 Title: title,
2472 Ts: float64(raisedTs),
2473 Context: metricsContext,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002474 DeviceId: mm.deviceID,
2475 LogicalDeviceId: mm.pDeviceHandler.GetLogicalDeviceID(),
2476 SerialNo: mm.pDeviceHandler.GetDevice().SerialNumber,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002477 }
2478
2479 // create slice of metrics given that there could be more than one VEIP instance
2480 metricInfo := voltha.MetricInformation{Metadata: &mmd, Metrics: data}
2481 return metricInfo
2482}
2483
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002484func (mm *OnuMetricsManager) updateAndValidateIntervalEndTime(ctx context.Context, entityID uint16, meAttributes me.AttributeValueMap, intervalEndTime *int) bool {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002485 valid := false
2486 if *intervalEndTime == -1 { // first time
2487 // Update the interval end time
2488 if val, ok := meAttributes["IntervalEndTime"]; ok && val != nil {
2489 *intervalEndTime = int(meAttributes["IntervalEndTime"].(uint8))
2490 valid = true
2491 }
2492 } else {
2493 var currIntervalEndTime int
2494 if val, ok := meAttributes["IntervalEndTime"]; ok && val != nil {
2495 currIntervalEndTime = int(meAttributes["IntervalEndTime"].(uint8))
2496 }
2497 if currIntervalEndTime != *intervalEndTime { // interval end time changed during metric collection
2498 logger.Errorw(ctx, "interval end time changed during metrics collection for ethernet pm history data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002499 log.Fields{"device-id": mm.deviceID, "entityID": entityID,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002500 "currIntervalEndTime": *intervalEndTime, "newIntervalEndTime": currIntervalEndTime})
2501 } else {
2502 valid = true
2503 }
2504 }
2505 return valid
2506}
2507
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002508func (mm *OnuMetricsManager) waitForResponseOrTimeout(ctx context.Context, create bool, instID uint16, meClassName string) bool {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002509 logger.Debugw(ctx, "waitForResponseOrTimeout", log.Fields{"create": create, "instID": instID, "meClassName": meClassName})
2510 select {
2511 case resp := <-mm.l2PmCreateOrDeleteResponseChan:
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002512 logger.Debugw(ctx, "received l2 pm me response",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002513 log.Fields{"device-id": mm.deviceID, "resp": resp, "create": create, "meClassName": meClassName, "instID": instID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002514 return resp
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002515 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002516 logger.Errorw(ctx, "timeout waiting for l2 pm me response",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002517 log.Fields{"device-id": mm.deviceID, "resp": false, "create": create, "meClassName": meClassName, "instID": instID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002518 }
2519 return false
2520}
2521
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002522func (mm *OnuMetricsManager) initializeGroupMetric(grpMtrcs map[string]voltha.PmConfig_PmType, grpName string, grpEnabled bool, grpFreq uint32) {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002523 var pmConfigSlice []*voltha.PmConfig
2524 for k, v := range grpMtrcs {
Girish Gowdra0e533642021-03-02 22:02:51 -08002525 pmConfigSlice = append(pmConfigSlice,
2526 &voltha.PmConfig{
2527 Name: k,
2528 Type: v,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002529 Enabled: grpEnabled && mm.pDeviceHandler.GetMetricsEnabled(),
Girish Gowdra0e533642021-03-02 22:02:51 -08002530 SampleFreq: grpFreq})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002531 }
2532 groupMetric := voltha.PmGroupConfig{
2533 GroupName: grpName,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002534 Enabled: grpEnabled && mm.pDeviceHandler.GetMetricsEnabled(),
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002535 GroupFreq: grpFreq,
2536 Metrics: pmConfigSlice,
2537 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002538 mm.pDeviceHandler.GetPmConfigs().Groups = append(mm.pDeviceHandler.GetPmConfigs().Groups, &groupMetric)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002539
2540}
2541
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002542func (mm *OnuMetricsManager) initializeL2PmFsm(ctx context.Context, aCommChannel chan cmn.Message) error {
2543 mm.PAdaptFsm = cmn.NewAdapterFsm("L2PmFSM", mm.deviceID, aCommChannel)
2544 if mm.PAdaptFsm == nil {
2545 logger.Errorw(ctx, "L2PMFsm cmn.AdapterFsm could not be instantiated!!", log.Fields{
2546 "device-id": mm.deviceID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002547 return fmt.Errorf("nil-adapter-fsm")
2548 }
2549 // L2 PM FSM related state machine
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002550 mm.PAdaptFsm.PFsm = fsm.NewFSM(
2551 L2PmStNull,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002552 fsm.Events{
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002553 {Name: L2PmEventInit, Src: []string{L2PmStNull}, Dst: L2PmStStarting},
2554 {Name: L2PmEventTick, Src: []string{L2PmStStarting}, Dst: L2PmStSyncTime},
2555 {Name: L2PmEventTick, Src: []string{L2PmStIdle, L2PmStCreatePmMe, L2PmStDeletePmMe}, Dst: L2PmStCollectData},
2556 {Name: L2PmEventSuccess, Src: []string{L2PmStSyncTime, L2PmStCreatePmMe, L2PmStDeletePmMe, L2PmStCollectData}, Dst: L2PmStIdle},
2557 {Name: L2PmEventFailure, Src: []string{L2PmStCreatePmMe, L2PmStDeletePmMe, L2PmStCollectData}, Dst: L2PmStIdle},
2558 {Name: L2PmEventFailure, Src: []string{L2PmStSyncTime}, Dst: L2PmStSyncTime},
2559 {Name: L2PmEventAddMe, Src: []string{L2PmStIdle}, Dst: L2PmStCreatePmMe},
2560 {Name: L2PmEventDeleteMe, Src: []string{L2PmStIdle}, Dst: L2PmStDeletePmMe},
2561 {Name: L2PmEventStop, Src: []string{L2PmStNull, L2PmStStarting, L2PmStSyncTime, L2PmStIdle, L2PmStCreatePmMe, L2PmStDeletePmMe, L2PmStCollectData}, Dst: L2PmStNull},
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002562 },
2563 fsm.Callbacks{
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002564 "enter_state": func(e *fsm.Event) { mm.PAdaptFsm.LogFsmStateChange(ctx, e) },
2565 "enter_" + L2PmStNull: func(e *fsm.Event) { mm.l2PMFsmNull(ctx, e) },
2566 "enter_" + L2PmStIdle: func(e *fsm.Event) { mm.l2PMFsmIdle(ctx, e) },
2567 "enter_" + L2PmStStarting: func(e *fsm.Event) { mm.l2PMFsmStarting(ctx, e) },
2568 "enter_" + L2PmStSyncTime: func(e *fsm.Event) { mm.l2PMFsmSyncTime(ctx, e) },
2569 "enter_" + L2PmStCollectData: func(e *fsm.Event) { mm.l2PmFsmCollectData(ctx, e) },
2570 "enter_" + L2PmStCreatePmMe: func(e *fsm.Event) { _ = mm.l2PmFsmCreatePM(ctx, e) },
2571 "enter_" + L2PmStDeletePmMe: func(e *fsm.Event) { _ = mm.l2PmFsmDeletePM(ctx, e) },
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002572 },
2573 )
2574 return nil
2575}
2576
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002577func (mm *OnuMetricsManager) initializeAllGroupMetrics() {
2578 mm.pDeviceHandler.InitPmConfigs()
2579 mm.pDeviceHandler.GetPmConfigs().Id = mm.deviceID
2580 mm.pDeviceHandler.GetPmConfigs().DefaultFreq = DefaultMetricCollectionFrequency
2581 mm.pDeviceHandler.GetPmConfigs().Grouped = GroupMetricEnabled
2582 mm.pDeviceHandler.GetPmConfigs().FreqOverride = DefaultFrequencyOverrideEnabled
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002583
2584 // Populate group metrics.
2585 // Lets populate irrespective of GroupMetricEnabled is true or not.
2586 // The group metrics collection will decided on this flag later
2587
2588 mm.initializeGroupMetric(OpticalPowerGroupMetrics, OpticalPowerGroupMetricName,
2589 OpticalPowerGroupMetricEnabled, OpticalPowerMetricGroupCollectionFrequency)
2590
2591 mm.initializeGroupMetric(UniStatusGroupMetrics, UniStatusGroupMetricName,
2592 UniStatusGroupMetricEnabled, UniStatusMetricGroupCollectionFrequency)
2593
2594 // classical l2 pm counter start
2595
2596 mm.initializeGroupMetric(EthernetBridgeHistory, EthernetBridgeHistoryName,
2597 EthernetBridgeHistoryEnabled, EthernetBridgeHistoryFrequency)
2598
2599 mm.initializeGroupMetric(EthernetUniHistory, EthernetUniHistoryName,
2600 EthernetUniHistoryEnabled, EthernetUniHistoryFrequency)
2601
2602 mm.initializeGroupMetric(FecHistory, FecHistoryName,
2603 FecHistoryEnabled, FecHistoryFrequency)
2604
2605 mm.initializeGroupMetric(GemPortHistory, GemPortHistoryName,
2606 GemPortHistoryEnabled, GemPortHistoryFrequency)
2607
2608 // classical l2 pm counter end
2609
2610 // Add standalone metric (if present) after this (will be added to dh.pmConfigs.Metrics)
2611}
2612
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002613func (mm *OnuMetricsManager) populateLocalGroupMetricData(ctx context.Context) {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002614 // Populate local group metric structures
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002615 for _, g := range mm.pDeviceHandler.GetPmConfigs().Groups {
2616 mm.GroupMetricMap[g.GroupName] = &groupMetric{
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002617 groupName: g.GroupName,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002618 Enabled: g.Enabled,
2619 Frequency: g.GroupFreq,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002620 }
2621 switch g.GroupName {
2622 case OpticalPowerGroupMetricName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002623 mm.GroupMetricMap[g.GroupName].metricMap = OpticalPowerGroupMetrics
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002624 case UniStatusGroupMetricName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002625 mm.GroupMetricMap[g.GroupName].metricMap = UniStatusGroupMetrics
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002626 case EthernetBridgeHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002627 mm.GroupMetricMap[g.GroupName].metricMap = EthernetBridgeHistory
2628 mm.GroupMetricMap[g.GroupName].IsL2PMCounter = true
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002629 case EthernetUniHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002630 mm.GroupMetricMap[g.GroupName].metricMap = EthernetUniHistory
2631 mm.GroupMetricMap[g.GroupName].IsL2PMCounter = true
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002632 case FecHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002633 mm.GroupMetricMap[g.GroupName].metricMap = FecHistory
2634 mm.GroupMetricMap[g.GroupName].IsL2PMCounter = true
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002635 case GemPortHistoryName:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002636 mm.GroupMetricMap[g.GroupName].metricMap = GemPortHistory
2637 mm.GroupMetricMap[g.GroupName].IsL2PMCounter = true
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002638 default:
2639 logger.Errorw(ctx, "unhandled-group-name", log.Fields{"groupName": g.GroupName})
2640 }
2641 }
2642
2643 // Populate local standalone metric structures
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002644 for _, m := range mm.pDeviceHandler.GetPmConfigs().Metrics {
2645 mm.StandaloneMetricMap[m.Name] = &standaloneMetric{
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002646 metricName: m.Name,
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002647 Enabled: m.Enabled,
2648 Frequency: m.SampleFreq,
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002649 }
2650 switch m.Name {
2651 // None exist as of now. Add when available.
2652 default:
2653 logger.Errorw(ctx, "unhandled-metric-name", log.Fields{"metricName": m.Name})
2654 }
2655 }
2656}
2657
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002658// AddGemPortForPerfMonitoring - TODO: add comment
2659func (mm *OnuMetricsManager) AddGemPortForPerfMonitoring(ctx context.Context, gemPortNTPInstID uint16) {
2660 mm.OnuMetricsManagerLock.Lock()
2661 defer mm.OnuMetricsManagerLock.Unlock()
2662 logger.Debugw(ctx, "add gemport for perf monitoring - start", log.Fields{"device-id": mm.deviceID, "gemPortID": gemPortNTPInstID})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002663 // mark the instance for addition
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002664 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd = mm.appendIfMissingUnt16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, gemPortNTPInstID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002665 // If the instance presence toggles too soon, we need to remove it from gemPortNCTPPerfHistInstToDelete slice
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002666 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete = mm.removeIfFoundUint16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete, gemPortNTPInstID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002667
2668 mm.l2PmToAdd = mm.appendIfMissingString(mm.l2PmToAdd, GemPortHistoryName)
2669 // We do not need to remove from l2PmToDelete slice as we could have Add and Delete of
2670 // GemPortPerfHistory ME simultaneously for different instances of the ME.
2671 // The creation or deletion of an instance is decided based on its presence in gemPortNCTPPerfHistInstToDelete or
2672 // gemPortNCTPPerfHistInstToAdd slice
Girish Gowdra69570d92021-04-22 18:26:20 -07002673
2674 logger.Debugw(ctx, "add gemport for perf monitoring - end",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002675 log.Fields{"device-id": mm.deviceID, "pms-to-add": mm.l2PmToAdd,
2676 "instances-to-add": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd})
Girish Gowdra69570d92021-04-22 18:26:20 -07002677 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002678 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventAddMe); err != nil {
Girish Gowdra69570d92021-04-22 18:26:20 -07002679 // log at warn level as the gem port for monitoring is going to be added eventually
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002680 logger.Warnw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdra69570d92021-04-22 18:26:20 -07002681 }
2682 }()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002683}
2684
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002685// RemoveGemPortForPerfMonitoring - TODO: add comment
2686func (mm *OnuMetricsManager) RemoveGemPortForPerfMonitoring(ctx context.Context, gemPortNTPInstID uint16) {
2687 mm.OnuMetricsManagerLock.Lock()
2688 defer mm.OnuMetricsManagerLock.Unlock()
2689 logger.Debugw(ctx, "remove gemport for perf monitoring - start", log.Fields{"device-id": mm.deviceID, "gemPortID": gemPortNTPInstID})
2690 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete = mm.appendIfMissingUnt16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete, gemPortNTPInstID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002691 // If the instance presence toggles too soon, we need to remove it from gemPortNCTPPerfHistInstToAdd slice
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002692 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd = mm.removeIfFoundUint16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, gemPortNTPInstID)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002693
2694 mm.l2PmToDelete = mm.appendIfMissingString(mm.l2PmToDelete, GemPortHistoryName)
2695 // We do not need to remove from l2PmToAdd slice as we could have Add and Delete of
2696 // GemPortPerfHistory ME simultaneously for different instances of the ME.
2697 // The creation or deletion of an instance is decided based on its presence in gemPortNCTPPerfHistInstToDelete or
2698 // gemPortNCTPPerfHistInstToAdd slice
Girish Gowdra69570d92021-04-22 18:26:20 -07002699
2700 logger.Debugw(ctx, "remove gemport from perf monitoring - end",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002701 log.Fields{"device-id": mm.deviceID, "pms-to-delete": mm.l2PmToDelete,
2702 "instances-to-delete": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete})
Girish Gowdra69570d92021-04-22 18:26:20 -07002703 go func() {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002704 if err := mm.PAdaptFsm.PFsm.Event(L2PmEventDeleteMe); err != nil {
Girish Gowdra69570d92021-04-22 18:26:20 -07002705 // log at warn level as the gem port for monitoring is going to be removed eventually
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002706 logger.Warnw(ctx, "error calling event", log.Fields{"device-id": mm.deviceID, "err": err})
Girish Gowdra69570d92021-04-22 18:26:20 -07002707 }
2708 }()
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002709}
2710
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002711func (mm *OnuMetricsManager) updateGemPortNTPInstanceToAddForPerfMonitoring(ctx context.Context) {
2712 if mm.pDeviceHandler.GetOnuTP() != nil {
2713 gemPortInstIDs := mm.pDeviceHandler.GetOnuTP().GetAllBidirectionalGemPortIDsForOnu()
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002714 // NOTE: It is expected that caller of this function has acquired the required mutex for synchronization purposes
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002715 for _, v := range gemPortInstIDs {
2716 // mark the instance for addition
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002717 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd = mm.appendIfMissingUnt16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, v)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002718 // If the instance presence toggles too soon, we need to remove it from gemPortNCTPPerfHistInstToDelete slice
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002719 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete = mm.removeIfFoundUint16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete, v)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002720 }
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002721 logger.Debugw(ctx, "updateGemPortNTPInstanceToAddForPerfMonitoring",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002722 log.Fields{"deviceID": mm.deviceID, "gemToAdd": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, "gemToDel": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete})
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002723 }
2724}
2725
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002726func (mm *OnuMetricsManager) updateGemPortNTPInstanceToDeleteForPerfMonitoring(ctx context.Context) {
2727 if mm.pDeviceHandler.GetOnuTP() != nil {
2728 gemPortInstIDs := mm.pDeviceHandler.GetOnuTP().GetAllBidirectionalGemPortIDsForOnu()
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002729 // NOTE: It is expected that caller of this function has acquired the required mutex for synchronization purposes
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002730 for _, v := range gemPortInstIDs {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002731 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete = mm.appendIfMissingUnt16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete, v)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002732 // If the instance presence toggles too soon, we need to remove it from gemPortNCTPPerfHistInstToAdd slice
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002733 mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd = mm.removeIfFoundUint16(mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, v)
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002734 }
2735 }
Girish Gowdra36ccf7d2021-02-25 20:42:51 -08002736 logger.Debugw(ctx, "updateGemPortNTPInstanceToDeleteForPerfMonitoring",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002737 log.Fields{"deviceID": mm.deviceID, "gemToAdd": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToAdd, "gemToDel": mm.GroupMetricMap[GemPortHistoryName].pmMEData.InstancesToDelete})
Girish Gowdra0e533642021-03-02 22:02:51 -08002738}
2739
2740// restorePmData restores any PM data available on the KV store to local cache
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002741func (mm *OnuMetricsManager) restorePmData(ctx context.Context) error {
2742 logger.Debugw(ctx, "restorePmData - start", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002743 if mm.pmKvStore == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002744 logger.Errorw(ctx, "pmKvStore not set - abort", log.Fields{"device-id": mm.deviceID})
2745 return fmt.Errorf(fmt.Sprintf("pmKvStore-not-set-abort-%s", mm.deviceID))
Girish Gowdra0e533642021-03-02 22:02:51 -08002746 }
2747 var errorsList []error
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002748 for groupName, group := range mm.GroupMetricMap {
Girish Gowdra0e533642021-03-02 22:02:51 -08002749 group.pmMEData = &pmMEData{}
2750 Value, err := mm.pmKvStore.Get(ctx, groupName)
2751 if err == nil {
2752 if Value != nil {
2753 logger.Debugw(ctx, "PM data read",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002754 log.Fields{"Key": Value.Key, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002755 tmpBytes, _ := kvstore.ToByte(Value.Value)
2756
2757 if err = json.Unmarshal(tmpBytes, &group.pmMEData); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002758 logger.Errorw(ctx, "unable to unmarshal PM data", log.Fields{"error": err, "device-id": mm.deviceID})
2759 errorsList = append(errorsList, fmt.Errorf(fmt.Sprintf("unable-to-unmarshal-PM-data-%s-for-group-%s", mm.deviceID, groupName)))
Girish Gowdra0e533642021-03-02 22:02:51 -08002760 continue
2761 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002762 logger.Debugw(ctx, "restorePmData - success", log.Fields{"pmData": group.pmMEData, "groupName": groupName, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002763 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002764 logger.Debugw(ctx, "no PM data found", log.Fields{"groupName": groupName, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002765 continue
2766 }
2767 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002768 logger.Errorw(ctx, "restorePmData - fail", log.Fields{"device-id": mm.deviceID, "groupName": groupName, "err": err})
2769 errorsList = append(errorsList, fmt.Errorf(fmt.Sprintf("unable-to-read-from-KVstore-%s-for-group-%s", mm.deviceID, groupName)))
Girish Gowdra0e533642021-03-02 22:02:51 -08002770 continue
2771 }
2772 }
2773 if len(errorsList) > 0 {
2774 return fmt.Errorf("errors-restoring-pm-data-for-one-or-more-groups--errors:%v", errorsList)
2775 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002776 logger.Debugw(ctx, "restorePmData - complete success", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002777 return nil
2778}
2779
2780// getPmData gets pmMEData from cache. Since we have write through cache implementation for pmMEData,
2781// the data must be available in cache.
2782// Note, it is expected that caller of this function manages the required synchronization (like using locks etc.).
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002783func (mm *OnuMetricsManager) getPmData(ctx context.Context, groupName string) (*pmMEData, error) {
2784 if grp, ok := mm.GroupMetricMap[groupName]; ok {
Girish Gowdra0e533642021-03-02 22:02:51 -08002785 return grp.pmMEData, nil
2786 }
2787 // Data not in cache, try to fetch from kv store.
2788 data := &pmMEData{}
2789 if mm.pmKvStore == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002790 logger.Errorw(ctx, "pmKvStore not set - abort", log.Fields{"device-id": mm.deviceID})
2791 return data, fmt.Errorf("pmKvStore not set. device-id - %s", mm.deviceID)
Girish Gowdra0e533642021-03-02 22:02:51 -08002792 }
2793 Value, err := mm.pmKvStore.Get(ctx, groupName)
2794 if err == nil {
2795 if Value != nil {
2796 logger.Debugw(ctx, "PM data read",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002797 log.Fields{"Key": Value.Key, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002798 tmpBytes, _ := kvstore.ToByte(Value.Value)
2799
2800 if err = json.Unmarshal(tmpBytes, data); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002801 logger.Errorw(ctx, "unable to unmarshal PM data", log.Fields{"error": err, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002802 return data, err
2803 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002804 logger.Debugw(ctx, "PM data", log.Fields{"pmData": data, "groupName": groupName, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002805 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002806 logger.Debugw(ctx, "no PM data found", log.Fields{"groupName": groupName, "device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002807 return data, err
2808 }
2809 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002810 logger.Errorw(ctx, "unable to read from KVstore", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002811 return data, err
2812 }
2813
2814 return data, nil
2815}
2816
2817// updatePmData update pmMEData to store. It is write through cache, i.e., write to cache first and then update store
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002818func (mm *OnuMetricsManager) updatePmData(ctx context.Context, groupName string, meInstanceID uint16, pmAction string) error {
2819 logger.Debugw(ctx, "updatePmData - start", log.Fields{"device-id": mm.deviceID, "groupName": groupName, "entityID": meInstanceID, "pmAction": pmAction})
2820 mm.OnuMetricsManagerLock.Lock()
2821 defer mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra0e533642021-03-02 22:02:51 -08002822
2823 if mm.pmKvStore == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002824 logger.Errorw(ctx, "pmKvStore not set - abort", log.Fields{"device-id": mm.deviceID})
2825 return fmt.Errorf(fmt.Sprintf("pmKvStore-not-set-abort-%s", mm.deviceID))
Girish Gowdra0e533642021-03-02 22:02:51 -08002826 }
2827
2828 pmMEData, err := mm.getPmData(ctx, groupName)
2829 if err != nil || pmMEData == nil {
2830 // error already logged in called function.
2831 return err
2832 }
2833 switch pmAction {
2834 case cPmAdd:
2835 pmMEData.InstancesToAdd = mm.appendIfMissingUnt16(pmMEData.InstancesToAdd, meInstanceID)
2836 pmMEData.InstancesToDelete = mm.removeIfFoundUint16(pmMEData.InstancesToDelete, meInstanceID)
2837 pmMEData.InstancesActive = mm.removeIfFoundUint16(pmMEData.InstancesActive, meInstanceID)
2838 case cPmAdded:
2839 pmMEData.InstancesActive = mm.appendIfMissingUnt16(pmMEData.InstancesActive, meInstanceID)
2840 pmMEData.InstancesToAdd = mm.removeIfFoundUint16(pmMEData.InstancesToAdd, meInstanceID)
2841 pmMEData.InstancesToDelete = mm.removeIfFoundUint16(pmMEData.InstancesToDelete, meInstanceID)
2842 case cPmRemove:
2843 pmMEData.InstancesToDelete = mm.appendIfMissingUnt16(pmMEData.InstancesToDelete, meInstanceID)
2844 pmMEData.InstancesToAdd = mm.removeIfFoundUint16(pmMEData.InstancesToAdd, meInstanceID)
2845 pmMEData.InstancesActive = mm.removeIfFoundUint16(pmMEData.InstancesActive, meInstanceID)
2846 case cPmRemoved:
2847 pmMEData.InstancesToDelete = mm.removeIfFoundUint16(pmMEData.InstancesToDelete, meInstanceID)
2848 pmMEData.InstancesToAdd = mm.removeIfFoundUint16(pmMEData.InstancesToAdd, meInstanceID)
2849 pmMEData.InstancesActive = mm.removeIfFoundUint16(pmMEData.InstancesActive, meInstanceID)
2850 default:
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002851 logger.Errorw(ctx, "unknown pm action", log.Fields{"device-id": mm.deviceID, "pmAction": pmAction, "groupName": groupName})
2852 return fmt.Errorf(fmt.Sprintf("unknown-pm-action-deviceid-%s-groupName-%s-pmaction-%s", mm.deviceID, groupName, pmAction))
Girish Gowdra0e533642021-03-02 22:02:51 -08002853 }
2854 // write through cache
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002855 mm.GroupMetricMap[groupName].pmMEData = pmMEData
Girish Gowdra0e533642021-03-02 22:02:51 -08002856
2857 Value, err := json.Marshal(*pmMEData)
2858 if err != nil {
2859 logger.Errorw(ctx, "unable to marshal PM data", log.Fields{"groupName": groupName, "pmAction": pmAction, "pmData": *pmMEData, "err": err})
2860 return err
2861 }
2862 // Update back to kv store
2863 if err = mm.pmKvStore.Put(ctx, groupName, Value); err != nil {
2864 logger.Errorw(ctx, "unable to put PM data to kv store", log.Fields{"groupName": groupName, "pmData": *pmMEData, "pmAction": pmAction, "err": err})
2865 return err
2866 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002867 logger.Debugw(ctx, "updatePmData - success", log.Fields{"device-id": mm.deviceID, "groupName": groupName, "pmData": *pmMEData, "pmAction": pmAction})
Girish Gowdra0e533642021-03-02 22:02:51 -08002868
2869 return nil
2870}
2871
2872// clearPmGroupData cleans PM Group data from store
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002873func (mm *OnuMetricsManager) clearPmGroupData(ctx context.Context) error {
2874 mm.OnuMetricsManagerLock.Lock()
2875 defer mm.OnuMetricsManagerLock.Unlock()
2876 logger.Debugw(ctx, "clearPmGroupData - start", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002877 if mm.pmKvStore == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002878 logger.Errorw(ctx, "pmKvStore not set - abort", log.Fields{"device-id": mm.deviceID})
2879 return fmt.Errorf(fmt.Sprintf("pmKvStore-not-set-abort-%s", mm.deviceID))
Girish Gowdra0e533642021-03-02 22:02:51 -08002880 }
2881
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002882 for n := range mm.GroupMetricMap {
Girish Gowdra0e533642021-03-02 22:02:51 -08002883 if err := mm.pmKvStore.Delete(ctx, n); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002884 logger.Errorw(ctx, "clearPmGroupData - fail", log.Fields{"deviceID": mm.deviceID, "groupName": n, "err": err})
Girish Gowdra0e533642021-03-02 22:02:51 -08002885 // do not abort this procedure. continue to delete next group.
2886 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002887 logger.Debugw(ctx, "clearPmGroupData - success", log.Fields{"device-id": mm.deviceID, "groupName": n})
Girish Gowdra0e533642021-03-02 22:02:51 -08002888 }
2889 }
2890
2891 return nil
2892}
2893
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002894// ClearAllPmData clears all PM data associated with the device from KV store
2895func (mm *OnuMetricsManager) ClearAllPmData(ctx context.Context) error {
2896 mm.OnuMetricsManagerLock.Lock()
2897 defer mm.OnuMetricsManagerLock.Unlock()
2898 logger.Debugw(ctx, "ClearAllPmData - start", log.Fields{"device-id": mm.deviceID})
Girish Gowdra0e533642021-03-02 22:02:51 -08002899 if mm.pmKvStore == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002900 logger.Errorw(ctx, "pmKvStore not set - abort", log.Fields{"device-id": mm.deviceID})
2901 return fmt.Errorf(fmt.Sprintf("pmKvStore-not-set-abort-%s", mm.deviceID))
Girish Gowdra0e533642021-03-02 22:02:51 -08002902 }
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002903 var value error
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002904 for n := range mm.GroupMetricMap {
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002905 if err := mm.pmKvStore.Delete(ctx, n); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002906 logger.Errorw(ctx, "clearPmGroupData - fail", log.Fields{"deviceID": mm.deviceID, "groupName": n, "err": err})
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002907 value = err
2908 // do not abort this procedure - continue to delete next group.
2909 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002910 logger.Debugw(ctx, "clearPmGroupData - success", log.Fields{"device-id": mm.deviceID, "groupName": n})
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002911 }
Girish Gowdra0e533642021-03-02 22:02:51 -08002912 }
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002913 if value == nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002914 logger.Debugw(ctx, "ClearAllPmData - success", log.Fields{"device-id": mm.deviceID})
Holger Hildebrandt44a0d4f2021-03-18 14:00:54 +00002915 }
2916 return value
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002917}
2918
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002919func (mm *OnuMetricsManager) updateOmciProcessingStatus(status bool) {
2920 mm.OnuMetricsManagerLock.Lock()
2921 defer mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002922 mm.omciProcessingActive = status
2923}
2924
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002925// updateTickGenerationStatus - TODO: add comment
2926func (mm *OnuMetricsManager) updateTickGenerationStatus(status bool) {
2927 mm.OnuMetricsManagerLock.Lock()
2928 defer mm.OnuMetricsManagerLock.Unlock()
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002929 mm.tickGenerationActive = status
2930}
2931
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002932// GetOmciProcessingStatus - TODO: add comment
2933func (mm *OnuMetricsManager) GetOmciProcessingStatus() bool {
2934 mm.OnuMetricsManagerLock.RLock()
2935 defer mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002936 return mm.omciProcessingActive
2937}
2938
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002939// GetTickGenerationStatus - TODO: add comment
2940func (mm *OnuMetricsManager) GetTickGenerationStatus() bool {
2941 mm.OnuMetricsManagerLock.RLock()
2942 defer mm.OnuMetricsManagerLock.RUnlock()
Girish Gowdra7b0ee5c2021-03-19 21:48:15 -07002943 return mm.tickGenerationActive
2944}
2945
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002946func (mm *OnuMetricsManager) appendIfMissingString(slice []string, n string) []string {
Girish Gowdrae0140f02021-02-02 16:55:09 -08002947 for _, ele := range slice {
2948 if ele == n {
2949 return slice
2950 }
2951 }
2952 return append(slice, n)
2953}
2954
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002955func (mm *OnuMetricsManager) removeIfFoundString(slice []string, n string) []string {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002956 for i, ele := range slice {
2957 if ele == n {
2958 return append(slice[:i], slice[i+1:]...)
2959 }
2960 }
2961 return slice
2962}
2963
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002964func (mm *OnuMetricsManager) appendIfMissingUnt16(slice []uint16, n uint16) []uint16 {
Girish Gowdra5c5aaf42021-02-17 19:40:50 -08002965 for _, ele := range slice {
2966 if ele == n {
2967 return slice
2968 }
2969 }
2970 return append(slice, n)
2971}
2972
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002973func (mm *OnuMetricsManager) removeIfFoundUint16(slice []uint16, n uint16) []uint16 {
Girish Gowdrae0140f02021-02-02 16:55:09 -08002974 for i, ele := range slice {
2975 if ele == n {
2976 return append(slice[:i], slice[i+1:]...)
2977 }
2978 }
2979 return slice
Girish Gowdrae09a6202021-01-12 18:10:59 -08002980}
Himani Chawla43f95ff2021-06-03 00:24:12 +05302981
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002982func (mm *OnuMetricsManager) getEthernetFrameExtendedMETypeFromKvStore(ctx context.Context) (bool, error) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05302983 // Check if the data is already available in KV store, if yes, do not send the request for get me.
2984 var data me.ClassID
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002985 key := fmt.Sprintf("%s/%s/%s", mm.pOnuDeviceEntry.GetPersVendorID(),
2986 mm.pOnuDeviceEntry.GetPersEquipmentID(),
2987 mm.pOnuDeviceEntry.GetPersActiveSwVersion())
Himani Chawla43f95ff2021-06-03 00:24:12 +05302988 Value, err := mm.extPmKvStore.Get(ctx, key)
2989 if err == nil {
2990 if Value != nil {
2991 logger.Debugw(ctx, "me-type-read",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002992 log.Fields{"key": Value.Key, "device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05302993 tmpBytes, _ := kvstore.ToByte(Value.Value)
2994
2995 if err = json.Unmarshal(tmpBytes, &data); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002996 logger.Errorw(ctx, "unable-to-unmarshal-data", log.Fields{"error": err, "device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05302997 return false, err
2998 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00002999 logger.Debugw(ctx, "me-ext-pm-class-data", log.Fields{"class-id": data, "device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303000 // We have found the data from db, no need to get through omci get message.
3001 mm.supportedEthernetFrameExtendedPMClass = data
3002 return true, nil
3003 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003004 logger.Debugw(ctx, "no-me-ext-pm-class-data-found", log.Fields{"device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303005 return false, nil
3006 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003007 logger.Errorw(ctx, "unable-to-read-from-kv-store", log.Fields{"device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303008 return false, err
3009}
3010
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003011func (mm *OnuMetricsManager) waitForEthernetFrameCreateOrDeleteResponseOrTimeout(ctx context.Context, create bool, instID uint16, meClassID me.ClassID, upstream bool) (bool, error) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303012 logger.Debugw(ctx, "wait-for-ethernet-frame-create-or-delete-response-or-timeout", log.Fields{"create": create, "instID": instID, "meClassID": meClassID})
3013 select {
3014 case resp := <-mm.extendedPMCreateOrDeleteResponseChan:
3015 logger.Debugw(ctx, "received-extended-pm-me-response",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003016 log.Fields{"device-id": mm.deviceID, "resp": resp, "create": create, "meClassID": meClassID, "instID": instID, "upstream": upstream})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303017 // If the result is me.InstanceExists it means the entity was already created. It is ok handled that as success
3018 if resp == me.Success || resp == me.InstanceExists {
3019 return true, nil
3020 } else if resp == me.UnknownEntity || resp == me.ParameterError ||
3021 resp == me.ProcessingError || resp == me.NotSupported || resp == me.AttributeFailure {
3022 return false, fmt.Errorf("not-supported-me--resp-code-%v", resp)
3023 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003024 logger.Warnw(ctx, "failed to create me", log.Fields{"device-id": mm.deviceID, "resp": resp, "class-id": meClassID, "instID": instID, "upstream": upstream})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303025 return true, fmt.Errorf("error-while-creating-me--resp-code-%v", resp)
3026 }
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003027 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Himani Chawla43f95ff2021-06-03 00:24:12 +05303028 logger.Errorw(ctx, "timeout-waiting-for-ext-pm-me-response",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003029 log.Fields{"device-id": mm.deviceID, "resp": false, "create": create, "meClassID": meClassID, "instID": instID, "upstream": upstream})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303030 }
3031 return false, fmt.Errorf("timeout-while-waiting-for-response")
3032}
3033
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003034func (mm *OnuMetricsManager) tryCreateExtPmMe(ctx context.Context, meType me.ClassID) (bool, error) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303035 cnt := 0
Himani Chawla43f95ff2021-06-03 00:24:12 +05303036 // Create ME twice, one for each direction. Boolean true is used to indicate upstream and false for downstream.
Himani Chawlac77d5372021-07-12 15:42:26 +05303037 for _, direction := range []bool{true, false} {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003038 for _, uniPort := range *mm.pDeviceHandler.GetUniEntityMap() {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303039 var entityID uint16
3040 if direction {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003041 entityID = uniPort.EntityID + 0x100
Himani Chawla43f95ff2021-06-03 00:24:12 +05303042 } else {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003043 entityID = uniPort.EntityID
Himani Chawla43f95ff2021-06-03 00:24:12 +05303044 }
3045
3046 // parent entity id will be same for both direction
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003047 controlBlock := mm.getControlBlockForExtendedPMDirection(ctx, direction, uniPort.EntityID)
Himani Chawla43f95ff2021-06-03 00:24:12 +05303048
3049 inner1:
3050 // retry ExtendedPmCreateAttempts times to create the instance of PM
3051 for cnt = 0; cnt < ExtendedPmCreateAttempts; cnt++ {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003052 meEnt, err := mm.pOnuDeviceEntry.GetDevOmciCC().SendCreateOrDeleteEthernetFrameExtendedPMME(
3053 ctx, mm.pDeviceHandler.GetOmciTimeout(), true, direction, true,
3054 mm.PAdaptFsm.CommChan, entityID, meType, controlBlock)
Himani Chawla43f95ff2021-06-03 00:24:12 +05303055 if err != nil {
3056 logger.Errorw(ctx, "EthernetFrameExtendedPMME-create-or-delete-failed",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003057 log.Fields{"device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303058 return false, err
3059 }
3060 if supported, err := mm.waitForEthernetFrameCreateOrDeleteResponseOrTimeout(ctx, true, entityID, meType, direction); err == nil && supported {
3061 if direction {
Himani Chawlac77d5372021-07-12 15:42:26 +05303062 mm.ethernetFrameExtendedPmUpStreamMEByEntityID[entityID] = meEnt
Himani Chawla43f95ff2021-06-03 00:24:12 +05303063 } else {
Himani Chawlac77d5372021-07-12 15:42:26 +05303064 mm.ethernetFrameExtendedPmDownStreamMEByEntityID[entityID] = meEnt
Himani Chawla43f95ff2021-06-03 00:24:12 +05303065 }
3066 break inner1
3067 } else if err != nil {
3068 if !supported {
3069 // Need to return immediately
3070 return false, err
3071 }
3072 //In case of failure, go for a retry
3073 }
3074 }
3075 if cnt == ExtendedPmCreateAttempts {
3076 logger.Error(ctx, "exceeded-attempts-while-creating-me-for-ethernet-frame-extended-pm")
3077 return true, fmt.Errorf("unable-to-create-me")
3078 }
3079 }
3080 }
3081 return true, nil
3082}
3083
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003084func (mm *OnuMetricsManager) putExtPmMeKvStore(ctx context.Context) {
3085 key := fmt.Sprintf("%s/%s/%s", mm.pOnuDeviceEntry.GetPersVendorID(),
3086 mm.pOnuDeviceEntry.GetPersEquipmentID(),
3087 mm.pOnuDeviceEntry.GetPersActiveSwVersion())
Himani Chawla43f95ff2021-06-03 00:24:12 +05303088 // check if we get the supported type me for ethernet frame extended pm class id
3089 if mm.supportedEthernetFrameExtendedPMClass == 0 {
3090 logger.Error(ctx, "unable-to-get-any-supported-extended-pm-me-class")
3091 }
3092 classSupported, err := json.Marshal(mm.supportedEthernetFrameExtendedPMClass)
3093 if err != nil {
3094 logger.Errorw(ctx, "unable-to-marshal-data", log.Fields{"err": err})
3095 }
3096 if err := mm.extPmKvStore.Put(ctx, key, classSupported); err != nil {
3097 logger.Errorw(ctx, "unable-to-add-data-in-db", log.Fields{"err": err})
3098 }
3099}
3100
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003101func (mm *OnuMetricsManager) setAllExtPmMeCreatedFlag() {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303102 mm.onuEthernetFrameExtendedPmLock.Lock()
3103 mm.isDeviceReadyToCollectExtendedPmStats = true
3104 mm.onuEthernetFrameExtendedPmLock.Unlock()
3105}
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003106
3107// CreateEthernetFrameExtendedPMME - TODO: add comment
3108func (mm *OnuMetricsManager) CreateEthernetFrameExtendedPMME(ctx context.Context) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303109 //get the type of extended frame pm me supported by onu first
3110 exist, err := mm.getEthernetFrameExtendedMETypeFromKvStore(ctx)
3111 if err != nil {
3112 logger.Error(ctx, "unable-to-get-supported-me-for-ethernet-frame-extended-pm")
3113 return
3114 }
3115 if exist {
3116 // we have the me type, go ahead with the me type supported.
3117 if _, err := mm.tryCreateExtPmMe(ctx, mm.supportedEthernetFrameExtendedPMClass); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003118 logger.Errorw(ctx, "unable-to-create-me-type", log.Fields{"device-id": mm.deviceID,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303119 "meClassID": mm.supportedEthernetFrameExtendedPMClass})
3120 return
3121 }
3122 mm.setAllExtPmMeCreatedFlag()
3123 return
3124 }
3125 // First try with 64 bit me
3126 // we have the me type, go ahead with the me type supported.
3127 supported64Bit, err := mm.tryCreateExtPmMe(ctx, me.EthernetFrameExtendedPm64BitClassID)
3128 if err != nil && !supported64Bit {
3129 logger.Errorw(ctx, "unable-to-create-me-type-as-it-is-not-supported",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003130 log.Fields{"device-id": mm.deviceID, "meClassID": me.EthernetFrameExtendedPm64BitClassID,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303131 "supported": supported64Bit})
3132 // Then Try with 32 bit type
3133 if supported32Bit, err := mm.tryCreateExtPmMe(ctx, me.EthernetFrameExtendedPmClassID); err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003134 logger.Errorw(ctx, "unable-to-create-me-type", log.Fields{"device-id": mm.deviceID,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303135 "meClassID": me.EthernetFrameExtendedPmClassID, "supported": supported32Bit})
3136 } else if supported32Bit {
3137 mm.supportedEthernetFrameExtendedPMClass = me.EthernetFrameExtendedPmClassID
3138 mm.putExtPmMeKvStore(ctx)
3139 mm.setAllExtPmMeCreatedFlag()
3140 }
3141 } else if err == nil && supported64Bit {
3142 mm.supportedEthernetFrameExtendedPMClass = me.EthernetFrameExtendedPm64BitClassID
3143 mm.putExtPmMeKvStore(ctx)
3144 mm.setAllExtPmMeCreatedFlag()
3145 }
3146}
3147
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003148// CollectEthernetFrameExtendedPMCounters - TODO: add comment
3149func (mm *OnuMetricsManager) CollectEthernetFrameExtendedPMCounters(ctx context.Context) *extension.SingleGetValueResponse {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303150 errFunc := func(reason extension.GetValueResponse_ErrorReason) *extension.SingleGetValueResponse {
3151 return &extension.SingleGetValueResponse{
3152 Response: &extension.GetValueResponse{
3153 Status: extension.GetValueResponse_ERROR,
3154 ErrReason: reason,
3155 },
3156 }
3157 }
3158 mm.onuEthernetFrameExtendedPmLock.RLock()
3159 if !mm.isDeviceReadyToCollectExtendedPmStats {
3160 mm.onuEthernetFrameExtendedPmLock.RUnlock()
3161 return errFunc(extension.GetValueResponse_INTERNAL_ERROR)
3162 }
3163 mm.onuEthernetFrameExtendedPmLock.RUnlock()
3164 // Collect metrics for upstream for all the PM Mes per uni port and aggregate
3165 var pmUpstream extension.OmciEthernetFrameExtendedPm
3166 var pmDownstream extension.OmciEthernetFrameExtendedPm
Himani Chawla89ea9e62021-08-25 17:01:03 +05303167 counterFormat := extension.GetOmciEthernetFrameExtendedPmResponse_SIXTY_FOUR_BIT
3168 if mm.supportedEthernetFrameExtendedPMClass == me.EthernetFrameExtendedPmClassID {
3169 counterFormat = extension.GetOmciEthernetFrameExtendedPmResponse_THIRTY_TWO_BIT
3170 }
Himani Chawlac77d5372021-07-12 15:42:26 +05303171 for entityID, meEnt := range mm.ethernetFrameExtendedPmUpStreamMEByEntityID {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303172 var receivedMask uint16
3173 if metricInfo, errResp := mm.collectEthernetFrameExtendedPMData(ctx, meEnt, entityID, true, &receivedMask); metricInfo != nil { // upstream
3174 if receivedMask == 0 {
3175 pmUpstream = mm.aggregateEthernetFrameExtendedPM(metricInfo, pmUpstream, false)
3176 logger.Error(ctx, "all-the-attributes-of-ethernet-frame-extended-pm-counters-are-unsupported")
3177 pmDownstream = pmUpstream
3178 singleValResp := extension.SingleGetValueResponse{
3179 Response: &extension.GetValueResponse{
3180 Status: extension.GetValueResponse_OK,
3181 Response: &extension.GetValueResponse_OnuCounters{
3182 OnuCounters: &extension.GetOmciEthernetFrameExtendedPmResponse{
Himani Chawla89ea9e62021-08-25 17:01:03 +05303183 Upstream: &pmUpstream,
3184 Downstream: &pmDownstream,
3185 OmciEthernetFrameExtendedPmFormat: counterFormat,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303186 },
3187 },
3188 },
3189 }
3190 return &singleValResp
3191 }
3192 // Aggregate the result for upstream
3193 pmUpstream = mm.aggregateEthernetFrameExtendedPM(metricInfo, pmUpstream, true)
3194 } else {
3195 return errFunc(errResp)
3196 }
3197 }
3198
Himani Chawlac77d5372021-07-12 15:42:26 +05303199 for entityID, meEnt := range mm.ethernetFrameExtendedPmDownStreamMEByEntityID {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303200 var receivedMask uint16
3201 if metricInfo, errResp := mm.collectEthernetFrameExtendedPMData(ctx, meEnt, entityID, false, &receivedMask); metricInfo != nil { // downstream
3202 // Aggregate the result for downstream
3203 pmDownstream = mm.aggregateEthernetFrameExtendedPM(metricInfo, pmDownstream, true)
3204 } else {
3205 return errFunc(errResp)
3206 }
3207 }
3208 singleValResp := extension.SingleGetValueResponse{
3209 Response: &extension.GetValueResponse{
3210 Status: extension.GetValueResponse_OK,
3211 Response: &extension.GetValueResponse_OnuCounters{
3212 OnuCounters: &extension.GetOmciEthernetFrameExtendedPmResponse{
Himani Chawla89ea9e62021-08-25 17:01:03 +05303213 Upstream: &pmUpstream,
3214 Downstream: &pmDownstream,
3215 OmciEthernetFrameExtendedPmFormat: counterFormat,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303216 },
3217 },
3218 },
3219 }
3220 return &singleValResp
3221}
3222
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003223func (mm *OnuMetricsManager) collectEthernetFrameExtendedPMData(ctx context.Context, meEnt *me.ManagedEntity, entityID uint16, upstream bool, receivedMask *uint16) (map[string]uint64, extension.GetValueResponse_ErrorReason) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303224 var classID me.ClassID
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003225 logger.Debugw(ctx, "collecting-data-for-ethernet-frame-extended-pm", log.Fields{"device-id": mm.deviceID, "entityID": entityID, "upstream": upstream})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303226
3227 classID = mm.supportedEthernetFrameExtendedPMClass
3228 attributeMaskList := maskToEthernetFrameExtendedPM64Bit
3229 if classID == me.EthernetFrameExtendedPmClassID {
3230 attributeMaskList = maskToEthernetFrameExtendedPM32Bit
3231 }
3232 ethPMData := make(map[string]uint64)
3233 var sumReceivedMask uint16
3234 for mask := range attributeMaskList {
3235 if errResp, err := mm.populateEthernetFrameExtendedPMMetrics(ctx, classID, entityID, mask, ethPMData, upstream, &sumReceivedMask); err != nil {
3236 logger.Errorw(ctx, "error-during-metric-collection",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003237 log.Fields{"device-id": mm.deviceID, "entityID": entityID, "err": err})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303238 return nil, errResp
3239 }
3240 if (mask == 0x3F00 || mask == 0x3800) && sumReceivedMask == 0 {
3241 //It means the first attributes fetch was a failure, hence instead of sending multiple failure get requests
3242 //populate all counters as failure and return
3243 mm.fillAllErrorCountersEthernetFrameExtendedPM(ethPMData)
3244 break
3245 }
3246 }
3247 *receivedMask = sumReceivedMask
3248 return ethPMData, extension.GetValueResponse_REASON_UNDEFINED
3249}
3250
3251// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003252func (mm *OnuMetricsManager) populateEthernetFrameExtendedPMMetrics(ctx context.Context, classID me.ClassID, entityID uint16,
Himani Chawla43f95ff2021-06-03 00:24:12 +05303253 requestedAttributesMask uint16, ethFrameExtPMData map[string]uint64, upstream bool, sumReceivedMask *uint16) (extension.GetValueResponse_ErrorReason, error) {
3254 var meAttributes me.AttributeValueMap
3255 logger.Debugw(ctx, "requesting-attributes", log.Fields{"attributes-mask": requestedAttributesMask, "entityID": entityID, "classID": classID})
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003256 err := mm.pOnuDeviceEntry.GetDevOmciCC().SendGetMeWithAttributeMask(ctx, classID, entityID, requestedAttributesMask, mm.pDeviceHandler.GetOmciTimeout(), true, mm.PAdaptFsm.CommChan)
Himani Chawla43f95ff2021-06-03 00:24:12 +05303257 if err != nil {
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003258 logger.Errorw(ctx, "get-me-failed", log.Fields{"device-id": mm.deviceID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303259 return extension.GetValueResponse_INTERNAL_ERROR, err
3260 }
3261 select {
3262 case meAttributes = <-mm.extendedPmMeChan:
3263 logger.Debugw(ctx, "received-extended-pm-data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003264 log.Fields{"device-id": mm.deviceID, "upstream": upstream, "entityID": entityID})
3265 case <-time.After(mm.pOnuDeviceEntry.GetDevOmciCC().GetMaxOmciTimeoutWithRetries() * time.Second):
Himani Chawla43f95ff2021-06-03 00:24:12 +05303266 logger.Errorw(ctx, "timeout-waiting-for-omci-get-response-for-received-extended-pm-data",
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003267 log.Fields{"device-id": mm.deviceID, "upstream": upstream, "entityID": entityID})
Himani Chawla43f95ff2021-06-03 00:24:12 +05303268 return extension.GetValueResponse_TIMEOUT, fmt.Errorf("timeout-waiting-for-omci-get-response-for-received-extended-pm-data")
3269 }
3270 if mm.supportedEthernetFrameExtendedPMClass == me.EthernetFrameExtendedPmClassID {
3271 mask := mm.getEthFrameExtPMDataFromResponse(ctx, ethFrameExtPMData, meAttributes, requestedAttributesMask)
3272 *sumReceivedMask += mask
3273 logger.Debugw(ctx, "data-received-for-ethernet-frame-ext-pm", log.Fields{"data": ethFrameExtPMData, "entityID": entityID})
3274 } else {
3275 mask := mm.getEthFrameExtPM64BitDataFromResponse(ctx, ethFrameExtPMData, meAttributes, requestedAttributesMask)
3276 *sumReceivedMask += mask
3277 logger.Debugw(ctx, "data-received-for-ethernet-frame-ext-pm", log.Fields{"data": ethFrameExtPMData, "entityID": entityID})
3278 }
3279
3280 return extension.GetValueResponse_REASON_UNDEFINED, nil
3281}
3282
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003283func (mm *OnuMetricsManager) fillAllErrorCountersEthernetFrameExtendedPM(ethFrameExtPMData map[string]uint64) {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303284 sourceMap := maskToEthernetFrameExtendedPM64Bit
3285 errorCounterValue := UnsupportedCounterValue64bit
3286 if mm.supportedEthernetFrameExtendedPMClass == me.EthernetFrameExtendedPmClassID {
3287 sourceMap = maskToEthernetFrameExtendedPM32Bit
3288 errorCounterValue = UnsupportedCounterValue32bit
3289 }
3290 for _, value := range sourceMap {
3291 for _, k := range value {
3292 if _, ok := ethFrameExtPMData[k]; !ok {
3293 ethFrameExtPMData[k] = errorCounterValue
3294 }
3295 }
3296 }
3297}
3298
3299// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003300func (mm *OnuMetricsManager) getEthFrameExtPMDataFromResponse(ctx context.Context, ethFrameExtPMData map[string]uint64, meAttributes me.AttributeValueMap, requestedAttributesMask uint16) uint16 {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303301 receivedMask := uint16(0)
3302 switch requestedAttributesMask {
3303 case 0x3F00:
3304 for _, k := range maskToEthernetFrameExtendedPM32Bit[requestedAttributesMask] {
3305 if _, ok := ethFrameExtPMData[k]; !ok {
3306 switch k {
3307 case "drop_events":
3308 if val, ok := meAttributes[dropEvents]; ok && val != nil {
3309 ethFrameExtPMData[k] = uint64(val.(uint32))
3310 receivedMask |= 0x2000
3311 } else if !ok {
3312 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3313 }
3314 case "octets":
3315 if val, ok := meAttributes[octets]; ok && val != nil {
3316 ethFrameExtPMData[k] = uint64(val.(uint32))
3317 receivedMask |= 0x1000
3318 } else if !ok {
3319 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3320 }
3321 case "frames":
3322 if val, ok := meAttributes[frames]; ok && val != nil {
3323 ethFrameExtPMData[k] = uint64(val.(uint32))
3324 receivedMask |= 0x800
3325 } else if !ok {
3326 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3327 }
3328 case "broadcast_frames":
3329 if val, ok := meAttributes[broadcastFrames]; ok && val != nil {
3330 ethFrameExtPMData[k] = uint64(val.(uint32))
3331 receivedMask |= 0x400
3332 } else if !ok {
3333 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3334 }
3335 case "multicast_frames":
3336 if val, ok := meAttributes[multicastFrames]; ok && val != nil {
3337 ethFrameExtPMData[k] = uint64(val.(uint32))
3338 receivedMask |= 0x200
3339 } else if !ok {
3340 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3341 }
3342 case "crc_errored_frames":
3343 if val, ok := meAttributes[crcErroredFrames]; ok && val != nil {
3344 ethFrameExtPMData[k] = uint64(val.(uint32))
3345 receivedMask |= 0x100
3346 } else if !ok {
3347 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3348 }
3349 default:
3350 //do nothing
3351 }
3352 }
3353 }
3354 case 0x00FC:
3355 for _, k := range maskToEthernetFrameExtendedPM32Bit[requestedAttributesMask] {
3356 if _, ok := ethFrameExtPMData[k]; !ok {
3357 switch k {
3358 case "undersize_frames":
3359 if val, ok := meAttributes[undersizeFrames]; ok && val != nil {
3360 ethFrameExtPMData[k] = uint64(val.(uint32))
3361 receivedMask |= 0x80
3362 } else if !ok {
3363 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3364 }
3365 case "oversize_frames":
3366 if val, ok := meAttributes[oversizeFrames]; ok && val != nil {
3367 ethFrameExtPMData[k] = uint64(val.(uint32))
3368 receivedMask |= 0x40
3369 } else if !ok {
3370 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3371 }
3372 case "64_octets":
3373 if val, ok := meAttributes[frames64Octets]; ok && val != nil {
3374 ethFrameExtPMData[k] = uint64(val.(uint32))
3375 receivedMask |= 0x20
3376 } else if !ok {
3377 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3378 }
3379 case "65_to_127_octets":
3380 if val, ok := meAttributes[frames65To127Octets]; ok && val != nil {
3381 ethFrameExtPMData[k] = uint64(val.(uint32))
3382 receivedMask |= 0x10
3383 } else if !ok {
3384 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3385 }
3386 case "128_to_255_octets":
3387 if val, ok := meAttributes[frames128To255Octets]; ok && val != nil {
3388 ethFrameExtPMData[k] = uint64(val.(uint32))
3389 receivedMask |= 0x8
3390 } else if !ok {
3391 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3392 }
3393 case "256_to_511_octets":
3394 if val, ok := meAttributes[frames256To511Octets]; ok && val != nil {
3395 ethFrameExtPMData[k] = uint64(val.(uint32))
3396 receivedMask |= 0x4
3397 } else if !ok {
3398 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3399 }
3400 default:
3401 //do nothing
3402 }
3403 }
3404 }
3405 case 0x0003:
3406 for _, k := range maskToEthernetFrameExtendedPM32Bit[requestedAttributesMask] {
3407 if _, ok := ethFrameExtPMData[k]; !ok {
3408 switch k {
3409 case "512_to_1023_octets":
3410 if val, ok := meAttributes[frames512To1023Octets]; ok && val != nil {
3411 ethFrameExtPMData[k] = uint64(val.(uint32))
3412 receivedMask |= 0x2
3413 } else if !ok {
3414 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3415 }
3416 case "1024_to_1518_octets":
3417 if val, ok := meAttributes[frames1024To1518Octets]; ok && val != nil {
3418 ethFrameExtPMData[k] = uint64(val.(uint32))
3419 receivedMask |= 0x1
3420 } else if !ok {
3421 ethFrameExtPMData[k] = UnsupportedCounterValue32bit
3422 }
3423 default:
3424 //do nothing
3425 }
3426 }
3427 }
3428 default:
3429 //do nothing
3430 }
3431 return receivedMask
3432}
3433
3434// nolint: gocyclo
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003435func (mm *OnuMetricsManager) getEthFrameExtPM64BitDataFromResponse(ctx context.Context, ethFrameExtPMData map[string]uint64, meAttributes me.AttributeValueMap, requestedAttributesMask uint16) uint16 {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303436 receivedMask := uint16(0)
3437 switch requestedAttributesMask {
3438 case 0x3800:
3439 for _, k := range maskToEthernetFrameExtendedPM64Bit[requestedAttributesMask] {
3440 if _, ok := ethFrameExtPMData[k]; !ok {
3441 switch k {
3442 case "drop_events":
3443 if val, ok := meAttributes[dropEvents]; ok && val != nil {
3444 ethFrameExtPMData[k] = val.(uint64)
3445 receivedMask |= 0x2000
3446 } else if !ok {
3447 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3448 }
3449 case "octets":
3450 if val, ok := meAttributes[octets]; ok && val != nil {
3451 ethFrameExtPMData[k] = val.(uint64)
3452 receivedMask |= 0x1000
3453 } else if !ok {
3454 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3455 }
3456 case "frames":
3457 if val, ok := meAttributes[frames]; ok && val != nil {
3458 ethFrameExtPMData[k] = val.(uint64)
3459 receivedMask |= 0x800
3460 } else if !ok {
3461 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3462 }
3463 }
3464 }
3465 }
3466 case 0x0700:
3467 for _, k := range maskToEthernetFrameExtendedPM64Bit[requestedAttributesMask] {
3468 if _, ok := ethFrameExtPMData[k]; !ok {
3469 switch k {
3470 case "broadcast_frames":
3471 if val, ok := meAttributes[broadcastFrames]; ok && val != nil {
3472 ethFrameExtPMData[k] = val.(uint64)
3473 receivedMask |= 0x400
3474 } else if !ok {
3475 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3476 }
3477 case "multicast_frames":
3478 if val, ok := meAttributes[multicastFrames]; ok && val != nil {
3479 ethFrameExtPMData[k] = val.(uint64)
3480 receivedMask |= 0x200
3481 } else if !ok {
3482 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3483 }
3484 case "crc_errored_frames":
3485 if val, ok := meAttributes[crcErroredFrames]; ok && val != nil {
3486 ethFrameExtPMData[k] = val.(uint64)
3487 receivedMask |= 0x100
3488 } else if !ok {
3489 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3490 }
3491 }
3492 }
3493 }
3494 case 0x00E0:
3495 for _, k := range maskToEthernetFrameExtendedPM64Bit[requestedAttributesMask] {
3496 if _, ok := ethFrameExtPMData[k]; !ok {
3497 switch k {
3498 case "undersize_frames":
3499 if val, ok := meAttributes[undersizeFrames]; ok && val != nil {
3500 ethFrameExtPMData[k] = val.(uint64)
3501 receivedMask |= 0x80
3502 } else if !ok {
3503 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3504 }
3505 case "oversize_frames":
3506 if val, ok := meAttributes[oversizeFrames]; ok && val != nil {
3507 ethFrameExtPMData[k] = val.(uint64)
3508 receivedMask |= 0x40
3509 } else if !ok {
3510 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3511 }
3512 case "64_octets":
3513 if val, ok := meAttributes[frames64Octets]; ok && val != nil {
3514 ethFrameExtPMData[k] = val.(uint64)
3515 receivedMask |= 0x20
3516 } else if !ok {
3517 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3518 }
3519 }
3520 }
3521 }
3522 case 0x001C:
3523 for _, k := range maskToEthernetFrameExtendedPM64Bit[requestedAttributesMask] {
3524 if _, ok := ethFrameExtPMData[k]; !ok {
3525 switch k {
3526 case "65_to_127_octets":
3527 if val, ok := meAttributes[frames65To127Octets]; ok && val != nil {
3528 ethFrameExtPMData[k] = val.(uint64)
3529 receivedMask |= 0x10
3530 } else if !ok {
3531 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3532 }
3533 case "128_to_255_octets":
3534 if val, ok := meAttributes[frames128To255Octets]; ok && val != nil {
3535 ethFrameExtPMData[k] = val.(uint64)
3536 receivedMask |= 0x8
3537 } else if !ok {
3538 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3539 }
3540 case "256_to_511_octets":
3541 if val, ok := meAttributes[frames256To511Octets]; ok && val != nil {
3542 ethFrameExtPMData[k] = val.(uint64)
3543 receivedMask |= 0x4
3544 } else if !ok {
3545 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3546 }
3547 default:
3548 //do nothing
3549 }
3550 }
3551 }
3552 case 0x0003:
3553 for _, k := range maskToEthernetFrameExtendedPM64Bit[requestedAttributesMask] {
3554 if _, ok := ethFrameExtPMData[k]; !ok {
3555 switch k {
3556 case "512_to_1023_octets":
3557 if val, ok := meAttributes[frames512To1023Octets]; ok && val != nil {
3558 ethFrameExtPMData[k] = val.(uint64)
3559 receivedMask |= 0x2
3560 } else if !ok {
3561 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3562 }
3563 case "1024_to_1518_octets":
3564 if val, ok := meAttributes[frames1024To1518Octets]; ok && val != nil {
3565 ethFrameExtPMData[k] = val.(uint64)
3566 receivedMask |= 0x1
3567 } else if !ok {
3568 ethFrameExtPMData[k] = UnsupportedCounterValue64bit
3569 }
3570 default:
3571 //do nothing
3572 }
3573 }
3574 }
3575 }
3576 return receivedMask
3577}
3578
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003579func (mm *OnuMetricsManager) aggregateEthernetFrameExtendedPM(pmDataIn map[string]uint64, pmData extension.OmciEthernetFrameExtendedPm, aggregate bool) extension.OmciEthernetFrameExtendedPm {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303580 mm.onuEthernetFrameExtendedPmLock.Lock()
3581 defer mm.onuEthernetFrameExtendedPmLock.Unlock()
3582 errorCounterValue := UnsupportedCounterValue64bit
3583 if mm.supportedEthernetFrameExtendedPMClass == me.EthernetFrameExtendedPmClassID {
3584 errorCounterValue = UnsupportedCounterValue32bit
3585 }
3586 var pmDataOut extension.OmciEthernetFrameExtendedPm
3587 if aggregate {
3588 if pmData.DropEvents != errorCounterValue {
3589 pmDataOut.DropEvents = pmData.DropEvents + pmDataIn["drop_events"]
3590 } else {
3591 pmDataOut.DropEvents = pmData.DropEvents
3592 }
3593 if pmData.Octets != errorCounterValue {
3594 pmDataOut.Octets = pmData.Octets + pmDataIn["octets"]
3595 } else {
3596 pmDataOut.Octets = pmData.Octets
3597 }
3598 if pmData.Frames != errorCounterValue {
3599 pmDataOut.Frames = pmData.Frames + pmDataIn["frames"]
3600 } else {
3601 pmDataOut.Frames = pmData.Frames
3602 }
3603 if pmData.BroadcastFrames != errorCounterValue {
3604 pmDataOut.BroadcastFrames = pmData.BroadcastFrames + pmDataIn["broadcast_frames"]
3605 } else {
3606 pmDataOut.BroadcastFrames = pmData.BroadcastFrames
3607 }
3608 if pmData.MulticastFrames != errorCounterValue {
3609 pmDataOut.MulticastFrames = pmData.MulticastFrames + pmDataIn["multicast_frames"]
3610 } else {
3611 pmDataOut.MulticastFrames = pmData.MulticastFrames
3612 }
3613 if pmData.CrcErroredFrames != errorCounterValue {
3614 pmDataOut.CrcErroredFrames = pmData.CrcErroredFrames + pmDataIn["crc_errored_frames"]
3615 } else {
3616 pmDataOut.CrcErroredFrames = pmData.CrcErroredFrames
3617 }
3618 if pmData.UndersizeFrames != errorCounterValue {
3619 pmDataOut.UndersizeFrames = pmData.UndersizeFrames + pmDataIn["undersize_frames"]
3620 } else {
3621 pmDataOut.UndersizeFrames = pmData.UndersizeFrames
3622 }
3623 if pmData.OversizeFrames != errorCounterValue {
3624 pmDataOut.OversizeFrames = pmData.OversizeFrames + pmDataIn["oversize_frames"]
3625 } else {
3626 pmDataOut.OversizeFrames = pmData.OversizeFrames
3627 }
3628 if pmData.Frames_64Octets != errorCounterValue {
3629 pmDataOut.Frames_64Octets = pmData.Frames_64Octets + pmDataIn["64_octets"]
3630 } else {
3631 pmDataOut.Frames_64Octets = pmData.Frames_64Octets
3632 }
3633 if pmData.Frames_65To_127Octets != errorCounterValue {
3634 pmDataOut.Frames_65To_127Octets = pmData.Frames_65To_127Octets + pmDataIn["65_to_127_octets"]
3635 } else {
3636 pmDataOut.Frames_65To_127Octets = pmData.Frames_65To_127Octets
3637 }
3638 if pmData.Frames_128To_255Octets != errorCounterValue {
3639 pmDataOut.Frames_128To_255Octets = pmData.Frames_128To_255Octets + pmDataIn["128_to_255_octets"]
3640 } else {
3641 pmDataOut.Frames_128To_255Octets = pmData.Frames_128To_255Octets
3642 }
3643 if pmData.Frames_256To_511Octets != errorCounterValue {
3644 pmDataOut.Frames_256To_511Octets = pmData.Frames_256To_511Octets + pmDataIn["256_to_511_octets"]
3645 } else {
3646 pmDataOut.Frames_256To_511Octets = pmData.Frames_256To_511Octets
3647 }
3648 if pmData.Frames_512To_1023Octets != errorCounterValue {
3649 pmDataOut.Frames_512To_1023Octets = pmData.Frames_512To_1023Octets + pmDataIn["512_to_1023_octets"]
3650 } else {
3651 pmDataOut.Frames_512To_1023Octets = pmData.Frames_512To_1023Octets
3652 }
3653 if pmData.Frames_1024To_1518Octets != errorCounterValue {
3654 pmDataOut.Frames_1024To_1518Octets = pmData.Frames_1024To_1518Octets + pmDataIn["1024_to_1518_octets"]
3655 } else {
3656 pmDataOut.Frames_1024To_1518Octets = pmData.Frames_1024To_1518Octets
3657 }
3658 } else {
3659 pmDataOut.DropEvents = pmDataIn["drop_events"]
3660 pmDataOut.Octets = pmDataIn["octets"]
3661 pmDataOut.Frames = pmDataIn["frames"]
3662 pmDataOut.BroadcastFrames = pmDataIn["broadcast_frames"]
3663 pmDataOut.MulticastFrames = pmDataIn["multicast_frames"]
3664 pmDataOut.CrcErroredFrames = pmDataIn["crc_errored_frames"]
3665 pmDataOut.UndersizeFrames = pmDataIn["undersize_frames"]
3666 pmDataOut.OversizeFrames = pmDataIn["oversize_frames"]
3667 pmDataOut.Frames_64Octets = pmDataIn["64_octets"]
3668 pmDataOut.Frames_65To_127Octets = pmDataIn["65_to_127_octets"]
3669 pmDataOut.Frames_128To_255Octets = pmDataIn["128_to_255_octets"]
3670 pmDataOut.Frames_256To_511Octets = pmDataIn["256_to_511_octets"]
3671 pmDataOut.Frames_512To_1023Octets = pmDataIn["512_to_1023_octets"]
3672 pmDataOut.Frames_1024To_1518Octets = pmDataIn["1024_to_1518_octets"]
3673 }
3674 return pmDataOut
3675}
3676
Holger Hildebrandt4b5e73f2021-08-19 06:51:21 +00003677func (mm *OnuMetricsManager) getControlBlockForExtendedPMDirection(ctx context.Context, upstream bool, entityID uint16) []uint16 {
Himani Chawla43f95ff2021-06-03 00:24:12 +05303678 controlBlock := make([]uint16, 8)
3679 // Control Block First two bytes are for threshold data 1/2 id - does not matter here
3680 controlBlock[0] = 0
3681 // Next two bytes are for the parent class ID
3682 controlBlock[1] = (uint16)(me.PhysicalPathTerminationPointEthernetUniClassID)
3683 // Next two bytes are for the parent me instance id
3684 controlBlock[2] = entityID
3685 // Next two bytes are for accumulation disable
3686 controlBlock[3] = 0
3687 // Next two bytes are for tca disable
3688 controlBlock[4] = 0x4000 //tca global disable
3689 // Next two bytes are for control fields - bit 1(lsb) as 1 for continuous accumulation and bit 2(0 for upstream)
3690 if upstream {
3691 controlBlock[5] = 1 << 0
3692 } else {
3693 controlBlock[5] = (1 << 0) | (1 << 1)
3694 }
3695 // Next two bytes are for tci - does not matter here
3696 controlBlock[6] = 0
3697 // Next two bytes are for reserved bits - does not matter here
3698 controlBlock[7] = 0
3699 return controlBlock
3700}