Matteo Scandolo | 1100699 | 2019-08-28 11:29:46 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 17 | package devices |
| 18 | |
| 19 | import ( |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 20 | "context" |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 21 | "errors" |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 22 | "fmt" |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 23 | "net" |
| 24 | |
| 25 | "time" |
| 26 | |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 27 | "github.com/cboling/omci" |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 28 | "github.com/google/gopacket/layers" |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 29 | "github.com/looplab/fsm" |
Matteo Scandolo | 075b189 | 2019-10-07 12:11:07 -0700 | [diff] [blame] | 30 | "github.com/opencord/bbsim/internal/bbsim/packetHandlers" |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 31 | "github.com/opencord/bbsim/internal/bbsim/responders/dhcp" |
| 32 | "github.com/opencord/bbsim/internal/bbsim/responders/eapol" |
Arjun E K | 57a7fcb | 2020-01-30 06:44:45 +0000 | [diff] [blame] | 33 | "github.com/opencord/bbsim/internal/bbsim/responders/igmp" |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 34 | "github.com/opencord/bbsim/internal/common" |
| 35 | omcilib "github.com/opencord/bbsim/internal/common/omci" |
| 36 | omcisim "github.com/opencord/omci-sim" |
Matteo Scandolo | 3de9de0 | 2019-11-14 13:40:03 -0800 | [diff] [blame] | 37 | "github.com/opencord/voltha-protos/v2/go/openolt" |
Anand S Katti | 0954135 | 2020-01-29 15:54:01 +0530 | [diff] [blame^] | 38 | tech_profile "github.com/opencord/voltha-protos/v2/go/tech_profile" |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 39 | log "github.com/sirupsen/logrus" |
| 40 | ) |
| 41 | |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 42 | var onuLogger = log.WithFields(log.Fields{ |
| 43 | "module": "ONU", |
| 44 | }) |
| 45 | |
Matteo Scandolo | 86e8ce6 | 2019-10-11 12:03:10 -0700 | [diff] [blame] | 46 | type Onu struct { |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 47 | ID uint32 |
| 48 | PonPortID uint32 |
| 49 | PonPort PonPort |
| 50 | STag int |
| 51 | CTag int |
| 52 | Auth bool // automatically start EAPOL if set to true |
| 53 | Dhcp bool // automatically start DHCP if set to true |
| 54 | HwAddress net.HardwareAddr |
| 55 | InternalState *fsm.FSM |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 56 | DiscoveryRetryDelay time.Duration // this is the time between subsequent Discovery Indication |
| 57 | DiscoveryDelay time.Duration // this is the time to send the first Discovery Indication |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 58 | |
| 59 | // ONU State |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 60 | // PortNo comes with flows and it's used when sending packetIndications, |
| 61 | // There is one PortNo per UNI Port, for now we're only storing the first one |
| 62 | // FIXME add support for multiple UNIs |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 63 | PortNo uint32 |
| 64 | DhcpFlowReceived bool |
| 65 | |
Matteo Scandolo | 86e8ce6 | 2019-10-11 12:03:10 -0700 | [diff] [blame] | 66 | OperState *fsm.FSM |
| 67 | SerialNumber *openolt.SerialNumber |
| 68 | |
| 69 | Channel chan Message // this Channel is to track state changes OMCI messages, EAPOL and DHCP packets |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 70 | |
| 71 | // OMCI params |
| 72 | tid uint16 |
| 73 | hpTid uint16 |
| 74 | seqNumber uint16 |
| 75 | HasGemPort bool |
| 76 | |
Anand S Katti | 0954135 | 2020-01-29 15:54:01 +0530 | [diff] [blame^] | 77 | DoneChannel chan bool // this channel is used to signal once the onu is complete (when the struct is used by BBR) |
| 78 | TrafficSchedulers *tech_profile.TrafficSchedulers |
Matteo Scandolo | 86e8ce6 | 2019-10-11 12:03:10 -0700 | [diff] [blame] | 79 | } |
| 80 | |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 81 | func (o *Onu) Sn() string { |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 82 | return common.OnuSnToString(o.SerialNumber) |
Matteo Scandolo | 86e8ce6 | 2019-10-11 12:03:10 -0700 | [diff] [blame] | 83 | } |
| 84 | |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 85 | func CreateONU(olt *OltDevice, pon PonPort, id uint32, sTag int, cTag int, auth bool, dhcp bool, delay time.Duration, isMock bool) *Onu { |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 86 | |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 87 | o := Onu{ |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 88 | ID: 0, |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 89 | PonPortID: pon.ID, |
| 90 | PonPort: pon, |
| 91 | STag: sTag, |
| 92 | CTag: cTag, |
| 93 | Auth: auth, |
| 94 | Dhcp: dhcp, |
| 95 | HwAddress: net.HardwareAddr{0x2e, 0x60, 0x70, 0x13, byte(pon.ID), byte(id)}, |
| 96 | PortNo: 0, |
| 97 | tid: 0x1, |
| 98 | hpTid: 0x8000, |
| 99 | seqNumber: 0, |
| 100 | DoneChannel: make(chan bool, 1), |
| 101 | DhcpFlowReceived: false, |
| 102 | DiscoveryRetryDelay: 60 * time.Second, // this is used to send OnuDiscoveryIndications until an activate call is received |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 103 | DiscoveryDelay: delay, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 104 | } |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 105 | o.SerialNumber = o.NewSN(olt.ID, pon.ID, id) |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 106 | |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 107 | // NOTE this state machine is used to track the operational |
| 108 | // state as requested by VOLTHA |
| 109 | o.OperState = getOperStateFSM(func(e *fsm.Event) { |
| 110 | onuLogger.WithFields(log.Fields{ |
| 111 | "ID": o.ID, |
| 112 | }).Debugf("Changing ONU OperState from %s to %s", e.Src, e.Dst) |
| 113 | }) |
| 114 | |
| 115 | // NOTE this state machine is used to activate the OMCI, EAPOL and DHCP clients |
| 116 | o.InternalState = fsm.NewFSM( |
| 117 | "created", |
| 118 | fsm.Events{ |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 119 | // DEVICE Lifecycle |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 120 | {Name: "initialize", Src: []string{"created", "disabled"}, Dst: "initialized"}, |
Pragya Arya | 6a708d6 | 2020-01-01 17:17:20 +0530 | [diff] [blame] | 121 | {Name: "discover", Src: []string{"initialized", "pon_disabled"}, Dst: "discovered"}, |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 122 | {Name: "enable", Src: []string{"discovered", "disabled"}, Dst: "enabled"}, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 123 | {Name: "receive_eapol_flow", Src: []string{"enabled", "gem_port_added"}, Dst: "eapol_flow_received"}, |
| 124 | {Name: "add_gem_port", Src: []string{"enabled", "eapol_flow_received"}, Dst: "gem_port_added"}, |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 125 | // NOTE should disabled state be different for oper_disabled (emulating an error) and admin_disabled (received a disabled call via VOLTHA)? |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 126 | {Name: "disable", Src: []string{"enabled", "eapol_flow_received", "gem_port_added", "eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed", "pon_disabled"}, Dst: "disabled"}, |
Pragya Arya | 6a708d6 | 2020-01-01 17:17:20 +0530 | [diff] [blame] | 127 | // ONU state when PON port is disabled but ONU is power ON(more states should be added in src?) |
Matteo Scandolo | df3f85d | 2020-01-15 12:50:48 -0800 | [diff] [blame] | 128 | {Name: "pon_disabled", Src: []string{"enabled", "gem_port_added", "eapol_flow_received", "eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed"}, Dst: "pon_disabled"}, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 129 | // EAPOL |
Shubham Sharma | bd4b657 | 2020-02-12 13:00:44 +0000 | [diff] [blame] | 130 | {Name: "start_auth", Src: []string{"eapol_flow_received", "gem_port_added", "eap_start_sent", "eap_response_identity_sent", "eap_response_challenge_sent", "eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed", "igmp_join_started", "igmp_left", "igmp_join_error"}, Dst: "auth_started"}, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 131 | {Name: "eap_start_sent", Src: []string{"auth_started"}, Dst: "eap_start_sent"}, |
| 132 | {Name: "eap_response_identity_sent", Src: []string{"eap_start_sent"}, Dst: "eap_response_identity_sent"}, |
| 133 | {Name: "eap_response_challenge_sent", Src: []string{"eap_response_identity_sent"}, Dst: "eap_response_challenge_sent"}, |
| 134 | {Name: "eap_response_success_received", Src: []string{"eap_response_challenge_sent"}, Dst: "eap_response_success_received"}, |
| 135 | {Name: "auth_failed", Src: []string{"auth_started", "eap_start_sent", "eap_response_identity_sent", "eap_response_challenge_sent"}, Dst: "auth_failed"}, |
| 136 | // DHCP |
Shubham Sharma | bd4b657 | 2020-02-12 13:00:44 +0000 | [diff] [blame] | 137 | {Name: "start_dhcp", Src: []string{"eap_response_success_received", "dhcp_discovery_sent", "dhcp_request_sent", "dhcp_ack_received", "dhcp_failed", "igmp_join_started", "igmp_left", "igmp_join_error"}, Dst: "dhcp_started"}, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 138 | {Name: "dhcp_discovery_sent", Src: []string{"dhcp_started"}, Dst: "dhcp_discovery_sent"}, |
| 139 | {Name: "dhcp_request_sent", Src: []string{"dhcp_discovery_sent"}, Dst: "dhcp_request_sent"}, |
| 140 | {Name: "dhcp_ack_received", Src: []string{"dhcp_request_sent"}, Dst: "dhcp_ack_received"}, |
| 141 | {Name: "dhcp_failed", Src: []string{"dhcp_started", "dhcp_discovery_sent", "dhcp_request_sent"}, Dst: "dhcp_failed"}, |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 142 | // BBR States |
| 143 | // TODO add start OMCI state |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 144 | {Name: "send_eapol_flow", Src: []string{"initialized"}, Dst: "eapol_flow_sent"}, |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 145 | {Name: "send_dhcp_flow", Src: []string{"eapol_flow_sent"}, Dst: "dhcp_flow_sent"}, |
Arjun E K | 57a7fcb | 2020-01-30 06:44:45 +0000 | [diff] [blame] | 146 | // IGMP |
Shubham Sharma | bd4b657 | 2020-02-12 13:00:44 +0000 | [diff] [blame] | 147 | {Name: "igmp_join_start", Src: []string{"eap_response_success_received", "gem_port_added", "eapol_flow_received", "dhcp_ack_received", "igmp_left", "igmp_join_error"}, Dst: "igmp_join_started"}, |
Arjun E K | dd443f0 | 2020-02-07 15:24:01 +0000 | [diff] [blame] | 148 | {Name: "igmp_join_startv3", Src: []string{"eap_response_success_received", "gem_port_added", "eapol_flow_received", "dhcp_ack_received", "igmp_left", "igmp_join_error"}, Dst: "igmp_join_started"}, |
Shubham Sharma | bd4b657 | 2020-02-12 13:00:44 +0000 | [diff] [blame] | 149 | {Name: "igmp_join_error", Src: []string{"igmp_join_started"}, Dst: "igmp_join_error"}, |
| 150 | {Name: "igmp_leave", Src: []string{"igmp_join_started", "gem_port_added", "eapol_flow_received", "eap_response_success_received", "dhcp_ack_received"}, Dst: "igmp_left"}, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 151 | }, |
| 152 | fsm.Callbacks{ |
| 153 | "enter_state": func(e *fsm.Event) { |
| 154 | o.logStateChange(e.Src, e.Dst) |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 155 | }, |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 156 | "enter_initialized": func(e *fsm.Event) { |
| 157 | // create new channel for ProcessOnuMessages Go routine |
| 158 | o.Channel = make(chan Message, 2048) |
Pragya Arya | 1cbefa4 | 2020-01-13 12:15:29 +0530 | [diff] [blame] | 159 | if !isMock { |
| 160 | // start ProcessOnuMessages Go routine |
| 161 | go o.ProcessOnuMessages(olt.enableContext, *olt.OpenoltStream, nil) |
| 162 | } |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 163 | }, |
| 164 | "enter_discovered": func(e *fsm.Event) { |
| 165 | msg := Message{ |
| 166 | Type: OnuDiscIndication, |
| 167 | Data: OnuDiscIndicationMessage{ |
| 168 | Onu: &o, |
| 169 | OperState: UP, |
| 170 | }, |
| 171 | } |
| 172 | o.Channel <- msg |
| 173 | }, |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 174 | "enter_enabled": func(event *fsm.Event) { |
| 175 | msg := Message{ |
| 176 | Type: OnuIndication, |
| 177 | Data: OnuIndicationMessage{ |
| 178 | OnuSN: o.SerialNumber, |
| 179 | PonPortID: o.PonPortID, |
| 180 | OperState: UP, |
| 181 | }, |
| 182 | } |
| 183 | o.Channel <- msg |
| 184 | }, |
| 185 | "enter_disabled": func(event *fsm.Event) { |
| 186 | msg := Message{ |
| 187 | Type: OnuIndication, |
| 188 | Data: OnuIndicationMessage{ |
| 189 | OnuSN: o.SerialNumber, |
| 190 | PonPortID: o.PonPortID, |
| 191 | OperState: DOWN, |
| 192 | }, |
| 193 | } |
| 194 | o.Channel <- msg |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 195 | // terminate the ONU's ProcessOnuMessages Go routine |
| 196 | close(o.Channel) |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 197 | }, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 198 | "enter_auth_started": func(e *fsm.Event) { |
| 199 | o.logStateChange(e.Src, e.Dst) |
| 200 | msg := Message{ |
| 201 | Type: StartEAPOL, |
| 202 | Data: PacketMessage{ |
| 203 | PonPortID: o.PonPortID, |
| 204 | OnuID: o.ID, |
| 205 | }, |
| 206 | } |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 207 | o.Channel <- msg |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 208 | }, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 209 | "enter_auth_failed": func(e *fsm.Event) { |
| 210 | onuLogger.WithFields(log.Fields{ |
| 211 | "OnuId": o.ID, |
| 212 | "IntfId": o.PonPortID, |
| 213 | "OnuSn": o.Sn(), |
| 214 | }).Errorf("ONU failed to authenticate!") |
| 215 | }, |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 216 | "before_start_dhcp": func(e *fsm.Event) { |
| 217 | if o.DhcpFlowReceived == false { |
| 218 | e.Cancel(errors.New("cannot-go-to-dhcp-started-as-dhcp-flow-is-missing")) |
| 219 | } |
| 220 | }, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 221 | "enter_dhcp_started": func(e *fsm.Event) { |
| 222 | msg := Message{ |
| 223 | Type: StartDHCP, |
| 224 | Data: PacketMessage{ |
| 225 | PonPortID: o.PonPortID, |
| 226 | OnuID: o.ID, |
| 227 | }, |
| 228 | } |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 229 | o.Channel <- msg |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 230 | }, |
| 231 | "enter_dhcp_failed": func(e *fsm.Event) { |
| 232 | onuLogger.WithFields(log.Fields{ |
| 233 | "OnuId": o.ID, |
| 234 | "IntfId": o.PonPortID, |
| 235 | "OnuSn": o.Sn(), |
| 236 | }).Errorf("ONU failed to DHCP!") |
| 237 | }, |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 238 | "enter_eapol_flow_sent": func(e *fsm.Event) { |
| 239 | msg := Message{ |
| 240 | Type: SendEapolFlow, |
| 241 | } |
| 242 | o.Channel <- msg |
| 243 | }, |
| 244 | "enter_dhcp_flow_sent": func(e *fsm.Event) { |
| 245 | msg := Message{ |
| 246 | Type: SendDhcpFlow, |
| 247 | } |
| 248 | o.Channel <- msg |
| 249 | }, |
Arjun E K | 57a7fcb | 2020-01-30 06:44:45 +0000 | [diff] [blame] | 250 | "igmp_join_start": func(e *fsm.Event) { |
| 251 | msg := Message{ |
| 252 | Type: IGMPMembershipReportV2, |
| 253 | } |
| 254 | o.Channel <- msg |
| 255 | }, |
| 256 | "igmp_leave": func(e *fsm.Event) { |
| 257 | msg := Message{ |
| 258 | Type: IGMPLeaveGroup} |
| 259 | o.Channel <- msg |
| 260 | }, |
Anand S Katti | 0954135 | 2020-01-29 15:54:01 +0530 | [diff] [blame^] | 261 | "igmp_join_startv3": func(e *fsm.Event) { |
| 262 | msg := Message{ |
| 263 | Type: IGMPMembershipReportV3, |
| 264 | } |
| 265 | o.Channel <- msg |
| 266 | }, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 267 | }, |
| 268 | ) |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 269 | |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 270 | return &o |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 271 | } |
| 272 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 273 | func (o *Onu) logStateChange(src string, dst string) { |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 274 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 275 | "OnuId": o.ID, |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 276 | "IntfId": o.PonPortID, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 277 | "OnuSn": o.Sn(), |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 278 | }).Debugf("Changing ONU InternalState from %s to %s", src, dst) |
| 279 | } |
| 280 | |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 281 | // ProcessOnuMessages starts indication channel for each ONU |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 282 | func (o *Onu) ProcessOnuMessages(ctx context.Context, stream openolt.Openolt_EnableIndicationServer, client openolt.OpenoltClient) { |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 283 | onuLogger.WithFields(log.Fields{ |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 284 | "onuID": o.ID, |
| 285 | "onuSN": o.Sn(), |
| 286 | "ponPort": o.PonPortID, |
| 287 | }).Debug("Starting ONU Indication Channel") |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 288 | |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 289 | loop: |
| 290 | for { |
| 291 | select { |
| 292 | case <-ctx.Done(): |
| 293 | onuLogger.WithFields(log.Fields{ |
| 294 | "onuID": o.ID, |
| 295 | "onuSN": o.Sn(), |
| 296 | }).Tracef("ONU message handling canceled via context") |
| 297 | break loop |
| 298 | case message, ok := <-o.Channel: |
| 299 | if !ok || ctx.Err() != nil { |
| 300 | onuLogger.WithFields(log.Fields{ |
| 301 | "onuID": o.ID, |
| 302 | "onuSN": o.Sn(), |
| 303 | }).Tracef("ONU message handling canceled via channel close") |
| 304 | break loop |
Matteo Scandolo | 075b189 | 2019-10-07 12:11:07 -0700 | [diff] [blame] | 305 | } |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 306 | onuLogger.WithFields(log.Fields{ |
| 307 | "onuID": o.ID, |
| 308 | "onuSN": o.Sn(), |
| 309 | "messageType": message.Type, |
| 310 | }).Tracef("Received message on ONU Channel") |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 311 | |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 312 | switch message.Type { |
| 313 | case OnuDiscIndication: |
| 314 | msg, _ := message.Data.(OnuDiscIndicationMessage) |
| 315 | // NOTE we need to slow down and send ONU Discovery Indication in batches to better emulate a real scenario |
Pragya Arya | 2225f20 | 2020-01-29 18:05:01 +0530 | [diff] [blame] | 316 | time.Sleep(o.DiscoveryDelay) |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 317 | o.sendOnuDiscIndication(msg, stream) |
| 318 | case OnuIndication: |
| 319 | msg, _ := message.Data.(OnuIndicationMessage) |
| 320 | o.sendOnuIndication(msg, stream) |
| 321 | case OMCI: |
| 322 | msg, _ := message.Data.(OmciMessage) |
| 323 | o.handleOmciMessage(msg, stream) |
| 324 | case FlowUpdate: |
| 325 | msg, _ := message.Data.(OnuFlowUpdateMessage) |
| 326 | o.handleFlowUpdate(msg) |
| 327 | case StartEAPOL: |
| 328 | log.Infof("Receive StartEAPOL message on ONU Channel") |
| 329 | eapol.SendEapStart(o.ID, o.PonPortID, o.Sn(), o.PortNo, o.HwAddress, o.InternalState, stream) |
| 330 | case StartDHCP: |
| 331 | log.Infof("Receive StartDHCP message on ONU Channel") |
| 332 | // FIXME use id, ponId as SendEapStart |
| 333 | dhcp.SendDHCPDiscovery(o.PonPortID, o.ID, o.Sn(), o.PortNo, o.InternalState, o.HwAddress, o.CTag, stream) |
| 334 | case OnuPacketOut: |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 335 | |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 336 | msg, _ := message.Data.(OnuPacketMessage) |
| 337 | |
| 338 | log.WithFields(log.Fields{ |
| 339 | "IntfId": msg.IntfId, |
| 340 | "OnuId": msg.OnuId, |
| 341 | "pktType": msg.Type, |
| 342 | }).Trace("Received OnuPacketOut Message") |
| 343 | |
| 344 | if msg.Type == packetHandlers.EAPOL { |
| 345 | eapol.HandleNextPacket(msg.OnuId, msg.IntfId, o.Sn(), o.PortNo, o.InternalState, msg.Packet, stream, client) |
| 346 | } else if msg.Type == packetHandlers.DHCP { |
| 347 | // NOTE here we receive packets going from the DHCP Server to the ONU |
| 348 | // for now we expect them to be double-tagged, but ideally the should be single tagged |
| 349 | dhcp.HandleNextPacket(o.ID, o.PonPortID, o.Sn(), o.PortNo, o.HwAddress, o.CTag, o.InternalState, msg.Packet, stream) |
| 350 | } |
| 351 | case OnuPacketIn: |
| 352 | // NOTE we only receive BBR packets here. |
| 353 | // Eapol.HandleNextPacket can handle both BBSim and BBr cases so the call is the same |
| 354 | // in the DHCP case VOLTHA only act as a proxy, the behaviour is completely different thus we have a dhcp.HandleNextBbrPacket |
| 355 | msg, _ := message.Data.(OnuPacketMessage) |
| 356 | |
| 357 | log.WithFields(log.Fields{ |
| 358 | "IntfId": msg.IntfId, |
| 359 | "OnuId": msg.OnuId, |
| 360 | "pktType": msg.Type, |
| 361 | }).Trace("Received OnuPacketIn Message") |
| 362 | |
| 363 | if msg.Type == packetHandlers.EAPOL { |
| 364 | eapol.HandleNextPacket(msg.OnuId, msg.IntfId, o.Sn(), o.PortNo, o.InternalState, msg.Packet, stream, client) |
| 365 | } else if msg.Type == packetHandlers.DHCP { |
| 366 | dhcp.HandleNextBbrPacket(o.ID, o.PonPortID, o.Sn(), o.STag, o.HwAddress, o.DoneChannel, msg.Packet, client) |
| 367 | } |
| 368 | case DyingGaspIndication: |
| 369 | msg, _ := message.Data.(DyingGaspIndicationMessage) |
| 370 | o.sendDyingGaspInd(msg, stream) |
| 371 | case OmciIndication: |
| 372 | msg, _ := message.Data.(OmciIndicationMessage) |
| 373 | o.handleOmci(msg, client) |
| 374 | case SendEapolFlow: |
| 375 | o.sendEapolFlow(client) |
| 376 | case SendDhcpFlow: |
| 377 | o.sendDhcpFlow(client) |
Arjun E K | 57a7fcb | 2020-01-30 06:44:45 +0000 | [diff] [blame] | 378 | case IGMPMembershipReportV2: |
| 379 | log.Infof("Recieved IGMPMembershipReportV2 message on ONU channel") |
| 380 | igmp.SendIGMPMembershipReportV2(o.PonPortID, o.ID, o.Sn(), o.PortNo, o.HwAddress, stream) |
| 381 | case IGMPLeaveGroup: |
| 382 | log.Infof("Recieved IGMPLeaveGroupV2 message on ONU channel") |
| 383 | igmp.SendIGMPLeaveGroupV2(o.PonPortID, o.ID, o.Sn(), o.PortNo, o.HwAddress, stream) |
Anand S Katti | 0954135 | 2020-01-29 15:54:01 +0530 | [diff] [blame^] | 384 | case IGMPMembershipReportV3: |
| 385 | log.Infof("Recieved IGMPMembershipReportV3 message on ONU channel") |
| 386 | igmp.SendIGMPMembershipReportV3(o.PonPortID, o.ID, o.Sn(), o.PortNo, o.HwAddress, stream) |
David Bainbridge | 103cf02 | 2019-12-16 20:11:35 +0000 | [diff] [blame] | 387 | default: |
| 388 | onuLogger.Warnf("Received unknown message data %v for type %v in OLT Channel", message.Data, message.Type) |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 389 | } |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 390 | } |
| 391 | } |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 392 | onuLogger.WithFields(log.Fields{ |
| 393 | "onuID": o.ID, |
| 394 | "onuSN": o.Sn(), |
| 395 | }).Debug("Stopped handling ONU Indication Channel") |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 396 | } |
| 397 | |
Matteo Scandolo | df3f85d | 2020-01-15 12:50:48 -0800 | [diff] [blame] | 398 | func (o *Onu) processOmciMessage(message omcisim.OmciChMessage, stream openolt.Openolt_EnableIndicationServer) { |
William Kurkian | 9dadc5b | 2019-10-22 13:51:57 -0400 | [diff] [blame] | 399 | switch message.Type { |
Matteo Scandolo | df3f85d | 2020-01-15 12:50:48 -0800 | [diff] [blame] | 400 | case omcisim.UniLinkUp, omcisim.UniLinkDown: |
| 401 | onuLogger.WithFields(log.Fields{ |
| 402 | "OnuId": message.Data.OnuId, |
| 403 | "IntfId": message.Data.IntfId, |
| 404 | "Type": message.Type, |
| 405 | }).Infof("UNI Link Alarm") |
| 406 | // TODO send to OLT |
| 407 | |
| 408 | omciInd := openolt.OmciIndication{ |
| 409 | IntfId: message.Data.IntfId, |
| 410 | OnuId: message.Data.OnuId, |
| 411 | Pkt: message.Packet, |
| 412 | } |
| 413 | |
| 414 | omci := &openolt.Indication_OmciInd{OmciInd: &omciInd} |
| 415 | if err := stream.Send(&openolt.Indication{Data: omci}); err != nil { |
| 416 | onuLogger.WithFields(log.Fields{ |
| 417 | "IntfId": o.PonPortID, |
| 418 | "SerialNumber": o.Sn(), |
| 419 | "Type": message.Type, |
| 420 | "omciPacket": omciInd.Pkt, |
| 421 | }).Errorf("Failed to send UNI Link Alarm: %v", err) |
| 422 | return |
| 423 | } |
| 424 | |
| 425 | onuLogger.WithFields(log.Fields{ |
| 426 | "IntfId": o.PonPortID, |
| 427 | "SerialNumber": o.Sn(), |
| 428 | "Type": message.Type, |
| 429 | "omciPacket": omciInd.Pkt, |
| 430 | }).Info("UNI Link alarm sent") |
| 431 | |
William Kurkian | 9dadc5b | 2019-10-22 13:51:57 -0400 | [diff] [blame] | 432 | case omcisim.GemPortAdded: |
| 433 | log.WithFields(log.Fields{ |
| 434 | "OnuId": message.Data.OnuId, |
| 435 | "IntfId": message.Data.IntfId, |
| 436 | }).Infof("GemPort Added") |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 437 | |
William Kurkian | 9dadc5b | 2019-10-22 13:51:57 -0400 | [diff] [blame] | 438 | // NOTE if we receive the GemPort but we don't have EAPOL flows |
| 439 | // go an intermediate state, otherwise start auth |
| 440 | if o.InternalState.Is("enabled") { |
| 441 | if err := o.InternalState.Event("add_gem_port"); err != nil { |
| 442 | log.Errorf("Can't go to gem_port_added: %v", err) |
| 443 | } |
| 444 | } else if o.InternalState.Is("eapol_flow_received") { |
Matteo Scandolo | 3c656a1 | 2019-12-10 09:54:51 -0800 | [diff] [blame] | 445 | if o.Auth == true { |
| 446 | if err := o.InternalState.Event("start_auth"); err != nil { |
| 447 | log.Warnf("Can't go to auth_started: %v", err) |
| 448 | } |
| 449 | } else { |
| 450 | onuLogger.WithFields(log.Fields{ |
| 451 | "IntfId": o.PonPortID, |
| 452 | "OnuId": o.ID, |
| 453 | "SerialNumber": o.Sn(), |
| 454 | }).Warn("Not starting authentication as Auth bit is not set in CLI parameters") |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 455 | } |
| 456 | } |
| 457 | } |
| 458 | } |
| 459 | |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 460 | func (o Onu) NewSN(oltid int, intfid uint32, onuid uint32) *openolt.SerialNumber { |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 461 | |
| 462 | sn := new(openolt.SerialNumber) |
| 463 | |
Matteo Scandolo | 47e69bb | 2019-08-28 15:41:12 -0700 | [diff] [blame] | 464 | //sn = new(openolt.SerialNumber) |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 465 | sn.VendorId = []byte("BBSM") |
| 466 | sn.VendorSpecific = []byte{0, byte(oltid % 256), byte(intfid), byte(onuid)} |
| 467 | |
| 468 | return sn |
| 469 | } |
| 470 | |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 471 | // NOTE handle_/process methods can change the ONU internal state as they are receiving messages |
| 472 | // send method should not change the ONU state |
| 473 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 474 | func (o *Onu) sendDyingGaspInd(msg DyingGaspIndicationMessage, stream openolt.Openolt_EnableIndicationServer) error { |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 475 | alarmData := &openolt.AlarmIndication_DyingGaspInd{ |
| 476 | DyingGaspInd: &openolt.DyingGaspIndication{ |
| 477 | IntfId: msg.PonPortID, |
| 478 | OnuId: msg.OnuID, |
| 479 | Status: "on", |
| 480 | }, |
| 481 | } |
| 482 | data := &openolt.Indication_AlarmInd{AlarmInd: &openolt.AlarmIndication{Data: alarmData}} |
| 483 | |
| 484 | if err := stream.Send(&openolt.Indication{Data: data}); err != nil { |
| 485 | onuLogger.Errorf("Failed to send DyingGaspInd : %v", err) |
| 486 | return err |
| 487 | } |
| 488 | onuLogger.WithFields(log.Fields{ |
| 489 | "IntfId": msg.PonPortID, |
| 490 | "OnuSn": o.Sn(), |
| 491 | "OnuId": msg.OnuID, |
| 492 | }).Info("sendDyingGaspInd") |
| 493 | return nil |
| 494 | } |
| 495 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 496 | func (o *Onu) sendOnuDiscIndication(msg OnuDiscIndicationMessage, stream openolt.Openolt_EnableIndicationServer) { |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 497 | discoverData := &openolt.Indication_OnuDiscInd{OnuDiscInd: &openolt.OnuDiscIndication{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 498 | IntfId: msg.Onu.PonPortID, |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 499 | SerialNumber: msg.Onu.SerialNumber, |
| 500 | }} |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 501 | |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 502 | if err := stream.Send(&openolt.Indication{Data: discoverData}); err != nil { |
Matteo Scandolo | 1100699 | 2019-08-28 11:29:46 -0700 | [diff] [blame] | 503 | log.Errorf("Failed to send Indication_OnuDiscInd: %v", err) |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 504 | return |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 505 | } |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 506 | |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 507 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 508 | "IntfId": msg.Onu.PonPortID, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 509 | "OnuSn": msg.Onu.Sn(), |
| 510 | "OnuId": o.ID, |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 511 | }).Debug("Sent Indication_OnuDiscInd") |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 512 | |
| 513 | // after DiscoveryRetryDelay check if the state is the same and in case send a new OnuDiscIndication |
| 514 | go func(delay time.Duration) { |
Matteo Scandolo | 569e717 | 2019-12-20 11:51:51 -0800 | [diff] [blame] | 515 | time.Sleep(delay) |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 516 | if o.InternalState.Current() == "discovered" { |
Matteo Scandolo | e811ae9 | 2019-12-10 17:50:14 -0800 | [diff] [blame] | 517 | o.sendOnuDiscIndication(msg, stream) |
| 518 | } |
| 519 | }(o.DiscoveryRetryDelay) |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 520 | } |
| 521 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 522 | func (o *Onu) sendOnuIndication(msg OnuIndicationMessage, stream openolt.Openolt_EnableIndicationServer) { |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 523 | // NOTE voltha returns an ID, but if we use that ID then it complains: |
| 524 | // expected_onu_id: 1, received_onu_id: 1024, event: ONU-id-mismatch, can happen if both voltha and the olt rebooted |
| 525 | // so we're using the internal ID that is 1 |
| 526 | // o.ID = msg.OnuID |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 527 | |
| 528 | indData := &openolt.Indication_OnuInd{OnuInd: &openolt.OnuIndication{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 529 | IntfId: o.PonPortID, |
| 530 | OnuId: o.ID, |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 531 | OperState: msg.OperState.String(), |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 532 | AdminState: o.OperState.Current(), |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 533 | SerialNumber: o.SerialNumber, |
| 534 | }} |
| 535 | if err := stream.Send(&openolt.Indication{Data: indData}); err != nil { |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 536 | // TODO do we need to transition to a broken state? |
Matteo Scandolo | 1100699 | 2019-08-28 11:29:46 -0700 | [diff] [blame] | 537 | log.Errorf("Failed to send Indication_OnuInd: %v", err) |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 538 | } |
Matteo Scandolo | 9a3518c | 2019-08-13 14:36:01 -0700 | [diff] [blame] | 539 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 540 | "IntfId": o.PonPortID, |
| 541 | "OnuId": o.ID, |
| 542 | "OperState": msg.OperState.String(), |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 543 | "AdminState": msg.OperState.String(), |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 544 | "OnuSn": o.Sn(), |
Matteo Scandolo | 4747d29 | 2019-08-05 11:50:18 -0700 | [diff] [blame] | 545 | }).Debug("Sent Indication_OnuInd") |
Matteo Scandolo | 10f965c | 2019-09-24 10:40:46 -0700 | [diff] [blame] | 546 | |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 547 | } |
| 548 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 549 | func (o *Onu) handleOmciMessage(msg OmciMessage, stream openolt.Openolt_EnableIndicationServer) { |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 550 | |
| 551 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 552 | "IntfId": o.PonPortID, |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 553 | "SerialNumber": o.Sn(), |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 554 | "omciPacket": msg.omciMsg.Pkt, |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 555 | }).Tracef("Received OMCI message") |
| 556 | |
| 557 | var omciInd openolt.OmciIndication |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 558 | respPkt, err := omcisim.OmciSim(o.PonPortID, o.ID, HexDecode(msg.omciMsg.Pkt)) |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 559 | if err != nil { |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 560 | onuLogger.WithFields(log.Fields{ |
| 561 | "IntfId": o.PonPortID, |
| 562 | "SerialNumber": o.Sn(), |
| 563 | "omciPacket": omciInd.Pkt, |
| 564 | "msg": msg, |
| 565 | }).Errorf("Error handling OMCI message %v", msg) |
| 566 | return |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 567 | } |
| 568 | |
| 569 | omciInd.IntfId = o.PonPortID |
| 570 | omciInd.OnuId = o.ID |
| 571 | omciInd.Pkt = respPkt |
| 572 | |
| 573 | omci := &openolt.Indication_OmciInd{OmciInd: &omciInd} |
| 574 | if err := stream.Send(&openolt.Indication{Data: omci}); err != nil { |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 575 | onuLogger.WithFields(log.Fields{ |
| 576 | "IntfId": o.PonPortID, |
| 577 | "SerialNumber": o.Sn(), |
| 578 | "omciPacket": omciInd.Pkt, |
| 579 | "msg": msg, |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 580 | }).Errorf("send omcisim indication failed: %v", err) |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 581 | return |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 582 | } |
| 583 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 584 | "IntfId": o.PonPortID, |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 585 | "SerialNumber": o.Sn(), |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 586 | "omciPacket": omciInd.Pkt, |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 587 | }).Tracef("Sent OMCI message") |
| 588 | } |
| 589 | |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 590 | func (o *Onu) storePortNumber(portNo uint32) { |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 591 | // NOTE this needed only as long as we don't support multiple UNIs |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 592 | // we need to add support for multiple UNIs |
| 593 | // the action plan is: |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 594 | // - refactor the omcisim-sim library to use https://github.com/cboling/omci instead of canned messages |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 595 | // - change the library so that it reports a single UNI and remove this workaroung |
| 596 | // - add support for multiple UNIs in BBSim |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 597 | if o.PortNo == 0 || portNo < o.PortNo { |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 598 | onuLogger.WithFields(log.Fields{ |
| 599 | "IntfId": o.PonPortID, |
| 600 | "OnuId": o.ID, |
| 601 | "SerialNumber": o.Sn(), |
| 602 | "OnuPortNo": o.PortNo, |
| 603 | "FlowPortNo": portNo, |
| 604 | }).Debug("Storing ONU portNo") |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 605 | o.PortNo = portNo |
| 606 | } |
| 607 | } |
| 608 | |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 609 | func (o *Onu) SetID(id uint32) { |
Matteo Scandolo | 583f17d | 2020-02-13 10:35:17 -0800 | [diff] [blame] | 610 | onuLogger.WithFields(log.Fields{ |
| 611 | "IntfId": o.PonPortID, |
| 612 | "OnuId": id, |
| 613 | "SerialNumber": o.Sn(), |
| 614 | }).Debug("Storing OnuId ") |
William Kurkian | 0418bc8 | 2019-11-06 12:16:24 -0500 | [diff] [blame] | 615 | o.ID = id |
| 616 | } |
| 617 | |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 618 | func (o *Onu) handleFlowUpdate(msg OnuFlowUpdateMessage) { |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 619 | onuLogger.WithFields(log.Fields{ |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 620 | "DstPort": msg.Flow.Classifier.DstPort, |
| 621 | "EthType": fmt.Sprintf("%x", msg.Flow.Classifier.EthType), |
| 622 | "FlowId": msg.Flow.FlowId, |
| 623 | "FlowType": msg.Flow.FlowType, |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 624 | "InnerVlan": msg.Flow.Classifier.IVid, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 625 | "IntfId": msg.Flow.AccessIntfId, |
| 626 | "IpProto": msg.Flow.Classifier.IpProto, |
| 627 | "OnuId": msg.Flow.OnuId, |
| 628 | "OnuSn": o.Sn(), |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 629 | "OuterVlan": msg.Flow.Classifier.OVid, |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 630 | "PortNo": msg.Flow.PortNo, |
| 631 | "SrcPort": msg.Flow.Classifier.SrcPort, |
| 632 | "UniID": msg.Flow.UniId, |
| 633 | }).Debug("ONU receives Flow") |
| 634 | |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 635 | if msg.Flow.UniId != 0 { |
| 636 | // as of now BBSim only support a single UNI, so ignore everything that is not targeted to it |
| 637 | onuLogger.WithFields(log.Fields{ |
| 638 | "IntfId": o.PonPortID, |
| 639 | "OnuId": o.ID, |
| 640 | "SerialNumber": o.Sn(), |
| 641 | }).Debug("Ignoring flow as it's not for the first UNI") |
| 642 | return |
| 643 | } |
| 644 | |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 645 | if msg.Flow.Classifier.EthType == uint32(layers.EthernetTypeEAPOL) && msg.Flow.Classifier.OVid == 4091 { |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 646 | // NOTE storing the PortNO, it's needed when sending PacketIndications |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 647 | o.storePortNumber(uint32(msg.Flow.PortNo)) |
Matteo Scandolo | 2742870 | 2019-10-11 16:21:16 -0700 | [diff] [blame] | 648 | |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 649 | // NOTE if we receive the EAPOL flows but we don't have GemPorts |
| 650 | // go an intermediate state, otherwise start auth |
| 651 | if o.InternalState.Is("enabled") { |
| 652 | if err := o.InternalState.Event("receive_eapol_flow"); err != nil { |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 653 | log.Warnf("Can't go to eapol_flow_received: %v", err) |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 654 | } |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 655 | } else if o.InternalState.Is("gem_port_added") { |
Matteo Scandolo | c114709 | 2019-10-29 09:38:33 -0700 | [diff] [blame] | 656 | |
| 657 | if o.Auth == true { |
| 658 | if err := o.InternalState.Event("start_auth"); err != nil { |
| 659 | log.Warnf("Can't go to auth_started: %v", err) |
| 660 | } |
| 661 | } else { |
| 662 | onuLogger.WithFields(log.Fields{ |
| 663 | "IntfId": o.PonPortID, |
| 664 | "OnuId": o.ID, |
| 665 | "SerialNumber": o.Sn(), |
| 666 | }).Warn("Not starting authentication as Auth bit is not set in CLI parameters") |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 667 | } |
Matteo Scandolo | c114709 | 2019-10-29 09:38:33 -0700 | [diff] [blame] | 668 | |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 669 | } |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 670 | } else if msg.Flow.Classifier.EthType == uint32(layers.EthernetTypeIPv4) && |
| 671 | msg.Flow.Classifier.SrcPort == uint32(68) && |
| 672 | msg.Flow.Classifier.DstPort == uint32(67) { |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 673 | |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 674 | // keep track that we received the DHCP Flows so that we can transition the state to dhcp_started |
Matteo Scandolo | 99f1846 | 2019-10-28 14:14:28 -0700 | [diff] [blame] | 675 | o.DhcpFlowReceived = true |
Matteo Scandolo | c114709 | 2019-10-29 09:38:33 -0700 | [diff] [blame] | 676 | |
| 677 | if o.Dhcp == true { |
Zdravko Bozakov | 681364d | 2019-11-10 14:28:46 +0100 | [diff] [blame] | 678 | // NOTE we are receiving multiple DHCP flows but we shouldn't call the transition multiple times |
Matteo Scandolo | c114709 | 2019-10-29 09:38:33 -0700 | [diff] [blame] | 679 | if err := o.InternalState.Event("start_dhcp"); err != nil { |
| 680 | log.Errorf("Can't go to dhcp_started: %v", err) |
| 681 | } |
| 682 | } else { |
| 683 | onuLogger.WithFields(log.Fields{ |
| 684 | "IntfId": o.PonPortID, |
| 685 | "OnuId": o.ID, |
| 686 | "SerialNumber": o.Sn(), |
| 687 | }).Warn("Not starting DHCP as Dhcp bit is not set in CLI parameters") |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 688 | } |
Matteo Scandolo | 3bc7374 | 2019-08-20 14:04:04 -0700 | [diff] [blame] | 689 | } |
| 690 | } |
| 691 | |
Matteo Scandolo | c559ef1 | 2019-08-20 13:24:21 -0700 | [diff] [blame] | 692 | // HexDecode converts the hex encoding to binary |
| 693 | func HexDecode(pkt []byte) []byte { |
| 694 | p := make([]byte, len(pkt)/2) |
| 695 | for i, j := 0, 0; i < len(pkt); i, j = i+2, j+1 { |
| 696 | // Go figure this ;) |
| 697 | u := (pkt[i] & 15) + (pkt[i]>>6)*9 |
| 698 | l := (pkt[i+1] & 15) + (pkt[i+1]>>6)*9 |
| 699 | p[j] = u<<4 + l |
| 700 | } |
| 701 | onuLogger.Tracef("Omci decoded: %x.", p) |
| 702 | return p |
Matteo Scandolo | 4b3fc7e | 2019-09-17 16:49:54 -0700 | [diff] [blame] | 703 | } |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 704 | |
| 705 | // BBR methods |
| 706 | |
| 707 | func sendOmciMsg(pktBytes []byte, intfId uint32, onuId uint32, sn *openolt.SerialNumber, msgType string, client openolt.OpenoltClient) { |
| 708 | omciMsg := openolt.OmciMsg{ |
| 709 | IntfId: intfId, |
| 710 | OnuId: onuId, |
| 711 | Pkt: pktBytes, |
| 712 | } |
| 713 | |
| 714 | if _, err := client.OmciMsgOut(context.Background(), &omciMsg); err != nil { |
| 715 | log.WithFields(log.Fields{ |
| 716 | "IntfId": intfId, |
| 717 | "OnuId": onuId, |
| 718 | "SerialNumber": common.OnuSnToString(sn), |
| 719 | "Pkt": omciMsg.Pkt, |
| 720 | }).Fatalf("Failed to send MIB Reset") |
| 721 | } |
| 722 | log.WithFields(log.Fields{ |
| 723 | "IntfId": intfId, |
| 724 | "OnuId": onuId, |
| 725 | "SerialNumber": common.OnuSnToString(sn), |
| 726 | "Pkt": omciMsg.Pkt, |
| 727 | }).Tracef("Sent OMCI message %s", msgType) |
| 728 | } |
| 729 | |
| 730 | func (onu *Onu) getNextTid(highPriority ...bool) uint16 { |
| 731 | var next uint16 |
| 732 | if len(highPriority) > 0 && highPriority[0] { |
| 733 | next = onu.hpTid |
| 734 | onu.hpTid += 1 |
| 735 | if onu.hpTid < 0x8000 { |
| 736 | onu.hpTid = 0x8000 |
| 737 | } |
| 738 | } else { |
| 739 | next = onu.tid |
| 740 | onu.tid += 1 |
| 741 | if onu.tid >= 0x8000 { |
| 742 | onu.tid = 1 |
| 743 | } |
| 744 | } |
| 745 | return next |
| 746 | } |
| 747 | |
| 748 | // TODO move this method in responders/omcisim |
| 749 | func (o *Onu) StartOmci(client openolt.OpenoltClient) { |
| 750 | mibReset, _ := omcilib.CreateMibResetRequest(o.getNextTid(false)) |
| 751 | sendOmciMsg(mibReset, o.PonPortID, o.ID, o.SerialNumber, "mibReset", client) |
| 752 | } |
| 753 | |
| 754 | func (o *Onu) handleOmci(msg OmciIndicationMessage, client openolt.OpenoltClient) { |
| 755 | msgType, packet := omcilib.DecodeOmci(msg.OmciInd.Pkt) |
| 756 | |
| 757 | log.WithFields(log.Fields{ |
| 758 | "IntfId": msg.OmciInd.IntfId, |
| 759 | "OnuId": msg.OmciInd.OnuId, |
| 760 | "OnuSn": common.OnuSnToString(o.SerialNumber), |
| 761 | "Pkt": msg.OmciInd.Pkt, |
| 762 | "msgType": msgType, |
Anand S Katti | 0954135 | 2020-01-29 15:54:01 +0530 | [diff] [blame^] | 763 | }).Trace("ONU Receives OMCI Msg") |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 764 | switch msgType { |
| 765 | default: |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 766 | log.WithFields(log.Fields{ |
| 767 | "IntfId": msg.OmciInd.IntfId, |
| 768 | "OnuId": msg.OmciInd.OnuId, |
| 769 | "OnuSn": common.OnuSnToString(o.SerialNumber), |
| 770 | "Pkt": msg.OmciInd.Pkt, |
| 771 | "msgType": msgType, |
| 772 | }).Fatalf("unexpected frame: %v", packet) |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 773 | case omci.MibResetResponseType: |
| 774 | mibUpload, _ := omcilib.CreateMibUploadRequest(o.getNextTid(false)) |
| 775 | sendOmciMsg(mibUpload, o.PonPortID, o.ID, o.SerialNumber, "mibUpload", client) |
| 776 | case omci.MibUploadResponseType: |
| 777 | mibUploadNext, _ := omcilib.CreateMibUploadNextRequest(o.getNextTid(false), o.seqNumber) |
| 778 | sendOmciMsg(mibUploadNext, o.PonPortID, o.ID, o.SerialNumber, "mibUploadNext", client) |
| 779 | case omci.MibUploadNextResponseType: |
| 780 | o.seqNumber++ |
| 781 | |
| 782 | if o.seqNumber > 290 { |
| 783 | // NOTE we are done with the MIB Upload (290 is the number of messages the omci-sim library will respond to) |
| 784 | galEnet, _ := omcilib.CreateGalEnetRequest(o.getNextTid(false)) |
| 785 | sendOmciMsg(galEnet, o.PonPortID, o.ID, o.SerialNumber, "CreateGalEnetRequest", client) |
| 786 | } else { |
| 787 | mibUploadNext, _ := omcilib.CreateMibUploadNextRequest(o.getNextTid(false), o.seqNumber) |
| 788 | sendOmciMsg(mibUploadNext, o.PonPortID, o.ID, o.SerialNumber, "mibUploadNext", client) |
| 789 | } |
| 790 | case omci.CreateResponseType: |
| 791 | // NOTE Creating a GemPort, |
| 792 | // BBsim actually doesn't care about the values, so we can do we want with the parameters |
| 793 | // In the same way we can create a GemPort even without setting up UNIs/TConts/... |
| 794 | // but we need the GemPort to trigger the state change |
| 795 | |
| 796 | if !o.HasGemPort { |
| 797 | // NOTE this sends a CreateRequestType and BBSim replies with a CreateResponseType |
| 798 | // thus we send this request only once |
| 799 | gemReq, _ := omcilib.CreateGemPortRequest(o.getNextTid(false)) |
| 800 | sendOmciMsg(gemReq, o.PonPortID, o.ID, o.SerialNumber, "CreateGemPortRequest", client) |
| 801 | o.HasGemPort = true |
| 802 | } else { |
| 803 | if err := o.InternalState.Event("send_eapol_flow"); err != nil { |
| 804 | onuLogger.WithFields(log.Fields{ |
| 805 | "OnuId": o.ID, |
| 806 | "IntfId": o.PonPortID, |
| 807 | "OnuSn": o.Sn(), |
| 808 | }).Errorf("Error while transitioning ONU State %v", err) |
| 809 | } |
| 810 | } |
| 811 | |
| 812 | } |
| 813 | } |
| 814 | |
| 815 | func (o *Onu) sendEapolFlow(client openolt.OpenoltClient) { |
| 816 | |
| 817 | classifierProto := openolt.Classifier{ |
| 818 | EthType: uint32(layers.EthernetTypeEAPOL), |
| 819 | OVid: 4091, |
| 820 | } |
| 821 | |
| 822 | actionProto := openolt.Action{} |
| 823 | |
| 824 | downstreamFlow := openolt.Flow{ |
| 825 | AccessIntfId: int32(o.PonPortID), |
| 826 | OnuId: int32(o.ID), |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 827 | UniId: int32(0), // NOTE do not hardcode this, we need to support multiple UNIs |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 828 | FlowId: uint32(o.ID), |
| 829 | FlowType: "downstream", |
| 830 | AllocId: int32(0), |
| 831 | NetworkIntfId: int32(0), |
| 832 | GemportId: int32(1), // FIXME use the same value as CreateGemPortRequest PortID, do not hardcode |
| 833 | Classifier: &classifierProto, |
| 834 | Action: &actionProto, |
| 835 | Priority: int32(100), |
| 836 | Cookie: uint64(o.ID), |
| 837 | PortNo: uint32(o.ID), // NOTE we are using this to map an incoming packetIndication to an ONU |
| 838 | } |
| 839 | |
| 840 | if _, err := client.FlowAdd(context.Background(), &downstreamFlow); err != nil { |
| 841 | log.WithFields(log.Fields{ |
| 842 | "IntfId": o.PonPortID, |
| 843 | "OnuId": o.ID, |
| 844 | "FlowId": downstreamFlow.FlowId, |
| 845 | "PortNo": downstreamFlow.PortNo, |
| 846 | "SerialNumber": common.OnuSnToString(o.SerialNumber), |
| 847 | }).Fatalf("Failed to EAPOL Flow") |
| 848 | } |
| 849 | log.WithFields(log.Fields{ |
| 850 | "IntfId": o.PonPortID, |
| 851 | "OnuId": o.ID, |
| 852 | "FlowId": downstreamFlow.FlowId, |
| 853 | "PortNo": downstreamFlow.PortNo, |
| 854 | "SerialNumber": common.OnuSnToString(o.SerialNumber), |
| 855 | }).Info("Sent EAPOL Flow") |
| 856 | } |
| 857 | |
| 858 | func (o *Onu) sendDhcpFlow(client openolt.OpenoltClient) { |
| 859 | classifierProto := openolt.Classifier{ |
| 860 | EthType: uint32(layers.EthernetTypeIPv4), |
| 861 | SrcPort: uint32(68), |
| 862 | DstPort: uint32(67), |
| 863 | } |
| 864 | |
| 865 | actionProto := openolt.Action{} |
| 866 | |
| 867 | downstreamFlow := openolt.Flow{ |
| 868 | AccessIntfId: int32(o.PonPortID), |
| 869 | OnuId: int32(o.ID), |
Matteo Scandolo | 813402b | 2019-10-23 19:24:52 -0700 | [diff] [blame] | 870 | UniId: int32(0), // FIXME do not hardcode this |
Matteo Scandolo | 40e067f | 2019-10-16 16:59:41 -0700 | [diff] [blame] | 871 | FlowId: uint32(o.ID), |
| 872 | FlowType: "downstream", |
| 873 | AllocId: int32(0), |
| 874 | NetworkIntfId: int32(0), |
| 875 | GemportId: int32(1), // FIXME use the same value as CreateGemPortRequest PortID, do not hardcode |
| 876 | Classifier: &classifierProto, |
| 877 | Action: &actionProto, |
| 878 | Priority: int32(100), |
| 879 | Cookie: uint64(o.ID), |
| 880 | PortNo: uint32(o.ID), // NOTE we are using this to map an incoming packetIndication to an ONU |
| 881 | } |
| 882 | |
| 883 | if _, err := client.FlowAdd(context.Background(), &downstreamFlow); err != nil { |
| 884 | log.WithFields(log.Fields{ |
| 885 | "IntfId": o.PonPortID, |
| 886 | "OnuId": o.ID, |
| 887 | "FlowId": downstreamFlow.FlowId, |
| 888 | "PortNo": downstreamFlow.PortNo, |
| 889 | "SerialNumber": common.OnuSnToString(o.SerialNumber), |
| 890 | }).Fatalf("Failed to send DHCP Flow") |
| 891 | } |
| 892 | log.WithFields(log.Fields{ |
| 893 | "IntfId": o.PonPortID, |
| 894 | "OnuId": o.ID, |
| 895 | "FlowId": downstreamFlow.FlowId, |
| 896 | "PortNo": downstreamFlow.PortNo, |
| 897 | "SerialNumber": common.OnuSnToString(o.SerialNumber), |
| 898 | }).Info("Sent DHCP Flow") |
| 899 | } |