[SEBA-824] Shutting of and Powering on ONU

Change-Id: I6d797ba1182eb46e9edc887e9c7d8994be4585a2
diff --git a/api/bbsim/bbsim.proto b/api/bbsim/bbsim.proto
index 8df10d5..59339cc 100644
--- a/api/bbsim/bbsim.proto
+++ b/api/bbsim/bbsim.proto
@@ -15,6 +15,8 @@
 syntax = "proto3";
 package bbsim;
 
+// Models
+
 message PONPort {
     int32 ID = 1;
     string OperState = 2;
@@ -49,6 +51,14 @@
     repeated ONU items = 1;
 }
 
+// Inputs
+
+message ONURequest {
+    string SerialNumber = 1;
+}
+
+// Utils
+
 message VersionNumber {
     string version = 1;
     string buildTime = 2;
@@ -61,6 +71,11 @@
     bool caller = 2;
 }
 
+message Response {
+    int32 status_code = 1;
+    string message = 2;
+}
+
 message Empty {}
 
 service BBSim {
@@ -68,4 +83,6 @@
     rpc GetOlt(Empty) returns (Olt) {}
     rpc GetONUs(Empty) returns (ONUs) {}
     rpc SetLogLevel(LogLevel) returns (LogLevel) {}
+    rpc ShutdownONU (ONURequest) returns (Response) {}
+    rpc PoweronONU (ONURequest) returns (Response) {}
 }
\ No newline at end of file
diff --git a/docs/onu-state-machine.md b/docs/onu-state-machine.md
index af33db1..6155ecf 100644
--- a/docs/onu-state-machine.md
+++ b/docs/onu-state-machine.md
@@ -4,13 +4,11 @@
 
 Here is a list of possible state transitions in BBSim:
 
-
-
 |Transition|Starting States|End State| Notes |
 | --- | --- | --- | --- |
 | - | - | created |
 | discover | created | discovered |
-| enable | discovered | enabled |
+| enable | discovered, disabled | enabled |
 | receive_eapol_flow | enabled, gem_port_added | eapol_flow_received |
 | add_gem_port | enabled, eapol_flow_received | gem_port_added | We need to wait for both the flow and the gem port to come before moving to `auth_started` |
 | start_auth | eapol_flow_received, gem_port_added | auth_started |
@@ -24,4 +22,10 @@
 | dhcp_request_sent | dhcp_discovery_sent | dhcp_request_sent |
 | dhcp_ack_received | dhcp_request_sent | dhcp_ack_received |
 | dhcp_failed | dhcp_started, dhcp_discovery_sent, dhcp_request_sent | dhcp_failed |
- 
\ No newline at end of file
+
+
+In addition some transition can be forced via the API:
+
+|Transition|Starting States|End State| Notes |
+| --- | --- | --- | --- |
+| disable | eap_response_success_received, auth_failed, dhcp_ack_received, dhcp_failed | disabled | Emulates a devide mulfunction. Sends a `DyingGaspInd` and then an `OnuIndication{OperState: 'down'}`|
\ No newline at end of file
diff --git a/internal/bbsim/api/grpc_api_server.go b/internal/bbsim/api/grpc_api_server.go
index 63f5f4a..caa60e8 100644
--- a/internal/bbsim/api/grpc_api_server.go
+++ b/internal/bbsim/api/grpc_api_server.go
@@ -80,30 +80,6 @@
 	return &res, nil
 }
 
-func (s BBSimServer) GetONUs(ctx context.Context, req *bbsim.Empty) (*bbsim.ONUs, error) {
-	olt := devices.GetOLT()
-	onus := bbsim.ONUs{
-		Items: []*bbsim.ONU{},
-	}
-
-	for _, pon := range olt.Pons {
-		for _, o := range pon.Onus {
-			onu := bbsim.ONU{
-				ID:            int32(o.ID),
-				SerialNumber:  o.Sn(),
-				OperState:     o.OperState.Current(),
-				InternalState: o.InternalState.Current(),
-				PonPortID:     int32(o.PonPortID),
-				STag:          int32(o.STag),
-				CTag:          int32(o.CTag),
-				HwAddress:     o.HwAddress.String(),
-			}
-			onus.Items = append(onus.Items, &onu)
-		}
-	}
-	return &onus, nil
-}
-
 func (s BBSimServer) SetLogLevel(ctx context.Context, req *bbsim.LogLevel) (*bbsim.LogLevel, error) {
 
 	bbsimLogger.SetLogLevel(log.StandardLogger(), req.Level, req.Caller)
diff --git a/internal/bbsim/api/onus_handler.go b/internal/bbsim/api/onus_handler.go
new file mode 100644
index 0000000..bd333b0
--- /dev/null
+++ b/internal/bbsim/api/onus_handler.go
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"context"
+	"fmt"
+	"github.com/opencord/bbsim/api/bbsim"
+	"github.com/opencord/bbsim/internal/bbsim/devices"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+)
+
+func (s BBSimServer) GetONUs(ctx context.Context, req *bbsim.Empty) (*bbsim.ONUs, error) {
+	olt := devices.GetOLT()
+	onus := bbsim.ONUs{
+		Items: []*bbsim.ONU{},
+	}
+
+	for _, pon := range olt.Pons {
+		for _, o := range pon.Onus {
+			onu := bbsim.ONU{
+				ID:            int32(o.ID),
+				SerialNumber:  o.Sn(),
+				OperState:     o.OperState.Current(),
+				InternalState: o.InternalState.Current(),
+				PonPortID:     int32(o.PonPortID),
+				STag:          int32(o.STag),
+				CTag:          int32(o.CTag),
+				HwAddress:     o.HwAddress.String(),
+			}
+			onus.Items = append(onus.Items, &onu)
+		}
+	}
+	return &onus, nil
+}
+
+func (s BBSimServer) ShutdownONU(ctx context.Context, req *bbsim.ONURequest) (*bbsim.Response, error) {
+	// NOTE this method is now sendying a Dying Gasp and then disabling the device (operState: down, adminState: up),
+	// is this the only way to do? Should we address other cases?
+	// Investigate what happens when:
+	// - a fiber is pulled
+	// - ONU malfunction
+	// - ONU shutdown
+	res := &bbsim.Response{}
+
+	logger.WithFields(log.Fields{
+		"OnuSn": req.SerialNumber,
+	}).Infof("Received request to shutdown ONU")
+
+	olt := devices.GetOLT()
+
+	onu, err := olt.FindOnu(req.SerialNumber)
+
+	if err != nil {
+		res.StatusCode = int32(codes.NotFound)
+		res.Message = err.Error()
+		return res, err
+	}
+
+	dyingGasp := devices.Message{
+		Type: devices.DyingGaspIndication,
+		Data: devices.DyingGaspIndicationMessage{
+			OnuID:     onu.ID,
+			PonPortID: onu.PonPortID,
+			Status:    "on", // TODO do we need a type for Dying Gasp Indication?
+		},
+	}
+
+	onu.Channel <- dyingGasp
+
+	if err := onu.InternalState.Event("disable"); err != nil {
+		logger.WithFields(log.Fields{
+			"OnuId":  onu.ID,
+			"IntfId": onu.PonPortID,
+			"OnuSn":  onu.Sn(),
+		}).Errorf("Cannot shutdown ONU: %s", err.Error())
+		res.StatusCode = int32(codes.FailedPrecondition)
+		res.Message = err.Error()
+		return res, err
+	}
+
+	res.StatusCode = int32(codes.OK)
+	res.Message = fmt.Sprintf("ONU %s successfully shut down.", onu.Sn())
+
+	return res, nil
+}
+
+func (s BBSimServer) PoweronONU(ctx context.Context, req *bbsim.ONURequest) (*bbsim.Response, error) {
+	res := &bbsim.Response{}
+
+	logger.WithFields(log.Fields{
+		"OnuSn": req.SerialNumber,
+	}).Infof("Received request to poweron ONU")
+
+	olt := devices.GetOLT()
+
+	onu, err := olt.FindOnu(req.SerialNumber)
+
+	if err != nil {
+		res.StatusCode = int32(codes.NotFound)
+		res.Message = err.Error()
+		return res, err
+	}
+
+	if err := onu.InternalState.Event("enable"); err != nil {
+		logger.WithFields(log.Fields{
+			"OnuId":  onu.ID,
+			"IntfId": onu.PonPortID,
+			"OnuSn":  onu.Sn(),
+		}).Errorf("Cannot poweron ONU: %s", err.Error())
+		res.StatusCode = int32(codes.FailedPrecondition)
+		res.Message = err.Error()
+		return res, err
+	}
+
+	res.StatusCode = int32(codes.OK)
+	res.Message = fmt.Sprintf("ONU %s successfully powered on.", onu.Sn())
+
+	return res, nil
+}
diff --git a/internal/bbsim/devices/helpers_test.go b/internal/bbsim/devices/helpers_test.go
index 7b3bf71..818009f 100644
--- a/internal/bbsim/devices/helpers_test.go
+++ b/internal/bbsim/devices/helpers_test.go
@@ -19,7 +19,6 @@
 import (
 	"github.com/looplab/fsm"
 	"gotest.tools/assert"
-	"os"
 	"testing"
 )
 
@@ -27,34 +26,28 @@
 	originalNewFSM func(initial string, events []fsm.EventDesc, callbacks map[string]fsm.Callback) *fsm.FSM
 )
 
-func setUp()  {
+func setUpHelpers() {
 	originalNewFSM = newFSM
 }
 
-
-func tearDown()  {
+func tearDownHelpers() {
 	newFSM = originalNewFSM
 }
 
-func TestMain(m *testing.M) {
-	setUp()
-	code := m.Run()
-	tearDown()
-	os.Exit(code)
-}
-
 func Test_Helpers(t *testing.T) {
 
+	setUpHelpers()
+
 	// feedback values for the mock
 	called := 0
 	args := struct {
-		initial string
-		events []fsm.EventDesc
+		initial   string
+		events    []fsm.EventDesc
 		callbacks map[string]fsm.Callback
 	}{}
 
 	// creating the mock function
-	mockFSM := func(initial string, events []fsm.EventDesc, callbacks map[string]fsm.Callback) *fsm.FSM  {
+	mockFSM := func(initial string, events []fsm.EventDesc, callbacks map[string]fsm.Callback) *fsm.FSM {
 		called++
 		args.initial = initial
 		args.events = events
@@ -89,4 +82,6 @@
 	sm.Event("enable")
 	assert.Equal(t, cb_called, 1)
 
-}
\ No newline at end of file
+	tearDownHelpers()
+
+}
diff --git a/internal/bbsim/devices/olt.go b/internal/bbsim/devices/olt.go
index c75f2d1..e6248e8 100644
--- a/internal/bbsim/devices/olt.go
+++ b/internal/bbsim/devices/olt.go
@@ -103,6 +103,7 @@
 			NumOnu: olt.NumOnuPerPon,
 			ID:     uint32(i),
 			Type:   "pon",
+			Olt:    olt,
 		}
 		p.OperState = getOperStateFSM(func(e *fsm.Event) {
 			oltLogger.WithFields(log.Fields{
@@ -146,7 +147,7 @@
 	for {
 		_, ok := <-*o.oltDoneChannel
 		if !ok {
-			// if the olt channel is closed, stop the gRPC server
+			// if the olt Channel is closed, stop the gRPC server
 			log.Warnf("Stopping OLT gRPC server")
 			grpcServer.Stop()
 			wg.Done()
@@ -168,7 +169,7 @@
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 
-	// create a channel for all the OLT events
+	// create a Channel for all the OLT events
 	go o.processOltMessages(stream)
 	go o.processNniPacketIns(stream)
 
@@ -207,6 +208,8 @@
 		for _, onu := range pon.Onus {
 			go onu.processOnuMessages(stream)
 			go onu.processOmciMessages(stream)
+			// FIXME move the message generation in the state transition
+			// from here only invoke the state transition
 			msg := Message{
 				Type: OnuDiscIndication,
 				Data: OnuDiscIndicationMessage{
@@ -214,7 +217,7 @@
 					OperState: UP,
 				},
 			}
-			onu.channel <- msg
+			onu.Channel <- msg
 		}
 	}
 
@@ -335,7 +338,7 @@
 			msg, _ := message.Data.(PonIndicationMessage)
 			o.sendPonIndication(msg, stream)
 		default:
-			oltLogger.Warnf("Received unknown message data %v for type %v in OLT channel", message.Data, message.Type)
+			oltLogger.Warnf("Received unknown message data %v for type %v in OLT Channel", message.Data, message.Type)
 		}
 
 	}
@@ -362,6 +365,19 @@
 	}
 }
 
+func (o OltDevice) FindOnu(serialNumber string) (*Onu, error) {
+
+	for _, pon := range o.Pons {
+		for _, onu := range pon.Onus {
+			if onu.Sn() == serialNumber {
+				return &onu, nil
+			}
+		}
+	}
+
+	return &Onu{}, errors.New(fmt.Sprintf("cannot-find-onu-%s", serialNumber))
+}
+
 // GRPC Endpoints
 
 func (o OltDevice) ActivateOnu(context context.Context, onu *openolt.Onu) (*openolt.Empty, error) {
@@ -372,16 +388,23 @@
 	pon, _ := o.getPonById(onu.IntfId)
 	_onu, _ := pon.getOnuBySn(onu.SerialNumber)
 
-	// NOTE we need to immediately activate the ONU or the OMCI state machine won't start
-	msg := Message{
-		Type: OnuIndication,
-		Data: OnuIndicationMessage{
-			OnuSN:     onu.SerialNumber,
-			PonPortID: onu.IntfId,
-			OperState: UP,
-		},
+	if err := _onu.OperState.Event("enable"); err != nil {
+		oltLogger.WithFields(log.Fields{
+			"IntfId": _onu.PonPortID,
+			"OnuSn":  _onu.Sn(),
+			"OnuId":  _onu.ID,
+		}).Infof("Failed to transition ONU.OperState to enabled state: %s", err.Error())
 	}
-	_onu.channel <- msg
+	if err := _onu.InternalState.Event("enable"); err != nil {
+		oltLogger.WithFields(log.Fields{
+			"IntfId": _onu.PonPortID,
+			"OnuSn":  _onu.Sn(),
+			"OnuId":  _onu.ID,
+		}).Infof("Failed to transition ONU to enabled state: %s", err.Error())
+	}
+
+	// NOTE we need to immediately activate the ONU or the OMCI state machine won't start
+
 	return new(openolt.Empty), nil
 }
 
@@ -453,7 +476,7 @@
 				Flow:      flow,
 			},
 		}
-		onu.channel <- msg
+		onu.Channel <- msg
 	}
 
 	return new(openolt.Empty), nil
@@ -499,6 +522,11 @@
 func (o OltDevice) OmciMsgOut(ctx context.Context, omci_msg *openolt.OmciMsg) (*openolt.Empty, error) {
 	pon, _ := o.getPonById(omci_msg.IntfId)
 	onu, _ := pon.getOnuById(omci_msg.OnuId)
+	oltLogger.WithFields(log.Fields{
+		"IntfId": onu.PonPortID,
+		"OnuId":  onu.ID,
+		"OnuSn":  onu.Sn(),
+	}).Tracef("Received OmciMsgOut")
 	msg := Message{
 		Type: OMCI,
 		Data: OmciMessage{
@@ -507,7 +535,7 @@
 			omciMsg: omci_msg,
 		},
 	}
-	onu.channel <- msg
+	onu.Channel <- msg
 	return new(openolt.Empty), nil
 }
 
diff --git a/internal/bbsim/devices/olt_test.go b/internal/bbsim/devices/olt_test.go
new file mode 100644
index 0000000..a022c7c
--- /dev/null
+++ b/internal/bbsim/devices/olt_test.go
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package devices
+
+import (
+	"gotest.tools/assert"
+	"testing"
+)
+
+func createMockOlt(numPon int, numOnu int) OltDevice {
+	olt := OltDevice{
+		ID: 0,
+	}
+
+	for i := 0; i < numPon; i++ {
+		pon := PonPort{
+			ID: uint32(i),
+		}
+
+		for j := 0; j < numOnu; j++ {
+			onu := Onu{
+				ID:        uint32(i + j),
+				PonPort:   pon,
+				PonPortID: uint32(i),
+			}
+			onu.SerialNumber = onu.NewSN(olt.ID, pon.ID, onu.ID)
+			pon.Onus = append(pon.Onus, onu)
+		}
+		olt.Pons = append(olt.Pons, pon)
+	}
+	return olt
+}
+
+func Test_Olt_FindOnu_Success(t *testing.T) {
+
+	numPon := 4
+	numOnu := 4
+
+	olt := createMockOlt(numPon, numOnu)
+
+	onu, err := olt.FindOnu("BBSM00000303")
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, onu.Sn(), "BBSM00000303")
+	assert.Equal(t, onu.ID, uint32(3))
+	assert.Equal(t, onu.PonPortID, uint32(3))
+}
+
+func Test_Olt_FindOnu_Error(t *testing.T) {
+
+	numPon := 1
+	numOnu := 4
+
+	olt := createMockOlt(numPon, numOnu)
+
+	_, err := olt.FindOnu("BBSM00000303")
+
+	assert.Equal(t, err.Error(), "cannot-find-onu-BBSM00000303")
+}
diff --git a/internal/bbsim/devices/onu.go b/internal/bbsim/devices/onu.go
index 77e912c..4587620 100644
--- a/internal/bbsim/devices/onu.go
+++ b/internal/bbsim/devices/onu.go
@@ -42,8 +42,8 @@
 		STag:      sTag,
 		CTag:      cTag,
 		HwAddress: net.HardwareAddr{0x2e, 0x60, 0x70, 0x13, byte(pon.ID), byte(id)},
-		// NOTE can we combine everything in a single channel?
-		channel:       make(chan Message, 2048),
+		// NOTE can we combine everything in a single Channel?
+		Channel:       make(chan Message, 2048),
 		eapolPktOutCh: make(chan *bbsim.ByteMsg, 1024),
 		dhcpPktOutCh:  make(chan *bbsim.ByteMsg, 1024),
 	}
@@ -61,11 +61,13 @@
 	o.InternalState = fsm.NewFSM(
 		"created",
 		fsm.Events{
-			// DEVICE Activation
+			// DEVICE Lifecycle
 			{Name: "discover", Src: []string{"created"}, Dst: "discovered"},
-			{Name: "enable", Src: []string{"discovered"}, Dst: "enabled"},
+			{Name: "enable", Src: []string{"discovered", "disabled"}, Dst: "enabled"},
 			{Name: "receive_eapol_flow", Src: []string{"enabled", "gem_port_added"}, Dst: "eapol_flow_received"},
 			{Name: "add_gem_port", Src: []string{"enabled", "eapol_flow_received"}, Dst: "gem_port_added"},
+			// NOTE should disabled state be diffente for oper_disabled (emulating an error) and admin_disabled (received a disabled call via VOLTHA)?
+			{Name: "disable", Src: []string{"eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed"}, Dst: "disabled"},
 			// EAPOL
 			{Name: "start_auth", Src: []string{"eapol_flow_received", "gem_port_added"}, Dst: "auth_started"},
 			{Name: "eap_start_sent", Src: []string{"auth_started"}, Dst: "eap_start_sent"},
@@ -84,6 +86,28 @@
 			"enter_state": func(e *fsm.Event) {
 				o.logStateChange(e.Src, e.Dst)
 			},
+			"enter_enabled": func(event *fsm.Event) {
+				msg := Message{
+					Type: OnuIndication,
+					Data: OnuIndicationMessage{
+						OnuSN:     o.SerialNumber,
+						PonPortID: o.PonPortID,
+						OperState: UP,
+					},
+				}
+				o.Channel <- msg
+			},
+			"enter_disabled": func(event *fsm.Event) {
+				msg := Message{
+					Type: OnuIndication,
+					Data: OnuIndicationMessage{
+						OnuSN:     o.SerialNumber,
+						PonPortID: o.PonPortID,
+						OperState: DOWN,
+					},
+				}
+				o.Channel <- msg
+			},
 			"enter_auth_started": func(e *fsm.Event) {
 				o.logStateChange(e.Src, e.Dst)
 				msg := Message{
@@ -93,7 +117,7 @@
 						OnuID:     o.ID,
 					},
 				}
-				o.channel <- msg
+				o.Channel <- msg
 			},
 			"enter_auth_failed": func(e *fsm.Event) {
 				onuLogger.WithFields(log.Fields{
@@ -110,7 +134,7 @@
 						OnuID:     o.ID,
 					},
 				}
-				o.channel <- msg
+				o.Channel <- msg
 			},
 			"enter_dhcp_failed": func(e *fsm.Event) {
 				onuLogger.WithFields(log.Fields{
@@ -138,10 +162,10 @@
 		"onuSN": o.Sn(),
 	}).Debug("Started ONU Indication Channel")
 
-	for message := range o.channel {
+	for message := range o.Channel {
 		onuLogger.WithFields(log.Fields{
 			"onuID":       o.ID,
-			"onuSN":       o.SerialNumber,
+			"onuSN":       o.Sn(),
 			"messageType": message.Type,
 		}).Tracef("Received message on ONU Channel")
 
@@ -159,7 +183,7 @@
 			msg, _ := message.Data.(OnuFlowUpdateMessage)
 			o.handleFlowUpdate(msg, stream)
 		case StartEAPOL:
-			log.Infof("Receive StartEAPOL message on ONU channel")
+			log.Infof("Receive StartEAPOL message on ONU Channel")
 			go func() {
 				// TODO kill this thread
 				eapol.CreateWPASupplicant(
@@ -172,7 +196,7 @@
 				)
 			}()
 		case StartDHCP:
-			log.Infof("Receive StartDHCP message on ONU channel")
+			log.Infof("Receive StartDHCP message on ONU Channel")
 			go func() {
 				// TODO kill this thread
 				dhcp.CreateDHCPClient(
@@ -186,8 +210,12 @@
 					o.dhcpPktOutCh,
 				)
 			}()
+
+		case DyingGaspIndication:
+			msg, _ := message.Data.(DyingGaspIndicationMessage)
+			o.sendDyingGaspInd(msg, stream)
 		default:
-			onuLogger.Warnf("Received unknown message data %v for type %v in OLT channel", message.Data, message.Type)
+			onuLogger.Warnf("Received unknown message data %v for type %v in OLT Channel", message.Data, message.Type)
 		}
 	}
 }
@@ -234,15 +262,49 @@
 	return sn
 }
 
+// NOTE handle_/process methods can change the ONU internal state as they are receiving messages
+// send method should not change the ONU state
+
+func (o Onu) sendDyingGaspInd(msg DyingGaspIndicationMessage, stream openolt.Openolt_EnableIndicationServer) error {
+	alarmData := &openolt.AlarmIndication_DyingGaspInd{
+		DyingGaspInd: &openolt.DyingGaspIndication{
+			IntfId: msg.PonPortID,
+			OnuId:  msg.OnuID,
+			Status: "on",
+		},
+	}
+	data := &openolt.Indication_AlarmInd{AlarmInd: &openolt.AlarmIndication{Data: alarmData}}
+
+	if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
+		onuLogger.Errorf("Failed to send DyingGaspInd : %v", err)
+		return err
+	}
+	onuLogger.WithFields(log.Fields{
+		"IntfId": msg.PonPortID,
+		"OnuSn":  o.Sn(),
+		"OnuId":  msg.OnuID,
+	}).Info("sendDyingGaspInd")
+	return nil
+}
+
 func (o Onu) sendOnuDiscIndication(msg OnuDiscIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
 	discoverData := &openolt.Indication_OnuDiscInd{OnuDiscInd: &openolt.OnuDiscIndication{
 		IntfId:       msg.Onu.PonPortID,
 		SerialNumber: msg.Onu.SerialNumber,
 	}}
+
 	if err := stream.Send(&openolt.Indication{Data: discoverData}); err != nil {
 		log.Errorf("Failed to send Indication_OnuDiscInd: %v", err)
 	}
-	o.InternalState.Event("discover")
+
+	if err := o.InternalState.Event("discover"); err != nil {
+		oltLogger.WithFields(log.Fields{
+			"IntfId": o.PonPortID,
+			"OnuSn":  o.Sn(),
+			"OnuId":  o.ID,
+		}).Infof("Failed to transition ONU to discovered state: %s", err.Error())
+	}
+
 	onuLogger.WithFields(log.Fields{
 		"IntfId": msg.Onu.PonPortID,
 		"OnuSn":  msg.Onu.Sn(),
@@ -255,19 +317,18 @@
 	// expected_onu_id: 1, received_onu_id: 1024, event: ONU-id-mismatch, can happen if both voltha and the olt rebooted
 	// so we're using the internal ID that is 1
 	// o.ID = msg.OnuID
-	o.OperState.Event("enable")
 
 	indData := &openolt.Indication_OnuInd{OnuInd: &openolt.OnuIndication{
 		IntfId:       o.PonPortID,
 		OnuId:        o.ID,
-		OperState:    o.OperState.Current(),
+		OperState:    msg.OperState.String(),
 		AdminState:   o.OperState.Current(),
 		SerialNumber: o.SerialNumber,
 	}}
 	if err := stream.Send(&openolt.Indication{Data: indData}); err != nil {
+		// TODO do we need to transition to a broken state?
 		log.Errorf("Failed to send Indication_OnuInd: %v", err)
 	}
-	o.InternalState.Event("enable")
 	onuLogger.WithFields(log.Fields{
 		"IntfId":     o.PonPortID,
 		"OnuId":      o.ID,
@@ -275,6 +336,7 @@
 		"AdminState": msg.OperState.String(),
 		"OnuSn":      o.Sn(),
 	}).Debug("Sent Indication_OnuInd")
+
 }
 
 func (o Onu) handleOmciMessage(msg OmciMessage, stream openolt.Openolt_EnableIndicationServer) {
diff --git a/internal/bbsim/devices/types.go b/internal/bbsim/devices/types.go
index e012858..93998e9 100644
--- a/internal/bbsim/devices/types.go
+++ b/internal/bbsim/devices/types.go
@@ -44,9 +44,9 @@
 	OperState    *fsm.FSM
 	SerialNumber *openolt.SerialNumber
 
-	channel       chan Message        // this channel is to track state changes and OMCI messages
-	eapolPktOutCh chan *bbsim.ByteMsg // this channel is for EAPOL Packet Outs (coming from the controller)
-	dhcpPktOutCh  chan *bbsim.ByteMsg // this channel is for DHCP Packet Outs (coming from the controller)
+	Channel       chan Message        // this Channel is to track state changes and OMCI messages
+	eapolPktOutCh chan *bbsim.ByteMsg // this Channel is for EAPOL Packet Outs (coming from the controller)
+	dhcpPktOutCh  chan *bbsim.ByteMsg // this Channel is for DHCP Packet Outs (coming from the controller)
 }
 
 func (o Onu) Sn() string {
@@ -67,6 +67,7 @@
 	ID     uint32
 	NumOnu int
 	Onus   []Onu
+	Olt    OltDevice
 
 	// PON Attributes
 	OperState *fsm.FSM
@@ -118,16 +119,16 @@
 type MessageType int
 
 const (
-	OltIndication     MessageType = 0
-	NniIndication     MessageType = 1
-	PonIndication     MessageType = 2
-	OnuDiscIndication MessageType = 3
-	OnuIndication     MessageType = 4
-	OMCI              MessageType = 5
-	FlowUpdate        MessageType = 6
-	StartEAPOL        MessageType = 7
-	DoneEAPOL         MessageType = 8
-	StartDHCP         MessageType = 9
+	OltIndication       MessageType = 0
+	NniIndication       MessageType = 1
+	PonIndication       MessageType = 2
+	OnuDiscIndication   MessageType = 3
+	OnuIndication       MessageType = 4
+	OMCI                MessageType = 5
+	FlowUpdate          MessageType = 6
+	StartEAPOL          MessageType = 7
+	StartDHCP           MessageType = 8
+	DyingGaspIndication MessageType = 9
 )
 
 func (m MessageType) String() string {
@@ -140,8 +141,8 @@
 		"OMCI",
 		"FlowUpdate",
 		"StartEAPOL",
-		"DoneEAPOL",
 		"StartDHCP",
+		"DyingGaspIndication",
 	}
 	return names[m]
 }
@@ -194,6 +195,12 @@
 	OnuID     uint32
 }
 
+type DyingGaspIndicationMessage struct {
+	PonPortID uint32
+	OnuID     uint32
+	Status    string
+}
+
 type OperState int
 
 const (
diff --git a/internal/bbsimctl/commands/onu.go b/internal/bbsimctl/commands/onu.go
index 2e8f887..f99cb5c 100644
--- a/internal/bbsimctl/commands/onu.go
+++ b/internal/bbsimctl/commands/onu.go
@@ -19,6 +19,7 @@
 
 import (
 	"context"
+	"fmt"
 	"github.com/jessevdk/go-flags"
 	pb "github.com/opencord/bbsim/api/bbsim"
 	"github.com/opencord/bbsim/internal/bbsimctl/config"
@@ -26,33 +27,57 @@
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"os"
+	"strings"
 )
 
 const (
 	DEFAULT_ONU_DEVICE_HEADER_FORMAT = "table{{ .PonPortID }}\t{{ .ID }}\t{{ .SerialNumber }}\t{{ .STag }}\t{{ .CTag }}\t{{ .OperState }}\t{{ .InternalState }}"
 )
 
-type ONUOptions struct{}
-
-func RegisterONUCommands(parser *flags.Parser) {
-	parser.AddCommand("onus", "List ONU Devices", "Commands to list the ONU devices and their internal state", &ONUOptions{})
+type OnuSnString string
+type ONUList struct{}
+type ONUShutDown struct {
+	Args struct {
+		OnuSn OnuSnString
+	} `positional-args:"yes" required:"yes"`
 }
 
-func getONUs() *pb.ONUs {
+type ONUPowerOn struct {
+	Args struct {
+		OnuSn OnuSnString
+	} `positional-args:"yes" required:"yes"`
+}
+
+type ONUOptions struct {
+	List     ONUList     `command:"list"`
+	ShutDown ONUShutDown `command:"shutdown"`
+	PowerOn  ONUPowerOn  `command:"poweron"`
+}
+
+func RegisterONUCommands(parser *flags.Parser) {
+	parser.AddCommand("onu", "ONU Commands", "Commands to query and manipulate ONU devices", &ONUOptions{})
+}
+
+func connect() (pb.BBSimClient, *grpc.ClientConn) {
 	conn, err := grpc.Dial(config.GlobalConfig.Server, grpc.WithInsecure())
 
 	if err != nil {
 		log.Fatalf("did not connect: %v", err)
-		return nil
+		return nil, conn
 	}
+	return pb.NewBBSimClient(conn), conn
+}
+
+func getONUs() *pb.ONUs {
+
+	client, conn := connect()
 	defer conn.Close()
-	c := pb.NewBBSimClient(conn)
 
 	// Contact the server and print out its response.
-
 	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
 	defer cancel()
-	onus, err := c.GetONUs(ctx, &pb.Empty{})
+
+	onus, err := client.GetONUs(ctx, &pb.Empty{})
 	if err != nil {
 		log.Fatalf("could not get OLT: %v", err)
 		return nil
@@ -60,7 +85,7 @@
 	return onus
 }
 
-func (options *ONUOptions) Execute(args []string) error {
+func (options *ONUList) Execute(args []string) error {
 	onus := getONUs()
 
 	// print out
@@ -71,3 +96,69 @@
 
 	return nil
 }
+
+func (options *ONUShutDown) Execute(args []string) error {
+
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+	req := pb.ONURequest{
+		SerialNumber: string(options.Args.OnuSn),
+	}
+	res, err := client.ShutdownONU(ctx, &req)
+
+	if err != nil {
+		log.Fatalf("Cannot not shutdown ONU %s: %v", options.Args.OnuSn, err)
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf("[Status: %d] %s", res.StatusCode, res.Message))
+
+	return nil
+}
+
+func (options *ONUPowerOn) Execute(args []string) error {
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+	req := pb.ONURequest{
+		SerialNumber: string(options.Args.OnuSn),
+	}
+	res, err := client.PoweronONU(ctx, &req)
+
+	if err != nil {
+		log.Fatalf("Cannot not power on ONU %s: %v", options.Args.OnuSn, err)
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf("[Status: %d] %s", res.StatusCode, res.Message))
+
+	return nil
+}
+
+func (onuSn *OnuSnString) Complete(match string) []flags.Completion {
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+
+	onus, err := client.GetONUs(ctx, &pb.Empty{})
+	if err != nil {
+		log.Fatal("could not get ONUs: %v", err)
+		return nil
+	}
+
+	list := make([]flags.Completion, 0)
+	for _, k := range onus.Items {
+		if strings.HasPrefix(k.SerialNumber, match) {
+			list = append(list, flags.Completion{Item: k.SerialNumber})
+		}
+	}
+
+	return list
+}