[SEBA-873] add reboot olt support

Change-Id: I1570d05313661a6d66e1596b9f9a1a1cc17d1a73
diff --git a/internal/bbsim/api/grpc_api_server.go b/internal/bbsim/api/grpc_api_server.go
index 9df725b..09e4080 100644
--- a/internal/bbsim/api/grpc_api_server.go
+++ b/internal/bbsim/api/grpc_api_server.go
@@ -18,11 +18,13 @@
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/opencord/bbsim/api/bbsim"
 	"github.com/opencord/bbsim/internal/bbsim/devices"
 	"github.com/opencord/bbsim/internal/common"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
 )
 
 var logger = log.WithFields(log.Fields{
@@ -81,6 +83,43 @@
 	return &res, nil
 }
 
+func (s BBSimServer) PoweronOlt(ctx context.Context, req *bbsim.Empty) (*bbsim.Response, error) {
+	res := &bbsim.Response{}
+	o := devices.GetOLT()
+
+	if err := o.InternalState.Event("initialize"); err != nil {
+		log.Errorf("Error initializing OLT: %v", err)
+		res.StatusCode = int32(codes.FailedPrecondition)
+		return res, err
+	}
+
+	res.StatusCode = int32(codes.OK)
+	return res, nil
+}
+
+func (s BBSimServer) ShutdownOlt(ctx context.Context, req *bbsim.Empty) (*bbsim.Response, error) {
+	res := &bbsim.Response{}
+	o := devices.GetOLT()
+
+	if err := o.InternalState.Event("disable"); err != nil {
+		log.Errorf("Error disabling OLT: %v", err)
+		res.StatusCode = int32(codes.FailedPrecondition)
+		return res, err
+	}
+
+	res.StatusCode = int32(codes.OK)
+	return res, nil
+}
+
+func (s BBSimServer) RebootOlt(ctx context.Context, req *bbsim.Empty) (*bbsim.Response, error) {
+	res := &bbsim.Response{}
+	o := devices.GetOLT()
+	go o.RestartOLT()
+	res.StatusCode = int32(codes.OK)
+	res.Message = fmt.Sprintf("OLT restart triggered.")
+	return res, nil
+}
+
 func (s BBSimServer) SetLogLevel(ctx context.Context, req *bbsim.LogLevel) (*bbsim.LogLevel, error) {
 
 	common.SetLogLevel(log.StandardLogger(), req.Level, req.Caller)
diff --git a/internal/bbsim/api/grpc_api_server_legacy.go b/internal/bbsim/api/grpc_api_server_legacy.go
index 0e9b67b..d4e9999 100644
--- a/internal/bbsim/api/grpc_api_server_legacy.go
+++ b/internal/bbsim/api/grpc_api_server_legacy.go
@@ -185,7 +185,7 @@
 	// Register REST endpoints
 	err := legacy.RegisterBBSimServiceHandlerFromEndpoint(ctx, mux, grpcAddress, opts)
 	if err != nil {
-		logger.Error("%v", err)
+		logger.Errorf("%v", err)
 		return
 	}
 
diff --git a/internal/bbsim/api/legacy_api_handler.go b/internal/bbsim/api/legacy_api_handler.go
index 8ae4854..018162f 100644
--- a/internal/bbsim/api/legacy_api_handler.go
+++ b/internal/bbsim/api/legacy_api_handler.go
@@ -123,7 +123,7 @@
 	var SerialNumberLength = 12
 
 	if len(SerialNumber) != SerialNumberLength {
-		logger.Error("Invalid serial number %s", SerialNumber)
+		logger.Errorf("Invalid serial number %s", SerialNumber)
 		return nil, errors.New("invalid serial number")
 	}
 	// First four characters are vendorId
diff --git a/internal/bbsim/devices/device_params.go b/internal/bbsim/devices/device_params.go
new file mode 100644
index 0000000..77421ca
--- /dev/null
+++ b/internal/bbsim/devices/device_params.go
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package devices
+
+const (
+	OltRebootDelay = 10
+)
diff --git a/internal/bbsim/devices/nni.go b/internal/bbsim/devices/nni.go
index 39430ef..6e5e34d 100644
--- a/internal/bbsim/devices/nni.go
+++ b/internal/bbsim/devices/nni.go
@@ -18,19 +18,18 @@
 
 import (
 	"bytes"
+	"os/exec"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/pcap"
 	"github.com/looplab/fsm"
 	"github.com/opencord/bbsim/internal/bbsim/packetHandlers"
 	"github.com/opencord/bbsim/internal/bbsim/types"
 	log "github.com/sirupsen/logrus"
-	"os/exec"
 )
 
 var (
 	nniLogger    = log.WithFields(log.Fields{"module": "NNI"})
-	nniVeth      = "nni"
-	upstreamVeth = "upstream"
 	dhcpServerIp = "192.168.254.1"
 )
 
@@ -52,7 +51,9 @@
 
 type NniPort struct {
 	// BBSIM Internals
-	ID uint32
+	ID           uint32
+	nniVeth      string
+	upstreamVeth string
 
 	// PON Attributes
 	OperState *fsm.FSM
@@ -61,19 +62,21 @@
 
 func CreateNNI(olt *OltDevice) (NniPort, error) {
 	nniPort := NniPort{
-		ID: uint32(0),
+		ID:           uint32(0),
+		nniVeth:      "nni",
+		upstreamVeth: "upstream",
 		OperState: getOperStateFSM(func(e *fsm.Event) {
 			oltLogger.Debugf("Changing NNI OperState from %s to %s", e.Src, e.Dst)
 		}),
 		Type: "nni",
 	}
-	createNNIPair(executor, olt)
+	createNNIPair(executor, olt, &nniPort)
 	return nniPort, nil
 }
 
 // sendNniPacket will send a packet out of the NNI interface.
 // We will send upstream only DHCP packets and drop anything else
-func sendNniPacket(packet gopacket.Packet) error {
+func (n *NniPort) sendNniPacket(packet gopacket.Packet) error {
 	isDhcp := packetHandlers.IsDhcpPacket(packet)
 	isLldp := packetHandlers.IsLldpPacket(packet)
 
@@ -93,7 +96,7 @@
 			return err
 		}
 
-		handle, err := getVethHandler(nniVeth)
+		handle, err := getVethHandler(n.nniVeth)
 		if err != nil {
 			return err
 		}
@@ -117,33 +120,47 @@
 //createNNIBridge will create a veth bridge to fake the connection between the NNI port
 //and something upstream, in this case a DHCP server.
 //It is also responsible to start the DHCP server itself
-func createNNIPair(executor Executor, olt *OltDevice) error {
+func createNNIPair(executor Executor, olt *OltDevice, nniPort *NniPort) error {
 
-	if err := executor.Command("ip", "link", "add", nniVeth, "type", "veth", "peer", "name", upstreamVeth).Run(); err != nil {
-		nniLogger.Errorf("Couldn't create veth pair between %s and %s", nniVeth, upstreamVeth)
+	if err := executor.Command("ip", "link", "add", nniPort.nniVeth, "type", "veth", "peer", "name", nniPort.upstreamVeth).Run(); err != nil {
+		nniLogger.Errorf("Couldn't create veth pair between %s and %s", nniPort.nniVeth, nniPort.upstreamVeth)
 		return err
 	}
 
-	if err := setVethUp(executor, nniVeth); err != nil {
+	if err := setVethUp(executor, nniPort.nniVeth); err != nil {
 		return err
 	}
 
-	if err := setVethUp(executor, upstreamVeth); err != nil {
+	if err := setVethUp(executor, nniPort.upstreamVeth); err != nil {
 		return err
 	}
 
-	if err := startDHCPServer(); err != nil {
+	// TODO should be moved out of this function in case there are multiple NNI interfaces.
+	// Only one DHCP server should be running and listening on all NNI interfaces
+	if err := startDHCPServer(nniPort.upstreamVeth, dhcpServerIp); err != nil {
 		return err
 	}
 
-	ch, err := listenOnVeth(nniVeth)
-	if err != nil {
-		return err
-	}
-	olt.nniPktInChannel = ch
 	return nil
 }
 
+func deleteNNIPair(executor Executor, nniPort *NniPort) error {
+	if err := executor.Command("ip", "link", "del", nniPort.nniVeth).Run(); err != nil {
+		nniLogger.Errorf("Couldn't delete veth pair between %s and %s", nniPort.nniVeth, nniPort.upstreamVeth)
+		return err
+	}
+	return nil
+}
+
+// NewVethChan returns a new channel for receiving packets over the NNI interface
+func (n *NniPort) NewVethChan() (chan *types.PacketMsg, error) {
+	ch, err := listenOnVeth(n.nniVeth)
+	if err != nil {
+		return nil, err
+	}
+	return ch, err
+}
+
 // setVethUp is responsible to activate a virtual interface
 func setVethUp(executor Executor, vethName string) error {
 	if err := executor.Command("ip", "link", "set", vethName, "up").Run(); err != nil {
@@ -153,7 +170,8 @@
 	return nil
 }
 
-var startDHCPServer = func() error {
+var startDHCPServer = func(upstreamVeth string, dhcpServerIp string) error {
+	// TODO the DHCP server should support multiple interfaces
 	if err := exec.Command("ip", "addr", "add", dhcpServerIp, "dev", upstreamVeth).Run(); err != nil {
 		nniLogger.Errorf("Couldn't assing ip %s to interface %s: %v", dhcpServerIp, upstreamVeth, err)
 		return err
diff --git a/internal/bbsim/devices/nni_test.go b/internal/bbsim/devices/nni_test.go
index 60856c6..348d40f 100644
--- a/internal/bbsim/devices/nni_test.go
+++ b/internal/bbsim/devices/nni_test.go
@@ -19,9 +19,10 @@
 
 import (
 	"errors"
+	"testing"
+
 	"github.com/opencord/bbsim/internal/bbsim/types"
 	"gotest.tools/assert"
-	"testing"
 )
 
 func TestSetVethUpSuccess(t *testing.T) {
@@ -49,7 +50,7 @@
 	startDHCPServerCalled := false
 	_startDHCPServer := startDHCPServer
 	defer func() { startDHCPServer = _startDHCPServer }()
-	startDHCPServer = func() error {
+	startDHCPServer = func(upstreamVeth string, dhcpServerIp string) error {
 		startDHCPServerCalled = true
 		return nil
 	}
@@ -67,8 +68,10 @@
 	}
 
 	olt := OltDevice{}
+	nni := NniPort{}
 
-	err := createNNIPair(spy, &olt)
+	err := createNNIPair(spy, &olt, &nni)
+	olt.nniPktInChannel, _ = nni.NewVethChan()
 
 	assert.Equal(t, spy.CommandCallCount, 3)
 	assert.Equal(t, startDHCPServerCalled, true)
diff --git a/internal/bbsim/devices/olt.go b/internal/bbsim/devices/olt.go
index e810ce1..3264bc3 100644
--- a/internal/bbsim/devices/olt.go
+++ b/internal/bbsim/devices/olt.go
@@ -22,6 +22,7 @@
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
@@ -33,6 +34,7 @@
 	"github.com/opencord/voltha-protos/v2/go/tech_profile"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
 )
 
 var oltLogger = log.WithFields(log.Fields{
@@ -48,9 +50,7 @@
 	NumOnuPerPon    int
 	InternalState   *fsm.FSM
 	channel         chan Message
-	oltDoneChannel  *chan bool
-	apiDoneChannel  *chan bool
-	nniPktInChannel chan *bbsim.PacketMsg
+	nniPktInChannel chan *bbsim.PacketMsg // packets coming in from the NNI and going to VOLTHA
 
 	Delay int
 
@@ -62,12 +62,13 @@
 }
 
 var olt OltDevice
+var oltServer *grpc.Server
 
 func GetOLT() *OltDevice {
 	return &olt
 }
 
-func CreateOLT(oltId int, nni int, pon int, onuPerPon int, sTag int, cTagInit int, oltDoneChannel *chan bool, apiDoneChannel *chan bool, auth bool, dhcp bool, delay int, isMock bool) *OltDevice {
+func CreateOLT(oltId int, nni int, pon int, onuPerPon int, sTag int, cTagInit int, auth bool, dhcp bool, delay int, isMock bool) *OltDevice {
 	oltLogger.WithFields(log.Fields{
 		"ID":           oltId,
 		"NumNni":       nni,
@@ -81,16 +82,12 @@
 		OperState: getOperStateFSM(func(e *fsm.Event) {
 			oltLogger.Debugf("Changing OLT OperState from %s to %s", e.Src, e.Dst)
 		}),
-		NumNni:          nni,
-		NumPon:          pon,
-		NumOnuPerPon:    onuPerPon,
-		Pons:            []*PonPort{},
-		Nnis:            []*NniPort{},
-		channel:         make(chan Message),
-		oltDoneChannel:  oltDoneChannel,
-		apiDoneChannel:  apiDoneChannel,
-		nniPktInChannel: make(chan *bbsim.PacketMsg, 1024), // packets coming in from the NNI and going to VOLTHA
-		Delay:           delay,
+		NumNni:       nni,
+		NumPon:       pon,
+		NumOnuPerPon: onuPerPon,
+		Pons:         []*PonPort{},
+		Nnis:         []*NniPort{},
+		Delay:        delay,
 	}
 
 	// OLT State machine
@@ -98,13 +95,16 @@
 	olt.InternalState = fsm.NewFSM(
 		"created",
 		fsm.Events{
-			{Name: "enable", Src: []string{"created"}, Dst: "enabled"},
+			{Name: "initialize", Src: []string{"disabled", "created"}, Dst: "initialized"},
+			{Name: "enable", Src: []string{"initialized", "disabled"}, Dst: "enabled"},
 			{Name: "disable", Src: []string{"enabled"}, Dst: "disabled"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) {
 				oltLogger.Debugf("Changing OLT InternalState from %s to %s", e.Src, e.Dst)
 			},
+			"enter_disabled":    func(e *fsm.Event) { olt.disableOlt() },
+			"enter_initialized": func(e *fsm.Event) { olt.InitOlt() },
 		},
 	)
 
@@ -143,16 +143,96 @@
 
 		olt.Pons = append(olt.Pons, &p)
 	}
+
+	if err := olt.InternalState.Event("initialize"); err != nil {
+		log.Errorf("Error initializing OLT: %v", err)
+		return nil
+	}
+
 	return &olt
 }
 
-// this function start the OLT gRPC server and blocks until it's done
-func StartOlt(olt *OltDevice, group *sync.WaitGroup) {
-	newOltServer(*olt)
-	group.Done()
+func (o *OltDevice) InitOlt() error {
+
+	if oltServer == nil {
+		oltServer, _ = newOltServer()
+	} else {
+		oltLogger.Warn("OLT server already running.")
+	}
+
+	// create new channel for processOltMessages Go routine
+	o.channel = make(chan Message)
+
+	o.nniPktInChannel = make(chan *bbsim.PacketMsg, 1024)
+	// FIXME we are assuming we have only one NNI
+	if o.Nnis[0] != nil {
+		ch, err := o.Nnis[0].NewVethChan()
+		if err == nil {
+			o.nniPktInChannel = ch
+		} else {
+			log.Errorf("Error getting NNI channel: %v", err)
+		}
+	}
+
+	for i := range olt.Pons {
+		for _, onu := range olt.Pons[i].Onus {
+			if err := onu.InternalState.Event("initialize"); err != nil {
+				log.Errorf("Error initializing ONU: %v", err)
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
-func newOltServer(o OltDevice) error {
+// callback for disable state entry
+func (o *OltDevice) disableOlt() error {
+
+	// disable all onus
+	for i := range o.Pons {
+		for _, onu := range o.Pons[i].Onus {
+			// NOTE order of these is important.
+			onu.OperState.Event("disable")
+			onu.InternalState.Event("disable")
+		}
+	}
+
+	// TODO handle hard poweroff (i.e. no indications sent to Voltha) vs soft poweroff
+	if err := StopOltServer(); err != nil {
+		return err
+	}
+
+	// terminate the OLT's processOltMessages go routine
+	close(o.channel)
+	// terminate the OLT's processNniPacketIns go routine
+	close(o.nniPktInChannel)
+	return nil
+}
+
+func (o *OltDevice) RestartOLT() error {
+	oltLogger.Infof("Simulating OLT restart... (%ds)", OltRebootDelay)
+
+	// transition internal state to disable
+	if !o.InternalState.Is("disabled") {
+		if err := o.InternalState.Event("disable"); err != nil {
+			log.Errorf("Error disabling OLT: %v", err)
+			return err
+		}
+	}
+
+	time.Sleep(OltRebootDelay * time.Second)
+
+	if err := o.InternalState.Event("initialize"); err != nil {
+		log.Errorf("Error initializing OLT: %v", err)
+		return err
+	}
+	oltLogger.Info("OLT restart completed")
+	return nil
+}
+
+// newOltServer launches a new grpc server for OpenOLT
+func newOltServer() (*grpc.Server, error) {
 	// TODO make configurable
 	address := "0.0.0.0:50060"
 	lis, err := net.Listen("tcp", address)
@@ -160,51 +240,50 @@
 		oltLogger.Fatalf("OLT failed to listen: %v", err)
 	}
 	grpcServer := grpc.NewServer()
+
+	o := GetOLT()
 	openolt.RegisterOpenoltServer(grpcServer, o)
 
-	wg := sync.WaitGroup{}
-	wg.Add(1)
+	reflection.Register(grpcServer)
 
 	go grpcServer.Serve(lis)
 	oltLogger.Debugf("OLT Listening on: %v", address)
 
-	for {
-		_, ok := <-*o.oltDoneChannel
-		if !ok {
-			// if the olt Channel is closed, stop the gRPC server
-			log.Warnf("Stopping OLT gRPC server")
-			grpcServer.Stop()
-			wg.Done()
-			break
-		}
+	return grpcServer, nil
+}
+
+// StopOltServer stops the OpenOLT grpc server
+func StopOltServer() error {
+	// TODO handle poweroff vs graceful shutdown
+	if oltServer != nil {
+		log.Warnf("Stopping OLT gRPC server")
+		oltServer.Stop()
+		oltServer = nil
 	}
-
-	wg.Wait()
-
 	return nil
 }
 
 // Device Methods
 
-func (o OltDevice) Enable(stream openolt.Openolt_EnableIndicationServer) error {
-
+// Enable implements the OpenOLT EnableIndicationServer functionality
+func (o *OltDevice) Enable(stream openolt.Openolt_EnableIndicationServer) error {
 	oltLogger.Debug("Enable OLT called")
 
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 
-	// create a Channel for all the OLT events
-	go o.processOltMessages(stream)
-	go o.processNniPacketIns(stream)
+	// create Go routine to process all OLT events
+	go o.processOltMessages(stream, &wg)
+	go o.processNniPacketIns(stream, &wg)
 
 	// enable the OLT
-	olt_msg := Message{
+	oltMsg := Message{
 		Type: OltIndication,
 		Data: OltIndicationMessage{
 			OperState: UP,
 		},
 	}
-	o.channel <- olt_msg
+	o.channel <- oltMsg
 
 	// send NNI Port Indications
 	for _, nni := range o.Nnis {
@@ -217,9 +296,11 @@
 		}
 		o.channel <- msg
 	}
+
 	go o.processOmciMessages()
+
 	// send PON Port indications
-	for _, pon := range o.Pons {
+	for i, pon := range o.Pons {
 		msg := Message{
 			Type: PonIndication,
 			Data: PonIndicationMessage{
@@ -229,29 +310,24 @@
 		}
 		o.channel <- msg
 
-		for _, onu := range pon.Onus {
+		for _, onu := range o.Pons[i].Onus {
 			go onu.ProcessOnuMessages(stream, nil)
-			// FIXME move the message generation in the state transition
-			// from here only invoke the state transition
-			msg := Message{
-				Type: OnuDiscIndication,
-				Data: OnuDiscIndicationMessage{
-					Onu:       onu,
-					OperState: UP,
-				},
+			if err := onu.InternalState.Event("discover"); err != nil {
+				log.Errorf("Error discover ONU: %v", err)
+				return err
 			}
-			onu.Channel <- msg
 		}
 	}
 
+	oltLogger.Warn("Enable OLT Done")
 	wg.Wait()
 	return nil
 }
 
-func (o OltDevice) processOmciMessages() {
+func (o *OltDevice) processOmciMessages() {
 	ch := omcisim.GetChannel()
 
-	oltLogger.Debug("Started OMCI Indication Channel")
+	oltLogger.Debug("Starting OMCI Indication Channel")
 
 	for message := range ch {
 		onuId := message.Data.OnuId
@@ -284,7 +360,7 @@
 	return nil, errors.New(fmt.Sprintf("Cannot find NniPort with id %d in OLT %d", id, o.ID))
 }
 
-func (o OltDevice) sendOltIndication(msg OltIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
+func (o *OltDevice) sendOltIndication(msg OltIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
 	data := &openolt.Indication_OltInd{OltInd: &openolt.OltIndication{OperState: msg.OperState.String()}}
 	if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
 		oltLogger.Errorf("Failed to send Indication_OltInd: %v", err)
@@ -295,7 +371,7 @@
 	}).Debug("Sent Indication_OltInd")
 }
 
-func (o OltDevice) sendNniIndication(msg NniIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
+func (o *OltDevice) sendNniIndication(msg NniIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
 	nni, _ := o.getNniById(msg.NniPortID)
 	nni.OperState.Event("enable")
 	// NOTE Operstate may need to be an integer
@@ -316,7 +392,7 @@
 	}).Debug("Sent Indication_IntfOperInd for NNI")
 }
 
-func (o OltDevice) sendPonIndication(msg PonIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
+func (o *OltDevice) sendPonIndication(msg PonIndicationMessage, stream openolt.Openolt_EnableIndicationServer) {
 	pon, _ := o.GetPonById(msg.PonPortID)
 	pon.OperState.Event("enable")
 	discoverData := &openolt.Indication_IntfInd{IntfInd: &openolt.IntfIndication{
@@ -350,8 +426,9 @@
 	}).Debug("Sent Indication_IntfOperInd for PON")
 }
 
-func (o OltDevice) processOltMessages(stream openolt.Openolt_EnableIndicationServer) {
-	oltLogger.Debug("Started OLT Indication Channel")
+// processOltMessages handles messages received over the OpenOLT interface
+func (o *OltDevice) processOltMessages(stream openolt.Openolt_EnableIndicationServer, wg *sync.WaitGroup) {
+	oltLogger.Debug("Starting OLT Indication Channel")
 	for message := range o.channel {
 
 		oltLogger.WithFields(log.Fields{
@@ -381,9 +458,12 @@
 		}
 
 	}
+	wg.Done()
+	oltLogger.Warn("Stopped handling OLT Indication Channel")
 }
 
-func (o OltDevice) processNniPacketIns(stream openolt.Openolt_EnableIndicationServer) {
+// processNniPacketIns handles messages received over the NNI interface
+func (o *OltDevice) processNniPacketIns(stream openolt.Openolt_EnableIndicationServer, wg *sync.WaitGroup) {
 	oltLogger.WithFields(log.Fields{
 		"nniChannel": o.nniPktInChannel,
 	}).Debug("Started NNI Channel")
@@ -436,6 +516,10 @@
 			"OnuSn":    onu.Sn(),
 		}).Tracef("Sent PktInd indication")
 	}
+	wg.Done()
+	oltLogger.WithFields(log.Fields{
+		"nniChannel": o.nniPktInChannel,
+	}).Warn("Stopped handling NNI Channel")
 }
 
 // returns an ONU with a given Serial Number
@@ -527,13 +611,13 @@
 
 func (o OltDevice) DisableOlt(context.Context, *openolt.Empty) (*openolt.Empty, error) {
 	// NOTE when we disable the OLT should we disable NNI, PONs and ONUs altogether?
-	olt_msg := Message{
+	oltMsg := Message{
 		Type: OltIndication,
 		Data: OltIndicationMessage{
 			OperState: DOWN,
 		},
 	}
-	o.channel <- olt_msg
+	o.channel <- oltMsg
 	return new(openolt.Empty), nil
 }
 
@@ -542,7 +626,7 @@
 	return new(openolt.Empty), nil
 }
 
-func (o OltDevice) EnableIndication(_ *openolt.Empty, stream openolt.Openolt_EnableIndicationServer) error {
+func (o *OltDevice) EnableIndication(_ *openolt.Empty, stream openolt.Openolt_EnableIndicationServer) error {
 	oltLogger.WithField("oltId", o.ID).Info("OLT receives EnableIndication call from VOLTHA")
 	o.Enable(stream)
 	return nil
@@ -703,11 +787,8 @@
 }
 
 func (o OltDevice) Reboot(context.Context, *openolt.Empty) (*openolt.Empty, error) {
-	defer func() {
-		oltLogger.Info("Shutting Down")
-		close(*o.oltDoneChannel)
-		close(*o.apiDoneChannel)
-	}()
+	oltLogger.Info("Shutting down")
+	o.RestartOLT()
 	return new(openolt.Empty), nil
 }
 
@@ -719,7 +800,7 @@
 func (o OltDevice) UplinkPacketOut(context context.Context, packet *openolt.UplinkPacket) (*openolt.Empty, error) {
 	pkt := gopacket.NewPacket(packet.Pkt, layers.LayerTypeEthernet, gopacket.Default)
 
-	sendNniPacket(pkt)
+	o.Nnis[0].sendNniPacket(pkt) // FIXME we are assuming we have only one NNI
 	// NOTE should we return an error if sendNniPakcet fails?
 	return new(openolt.Empty), nil
 }
diff --git a/internal/bbsim/devices/onu.go b/internal/bbsim/devices/onu.go
index bf5140b..a915962 100644
--- a/internal/bbsim/devices/onu.go
+++ b/internal/bbsim/devices/onu.go
@@ -20,6 +20,10 @@
 	"context"
 	"errors"
 	"fmt"
+	"net"
+
+	"time"
+
 	"github.com/cboling/omci"
 	"github.com/google/gopacket/layers"
 	"github.com/looplab/fsm"
@@ -31,8 +35,6 @@
 	omcisim "github.com/opencord/omci-sim"
 	"github.com/opencord/voltha-protos/v2/go/openolt"
 	log "github.com/sirupsen/logrus"
-	"net"
-	"time"
 )
 
 var onuLogger = log.WithFields(log.Fields{
@@ -87,7 +89,6 @@
 		Dhcp:             dhcp,
 		HwAddress:        net.HardwareAddr{0x2e, 0x60, 0x70, 0x13, byte(pon.ID), byte(id)},
 		PortNo:           0,
-		Channel:          make(chan Message, 2048),
 		tid:              0x1,
 		hpTid:            0x8000,
 		seqNumber:        0,
@@ -109,12 +110,13 @@
 		"created",
 		fsm.Events{
 			// DEVICE Lifecycle
-			{Name: "discover", Src: []string{"created"}, Dst: "discovered"},
+			{Name: "initialize", Src: []string{"created", "disabled"}, Dst: "initialized"},
+			{Name: "discover", Src: []string{"initialized"}, Dst: "discovered"},
 			{Name: "enable", Src: []string{"discovered", "disabled"}, Dst: "enabled"},
 			{Name: "receive_eapol_flow", Src: []string{"enabled", "gem_port_added"}, Dst: "eapol_flow_received"},
 			{Name: "add_gem_port", Src: []string{"enabled", "eapol_flow_received"}, Dst: "gem_port_added"},
-			// NOTE should disabled state be diffente for oper_disabled (emulating an error) and admin_disabled (received a disabled call via VOLTHA)?
-			{Name: "disable", Src: []string{"eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed"}, Dst: "disabled"},
+			// NOTE should disabled state be different for oper_disabled (emulating an error) and admin_disabled (received a disabled call via VOLTHA)?
+			{Name: "disable", Src: []string{"enabled", "eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed"}, Dst: "disabled"},
 			// EAPOL
 			{Name: "start_auth", Src: []string{"eapol_flow_received", "gem_port_added", "eap_start_sent", "eap_response_identity_sent", "eap_response_challenge_sent", "eap_response_success_received", "auth_failed", "dhcp_ack_received", "dhcp_failed"}, Dst: "auth_started"},
 			{Name: "eap_start_sent", Src: []string{"auth_started"}, Dst: "eap_start_sent"},
@@ -130,13 +132,27 @@
 			{Name: "dhcp_failed", Src: []string{"dhcp_started", "dhcp_discovery_sent", "dhcp_request_sent"}, Dst: "dhcp_failed"},
 			// BBR States
 			// TODO add start OMCI state
-			{Name: "send_eapol_flow", Src: []string{"created"}, Dst: "eapol_flow_sent"},
+			{Name: "send_eapol_flow", Src: []string{"initialized"}, Dst: "eapol_flow_sent"},
 			{Name: "send_dhcp_flow", Src: []string{"eapol_flow_sent"}, Dst: "dhcp_flow_sent"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) {
 				o.logStateChange(e.Src, e.Dst)
 			},
+			"enter_initialized": func(e *fsm.Event) {
+				// create new channel for ProcessOnuMessages Go routine
+				o.Channel = make(chan Message, 2048)
+			},
+			"enter_discovered": func(e *fsm.Event) {
+				msg := Message{
+					Type: OnuDiscIndication,
+					Data: OnuDiscIndicationMessage{
+						Onu:       &o,
+						OperState: UP,
+					},
+				}
+				o.Channel <- msg
+			},
 			"enter_enabled": func(event *fsm.Event) {
 				msg := Message{
 					Type: OnuIndication,
@@ -158,6 +174,8 @@
 					},
 				}
 				o.Channel <- msg
+				// terminate the ONU's ProcessOnuMessages Go routine
+				close(o.Channel)
 			},
 			"enter_auth_started": func(e *fsm.Event) {
 				o.logStateChange(e.Src, e.Dst)
@@ -213,6 +231,7 @@
 			},
 		},
 	)
+
 	return &o
 }
 
@@ -224,11 +243,13 @@
 	}).Debugf("Changing ONU InternalState from %s to %s", src, dst)
 }
 
+// ProcessOnuMessages starts indication channel for each ONU
 func (o *Onu) ProcessOnuMessages(stream openolt.Openolt_EnableIndicationServer, client openolt.OpenoltClient) {
 	onuLogger.WithFields(log.Fields{
-		"onuID": o.ID,
-		"onuSN": o.Sn(),
-	}).Debug("Started ONU Indication Channel")
+		"onuID":   o.ID,
+		"onuSN":   o.Sn(),
+		"ponPort": o.PonPortID,
+	}).Debug("Starting ONU Indication Channel")
 
 	for message := range o.Channel {
 		onuLogger.WithFields(log.Fields{
@@ -307,6 +328,10 @@
 			onuLogger.Warnf("Received unknown message data %v for type %v in OLT Channel", message.Data, message.Type)
 		}
 	}
+	onuLogger.WithFields(log.Fields{
+		"onuID": o.ID,
+		"onuSN": o.Sn(),
+	}).Debug("Stopped handling ONU Indication Channel")
 }
 
 func (o *Onu) processOmciMessage(message omcisim.OmciChMessage) {
@@ -331,7 +356,7 @@
 	}
 }
 
-func (o *Onu) NewSN(oltid int, intfid uint32, onuid uint32) *openolt.SerialNumber {
+func (o Onu) NewSN(oltid int, intfid uint32, onuid uint32) *openolt.SerialNumber {
 
 	sn := new(openolt.SerialNumber)
 
@@ -540,11 +565,11 @@
 		msg.Flow.Classifier.SrcPort == uint32(68) &&
 		msg.Flow.Classifier.DstPort == uint32(67) {
 
-		// keep track that we reveived the DHCP Flows so that we can transition the state to dhcp_started
+		// keep track that we received the DHCP Flows so that we can transition the state to dhcp_started
 		o.DhcpFlowReceived = true
 
 		if o.Dhcp == true {
-			// NOTE we are receiving mulitple DHCP flows but we shouldn't call the transition multiple times
+			// NOTE we are receiving multiple DHCP flows but we shouldn't call the transition multiple times
 			if err := o.InternalState.Event("start_dhcp"); err != nil {
 				log.Errorf("Can't go to dhcp_started: %v", err)
 			}
diff --git a/internal/bbsim/devices/onu_state_machine_test.go b/internal/bbsim/devices/onu_state_machine_test.go
index c169c12..4ceded9 100644
--- a/internal/bbsim/devices/onu_state_machine_test.go
+++ b/internal/bbsim/devices/onu_state_machine_test.go
@@ -17,14 +17,14 @@
 package devices
 
 import (
-	"gotest.tools/assert"
 	"testing"
+
+	"gotest.tools/assert"
 )
 
 func Test_Onu_StateMachine_enable(t *testing.T) {
 	onu := createTestOnu()
-
-	assert.Equal(t, onu.InternalState.Current(), "created")
+	assert.Equal(t, onu.InternalState.Current(), "initialized")
 	onu.InternalState.Event("discover")
 	assert.Equal(t, onu.InternalState.Current(), "discovered")
 	onu.InternalState.Event("enable")
diff --git a/internal/bbsim/devices/onu_test_helpers.go b/internal/bbsim/devices/onu_test_helpers.go
index 2f9926f..8e071c1 100644
--- a/internal/bbsim/devices/onu_test_helpers.go
+++ b/internal/bbsim/devices/onu_test_helpers.go
@@ -129,5 +129,7 @@
 		ID: 1,
 	}
 	onu := CreateONU(olt, pon, 1, 900, 900, false, false)
+	// NOTE we need this in order to create the OnuChannel
+	onu.InternalState.Event("initialize")
 	return onu
 }
diff --git a/internal/bbsimctl/commands/olt.go b/internal/bbsimctl/commands/olt.go
index fa847ff..9ff6a30 100644
--- a/internal/bbsimctl/commands/olt.go
+++ b/internal/bbsimctl/commands/olt.go
@@ -40,10 +40,19 @@
 
 type OltPONs struct{}
 
+type OltShutdown struct{}
+
+type OltPoweron struct{}
+
+type OltReboot struct{}
+
 type oltOptions struct {
-	Get OltGet  `command:"get"`
-	NNI OltNNIs `command:"nnis"`
-	PON OltPONs `command:"pons"`
+	Get      OltGet      `command:"get"`
+	NNI      OltNNIs     `command:"nnis"`
+	PON      OltPONs     `command:"pons"`
+	Shutdown OltShutdown `command:"shutdown"`
+	Poweron  OltPoweron  `command:"poweron"`
+	Reboot   OltReboot   `command:"reboot"`
 }
 
 func RegisterOltCommands(parser *flags.Parser) {
@@ -107,3 +116,57 @@
 
 	return nil
 }
+
+func (o *OltShutdown) Execute(args []string) error {
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+
+	res, err := client.ShutdownOlt(ctx, &pb.Empty{})
+
+	if err != nil {
+		log.Fatalf("Cannot shut down OLT: %v", err)
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf("[Status: %d] %s", res.StatusCode, res.Message))
+	return nil
+}
+
+func (o *OltPoweron) Execute(args []string) error {
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+
+	res, err := client.PoweronOlt(ctx, &pb.Empty{})
+
+	if err != nil {
+		log.Fatalf("Cannot power on OLT: %v", err)
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf("[Status: %d] %s", res.StatusCode, res.Message))
+	return nil
+}
+
+func (o *OltReboot) Execute(args []string) error {
+	client, conn := connect()
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), config.GlobalConfig.Grpc.Timeout)
+	defer cancel()
+
+	res, err := client.RebootOlt(ctx, &pb.Empty{})
+
+	if err != nil {
+		log.Fatalf("Cannot reboot OLT: %v", err)
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf("[Status: %d] %s", res.StatusCode, res.Message))
+	return nil
+}