[VOL-1914] Correctly rebooting the OLT

Change-Id: I4b97b8bc09c20452ec25fdbe93ed46b2b7f04fdf
diff --git a/VERSION b/VERSION
index 02cc5f6..d169b2f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.0.8-dev
+0.0.8
diff --git a/docs/source/index.rst b/docs/source/index.rst
index fff0bfd..3fb460f 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -12,6 +12,7 @@
 
    operations.rst
    onu-state-machine.rst
+   olt-state-machine.rst
    development-dependencies.rst
    bbr.rst
    bbsimctl.rst
diff --git a/docs/source/olt-state-machine.rst b/docs/source/olt-state-machine.rst
new file mode 100644
index 0000000..d92d6f8
--- /dev/null
+++ b/docs/source/olt-state-machine.rst
@@ -0,0 +1,42 @@
+.. _OLT State Machine:
+
+OLT State Machine
+=================
+
+In ``BBSim`` the device state is created using a state machine
+library: `fsm <https://github.com/looplab/fsm>`__.
+
+Here is a list of possible states for an OLT in BBSim:
+
+.. list-table:: OLT States
+    :header-rows: 1
+
+    * -
+      - Initialized
+      - Enabled
+      - Disabled
+      - Deleted
+    * - Data model is created for OLT, NNIs, PONs and ONUs
+      - Starts the listener on the NNI interface and the DHCP server,
+        Starts the OLT gRPC server,
+        Moves the ONUs to ``initialized`` state
+      - Sends OLT, NNIs and PONs ``UP`` indications
+        Transition the ONUs into ``discovered`` state
+      - Transition the ONUs into ``disabled`` state
+        Sends OLT, NNIs and PONs ``UP`` indications
+      - Stops the OLT gRPC Server
+
+Below is a diagram of the state machine allowed transitions:
+
+.. graphviz::
+
+    digraph {
+        rankdir=TB
+        newrank=true
+        graph [pad="1,1" bgcolor="#cccccc"]
+        node [style=filled]
+
+        created -> initialized -> enabled -> disabled -> deleted
+        disabled -> enabled
+        deleted -> initialized
+    }
\ No newline at end of file
diff --git a/docs/source/onu-state-machine.rst b/docs/source/onu-state-machine.rst
index 38d60ef..3af8f5a 100644
--- a/docs/source/onu-state-machine.rst
+++ b/docs/source/onu-state-machine.rst
@@ -6,7 +6,7 @@
 In ``BBSim`` the device state is created using a state machine
 library: `fsm <https://github.com/looplab/fsm>`__.
 
-Here is a list of possible state transitions in BBSim:
+Here is a list of possible state transitions for an ONU in BBSim:
 
 .. list-table:: ONU States
     :widths: 10 35 10 45
diff --git a/internal/bbsim/devices/olt.go b/internal/bbsim/devices/olt.go
index f17ecd4..bde98cb 100644
--- a/internal/bbsim/devices/olt.go
+++ b/internal/bbsim/devices/olt.go
@@ -96,15 +96,15 @@
 	olt.InternalState = fsm.NewFSM(
 		"created",
 		fsm.Events{
-			{Name: "initialize", Src: []string{"disabled", "created"}, Dst: "initialized"},
+			{Name: "initialize", Src: []string{"created", "deleted"}, Dst: "initialized"},
 			{Name: "enable", Src: []string{"initialized", "disabled"}, Dst: "enabled"},
 			{Name: "disable", Src: []string{"enabled"}, Dst: "disabled"},
+			{Name: "delete", Src: []string{"disabled"}, Dst: "deleted"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) {
 				oltLogger.Debugf("Changing OLT InternalState from %s to %s", e.Src, e.Dst)
 			},
-			"enter_disabled":    func(e *fsm.Event) { olt.disableOlt() },
 			"enter_initialized": func(e *fsm.Event) { olt.InitOlt() },
 		},
 	)
@@ -158,9 +158,10 @@
 func (o *OltDevice) InitOlt() error {
 
 	if oltServer == nil {
-		oltServer, _ = newOltServer()
+		oltServer, _ = o.newOltServer()
 	} else {
-		oltLogger.Warn("OLT server already running.")
+		// FIXME there should never be a server running if we are initializing the OLT
+		oltLogger.Fatal("OLT server already running.")
 	}
 
 	// create new channel for processOltMessages Go routine
@@ -189,20 +190,25 @@
 	return nil
 }
 
-// callback for disable state entry
-func (o *OltDevice) disableOlt() error {
+func (o *OltDevice) RestartOLT() error {
 
-	// disable all onus
-	for i := range o.Pons {
-		for _, onu := range o.Pons[i].Onus {
-			// NOTE order of these is important.
-			onu.OperState.Event("disable")
-			onu.InternalState.Event("disable")
-		}
+	rebootDelay := common.Options.Olt.OltRebootDelay
+
+	oltLogger.WithFields(log.Fields{
+		"oltId": o.ID,
+	}).Infof("Simulating OLT restart... (%ds)", rebootDelay)
+
+	// transition internal state to deleted
+	if err := o.InternalState.Event("delete"); err != nil {
+		oltLogger.WithFields(log.Fields{
+			"oltId": o.ID,
+		}).Errorf("Error deleting OLT: %v", err)
+		return err
 	}
 
 	// TODO handle hard poweroff (i.e. no indications sent to Voltha) vs soft poweroff
-	if err := StopOltServer(); err != nil {
+	time.Sleep(1 * time.Second) // we need to give the OLT the time to respond to all the pending gRPC request before stopping the server
+	if err := o.StopOltServer(); err != nil {
 		return err
 	}
 
@@ -210,33 +216,23 @@
 	close(o.channel)
 	// terminate the OLT's processNniPacketIns go routine
 	close(o.nniPktInChannel)
-	return nil
-}
-
-func (o *OltDevice) RestartOLT() error {
-	rebootDelay := common.Options.Olt.OltRebootDelay
-	oltLogger.Infof("Simulating OLT restart... (%ds)", rebootDelay)
-
-	// transition internal state to disable
-	if !o.InternalState.Is("disabled") {
-		if err := o.InternalState.Event("disable"); err != nil {
-			log.Errorf("Error disabling OLT: %v", err)
-			return err
-		}
-	}
 
 	time.Sleep(time.Duration(rebootDelay) * time.Second)
 
 	if err := o.InternalState.Event("initialize"); err != nil {
-		log.Errorf("Error initializing OLT: %v", err)
+		oltLogger.WithFields(log.Fields{
+			"oltId": o.ID,
+		}).Errorf("Error initializing OLT: %v", err)
 		return err
 	}
-	oltLogger.Info("OLT restart completed")
+	oltLogger.WithFields(log.Fields{
+		"oltId": o.ID,
+	}).Info("OLT restart completed")
 	return nil
 }
 
 // newOltServer launches a new grpc server for OpenOLT
-func newOltServer() (*grpc.Server, error) {
+func (o *OltDevice) newOltServer() (*grpc.Server, error) {
 	address := common.Options.BBSim.OpenOltAddress
 	lis, err := net.Listen("tcp", address)
 	if err != nil {
@@ -244,7 +240,6 @@
 	}
 	grpcServer := grpc.NewServer()
 
-	o := GetOLT()
 	openolt.RegisterOpenoltServer(grpcServer, o)
 
 	reflection.Register(grpcServer)
@@ -256,13 +251,16 @@
 }
 
 // StopOltServer stops the OpenOLT grpc server
-func StopOltServer() error {
+func (o *OltDevice) StopOltServer() error {
 	// TODO handle poweroff vs graceful shutdown
 	if oltServer != nil {
-		log.Warnf("Stopping OLT gRPC server")
+		oltLogger.WithFields(log.Fields{
+			"oltId": o.SerialNumber,
+		}).Warnf("Stopping OLT gRPC server")
 		oltServer.Stop()
 		oltServer = nil
 	}
+
 	return nil
 }
 
@@ -273,7 +271,7 @@
 	oltLogger.Debug("Enable OLT called")
 
 	wg := sync.WaitGroup{}
-	wg.Add(2)
+	wg.Add(3)
 
 	// create Go routine to process all OLT events
 	go o.processOltMessages(stream, &wg)
@@ -322,7 +320,7 @@
 		}
 	}
 
-	oltLogger.Warn("Enable OLT Done")
+	oltLogger.Debug("Enable OLT Done")
 	wg.Wait()
 	return nil
 }
@@ -614,6 +612,47 @@
 
 func (o OltDevice) DisableOlt(context.Context, *openolt.Empty) (*openolt.Empty, error) {
 	// NOTE when we disable the OLT should we disable NNI, PONs and ONUs altogether?
+	oltLogger.WithFields(log.Fields{
+		"oltId": o.ID,
+	}).Info("Disabling OLT")
+
+	for i, pon := range o.Pons {
+		// disable all onus
+		for _, onu := range o.Pons[i].Onus {
+			// NOTE order of these is important.
+			if err := onu.OperState.Event("disable"); err != nil {
+				log.Errorf("Error disabling ONU oper state: %v", err)
+			}
+			if err := onu.InternalState.Event("disable"); err != nil {
+				log.Errorf("Error disabling ONU: %v", err)
+			}
+		}
+
+		// disable PONs
+		msg := Message{
+			Type: PonIndication,
+			Data: PonIndicationMessage{
+				OperState: DOWN,
+				PonPortID: pon.ID,
+			},
+		}
+
+		o.channel <- msg
+	}
+
+	// disable NNI
+	for _, nni := range o.Nnis {
+		msg := Message{
+			Type: NniIndication,
+			Data: NniIndicationMessage{
+				OperState: DOWN,
+				NniPortID: nni.ID,
+			},
+		}
+		o.channel <- msg
+	}
+
+	// disable OLT
 	oltMsg := Message{
 		Type: OltIndication,
 		Data: OltIndicationMessage{
@@ -790,8 +829,10 @@
 }
 
 func (o OltDevice) Reboot(context.Context, *openolt.Empty) (*openolt.Empty, error) {
-	oltLogger.Info("Shutting down")
-	o.RestartOLT()
+	oltLogger.WithFields(log.Fields{
+		"oltId": o.ID,
+	}).Info("Shutting down")
+	go o.RestartOLT()
 	return new(openolt.Empty), nil
 }
 
diff --git a/internal/bbsim/devices/onu.go b/internal/bbsim/devices/onu.go
index ab3c6c2..a938433 100644
--- a/internal/bbsim/devices/onu.go
+++ b/internal/bbsim/devices/onu.go
@@ -411,14 +411,6 @@
 		return
 	}
 
-	if err := o.InternalState.Event("discover"); err != nil {
-		oltLogger.WithFields(log.Fields{
-			"IntfId": o.PonPortID,
-			"OnuSn":  o.Sn(),
-			"OnuId":  o.ID,
-		}).Infof("Failed to transition ONU to discovered state: %s", err.Error())
-	}
-
 	onuLogger.WithFields(log.Fields{
 		"IntfId": msg.Onu.PonPortID,
 		"OnuSn":  msg.Onu.Sn(),