[VOL-3548] Differentiate between OLT gRPC channel
disconnect and device reboot
Change-Id: Id52a3cf7b1a69f20e8b061d0655015857fa19ae5
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index 1e86007..8b3947e 100644
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -101,6 +101,9 @@
// Slice of channels. Each channel in slice, index by (mcast-group-id modulo MaxNumOfGroupHandlerChannels)
// A go routine per index, waits on a unique channel for incoming mcast flow or group (add/modify/remove).
incomingMcastFlowOrGroup []chan McastFlowOrGroupControlBlock
+
+ adapterPreviouslyConnected bool
+ agentPreviouslyConnected bool
}
//OnuDevice represents ONU related info
@@ -642,6 +645,18 @@
raisedTs := time.Now().UnixNano()
go dh.eventMgr.oltCommunicationEvent(ctx, dh.device, raisedTs)
+ //check adapter and agent reconcile status
+ //reboot olt if needed (olt disconnection case)
+ if dh.adapterPreviouslyConnected != dh.agentPreviouslyConnected {
+ logger.Warnw(ctx, "different-reconcile-status-between-adapter-and-agent-rebooting-device",
+ log.Fields{
+ "device-id": dh.device.Id,
+ "adapter-status": dh.adapterPreviouslyConnected,
+ "agent-status": dh.agentPreviouslyConnected,
+ })
+ _ = dh.RebootDevice(ctx, dh.device)
+ }
+
return nil
}
@@ -813,6 +828,7 @@
return olterrors.NewErrAdapter("populate-device-info-failed", log.Fields{"device-id": dh.device.Id}, err)
}
dh.totalPonPorts = deviceInfo.GetPonPorts()
+ dh.agentPreviouslyConnected = deviceInfo.PreviouslyConnected
// Instantiate resource manager
if dh.resourceMgr = rsrcMgr.NewResourceMgr(ctx, dh.device.Id, dh.openOLT.KVStoreAddress, dh.openOLT.KVStoreType, dh.device.Type, deviceInfo, dh.cm.Backend.PathPrefix); dh.resourceMgr == nil {
@@ -2044,6 +2060,9 @@
}
dh.lockDevice.RUnlock()
+ //reset adapter reconcile flag
+ dh.adapterPreviouslyConnected = false
+
dh.transitionMap.Handle(ctx, DeviceInit)
}
diff --git a/internal/pkg/core/device_handler_test.go b/internal/pkg/core/device_handler_test.go
index 9770f15..012ce51 100644
--- a/internal/pkg/core/device_handler_test.go
+++ b/internal/pkg/core/device_handler_test.go
@@ -1192,3 +1192,62 @@
})
}
}
+
+func TestDeviceHandler_TestReconcileStatus(t *testing.T) {
+
+ // olt disconnect (not reboot)
+ dh1 := newMockDeviceHandler()
+ dh1.adapterPreviouslyConnected = false
+ dh1.agentPreviouslyConnected = true
+
+ // adapter restart
+ dh2 := newMockDeviceHandler()
+ dh2.Client = &mocks.MockOpenoltClient{}
+ dh2.adapterPreviouslyConnected = true
+ dh2.agentPreviouslyConnected = true
+
+ // first connection or olt restart
+ dh3 := newMockDeviceHandler()
+ dh3.Client = &mocks.MockOpenoltClient{}
+ dh3.adapterPreviouslyConnected = false
+ dh3.agentPreviouslyConnected = false
+
+ // olt and adapter restart at the same time (first case)
+ dh4 := newMockDeviceHandler()
+ dh4.Client = &mocks.MockOpenoltClient{}
+ dh4.adapterPreviouslyConnected = true
+ dh4.agentPreviouslyConnected = false
+
+ // adapter restart and olt disconnect at the same time
+ dh5 := newMockDeviceHandler()
+ dh5.Client = &mocks.MockOpenoltClient{}
+ dh5.adapterPreviouslyConnected = true
+ dh5.agentPreviouslyConnected = true
+
+ tests := []struct {
+ name string
+ devicehandler *DeviceHandler
+ expectedRestart bool
+ wantErr bool
+ }{
+ {"dostateup-1", dh1, true, false},
+ {"dostateup-2", dh2, false, false},
+ {"dostateup-3", dh3, false, false},
+ {"dostateup-4", dh4, true, false},
+ {"dostateup-5", dh5, false, false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := tt.devicehandler.doStateUp(ctx); (err != nil) != tt.wantErr {
+ t.Logf("DeviceHandler.doStateUp() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ tt.devicehandler.stopCollector <- true //stop the stat collector invoked from doStateUp
+ isRestarted := tt.devicehandler.Client.(*mocks.MockOpenoltClient).IsRestarted
+ if tt.expectedRestart != isRestarted {
+ t.Errorf("olt-reboot-failed expected= %v, got= %v", tt.expectedRestart, isRestarted)
+ }
+ })
+ }
+}
diff --git a/internal/pkg/core/openolt.go b/internal/pkg/core/openolt.go
index a590846..d7e5b5a 100644
--- a/internal/pkg/core/openolt.go
+++ b/internal/pkg/core/openolt.go
@@ -183,6 +183,7 @@
var handler *DeviceHandler
if handler = oo.getDeviceHandler(device.Id); handler == nil {
handler := NewDeviceHandler(oo.coreProxy, oo.adapterProxy, oo.eventProxy, device, oo, oo.configManager)
+ handler.adapterPreviouslyConnected = true
oo.addDeviceHandlerToMap(handler)
handler.transitionMap = NewTransitionMap(handler)
handler.transitionMap.Handle(ctx, DeviceInit)