VOL-2362 - add exponential backoff around enabling indications
Change-Id: I212312f297a0ca5ae33a4250bac1008e9be2f2c2
diff --git a/adaptercore/device_handler.go b/adaptercore/device_handler.go
index 80d706a..7e09fc8 100644
--- a/adaptercore/device_handler.go
+++ b/adaptercore/device_handler.go
@@ -31,6 +31,7 @@
"google.golang.org/grpc/codes"
+ backoff "github.com/cenkalti/backoff/v3"
"github.com/gogo/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/opencord/voltha-lib-go/v2/pkg/adapters/adapterif"
@@ -278,10 +279,27 @@
dh.lockDevice.Unlock()
}
+ // Create an exponential backoff around re-enabling indications. The
+ // maximum elapsed time for the back off is set to 0 so that we will
+ // continue to retry. The max interval defaults to 1m, but is set
+ // here for code clarity
+ indicationBackoff := backoff.NewExponentialBackOff()
+ indicationBackoff.MaxElapsedTime = 0
+ indicationBackoff.MaxInterval = 1 * time.Minute
for {
indication, err := indications.Recv()
if err == io.EOF {
log.Infow("EOF for indications", log.Fields{"err": err})
+ // Use an exponential back off to prevent getting into a tight loop
+ duration := indicationBackoff.NextBackOff()
+ if duration == backoff.Stop {
+ // If we reach a maximum then warn and reset the backoff
+ // timer and keep attempting.
+ log.Warnw("Maximum indication backoff reached, resetting backoff timer",
+ log.Fields{"max_indication_backoff": indicationBackoff.MaxElapsedTime})
+ indicationBackoff.Reset()
+ }
+ time.Sleep(indicationBackoff.NextBackOff())
indications, err = dh.Client.EnableIndication(context.Background(), new(oop.Empty))
if err != nil {
log.Errorw("Failed to read indications", log.Fields{"err": err})
@@ -299,6 +317,8 @@
dh.transitionMap.Handle(DeviceInit)
break
}
+ // Reset backoff if we have a successful receive
+ indicationBackoff.Reset()
dh.lockDevice.RLock()
adminState := dh.adminState
dh.lockDevice.RUnlock()