[VOL-3716] Retry querying a service if the connections times out
Change-Id: Ieec109c04d42183b10f5c2c49cc3d26ed9cd4602
diff --git a/VERSION b/VERSION
index b1e80bb..845639e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1.3
+0.1.4
diff --git a/internal/core/watcher.go b/internal/core/watcher.go
index b80b619..eb328e1 100644
--- a/internal/core/watcher.go
+++ b/internal/core/watcher.go
@@ -28,8 +28,11 @@
"k8s.io/client-go/kubernetes"
"net/http"
"sync"
+ "time"
)
+const attemptLimit = 10
+
type Watcher struct {
client *kubernetes.Clientset
store *Store
@@ -111,19 +114,28 @@
func (w *Watcher) handleServices(ctx context.Context, services *v1.ServiceList) {
// TODO if a service is removed we'll want to remove the related entries
for _, service := range services.Items {
- if err := w.queryService(ctx, service); err != nil {
- logger.Errorw(ctx, "error-while-reading-from-service", log.Fields{"error": err.Error()})
- }
+ go func(service v1.Service) {
+ if err := w.queryService(ctx, service, 0); err != nil {
+ logger.Errorw(ctx, "error-while-reading-from-service", log.Fields{"error": err.Error()})
+ }
+ }(service)
}
}
-func (w *Watcher) queryService(ctx context.Context, service v1.Service) error {
+func (w *Watcher) queryService(ctx context.Context, service v1.Service, attempt int) error {
endpoint := fmt.Sprintf("%s.%s.svc:%d", service.Name, service.Namespace, w.config.BBsimSadisPort)
logger.Infow(ctx, "querying-service", log.Fields{"endpoint": endpoint})
res, err := http.Get(fmt.Sprintf("http://%s/v2/static", endpoint))
if err != nil {
+ if attempt < attemptLimit {
+ logger.Warnw(ctx, "error-while-reading-from-service-retrying", log.Fields{"error": err.Error()})
+ // if there is an error and we have attempt left just retry later
+ time.Sleep(1 * time.Second)
+ return w.queryService(ctx, service, attempt+1)
+ }
+
return err
}