VOL-3503 Add a device operational state of RECONCILING

Change-Id: I55dad67a24acdfac0af9448e6f19ec9d35edc39e
diff --git a/rw_core/core/device/agent_image.go b/rw_core/core/device/agent_image.go
index 3781540..94a8706 100644
--- a/rw_core/core/device/agent_image.go
+++ b/rw_core/core/device/agent_image.go
@@ -38,9 +38,15 @@
 	logger.Debugw(ctx, "download-image", log.Fields{"device-id": agent.deviceID})
 
 	if agent.device.Root {
+		agent.requestQueue.RequestComplete()
 		return nil, status.Errorf(codes.FailedPrecondition, "device-id:%s, is an OLT. Image update "+
 			"not supported by VOLTHA. Use Device Manager or other means", agent.deviceID)
 	}
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
+		return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+			agent.deviceID)
+	}
 
 	device := agent.cloneDeviceWithoutLock()
 	if device.ImageDownloads != nil {
@@ -99,6 +105,12 @@
 	}
 	logger.Debugw(ctx, "cancel-image-download", log.Fields{"device-id": agent.deviceID})
 
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
+		return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+			agent.deviceID)
+	}
+
 	// Update image download state
 	cloned := agent.cloneDeviceWithoutLock()
 	_, index, err := getImage(img, cloned)
@@ -137,6 +149,12 @@
 	}
 	logger.Debugw(ctx, "activate-image", log.Fields{"device-id": agent.deviceID})
 
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
+		return nil, status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+			agent.deviceID)
+	}
+
 	// Update image download state
 	cloned := agent.cloneDeviceWithoutLock()
 	image, index, err := getImage(img, cloned)
@@ -250,6 +268,12 @@
 	}
 	logger.Debugw(ctx, "updating-image-download", log.Fields{"device-id": agent.deviceID, "img": img})
 
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
+		return status.Errorf(codes.FailedPrecondition, "deviceId:%s, Device reconciling or deletion is in progress.",
+			agent.deviceID)
+	}
+
 	// Update the image as well as remove it if the download was cancelled
 	cloned := agent.cloneDeviceWithoutLock()
 	clonedImages := make([]*voltha.ImageDownload, len(cloned.ImageDownloads))
@@ -301,10 +325,14 @@
 	// original context has failed due to timeout , let's open a new one
 	subCtx, cancel := context.WithTimeout(log.WithSpanFromContext(context.Background(), ctx), agent.defaultTimeout)
 	subCtx = coreutils.WithRPCMetadataFromContext(subCtx, ctx)
+	defer cancel()
 
 	if err := agent.requestQueue.WaitForGreenLight(subCtx); err != nil {
 		logger.Errorw(subCtx, "can't obtain lock", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "error": err, "args": reqArgs})
-		cancel()
+		return
+	}
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
 		return
 	}
 	if res, ok := response.(error); ok {
@@ -325,7 +353,6 @@
 
 		if imageFailed == nil {
 			logger.Errorw(subCtx, "can't find image", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "args": reqArgs})
-			cancel()
 			return
 		}
 
@@ -341,10 +368,8 @@
 			logger.Errorw(subCtx, "failed-enable-device-after-image-failure",
 				log.Fields{"rpc": rpc, "device-id": agent.deviceID, "error": res, "args": reqArgs})
 		}
-		cancel()
 	} else {
 		logger.Errorw(subCtx, "rpc-failed-invalid-error", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "args": reqArgs})
-		cancel()
 		return
 	}
 	// TODO: Post failure message onto kafka
@@ -356,6 +381,10 @@
 		logger.Errorw(ctx, "cannot-obtain-lock", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "error": err, "args": reqArgs})
 		return
 	}
+	if !agent.proceedWithRequestNoLock() {
+		agent.requestQueue.RequestComplete()
+		return
+	}
 	logger.Infow(ctx, "rpc-successful", log.Fields{"rpc": rpc, "device-id": agent.deviceID, "response": response, "args": reqArgs})
 	cloned := agent.cloneDeviceWithoutLock()
 	//TODO base this on IMAGE ID when created