[VOL-4178, VOL-3952] This commit removes flows/groups/meters persistency

This commit removes flows/groups/meters persistency from rw-core.
As part of this change, it also fixes a bug where devices were not
being loaded on an rw-core restart.  This is a necessary condition
to allow the non-persistency of flows/groups/meters to work.

This commit also renames "loader" to "cache" for the flows/groups/
meters to differentiate between data that is loaded from the KV
store and the one in cache.

Change-Id: Ib14e1450021abe30b17673c2910768fb740dba51
diff --git a/rw_core/core/device/logical_agent_flow.go b/rw_core/core/device/logical_agent_flow.go
index 46d4ffb..808dfd3 100644
--- a/rw_core/core/device/logical_agent_flow.go
+++ b/rw_core/core/device/logical_agent_flow.go
@@ -36,10 +36,10 @@
 
 // listLogicalDeviceFlows returns logical device flows
 func (agent *LogicalAgent) listLogicalDeviceFlows() map[uint64]*ofp.OfpFlowStats {
-	flowIDs := agent.flowLoader.ListIDs()
+	flowIDs := agent.flowCache.ListIDs()
 	flows := make(map[uint64]*ofp.OfpFlowStats, len(flowIDs))
 	for flowID := range flowIDs {
-		if flowHandle, have := agent.flowLoader.Lock(flowID); have {
+		if flowHandle, have := agent.flowCache.Lock(flowID); have {
 			flows[flowID] = flowHandle.GetReadOnly()
 			flowHandle.Unlock()
 		}
@@ -104,7 +104,7 @@
 	var flowToReplace *ofp.OfpFlowStats
 
 	//if flow is not found in the map, create a new entry, otherwise get the existing one.
-	flowHandle, created, err := agent.flowLoader.LockOrCreate(ctx, flow)
+	flowHandle, created, err := agent.flowCache.LockOrCreate(ctx, flow)
 	if err != nil {
 		return changed, updated, err
 	}
@@ -146,10 +146,10 @@
 			return changed, updated, err
 		}
 
-		groupIDs := agent.groupLoader.ListIDs()
+		groupIDs := agent.groupCache.ListIDs()
 		groups := make(map[uint32]*ofp.OfpGroupEntry, len(groupIDs))
 		for groupID := range groupIDs {
-			if groupHandle, have := agent.groupLoader.Lock(groupID); have {
+			if groupHandle, have := agent.groupCache.Lock(groupID); have {
 				groups[groupID] = groupHandle.GetReadOnly()
 				groupHandle.Unlock()
 			}
@@ -214,7 +214,7 @@
 func (agent *LogicalAgent) revertAddedFlows(ctx context.Context, mod *ofp.OfpFlowMod, addedFlow *ofp.OfpFlowStats, replacedFlow *ofp.OfpFlowStats, deviceRules *fu.DeviceRules, metadata *voltha.FlowMetadata) error {
 	logger.Debugw(ctx, "revert-flow-add", log.Fields{"added-flow": addedFlow, "replaced-flow": replacedFlow, "device-rules": deviceRules, "metadata": metadata})
 
-	flowHandle, have := agent.flowLoader.Lock(addedFlow.Id)
+	flowHandle, have := agent.flowCache.Lock(addedFlow.Id)
 	if !have {
 		// Not found - do nothing
 		logger.Debugw(ctx, "flow-not-found", log.Fields{"added-flow": addedFlow})
@@ -271,14 +271,14 @@
 	if err != nil {
 		return err
 	}
-	if handle, have := agent.flowLoader.Lock(fs.Id); have {
+	if handle, have := agent.flowCache.Lock(fs.Id); have {
 		toDelete[fs.Id] = handle.GetReadOnly()
 		handle.Unlock()
 	}
 
 	// search through all the flows
-	for flowID := range agent.flowLoader.ListIDs() {
-		if flowHandle, have := agent.flowLoader.Lock(flowID); have {
+	for flowID := range agent.flowCache.ListIDs() {
+		if flowHandle, have := agent.flowCache.Lock(flowID); have {
 			if flow := flowHandle.GetReadOnly(); fu.FlowMatchesMod(flow, mod) {
 				toDelete[flow.Id] = flow
 			}
@@ -291,7 +291,7 @@
 		logger.Debugw(ctx, "flow-delete", log.Fields{"logical-device-id": agent.logicalDeviceID, "to-delete": len(toDelete)})
 
 		for _, flow := range toDelete {
-			if flowHandle, have := agent.flowLoader.Lock(flow.Id); have {
+			if flowHandle, have := agent.flowCache.Lock(flow.Id); have {
 				// TODO: Flow should only be updated if meter is updated, and meter should only be updated if flow is updated
 				//       currently an error while performing the second operation will leave an inconsistent state in kv.
 				//       This should be a single atomic operation down to the kv.
@@ -318,8 +318,8 @@
 		}
 
 		groups := make(map[uint32]*ofp.OfpGroupEntry)
-		for groupID := range agent.groupLoader.ListIDs() {
-			if groupHandle, have := agent.groupLoader.Lock(groupID); have {
+		for groupID := range agent.groupCache.ListIDs() {
+			if groupHandle, have := agent.groupCache.Lock(groupID); have {
 				groups[groupID] = groupHandle.GetReadOnly()
 				groupHandle.Unlock()
 			}
@@ -386,7 +386,7 @@
 		return err
 	}
 	logger.Debugw(ctx, "flow-id-in-flow-delete-strict", log.Fields{"flow-id": flow.Id})
-	flowHandle, have := agent.flowLoader.Lock(flow.Id)
+	flowHandle, have := agent.flowCache.Lock(flow.Id)
 	if !have {
 		logger.Debugw(ctx, "skipping-flow-delete-strict-request-no-flow-found", log.Fields{"flow-mod": mod})
 		return nil
@@ -394,8 +394,8 @@
 	defer flowHandle.Unlock()
 
 	groups := make(map[uint32]*ofp.OfpGroupEntry)
-	for groupID := range agent.groupLoader.ListIDs() {
-		if groupHandle, have := agent.groupLoader.Lock(groupID); have {
+	for groupID := range agent.groupCache.ListIDs() {
+		if groupHandle, have := agent.groupCache.Lock(groupID); have {
 			groups[groupID] = groupHandle.GetReadOnly()
 			groupHandle.Unlock()
 		}
@@ -487,8 +487,8 @@
 
 func (agent *LogicalAgent) deleteFlowsHavingMeter(ctx context.Context, meterID uint32) error {
 	logger.Infow(ctx, "delete-flows-matching-meter", log.Fields{"meter": meterID})
-	for flowID := range agent.flowLoader.ListIDs() {
-		if flowHandle, have := agent.flowLoader.Lock(flowID); have {
+	for flowID := range agent.flowCache.ListIDs() {
+		if flowHandle, have := agent.flowCache.Lock(flowID); have {
 			if flowMeterID := fu.GetMeterIdFromFlow(flowHandle.GetReadOnly()); flowMeterID != 0 && flowMeterID == meterID {
 				if err := flowHandle.Delete(ctx); err != nil {
 					//TODO: Think on carrying on and deleting the remaining flows, instead of returning.
@@ -506,8 +506,8 @@
 func (agent *LogicalAgent) deleteFlowsHavingGroup(ctx context.Context, groupID uint32) (map[uint64]*ofp.OfpFlowStats, error) {
 	logger.Infow(ctx, "delete-flows-matching-group", log.Fields{"group-id": groupID})
 	flowsRemoved := make(map[uint64]*ofp.OfpFlowStats)
-	for flowID := range agent.flowLoader.ListIDs() {
-		if flowHandle, have := agent.flowLoader.Lock(flowID); have {
+	for flowID := range agent.flowCache.ListIDs() {
+		if flowHandle, have := agent.flowCache.Lock(flowID); have {
 			if flow := flowHandle.GetReadOnly(); fu.FlowHasOutGroup(flow, groupID) {
 				if err := flowHandle.Delete(ctx); err != nil {
 					return nil, err