VOL-1774 Etcd Crash Handling

Change-Id: I1eeb726654c3972fd0a4fafae134607e5a810415
diff --git a/db/model/node.go b/db/model/node.go
index 264a9dd..7e703ff 100644
--- a/db/model/node.go
+++ b/db/model/node.go
@@ -41,11 +41,11 @@
 
 	// CRUD functions
 	Add(ctx context.Context, path string, data interface{}, txid string, makeBranch MakeBranchFunction) Revision
-	Get(ctx context.Context, path string, hash string, depth int, deep bool, txid string) interface{}
-	List(ctx context.Context, path string, hash string, depth int, deep bool, txid string) interface{}
+	Get(ctx context.Context, path string, hash string, depth int, deep bool, txid string) (interface{}, error)
+	List(ctx context.Context, path string, hash string, depth int, deep bool, txid string) (interface{}, error)
 	Update(ctx context.Context, path string, data interface{}, strict bool, txid string, makeBranch MakeBranchFunction) Revision
 	Remove(ctx context.Context, path string, txid string, makeBranch MakeBranchFunction) Revision
-	CreateProxy(ctx context.Context, path string, exclusive bool) *Proxy
+	CreateProxy(ctx context.Context, path string, exclusive bool) (*Proxy, error)
 
 	GetProxy() *Proxy
 
@@ -250,7 +250,7 @@
 }
 
 // Get retrieves the data from a node tree that resides at the specified path
-func (n *node) List(ctx context.Context, path string, hash string, depth int, deep bool, txid string) interface{} {
+func (n *node) List(ctx context.Context, path string, hash string, depth int, deep bool, txid string) (interface{}, error) {
 	n.mutex.Lock()
 	defer n.mutex.Unlock()
 
@@ -278,18 +278,23 @@
 
 	var result interface{}
 	var prList []interface{}
-	if pr := rev.LoadFromPersistence(ctx, path, txid, nil); pr != nil {
+
+	pr, err := rev.LoadFromPersistence(ctx, path, txid, nil)
+	if err != nil {
+		log.Errorf("failed-to-load-from-persistence")
+		return nil, err
+	}
+	if pr != nil {
 		for _, revEntry := range pr {
 			prList = append(prList, revEntry.GetData())
 		}
 		result = prList
 	}
-
-	return result
+	return result, nil
 }
 
 // Get retrieves the data from a node tree that resides at the specified path
-func (n *node) Get(ctx context.Context, path string, hash string, depth int, reconcile bool, txid string) interface{} {
+func (n *node) Get(ctx context.Context, path string, hash string, depth int, reconcile bool, txid string) (interface{}, error) {
 	n.mutex.Lock()
 	defer n.mutex.Unlock()
 
@@ -328,13 +333,13 @@
 					"hash": hash,
 					"age":  entryAge,
 				})
-				return proto.Clone(entry.(Revision).GetData().(proto.Message))
+				return proto.Clone(entry.(Revision).GetData().(proto.Message)), nil
 			} else {
 				log.Debugw("cache-entry-expired", log.Fields{"path": path, "hash": hash, "age": entryAge})
 			}
 		} else if result = n.getPath(ctx, rev.GetBranch().GetLatest(), path, depth); result != nil && reflect.ValueOf(result).IsValid() && !reflect.ValueOf(result).IsNil() {
 			log.Debugw("using-rev-tree-entry", log.Fields{"path": path, "hash": hash, "depth": depth, "reconcile": reconcile, "txid": txid})
-			return result
+			return result, nil
 		} else {
 			log.Debugw("not-using-cache-entry", log.Fields{
 				"path": path,
@@ -354,7 +359,10 @@
 	// If we got to this point, we are either trying to reconcile with the db
 	// or we simply failed at getting information from memory
 	if n.Root.KvStore != nil {
-		if pr := rev.LoadFromPersistence(ctx, path, txid, nil); pr != nil && len(pr) > 0 {
+		if pr, err := rev.LoadFromPersistence(ctx, path, txid, nil); err != nil {
+			log.Errorf("failed-to-load-from-persistence")
+			return nil, err
+		} else if len(pr) > 0 {
 			// Did we receive a single or multiple revisions?
 			if len(pr) > 1 {
 				var revs []interface{}
@@ -367,8 +375,7 @@
 			}
 		}
 	}
-
-	return result
+	return result, nil
 }
 
 //getPath traverses the specified path and retrieves the data associated to it
@@ -949,11 +956,11 @@
 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ node Proxy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 // CreateProxy returns a reference to a sub-tree of the data model
-func (n *node) CreateProxy(ctx context.Context, path string, exclusive bool) *Proxy {
+func (n *node) CreateProxy(ctx context.Context, path string, exclusive bool) (*Proxy, error) {
 	return n.createProxy(ctx, path, path, n, exclusive)
 }
 
-func (n *node) createProxy(ctx context.Context, path string, fullPath string, parentNode *node, exclusive bool) *Proxy {
+func (n *node) createProxy(ctx context.Context, path string, fullPath string, parentNode *node, exclusive bool) (*Proxy, error) {
 	log.Debugw("node-create-proxy", log.Fields{
 		"node-type":        reflect.ValueOf(n.Type).Type(),
 		"parent-node-type": reflect.ValueOf(parentNode.Type).Type(),
@@ -965,7 +972,7 @@
 		path = path[1:]
 	}
 	if path == "" {
-		return n.makeProxy(path, fullPath, parentNode, exclusive)
+		return n.makeProxy(path, fullPath, parentNode, exclusive), nil
 	}
 
 	rev := n.GetBranch(NONE).GetLatest()
@@ -998,7 +1005,7 @@
 					"name":             name,
 				})
 				newNode := n.MakeNode(reflect.New(field.ClassType.Elem()).Interface(), "")
-				return newNode.makeProxy(path, fullPath, parentNode, exclusive)
+				return newNode.makeProxy(path, fullPath, parentNode, exclusive), nil
 			} else if field.Key != "" {
 				log.Debugw("key-proxy", log.Fields{
 					"node-type":        reflect.ValueOf(n.Type).Type(),
@@ -1026,7 +1033,10 @@
 						"fullPath":         fullPath,
 						"name":             name,
 					})
-				} else if revs := n.GetBranch(NONE).GetLatest().LoadFromPersistence(ctx, fullPath, "", nil); revs != nil && len(revs) > 0 {
+				} else if revs, err := n.GetBranch(NONE).GetLatest().LoadFromPersistence(ctx, fullPath, "", nil); err != nil {
+					log.Errorf("failed-to-load-from-persistence")
+					return nil, err
+				} else if revs != nil && len(revs) > 0 {
 					log.Debugw("found-revision-matching-key-in-db", log.Fields{
 						"node-type":        reflect.ValueOf(n.Type).Type(),
 						"parent-node-type": reflect.ValueOf(parentNode.Type).Type(),
@@ -1081,7 +1091,7 @@
 		"fullPath":         fullPath,
 		"latest-rev":       rev.GetHash(),
 	})
-	return nil
+	return nil, nil
 }
 
 func (n *node) makeProxy(path string, fullPath string, parentNode *node, exclusive bool) *Proxy {