Re-enabled redundancy for binding routers.

In other words: Enabled communication to core pairs instead of communicating with just one core.

Change-Id: I5e926e94c55d60bf77a22a0ce053ca9c2765a834
diff --git a/afrouter/afrouter/backend.go b/afrouter/afrouter/backend.go
index 863652f..19133a3 100644
--- a/afrouter/afrouter/backend.go
+++ b/afrouter/afrouter/backend.go
@@ -69,7 +69,6 @@
 	activeAssoc      assoc
 	connFailCallback func(string, *backend) bool
 	connections      map[string]*beConnection
-	srtdConns        []*beConnection
 	opnConns         int
 }
 
@@ -265,18 +264,7 @@
 	var atLeastOne bool = false
 	var errStr strings.Builder
 	log.Debugf("There are %d connections to open", len(be.connections))
-	for _, cn := range be.srtdConns {
-		// TODO: THIS IS A HACK to suspend redundancy for binding routers for all calls
-		// and its very specific to a use case. There should really be a per method
-		// mechanism to select non-redundant calls for all router types. This needs
-		// to be fixed ASAP. The overrides should be used for this, the implementation
-		// is simple, and it can be done here.
-		if atLeastOne == true && f.metaKey != NoMeta {
-			// Don't open any more southbound streams
-			log.Debugf("Not opening any more SB streams, metaKey = %s", f.metaKey)
-			rtrn.strms[cn.name] = nil
-			continue
-		}
+	for _, cn := range be.connections {
 		// Copy in the metadata
 		if cn.getState() == connectivity.Ready && cn.getConn() != nil {
 			log.Debugf("Opening southbound stream for connection '%s'", cn.name)
@@ -598,17 +586,6 @@
 	}
 }
 
-func (be *backend) sortConns() {
-	var tmpKeys []string
-	for k, _ := range be.connections {
-		tmpKeys = append(tmpKeys, k)
-	}
-	sort.Strings(tmpKeys)
-	for _, v := range tmpKeys {
-		be.srtdConns = append(be.srtdConns, be.connections[v])
-	}
-}
-
 func newBackend(conf *BackendConfig, clusterName string) (*backend, error) {
 	var rtrn_err bool = false
 
@@ -701,9 +678,6 @@
 	if rtrn_err {
 		return nil, errors.New("Connection configuration failed")
 	}
-	// Create the sorted connection list for deterministic
-	// active-active call orders.
-	be.sortConns()
 	// All is well start the backend cluster connections
 	be.connectAll()
 
diff --git a/tests/afrouter/suites/test1/test1.json b/tests/afrouter/suites/test1/test1.json
index ad0d579..c0326b1 100644
--- a/tests/afrouter/suites/test1/test1.json
+++ b/tests/afrouter/suites/test1/test1.json
@@ -21,7 +21,7 @@
 			}
 		],
 		"imports": [
- 				 "github.com/opencord/voltha-protos/go/common"
+			"github.com/opencord/voltha-protos/go/common"
 		],
 		"protoDesc":"voltha.pb",
 		"protoSubst": [
@@ -32,9 +32,9 @@
 		],
 		"clients": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
- 				 "github.com/opencord/voltha-protos/go/common"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/common"
 			],
 			"endpoints": [
 				{
@@ -49,10 +49,10 @@
 		},
 		"servers": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
- 				 "github.com/opencord/voltha-protos/go/common",
- 				 "github.com/opencord/voltha-protos/go/omci"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/common",
+				"github.com/opencord/voltha-protos/go/omci"
 			],
 			"endpoints": [
 			{
@@ -190,6 +190,19 @@
 							"value":""
 						}
 					]
+				},
+				{
+					"name":"core12",
+					"meta": [
+						{
+							"key":"voltha_serial_number",
+							"value":"2"
+						},
+						{
+							"key":"voltha_backend_name",
+							"value":""
+						}
+					]
 				}
 			]
 		},
diff --git a/tests/afrouter/suites/test2/test2.tmpl.json b/tests/afrouter/suites/test2/test2.tmpl.json
index 6efa146..a15230d 100644
--- a/tests/afrouter/suites/test2/test2.tmpl.json
+++ b/tests/afrouter/suites/test2/test2.tmpl.json
@@ -14,8 +14,8 @@
 			}
 		],
 		"imports": [
-			 "github.com/golang/protobuf/ptypes/empty",
-             "github.com/opencord/voltha-protos/go/common"
+			"github.com/golang/protobuf/ptypes/empty",
+			"github.com/opencord/voltha-protos/go/common"
 		],
 		"protoDesc":"voltha.pb",
 		"protoSubst": [
@@ -26,9 +26,9 @@
 		],
 		"clients": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
-                 "github.com/opencord/voltha-protos/go/common"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/common"
 			],
 			"endpoints": [
 				{
@@ -43,10 +43,10 @@
 		},
 		"servers": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
-                 "github.com/opencord/voltha-protos/go/omci",
-                 "github.com/opencord/voltha-protos/go/common"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/omci",
+				"github.com/opencord/voltha-protos/go/common"
 			],
 			"endpoints": [
 			{
@@ -206,6 +206,19 @@
 							"value":""
 						}
 					]
+				},
+				{
+					"name":"core12",
+					"meta": [
+						{
+							"key":"voltha_serial_number",
+							"value":"110001"
+						},
+						{
+							"key":"voltha_backend_name",
+							"value":""
+						}
+					]
 				}
 			]
 		},
diff --git a/tests/afrouter/suites/test3/test3.tmpl.json b/tests/afrouter/suites/test3/test3.tmpl.json
index a3c5456..a536d60 100644
--- a/tests/afrouter/suites/test3/test3.tmpl.json
+++ b/tests/afrouter/suites/test3/test3.tmpl.json
@@ -16,7 +16,7 @@
 		"imports": [
 			"github.com/golang/protobuf/ptypes/empty",
 			"github.com/opencord/voltha-protos/go/openflow_13",
-            "github.com/opencord/voltha-protos/go/common"
+			"github.com/opencord/voltha-protos/go/common"
 		],
 		"protoDesc":"voltha.pb",
 		"protoSubst": [
@@ -27,9 +27,9 @@
 		],
 		"clients": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
-                 "github.com/opencord/voltha-protos/go/common"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/common"
 			],
 			"endpoints": [
 				{
@@ -44,10 +44,10 @@
 		},
 		"servers": {
 			"imports": [
-				 "github.com/golang/protobuf/ptypes/empty",
-				 "github.com/opencord/voltha-protos/go/openflow_13",
-                 "github.com/opencord/voltha-protos/go/omci",
-                 "github.com/opencord/voltha-protos/go/common"
+				"github.com/golang/protobuf/ptypes/empty",
+				"github.com/opencord/voltha-protos/go/openflow_13",
+				"github.com/opencord/voltha-protos/go/omci",
+				"github.com/opencord/voltha-protos/go/common"
 			],
 			"endpoints": [
 			{
@@ -219,6 +219,19 @@
 							"value":""
 						}
 					]
+				},
+				{
+					"name":"core12",
+					"meta": [
+						{
+							"key":"voltha_serial_number",
+							"value":"44"
+						},
+						{
+							"key":"voltha_backend_name",
+							"value":""
+						}
+					]
 				}
 			]
 		},
diff --git a/tests/afrouter/templates/runTests.go.tmpl b/tests/afrouter/templates/runTests.go.tmpl
index ac58220..3aa59ac 100644
--- a/tests/afrouter/templates/runTests.go.tmpl
+++ b/tests/afrouter/templates/runTests.go.tmpl
@@ -237,7 +237,7 @@
 				}
 			}
 		case <-glCtx.Done():
-			rtrn = errors.New(fmt.Sprintf("Timeout: no response data available for server %s", testName))
+			rtrn = errors.New(fmt.Sprintf("Timeout: no response data available for server %s (%s)", v, testName))
 			stats.testLog("%s\n", rtrn.Error())
 			log.Error(rtrn)
 		}