Changes to the test framework to support templating of
the json test files to enable the creation of very large
stress test suites. Still a work in progress.
Change-Id: I1a35e4143a2feb577c9ad6048a0339c7b9dc0f89
diff --git a/tests/afrouter/arouter_test.json b/tests/afrouter/arouter_test.json
index 0144478..0829737 100644
--- a/tests/afrouter/arouter_test.json
+++ b/tests/afrouter/arouter_test.json
@@ -143,7 +143,8 @@
"type":"active_active",
"association": {
"strategy":"serial_number",
- "location":"header"
+ "location":"header",
+ "key":"voltha_serial_number"
},
"connections": [ {
"name":"vcore21",
@@ -161,7 +162,8 @@
"type":"active_active",
"association": {
"strategy":"serial_number",
- "location":"header"
+ "location":"header",
+ "key":"voltha_serial_number"
},
"connections": [ {
"name":"vcore31",
diff --git a/tests/afrouter/suites/main.json b/tests/afrouter/suites/main.json
index e4862c6..a1bc09c 100644
--- a/tests/afrouter/suites/main.json
+++ b/tests/afrouter/suites/main.json
@@ -1,6 +1,7 @@
{
"__COMMENT":"Top level test driver file",
"suites": [
- "test1.json"
+ "test1",
+ "test2"
]
}
diff --git a/tests/afrouter/suites/test2.go b/tests/afrouter/suites/test2.go
new file mode 100644
index 0000000..fc0332e
--- /dev/null
+++ b/tests/afrouter/suites/test2.go
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+ "os"
+ //"fmt"
+ //"flag"
+ //"path"
+ //"bufio"
+ //"errors"
+ //"os/exec"
+ //"strconv"
+ //"io/ioutil"
+ //"encoding/json"
+ "text/template"
+ //"github.com/golang/protobuf/proto"
+ "github.com/opencord/voltha-go/common/log"
+ //pb "github.com/golang/protobuf/protoc-gen-go/descriptor"
+)
+
+type test struct {
+ Core int
+}
+
+func main() {
+
+ var ary []test
+
+ // Setup logging
+ if _, err := log.SetDefaultLogger(log.JSON, 0, nil); err != nil {
+ log.With(log.Fields{"error": err}).Fatal("Cannot setup logging")
+ }
+
+ for i :=0; i<10000; i++ {
+
+ ary = append(ary,test{Core:(i%3)+1})
+ }
+
+ // Load the template to execute
+ t := template.Must(template.New("").ParseFiles("./test2.tmpl.json"))
+ if f,err := os.Create("test2.json"); err == nil {
+ _=f
+ defer f.Close()
+ if err := t.ExecuteTemplate(f, "test2.tmpl.json", ary); err != nil {
+ log.Errorf("Unable to execute template for test2.tmpl.json: %v", err)
+ }
+ } else {
+ log.Errorf("Couldn't create file test2.json: %v", err)
+ }
+ return
+}
diff --git a/tests/afrouter/suites/test2.json b/tests/afrouter/suites/test2.tmpl.json
similarity index 81%
rename from tests/afrouter/suites/test2.json
rename to tests/afrouter/suites/test2.tmpl.json
index bdecd95..fc8ef1c 100644
--- a/tests/afrouter/suites/test2.json
+++ b/tests/afrouter/suites/test2.tmpl.json
@@ -1,6 +1,6 @@
{
"environment":{
- "cmdLine":"afrouter -config arouter_test.json",
+ "cmdLine":"afrouter -config arouter_test.json -grpclog",
"protoFiles": [
{
"importPath":"github.com/opencord/voltha-go/protos/",
@@ -53,43 +53,45 @@
]
},
"tests":[
+ {{range $k,$v := .}}
{
"name":"Test CreateDevice",
"send": {
"client":"client",
"method":"CreateDevice",
"param":"{Type:\"simulated_olt\"}",
- "expect":"{Id:\"abcd1234\",Type:\"simulated_olt\"}",
+ "expect":"{Id:\"abcd1234{{$k}}\",Type:\"simulated_olt\"}",
"_meta":""
},
"servers": [
{
- "name":"core11",
+ "name":"core{{$v.Core}}1",
"meta": [
{
"key":"voltha_serial_number",
- "value":"0"
+ "value":"{{$k}}"
}
]
},
{
- "name":"core12",
+ "name":"core{{$v.Core}}2",
"meta": [
{
"key":"voltha_serial_number",
- "value":"0"
+ "value":"{{$k}}"
}
]
}
]
},
+ {{end}}
{
"name":"Test GetDevice",
"send": {
"client":"client",
"method":"GetDevice",
- "param":"{Id:\"abcd1234\"}",
- "expect":"{Id:\"abcd1234\",Type:\"simulated_olt\"}",
+ "param":"{Id:\"abcd12340\"}",
+ "expect":"{Id:\"abcd12340\",Type:\"simulated_olt\"}",
"_meta":""
},
"servers": [
@@ -98,7 +100,7 @@
"meta": [
{
"key":"voltha_serial_number",
- "value":"1"
+ "value":"10000"
}
]
},
@@ -107,7 +109,7 @@
"meta": [
{
"key":"voltha_serial_number",
- "value":"1"
+ "value":"10000"
}
]
}
@@ -129,7 +131,7 @@
"expectMeta": [
{
"key":"voltha_backend_name",
- "value":"core11"
+ "value":"vcore1"
}
]
},
@@ -139,11 +141,11 @@
"meta": [
{
"key":"voltha_serial_number",
- "value":"2"
+ "value":"10001"
},
{
"key":"voltha_backend_name",
- "value":"a"
+ "value":""
}
]
}
diff --git a/tests/afrouter/templates/main.go b/tests/afrouter/templates/main.go
index f539a02..c2fddb9 100644
--- a/tests/afrouter/templates/main.go
+++ b/tests/afrouter/templates/main.go
@@ -26,9 +26,28 @@
"os/exec"
"strings"
"context"
+ slog "log"
+ "io/ioutil"
+ "encoding/json"
+ "google.golang.org/grpc/grpclog"
"github.com/opencord/voltha-go/common/log"
)
+type TestCase struct {
+ Title string `json:"title"`
+ Result bool `json:"result"`
+ Info []string `json:"info"`
+}
+
+type TestSuite struct {
+ Name string `json:"name"`
+ TestCases []TestCase `json:"testCases"`
+}
+
+type TestRun struct {
+ TestSuites []TestSuite
+}
+
var resFile *tstLog
type tstLog struct {
fp * os.File
@@ -95,6 +114,37 @@
//time.Sleep(1 * time.Second)
}
+func readStats(stats * TestRun) () {
+ // Check if the stats file exists
+ if _,err := os.Stat("stats.json"); err != nil {
+ // Nothing to do, just return
+ return
+ }
+ // The file is there, read it an unmarshal it into the stats struct
+ if statBytes, err := ioutil.ReadFile("stats.json"); err != nil {
+ log.Error(err)
+ return
+ } else if err := json.Unmarshal(statBytes, stats); err != nil {
+ log.Error(err)
+ return
+ }
+}
+
+func writeStats(stats * TestRun) () {
+ // Check if the stats file exists
+ // The file is there, read it an unmarshal it into the stats struct
+ if statBytes, err := json.MarshalIndent(stats, ""," "); err != nil {
+ log.Error(err)
+ return
+ } else if err := ioutil.WriteFile("stats.json.new", statBytes, 0644); err != nil {
+ log.Error(err)
+ return
+ }
+ os.Rename("stats.json", "stats.json~")
+ os.Rename("stats.json.new", "stats.json")
+}
+var stats TestRun
+
func main() {
var err error
@@ -102,9 +152,13 @@
if _, err = log.SetDefaultLogger(log.JSON, 0, nil); err != nil {
log.With(log.Fields{"error": err}).Fatal("Cannot setup logging")
}
-
defer log.CleanUp()
+ readStats(&stats)
+
+
+ grpclog.SetLogger(slog.New(os.Stderr, "grpc: ", slog.LstdFlags))
+
resFile = &tstLog{fn:os.Args[1]}
defer resFile.close()
@@ -133,5 +187,6 @@
// Run all the test cases now
log.Infof("Executing tests")
runTests()
+ writeStats(&stats)
}
diff --git a/tests/afrouter/templates/runAll.go b/tests/afrouter/templates/runAll.go
index b4079d1..03cdab1 100644
--- a/tests/afrouter/templates/runAll.go
+++ b/tests/afrouter/templates/runAll.go
@@ -83,7 +83,7 @@
}
tl := &tstLog{fn:"results.txt"}
{{range .}}
- cmdStr = "./"+"{{.}}"[:len("{{.}}")-5]
+ cmdStr = "./"+"{{.}}"+".e"
tl.testLogOnce("Running test suite '%s'\n", cmdStr[2:])
log.Infof("Running test suite %s",cmdStr)
diff --git a/tests/afrouter/templates/runTests.go b/tests/afrouter/templates/runTests.go
index 2c65f09..61ba5eb 100644
--- a/tests/afrouter/templates/runTests.go
+++ b/tests/afrouter/templates/runTests.go
@@ -18,9 +18,12 @@
package main
import (
+ "fmt"
+ "time"
"errors"
+ "context"
"encoding/json"
- "golang.org/x/net/context"
+ //"golang.org/x/net/context"
"google.golang.org/grpc/metadata"
"github.com/opencord/voltha-go/common/log"
{{range .Imports}}
@@ -31,6 +34,8 @@
{{end}}
)
+var glCtx context.Context
+
func resetChannels() {
// Drain all channels of data
for _,v := range servers {
@@ -56,15 +61,19 @@
} else {
resFile.testLog("\tTest Failed\n")
}
- resetChannels()
+ //resetChannels()
{{end}}
}
{{range $k,$v := .Tests}}
func test{{$k}}() error {
var rtrn error = nil
+ var cancel context.CancelFunc
+
+ glCtx, cancel = context.WithTimeout(context.Background(), 900*time.Millisecond)
+ defer cancel()
// Announce the test being run
- resFile.testLog("******************** Running test case: {{$v.Name}}\n")
+ resFile.testLog("******************** Running test case ({{$k}}): {{$v.Name}}\n")
// Acquire the client used to run the test
cl := clients["{{$v.Send.Client}}"]
// Create the server's reply data structure
@@ -76,13 +85,18 @@
log.Error(err)
return err
}
- select {
- case servers["{{$s.Name}}"].replyData <- repl:
- default:
- err := errors.New("Could not provide server {{$s.Name}} with reply data")
- log.Error(err)
- return err
- }
+ // Start a go routine to send the the reply data to the
+ // server. The go routine blocks until the server picks
+ // up the data or the timeout is exceeded.
+ go func (ctx context.Context) {
+ select {
+ case servers["{{$s.Name}}"].replyData <- repl:
+ case <-ctx.Done():
+ rtrn := errors.New("Could not provide server {{$s.Name}} with reply data")
+ log.Error(rtrn)
+ resFile.testLog("%s\n", rtrn.Error())
+ }
+ }(glCtx)
{{end}}
// Now call the RPC with the data provided
@@ -118,25 +132,27 @@
}
{{range $s := .Srvr}}
s = servers["{{$s.Name}}"]
+ // Oddly sometimes the data isn't in the channel yet when we come to read it.
select {
case i = <-s.incmg:
if i.payload != payload {
- log.Errorf("Mismatched payload for test {{$v.Name}}, %s:%s", i.payload, payload)
- resFile.testLog("Mismatched payload expected '%s', got '%s'\n", payload, i.payload)
- rtrn = errors.New("Failed")
+ rtrn = errors.New(fmt.Sprintf("Mismatched payload expected '%s', got '%s'", payload, i.payload))
+ log.Error(rtrn.Error())
+ resFile.testLog("%s\n", rtrn.Error())
}
{{range $m := $s.Meta}}
if mv,ok := i.meta["{{$m.Key}}"]; ok == true {
if "{{$m.Val}}" != mv[0] {
- log.Errorf("Mismatched metadata for test {{$v.Name}}, %s:%s", mv[0], "{{$m.Val}}")
- resFile.testLog("Mismatched metadata on server '%s' expected '%s', got '%s'\n", "{{$s.Name}}", "{{$m.Val}}", mv[0])
- rtrn = errors.New("Failed")
+ rtrn=errors.New(fmt.Sprintf("Mismatched metadata on server '%s' expected '%s', got '%s'", "{{$s.Name}}", "{{$m.Val}}", mv[0]))
+ log.Error(rtrn.Error())
+ resFile.testLog("%s\n", rtrn.Error())
}
}
{{end}}
- default:
- err := errors.New("No response data available for server {{$s.Name}}")
- log.Error(err)
+ case <-glCtx.Done():
+ rtrn = errors.New("Timeout: no response data available for server {{$s.Name}}")
+ resFile.testLog("%s\n", rtrn.Error())
+ log.Error(rtrn)
}
{{end}}
diff --git a/tests/afrouter/templates/server.go b/tests/afrouter/templates/server.go
index 85c3b93..ffea3d5 100644
--- a/tests/afrouter/templates/server.go
+++ b/tests/afrouter/templates/server.go
@@ -76,7 +76,7 @@
func {{.Name}}ListenAndServe() error {
var s {{.Name}}TestServer
- s.control = &serverCtl{replyData:make(chan *reply,1), incmg:make(chan *incoming,1)}
+ s.control = &serverCtl{replyData:make(chan *reply), incmg:make(chan *incoming)}
servers["{{.Name}}"] = s.control
log.Debugf("Starting server %s on port %d", "{{.Name}}", {{.Port}})
@@ -112,15 +112,18 @@
{{range .Methods}}
{{if .Ss}}
func (ts {{$.Name}}TestServer) {{.Name}}(in *{{.Param}}, srvr {{.Pkg}}.{{.Svc}}_{{.Name}}Server) error {
+ log.Debug("Serving server streaming {{$.Name}}")
return nil
}
{{else if .Cs}}
func (ts {{$.Name}}TestServer) {{.Name}}({{.Pkg}}.{{.Svc}}_{{.Name}}Server) error {
+ log.Debug("Serving client streaming {{$.Name}}")
return nil
}
{{else}}
func (ts {{$.Name}}TestServer) {{.Name}}(ctx context.Context, in *{{.Param}}) (*{{.Rtrn}}, error) {
var r * incoming = &incoming{}
+ log.Debug("Serving {{$.Name}}")
// Read the metadata
if md,ok := metadata.FromIncomingContext(ctx); ok == false {
log.Error("Getting matadata during call to {{.Name}}")
@@ -134,7 +137,15 @@
r.payload = string(parm)
}
// Send the server information back to the test framework
- ts.control.incmg <- r
+ go func(ctx context.Context) {
+ select {
+ case ts.control.incmg <- r:
+ return
+ case <-ctx.Done():
+ return
+
+ }
+ }(glCtx)
// Read the value that needs to be returned from the channel
select {
case d := <-ts.control.replyData:
@@ -144,8 +155,8 @@
default:
return nil, errors.New("Mismatched type in call to {{.Name}}")
}
- default:
- return nil, errors.New("Nothing in the reply data channel in call to {{.Name}}, sending nil")
+ case <-glCtx.Done():
+ return nil, errors.New("Timeout: nothing in the reply data channel in call to {{.Name}}, sending nil")
}
return &{{.Rtrn}}{},nil
}
diff --git a/tests/afrouter/tester.go b/tests/afrouter/tester.go
index 939cc24..f3d975e 100644
--- a/tests/afrouter/tester.go
+++ b/tests/afrouter/tester.go
@@ -17,7 +17,6 @@
package main
-// Command line parameters and parsing
import (
"os"
"fmt"
@@ -194,9 +193,31 @@
return nil
}
-func (suite * TestSuite) loadSuite(fileN string) error {
- suiteF, err := os.Open(fileN);
- log.Info("Loading test suite from: ", fileN)
+func (suite * TestSuite) loadSuite(suiteN string) error {
+ // Check if there's a corresponding go file for the suite.
+ // If it is present then the json test suite file is a
+ // template. Compile the go file and run it to process
+ // the template.
+ if _,err := os.Stat(suiteN+".go"); err == nil {
+ // Compile and run the the go file
+ log.Infof("Suite '%s' is a template, compiling '%s'", suiteN, suiteN+".go")
+ cmd := exec.Command("go", "build", "-o", suiteN+".te", suiteN+".go")
+ cmd.Stdin = os.Stdin
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Run(); err != nil {
+ log.Errorf("Error running the compile command:%v", err)
+ }
+ cmd = exec.Command("./"+suiteN+".te")
+ cmd.Stdin = os.Stdin
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Run(); err != nil {
+ log.Errorf("Error running the %s command:%v", suiteN+".te", err)
+ }
+ }
+ suiteF, err := os.Open(suiteN+".json");
+ log.Infof("Loading test suite from: %s", suiteN+".json")
if err != nil {
log.Error(err)
return err
@@ -520,7 +541,7 @@
if err := os.Chdir(suiteDir); err != nil {
log.Errorf("Could not change to directory '%s':%v",suiteDir, err)
}
- cmd := exec.Command("go", "build", "-o", outDir+"/"+v[:len(v)-5])
+ cmd := exec.Command("go", "build", "-o", outDir+"/"+v+".e")
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@@ -530,9 +551,6 @@
if err := os.Chdir("../../suites"); err != nil {
log.Errorf("Could not change to directory '%s':%v","../../suites", err)
}
-
- // Now generate the test cases
-
}
return nil