blob: 2d69041da0c0307f582e7190840d94842fc0ed49 [file] [log] [blame]
khenaidoo26721882021-08-11 17:42:52 -04001/*
2 * Copyright 2021-present Open Networking Foundation
3
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7
8 * http://www.apache.org/licenses/LICENSE-2.0
9
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
khenaidoo0927c722021-12-15 16:49:32 -050016package grpc_test
khenaidoo26721882021-08-11 17:42:52 -040017
18import (
19 "context"
20 "fmt"
21 "os"
22 "strconv"
23 "sync"
24 "testing"
25 "time"
26
khenaidoo0927c722021-12-15 16:49:32 -050027 vgrpc "github.com/opencord/voltha-lib-go/v7/pkg/grpc"
khenaidoo26721882021-08-11 17:42:52 -040028 "github.com/opencord/voltha-lib-go/v7/pkg/log"
29 "github.com/opencord/voltha-lib-go/v7/pkg/probe"
30 "github.com/opencord/voltha-protos/v5/go/common"
khenaidooa5feb8e2021-10-19 17:29:22 -040031 "github.com/opencord/voltha-protos/v5/go/core_service"
khenaidoo26721882021-08-11 17:42:52 -040032 "github.com/opencord/voltha-protos/v5/go/voltha"
33 "github.com/phayes/freeport"
34 "github.com/stretchr/testify/assert"
35 "google.golang.org/grpc"
36)
37
38const (
39 testGrpcServer = "test-grpc-server"
40 initialInterval = 100 * time.Millisecond
41 maxInterval = 5000 * time.Millisecond
42 maxElapsedTime = 0 * time.Millisecond
43 monitorInterval = 2 * time.Second
44 timeout = 10 * time.Second
45)
46
khenaidoo0927c722021-12-15 16:49:32 -050047var testForNoActivityCh = make(chan time.Time, 20)
48var testKeepAliveCh = make(chan time.Time, 20)
khenaidoo26721882021-08-11 17:42:52 -040049
50type testCoreServer struct {
51 apiEndPoint string
khenaidoo0927c722021-12-15 16:49:32 -050052 server *vgrpc.GrpcServer
khenaidoo26721882021-08-11 17:42:52 -040053 probe *probe.Probe
khenaidoo0927c722021-12-15 16:49:32 -050054 coreService *vgrpc.MockCoreServiceHandler
khenaidoo26721882021-08-11 17:42:52 -040055}
56
57func newTestCoreServer(apiEndpoint string) *testCoreServer {
58 return &testCoreServer{
59 apiEndPoint: apiEndpoint,
60 probe: &probe.Probe{},
61 }
62}
63
64func (s *testCoreServer) registerService(ctx context.Context, t *testing.T) {
65 assert.NotEqual(t, "", s.apiEndPoint)
66
67 probePort, err := freeport.GetFreePort()
68 assert.Nil(t, err)
khenaidoo0927c722021-12-15 16:49:32 -050069 probeEndpoint := ":" + strconv.Itoa(probePort)
khenaidoo26721882021-08-11 17:42:52 -040070 go s.probe.ListenAndServe(ctx, probeEndpoint)
71 s.probe.RegisterService(ctx, testGrpcServer)
72
khenaidoo0927c722021-12-15 16:49:32 -050073 s.server = vgrpc.NewGrpcServer(s.apiEndPoint, nil, false, s.probe)
74 s.coreService = vgrpc.NewMockCoreServiceHandler()
khenaidoo26721882021-08-11 17:42:52 -040075
76 s.server.AddService(func(server *grpc.Server) {
khenaidoo0927c722021-12-15 16:49:32 -050077 core_service.RegisterCoreServiceServer(server, s.coreService)
khenaidoo26721882021-08-11 17:42:52 -040078 })
79}
80
81func (s *testCoreServer) start(ctx context.Context, t *testing.T) {
82 assert.NotNil(t, s.server)
83 assert.NotEqual(t, "", s.apiEndPoint)
84
85 s.probe.UpdateStatus(ctx, testGrpcServer, probe.ServiceStatusRunning)
khenaidoo0927c722021-12-15 16:49:32 -050086 s.coreService.Start()
khenaidoo26721882021-08-11 17:42:52 -040087 s.server.Start(ctx)
88 s.probe.UpdateStatus(ctx, testGrpcServer, probe.ServiceStatusStopped)
89}
90
91func (s *testCoreServer) stop() {
khenaidoo0927c722021-12-15 16:49:32 -050092 s.coreService.Stop()
khenaidoo26721882021-08-11 17:42:52 -040093 if s.server != nil {
94 s.server.Stop()
95 }
96}
97
98type testClient struct {
99 apiEndPoint string
100 probe *probe.Probe
khenaidoo0927c722021-12-15 16:49:32 -0500101 client *vgrpc.Client
khenaidoo26721882021-08-11 17:42:52 -0400102}
103
104func serverRestarted(ctx context.Context, endPoint string) error {
105 logger.Infow(ctx, "remote-restarted", log.Fields{"endpoint": endPoint})
106 return nil
107}
108
khenaidoo0927c722021-12-15 16:49:32 -0500109func newTestClient(apiEndpoint string, handler vgrpc.RestartedHandler) *testClient {
khenaidoo26721882021-08-11 17:42:52 -0400110 tc := &testClient{
111 apiEndPoint: apiEndpoint,
112 probe: &probe.Probe{},
113 }
114 // Set the environment variables that this client will use
115 var err error
khenaidoo0927c722021-12-15 16:49:32 -0500116 err = os.Setenv("GRPC_BACKOFF_INITIAL_INTERVAL", initialInterval.String())
khenaidoo26721882021-08-11 17:42:52 -0400117 if err != nil {
khenaidoo0927c722021-12-15 16:49:32 -0500118 logger.Warnw(context.Background(), "setting-env-variable-failed", log.Fields{"error": err, "variable": "GRPC_BACKOFF_INITIAL_INTERVAL"})
khenaidoo26721882021-08-11 17:42:52 -0400119 return nil
120 }
khenaidoo0927c722021-12-15 16:49:32 -0500121 err = os.Setenv("GRPC_BACKOFF_MAX_INTERVAL", maxInterval.String())
khenaidoo26721882021-08-11 17:42:52 -0400122 if err != nil {
khenaidoo0927c722021-12-15 16:49:32 -0500123 logger.Warnw(context.Background(), "setting-env-variable-failed", log.Fields{"error": err, "variable": "GRPC_BACKOFF_MAX_INTERVAL"})
khenaidoo26721882021-08-11 17:42:52 -0400124 return nil
125 }
khenaidoo0927c722021-12-15 16:49:32 -0500126 err = os.Setenv("GRPC_BACKOFF_MAX_ELAPSED_TIME", maxElapsedTime.String())
khenaidoo26721882021-08-11 17:42:52 -0400127 if err != nil {
khenaidoo0927c722021-12-15 16:49:32 -0500128 logger.Warnw(context.Background(), "setting-env-variable-failed", log.Fields{"error": err, "variable": "GRPC_BACKOFF_MAX_ELAPSED_TIME"})
khenaidoo26721882021-08-11 17:42:52 -0400129 return nil
130 }
131
khenaidoo0927c722021-12-15 16:49:32 -0500132 err = os.Setenv("GRPC_MONITOR_INTERVAL", monitorInterval.String())
khenaidoo26721882021-08-11 17:42:52 -0400133 if err != nil {
khenaidoo0927c722021-12-15 16:49:32 -0500134 logger.Warnw(context.Background(), "setting-env-variable-failed", log.Fields{"error": err, "variable": "GRPC_MONITOR_INTERVAL"})
khenaidoo26721882021-08-11 17:42:52 -0400135 return nil
136 }
137
khenaidoo0927c722021-12-15 16:49:32 -0500138 tc.client, err = vgrpc.NewClient(
khenaidoob9503212021-12-08 14:22:21 -0500139 "test-endpoint",
140 apiEndpoint,
khenaidoo0927c722021-12-15 16:49:32 -0500141 "core_service.CoreService",
khenaidoob9503212021-12-08 14:22:21 -0500142 handler)
khenaidoo26721882021-08-11 17:42:52 -0400143 if err != nil {
144 return nil
145 }
146 return tc
147}
148
khenaidoo0927c722021-12-15 16:49:32 -0500149func getCoreServiceHandler(ctx context.Context, conn *grpc.ClientConn) interface{} {
khenaidoo26721882021-08-11 17:42:52 -0400150 if conn == nil {
151 return nil
152 }
khenaidoo0927c722021-12-15 16:49:32 -0500153 return core_service.NewCoreServiceClient(conn)
khenaidoo26721882021-08-11 17:42:52 -0400154}
155
khenaidoo0927c722021-12-15 16:49:32 -0500156func idleConnectionTest(ctx context.Context, conn *grpc.ClientConn) interface{} {
khenaidoo26721882021-08-11 17:42:52 -0400157 if conn == nil {
158 return nil
159 }
khenaidooa5feb8e2021-10-19 17:29:22 -0400160 svc := core_service.NewCoreServiceClient(conn)
khenaidoo0927c722021-12-15 16:49:32 -0500161
khenaidoo26721882021-08-11 17:42:52 -0400162 testForNoActivityCh <- time.Now()
163 return svc
164}
165
khenaidoo0927c722021-12-15 16:49:32 -0500166func (c *testClient) start(ctx context.Context, t *testing.T, handler vgrpc.GetServiceClient) {
khenaidoo26721882021-08-11 17:42:52 -0400167 assert.NotNil(t, c.client)
168
169 probePort, err := freeport.GetFreePort()
170 assert.Nil(t, err)
171 probeEndpoint := "127.0.0.1:" + strconv.Itoa(probePort)
172 go c.probe.ListenAndServe(ctx, probeEndpoint)
173
174 probeCtx := context.WithValue(ctx, probe.ProbeContextKey, c.probe)
175 c.client.Start(probeCtx, handler)
176}
177
khenaidooa5feb8e2021-10-19 17:29:22 -0400178func (c *testClient) getClient(t *testing.T) core_service.CoreServiceClient {
khenaidoo26721882021-08-11 17:42:52 -0400179 gc, err := c.client.GetClient()
180 assert.Nil(t, err)
khenaidooa5feb8e2021-10-19 17:29:22 -0400181 coreClient, ok := gc.(core_service.CoreServiceClient)
khenaidoo26721882021-08-11 17:42:52 -0400182 assert.True(t, ok)
183 return coreClient
184}
185
186func serverStartsFirstTest(t *testing.T) {
187 // Setup
188 ctx, cancel := context.WithCancel(context.Background())
189 defer cancel()
190
191 // Create and start the test server
192 grpcPort, err := freeport.GetFreePort()
193 assert.Nil(t, err)
194 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
195 ts := newTestCoreServer(apiEndpoint)
196 ts.registerService(ctx, t)
197 go ts.start(ctx, t)
198
199 // Create the test client and start it
200 tc := newTestClient(apiEndpoint, serverRestarted)
201 assert.NotNil(t, tc)
khenaidoo0927c722021-12-15 16:49:32 -0500202 go tc.start(ctx, t, getCoreServiceHandler)
khenaidoo26721882021-08-11 17:42:52 -0400203
204 // Test 1: Verify that probe status shows ready eventually
205 var servicesReady isConditionSatisfied = func() bool {
206 return ts.probe.IsReady() && tc.probe.IsReady()
207 }
208 err = waitUntilCondition(timeout, servicesReady)
209 assert.Nil(t, err)
210
211 // Test 2: Verify we get a valid client and can make grpc requests with it
212 coreClient := tc.getClient(t)
213 assert.NotNil(t, coreClient)
214
215 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
216 assert.Nil(t, err)
217 assert.NotNil(t, device)
218 assert.Equal(t, "test-1234", device.Type)
219}
220
221func clientStartsFirstTest(t *testing.T) {
222 ctx, cancel := context.WithCancel(context.Background())
223 defer cancel()
224
225 // Create a grpc endpoint for the server
226 grpcPort, err := freeport.GetFreePort()
227 assert.Nil(t, err)
228 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
229
230 // Create the test client and start it
231 tc := newTestClient(apiEndpoint, serverRestarted)
232 assert.NotNil(t, tc)
khenaidoo0927c722021-12-15 16:49:32 -0500233 go tc.start(ctx, t, getCoreServiceHandler)
khenaidoo26721882021-08-11 17:42:52 -0400234
235 // Verify client is not ready
236 var clientNotReady isConditionSatisfied = func() bool {
237 serviceStatus := tc.probe.GetStatus(apiEndpoint)
238 return serviceStatus == probe.ServiceStatusNotReady ||
239 serviceStatus == probe.ServiceStatusPreparing ||
240 serviceStatus == probe.ServiceStatusFailed
241 }
242 err = waitUntilCondition(timeout, clientNotReady)
243 assert.Nil(t, err)
244
245 // Create and start the test server
246 ts := newTestCoreServer(apiEndpoint)
247 ts.registerService(ctx, t)
248 go ts.start(ctx, t)
249
250 // Test 1: Verify that probe status shows ready eventually
251 var servicesReady isConditionSatisfied = func() bool {
252 return ts.probe.IsReady() && tc.probe.IsReady()
253 }
254 err = waitUntilCondition(timeout, servicesReady)
255 assert.Nil(t, err)
256
257 // Test 2: Verify we get a valid client and can make grpc requests with it
258 coreClient := tc.getClient(t)
259 assert.NotNil(t, coreClient)
260
261 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
262 assert.Nil(t, err)
263 assert.NotNil(t, device)
264 assert.Equal(t, "test-1234", device.Type)
265}
266
267// Liveness function
268func livessness(timestamp time.Time) {
269 logger.Debugw(context.Background(), "received-liveness", log.Fields{"timestamp": timestamp})
270}
271
272func serverRestarts(t *testing.T, numRestartRuns int) {
273 // Setup
274 ctx, cancel := context.WithCancel(context.Background())
275 defer cancel()
276
277 // Create and start the test server
278 grpcPort, err := freeport.GetFreePort()
279 assert.Nil(t, err)
280 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
281 ts := newTestCoreServer(apiEndpoint)
282 ts.registerService(ctx, t)
283 go ts.start(ctx, t)
284
285 // Create the test client and start it
286 tc := newTestClient(apiEndpoint, serverRestarted)
287 assert.NotNil(t, tc)
288
289 // Subscribe for liveness
290 tc.client.SubscribeForLiveness(livessness)
khenaidoo0927c722021-12-15 16:49:32 -0500291 go tc.start(ctx, t, getCoreServiceHandler)
khenaidoo26721882021-08-11 17:42:52 -0400292
293 // Test 1: Verify that probe status shows ready eventually
294 var servicesReady isConditionSatisfied = func() bool {
295 return ts.probe.IsReady() && tc.probe.IsReady()
296 }
297 err = waitUntilCondition(timeout, servicesReady)
298 assert.Nil(t, err)
299
300 // Test 2: Verify we get a valid client and can make grpc requests with it
301 coreClient := tc.getClient(t)
302 assert.NotNil(t, coreClient)
303
304 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
305 assert.Nil(t, err)
306 assert.NotNil(t, device)
307 assert.Equal(t, "test-1234", device.Type)
308
309 for i := 1; i <= numRestartRuns; i++ {
310 //Test 3: Stop server and verify server status
311 ts.stop()
312 var serverDown isConditionSatisfied = func() bool {
313 return ts.probe.GetStatus(testGrpcServer) == probe.ServiceStatusStopped
314 }
315 err = waitUntilCondition(timeout, serverDown)
316 assert.Nil(t, err)
317
khenaidoo26721882021-08-11 17:42:52 -0400318 // Wait until the client service shows as not ready. A wait is not needed. It's just to verify that the
319 // client changes connection state.
320 var clientNotReady isConditionSatisfied = func() bool {
321 serviceStatus := tc.probe.GetStatus(apiEndpoint)
322 return serviceStatus == probe.ServiceStatusNotReady ||
323 serviceStatus == probe.ServiceStatusPreparing ||
324 serviceStatus == probe.ServiceStatusFailed
325 }
326 err = waitUntilCondition(timeout, clientNotReady)
327
328 assert.Nil(t, err)
329
khenaidoo0927c722021-12-15 16:49:32 -0500330 // Test 4: Re-create the server and verify the server is back online
331 ts = newTestCoreServer(apiEndpoint)
332 ts.registerService(ctx, t)
khenaidoo26721882021-08-11 17:42:52 -0400333 go ts.start(ctx, t)
334 err = waitUntilCondition(timeout, servicesReady)
335 assert.Nil(t, err)
336
337 // Test 5: verify we can pull new device with a new client instance
338 coreClient = tc.getClient(t)
339 assert.NotNil(t, coreClient)
340 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
341 assert.Nil(t, err)
342 assert.Equal(t, "test-1234", device.Type)
343 }
344 // Stop the server
345 ts.stop()
346}
347
khenaidoo0927c722021-12-15 16:49:32 -0500348// Liveness function
349func keepAliveMonitor(timestamp time.Time) {
350 logger.Debugw(context.Background(), "received-liveness", log.Fields{"timestamp": timestamp})
351 testKeepAliveCh <- timestamp
352}
353
354func testKeepAlive(t *testing.T) {
khenaidoo26721882021-08-11 17:42:52 -0400355 ctx, cancel := context.WithCancel(context.Background())
356 defer cancel()
357
358 // Create a grpc endpoint for the server
359 grpcPort, err := freeport.GetFreePort()
360 assert.Nil(t, err)
361 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
362
363 // Create the test client and start it
364 tc := newTestClient(apiEndpoint, serverRestarted)
365 assert.NotNil(t, tc)
366 go tc.start(ctx, t, idleConnectionTest)
367
368 // Create and start the test server
369 ts := newTestCoreServer(apiEndpoint)
khenaidoo0927c722021-12-15 16:49:32 -0500370 tc.client.SubscribeForLiveness(keepAliveMonitor)
khenaidoo26721882021-08-11 17:42:52 -0400371 ts.registerService(ctx, t)
372 go ts.start(ctx, t)
khenaidoo0927c722021-12-15 16:49:32 -0500373 defer tc.client.Stop(context.Background())
khenaidoo26721882021-08-11 17:42:52 -0400374
375 // Test 1: Verify that probe status shows ready eventually
376 var servicesReady isConditionSatisfied = func() bool {
377 return ts.probe.IsReady() && tc.probe.IsReady()
378 }
379 err = waitUntilCondition(timeout, servicesReady)
380 assert.Nil(t, err)
381
382 // Test 2: Verify we get a valid client and can make grpc requests with it
383 coreClient := tc.getClient(t)
384 assert.NotNil(t, coreClient)
385
386 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
387 assert.Nil(t, err)
388 assert.NotNil(t, device)
389 assert.Equal(t, "test-1234", device.Type)
390
391 start := time.Now()
392 numChecks := 3 // Test for 3 checks
393 // Wait on the the idle channel - on no activity a connection probe will be attempted by the client
394 timer := time.NewTimer((monitorInterval + 1*time.Second) * time.Duration(numChecks))
395 defer timer.Stop()
396 count := 0
397loop:
398 for {
399 select {
khenaidoo0927c722021-12-15 16:49:32 -0500400 case timestamp := <-testKeepAliveCh:
khenaidoo26721882021-08-11 17:42:52 -0400401 if timestamp.After(start) {
402 count += 1
403 if count > numChecks {
404 break loop
405 }
406 }
407 case <-timer.C:
408 t.Fatal("no activity on the idle channel")
409 }
410 }
411}
412
khenaidoofe90ac32021-11-08 18:17:32 -0500413func testClientFailure(t *testing.T, numClientRestarts int) {
414 ctx, cancel := context.WithCancel(context.Background())
415 defer cancel()
416 // Create a grpc endpoint for the server
417 grpcPort, err := freeport.GetFreePort()
418 assert.Nil(t, err)
419 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
420 // Create the test client and start it
421 tc := newTestClient(apiEndpoint, serverRestarted)
422 assert.NotNil(t, tc)
423 go tc.start(ctx, t, idleConnectionTest)
424 // Create and start the test server
425 ts := newTestCoreServer(apiEndpoint)
426 ts.registerService(ctx, t)
427 go ts.start(ctx, t)
428 defer ts.stop()
429 // Test 1: Verify that probe status shows ready eventually
430 var servicesReady isConditionSatisfied = func() bool {
431 return ts.probe.IsReady() && tc.probe.IsReady()
432 }
433 err = waitUntilCondition(timeout, servicesReady)
434 assert.Nil(t, err)
435 // Test 2: Verify we get a valid client and can make grpc requests with it
436 coreClient := tc.getClient(t)
437 assert.NotNil(t, coreClient)
438 device, err := coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
439 assert.Nil(t, err)
440 assert.NotNil(t, device)
441 assert.Equal(t, "test-1234", device.Type)
442 for i := 1; i <= numClientRestarts; i++ {
443 // Kill grpc client
444 tc.client.Stop(context.Background())
445 var clientNotReady isConditionSatisfied = func() bool {
446 return !tc.probe.IsReady()
447 }
448 err = waitUntilCondition(timeout, clientNotReady)
449 assert.Nil(t, err)
khenaidoo0927c722021-12-15 16:49:32 -0500450
khenaidoofe90ac32021-11-08 18:17:32 -0500451 // Create a new client
khenaidoo0927c722021-12-15 16:49:32 -0500452 tc.client, err = vgrpc.NewClient(
453 "test-endpoint",
khenaidoofe90ac32021-11-08 18:17:32 -0500454 apiEndpoint,
khenaidoo0927c722021-12-15 16:49:32 -0500455 "core_service.CoreService",
khenaidoob9503212021-12-08 14:22:21 -0500456 serverRestarted)
khenaidoofe90ac32021-11-08 18:17:32 -0500457 assert.Nil(t, err)
458 probeCtx := context.WithValue(ctx, probe.ProbeContextKey, tc.probe)
459 go tc.client.Start(probeCtx, idleConnectionTest)
khenaidoo0927c722021-12-15 16:49:32 -0500460
khenaidoofe90ac32021-11-08 18:17:32 -0500461 //Verify that probe status shows ready eventually
462 err = waitUntilCondition(timeout, servicesReady)
463 assert.Nil(t, err)
khenaidoo0927c722021-12-15 16:49:32 -0500464
khenaidoofe90ac32021-11-08 18:17:32 -0500465 // Verify we get a valid client and can make grpc requests with it
466 coreClient = tc.getClient(t)
467 assert.NotNil(t, coreClient)
khenaidoo0927c722021-12-15 16:49:32 -0500468
khenaidoofe90ac32021-11-08 18:17:32 -0500469 device, err = coreClient.GetDevice(context.Background(), &common.ID{Id: "1234"})
470 assert.Nil(t, err)
471 assert.NotNil(t, device)
472 assert.Equal(t, "test-1234", device.Type)
473 }
474 tc.client.Stop(context.Background())
475}
476
khenaidoo26721882021-08-11 17:42:52 -0400477func testServerLimit(t *testing.T) {
478 t.Skip() // Not needed for regular unit tests
479
480 ctx, cancel := context.WithCancel(context.Background())
481 defer cancel()
482
483 // Create a grpc endpoint for the server
484 grpcPort, err := freeport.GetFreePort()
485 assert.Nil(t, err)
486 apiEndpoint := "127.0.0.1:" + strconv.Itoa(grpcPort)
487
488 // Create the test client and start it
489 tc := newTestClient(apiEndpoint, serverRestarted)
490 assert.NotNil(t, tc)
491 go tc.start(ctx, t, idleConnectionTest)
492
493 // Create and start the test server
494 ts := newTestCoreServer(apiEndpoint)
495 ts.registerService(ctx, t)
496 go ts.start(ctx, t)
497
498 // Test 1: Verify that probe status shows ready eventually
499 var servicesReady isConditionSatisfied = func() bool {
500 return ts.probe.IsReady() && tc.probe.IsReady()
501 }
502 err = waitUntilCondition(timeout, servicesReady)
503 assert.Nil(t, err)
504
505 // Test 2: Verify we get a valid client and can make grpc requests with it
506 coreClient := tc.getClient(t)
507 assert.NotNil(t, coreClient)
508
509 var lock sync.RWMutex
510 bad := []time.Duration{}
511 bad_err := []string{}
512 good := []time.Duration{}
513 var wg sync.WaitGroup
514 numRPCs := 10
515 total_good := time.Duration(0)
516 max_good := time.Duration(0)
517 total_bad := time.Duration(0)
518 max_bad := time.Duration(0)
519 order_received := []uint32{}
520 for i := 1; i <= numRPCs; i++ {
521 wg.Add(1)
522 go func(seq int) {
523 local := time.Now()
524 ctx, cancel := context.WithTimeout(context.Background(), 10000*time.Millisecond)
525 defer cancel()
526 var err error
527 var d *voltha.Device
528 d, err = coreClient.GetDevice(ctx, &common.ID{Id: strconv.Itoa(seq)})
529 if err != nil {
530 lock.Lock()
531 bad = append(bad, time.Since(local))
532 bad_err = append(bad_err, err.Error())
533 total_bad += time.Since(local)
534 if time.Since(local) > max_bad {
535 max_bad = time.Since(local)
536 }
537 logger.Errorw(ctx, "error produced", log.Fields{"error": err})
538 lock.Unlock()
539 } else {
540 lock.Lock()
541 good = append(good, time.Since(local))
542 total_good += time.Since(local)
543 if time.Since(local) > max_good {
544 max_good = time.Since(local)
545 }
546 if d != nil {
547 order_received = append(order_received, d.Vlan)
548 }
549 lock.Unlock()
550 }
551 wg.Done()
552 }(i)
553 }
554 wg.Wait()
555 assert.Equal(t, 0, len(bad))
556 assert.Equal(t, numRPCs, len(good))
557 //fmt.Println("Bad:", bad[:10])
558 if len(bad_err) > 0 {
559 fmt.Println("Bad Err Last:", bad_err[len(bad_err)-1:])
560 fmt.Println("Bad Err First:", bad_err[:1])
561 }
562 fmt.Println("Good:", good[len(good)-10:])
563 fmt.Println("Good average time:", total_good.Milliseconds()/int64(numRPCs))
564 fmt.Println("Bad average time:", total_bad.Milliseconds()/int64(numRPCs))
565 fmt.Println("Bad Max:", max_bad)
566 fmt.Println("Good Max:", max_good)
567 //fmt.Println("Order received:", order_received)
568
569 prev := order_received[0]
570
571 for i := 1; i < len(order_received); i++ {
572 if order_received[i] < prev {
573 fmt.Println("Prev:", prev, " curr:", order_received[i])
574 }
575 prev = order_received[i]
576 }
577}
578
khenaidoo0927c722021-12-15 16:49:32 -0500579func TestSuite(t *testing.T) {
khenaidoo26721882021-08-11 17:42:52 -0400580 // Setup
581 log.SetAllLogLevel(volthaTestLogLevel)
582
583 // Test starting server before client
584 serverStartsFirstTest(t)
585
586 // Test starting client before server
587 clientStartsFirstTest(t)
588
589 // Test server restarts
khenaidoo0927c722021-12-15 16:49:32 -0500590 serverRestarts(t, 10)
khenaidoo26721882021-08-11 17:42:52 -0400591
khenaidoo0927c722021-12-15 16:49:32 -0500592 // Test that the client test the grpc connection on no activity
593 testKeepAlive(t)
khenaidoo26721882021-08-11 17:42:52 -0400594
595 // Test client queueing with server limit
596 testServerLimit(t)
khenaidoofe90ac32021-11-08 18:17:32 -0500597
khenaidoo0927c722021-12-15 16:49:32 -0500598 // // Test the scenario where a client restarts
khenaidoofe90ac32021-11-08 18:17:32 -0500599 testClientFailure(t, 10)
khenaidoo26721882021-08-11 17:42:52 -0400600}