khenaidoo | bf6e7bb | 2018-08-14 22:27:29 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 16 | package main |
| 17 | |
| 18 | import ( |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 19 | "context" |
| 20 | "errors" |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 21 | "fmt" |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 22 | "github.com/opencord/voltha-go/rw_core/config" |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 23 | c "github.com/opencord/voltha-go/rw_core/core" |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 24 | "github.com/opencord/voltha-go/rw_core/utils" |
Scott Baker | 807addd | 2019-10-24 15:16:21 -0700 | [diff] [blame] | 25 | "github.com/opencord/voltha-lib-go/v2/pkg/db/kvstore" |
| 26 | grpcserver "github.com/opencord/voltha-lib-go/v2/pkg/grpc" |
| 27 | "github.com/opencord/voltha-lib-go/v2/pkg/kafka" |
| 28 | "github.com/opencord/voltha-lib-go/v2/pkg/log" |
| 29 | "github.com/opencord/voltha-lib-go/v2/pkg/probe" |
| 30 | "github.com/opencord/voltha-lib-go/v2/pkg/version" |
Scott Baker | 555307d | 2019-11-04 08:58:01 -0800 | [diff] [blame] | 31 | ic "github.com/opencord/voltha-protos/v2/go/inter_container" |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 32 | "os" |
| 33 | "os/signal" |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 34 | "strconv" |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 35 | "syscall" |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 36 | "time" |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 37 | ) |
| 38 | |
| 39 | type rwCore struct { |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 40 | kvClient kvstore.Client |
| 41 | config *config.RWCoreFlags |
| 42 | halted bool |
| 43 | exitChannel chan int |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 44 | //kmp *kafka.KafkaMessagingProxy |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 45 | grpcServer *grpcserver.GrpcServer |
| 46 | kafkaClient kafka.Client |
| 47 | core *c.Core |
khenaidoo | abad44c | 2018-08-03 16:58:35 -0400 | [diff] [blame] | 48 | //For test |
khenaidoo | 7923270 | 2018-12-04 11:00:41 -0500 | [diff] [blame] | 49 | receiverChannels []<-chan *ic.InterContainerMessage |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 50 | } |
| 51 | |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 52 | func init() { |
khenaidoo | 2c6f167 | 2018-09-20 23:14:41 -0400 | [diff] [blame] | 53 | log.AddPackage(log.JSON, log.DebugLevel, nil) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 54 | } |
| 55 | |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 56 | func newKVClient(storeType string, address string, timeout int) (kvstore.Client, error) { |
| 57 | |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 58 | log.Infow("kv-store-type", log.Fields{"store": storeType}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 59 | switch storeType { |
| 60 | case "consul": |
| 61 | return kvstore.NewConsulClient(address, timeout) |
| 62 | case "etcd": |
| 63 | return kvstore.NewEtcdClient(address, timeout) |
| 64 | } |
| 65 | return nil, errors.New("unsupported-kv-store") |
| 66 | } |
| 67 | |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 68 | func newKafkaClient(clientType string, host string, port int, instanceID string, livenessChannelInterval time.Duration) (kafka.Client, error) { |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 69 | |
| 70 | log.Infow("kafka-client-type", log.Fields{"client": clientType}) |
| 71 | switch clientType { |
| 72 | case "sarama": |
| 73 | return kafka.NewSaramaClient( |
| 74 | kafka.Host(host), |
khenaidoo | 9084792 | 2018-12-03 14:47:51 -0500 | [diff] [blame] | 75 | kafka.Port(port), |
khenaidoo | ca30132 | 2019-01-09 23:06:32 -0500 | [diff] [blame] | 76 | kafka.ConsumerType(kafka.GroupCustomer), |
khenaidoo | 9084792 | 2018-12-03 14:47:51 -0500 | [diff] [blame] | 77 | kafka.ProducerReturnOnErrors(true), |
| 78 | kafka.ProducerReturnOnSuccess(true), |
| 79 | kafka.ProducerMaxRetries(6), |
khenaidoo | ca30132 | 2019-01-09 23:06:32 -0500 | [diff] [blame] | 80 | kafka.NumPartitions(3), |
| 81 | kafka.ConsumerGroupName(instanceID), |
| 82 | kafka.ConsumerGroupPrefix(instanceID), |
khenaidoo | 54e0ddf | 2019-02-27 16:21:33 -0500 | [diff] [blame] | 83 | kafka.AutoCreateTopic(true), |
khenaidoo | ca30132 | 2019-01-09 23:06:32 -0500 | [diff] [blame] | 84 | kafka.ProducerFlushFrequency(5), |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 85 | kafka.ProducerRetryBackoff(time.Millisecond*30), |
| 86 | kafka.LivenessChannelInterval(livenessChannelInterval), |
| 87 | ), nil |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 88 | } |
| 89 | return nil, errors.New("unsupported-client-type") |
| 90 | } |
| 91 | |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 92 | func newRWCore(cf *config.RWCoreFlags) *rwCore { |
| 93 | var rwCore rwCore |
| 94 | rwCore.config = cf |
| 95 | rwCore.halted = false |
| 96 | rwCore.exitChannel = make(chan int, 1) |
khenaidoo | 7923270 | 2018-12-04 11:00:41 -0500 | [diff] [blame] | 97 | rwCore.receiverChannels = make([]<-chan *ic.InterContainerMessage, 0) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 98 | return &rwCore |
| 99 | } |
| 100 | |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 101 | func (rw *rwCore) setKVClient() error { |
| 102 | addr := rw.config.KVStoreHost + ":" + strconv.Itoa(rw.config.KVStorePort) |
| 103 | client, err := newKVClient(rw.config.KVStoreType, addr, rw.config.KVStoreTimeout) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 104 | if err != nil { |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 105 | rw.kvClient = nil |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 106 | log.Error(err) |
| 107 | return err |
| 108 | } |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 109 | rw.kvClient = client |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 110 | return nil |
| 111 | } |
| 112 | |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 113 | func toString(value interface{}) (string, error) { |
| 114 | switch t := value.(type) { |
| 115 | case []byte: |
| 116 | return string(value.([]byte)), nil |
| 117 | case string: |
| 118 | return value.(string), nil |
| 119 | default: |
| 120 | return "", fmt.Errorf("unexpected-type-%T", t) |
| 121 | } |
| 122 | } |
| 123 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 124 | func (rw *rwCore) start(ctx context.Context, instanceId string) { |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 125 | log.Info("Starting RW Core components") |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 126 | |
khenaidoo | 9084792 | 2018-12-03 14:47:51 -0500 | [diff] [blame] | 127 | // Setup KV Client |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 128 | log.Debugw("create-kv-client", log.Fields{"kvstore": rw.config.KVStoreType}) |
| 129 | err := rw.setKVClient() |
| 130 | if err == nil { |
| 131 | // Setup KV transaction context |
khenaidoo | 9cdc1a6 | 2019-01-24 21:57:40 -0500 | [diff] [blame] | 132 | txnPrefix := rw.config.KVStoreDataPrefix + "/transactions/" |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 133 | if err = c.SetTransactionContext(instanceId, |
khenaidoo | 9cdc1a6 | 2019-01-24 21:57:40 -0500 | [diff] [blame] | 134 | txnPrefix, |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 135 | rw.kvClient, |
khenaidoo | 09771ef | 2019-10-11 14:25:02 -0400 | [diff] [blame] | 136 | rw.config.KVStoreTimeout); err != nil { |
khenaidoo | 9cdc1a6 | 2019-01-24 21:57:40 -0500 | [diff] [blame] | 137 | log.Fatal("creating-transaction-context-failed") |
| 138 | } |
Richard Jankowski | e4d7766 | 2018-10-17 13:53:21 -0400 | [diff] [blame] | 139 | } |
| 140 | |
khenaidoo | 9084792 | 2018-12-03 14:47:51 -0500 | [diff] [blame] | 141 | // Setup Kafka Client |
Scott Baker | ee6a087 | 2019-10-29 15:59:52 -0700 | [diff] [blame] | 142 | if rw.kafkaClient, err = newKafkaClient("sarama", |
| 143 | rw.config.KafkaAdapterHost, |
| 144 | rw.config.KafkaAdapterPort, |
| 145 | instanceId, |
| 146 | time.Duration(rw.config.LiveProbeInterval)*time.Second/2); err != nil { |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 147 | log.Fatal("Unsupported-kafka-client") |
| 148 | } |
| 149 | |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 150 | // Create the core service |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 151 | rw.core = c.NewCore(instanceId, rw.config, rw.kvClient, rw.kafkaClient) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 152 | |
| 153 | // start the core |
| 154 | rw.core.Start(ctx) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 155 | } |
| 156 | |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 157 | func (rw *rwCore) stop(ctx context.Context) { |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 158 | // Stop leadership tracking |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 159 | rw.halted = true |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 160 | |
| 161 | // send exit signal |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 162 | rw.exitChannel <- 0 |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 163 | |
| 164 | // Cleanup - applies only if we had a kvClient |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 165 | if rw.kvClient != nil { |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 166 | // Release all reservations |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 167 | if err := rw.kvClient.ReleaseAllReservations(); err != nil { |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 168 | log.Infow("fail-to-release-all-reservations", log.Fields{"error": err}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 169 | } |
| 170 | // Close the DB connection |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 171 | rw.kvClient.Close() |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 172 | } |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 173 | |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 174 | rw.core.Stop(ctx) |
khenaidoo | 43c8212 | 2018-11-22 18:38:28 -0500 | [diff] [blame] | 175 | |
| 176 | //if rw.kafkaClient != nil { |
| 177 | // rw.kafkaClient.Stop() |
| 178 | //} |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 179 | } |
| 180 | |
| 181 | func waitForExit() int { |
| 182 | signalChannel := make(chan os.Signal, 1) |
| 183 | signal.Notify(signalChannel, |
| 184 | syscall.SIGHUP, |
| 185 | syscall.SIGINT, |
| 186 | syscall.SIGTERM, |
| 187 | syscall.SIGQUIT) |
| 188 | |
| 189 | exitChannel := make(chan int) |
| 190 | |
| 191 | go func() { |
| 192 | s := <-signalChannel |
| 193 | switch s { |
| 194 | case syscall.SIGHUP, |
| 195 | syscall.SIGINT, |
| 196 | syscall.SIGTERM, |
| 197 | syscall.SIGQUIT: |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 198 | log.Infow("closing-signal-received", log.Fields{"signal": s}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 199 | exitChannel <- 0 |
| 200 | default: |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 201 | log.Infow("unexpected-signal-received", log.Fields{"signal": s}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 202 | exitChannel <- 1 |
| 203 | } |
| 204 | }() |
| 205 | |
| 206 | code := <-exitChannel |
| 207 | return code |
| 208 | } |
| 209 | |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 210 | func printBanner() { |
| 211 | fmt.Println(" ") |
| 212 | fmt.Println(" ______ ______ ") |
| 213 | fmt.Println("| _ \\ \\ / / ___|___ _ __ ___ ") |
| 214 | fmt.Println("| |_) \\ \\ /\\ / / | / _ \\| '__/ _ \\ ") |
| 215 | fmt.Println("| _ < \\ V V /| |__| (_) | | | __/ ") |
| 216 | fmt.Println("|_| \\_\\ \\_/\\_/ \\____\\___/|_| \\___| ") |
| 217 | fmt.Println(" ") |
| 218 | } |
| 219 | |
David K. Bainbridge | f430cd5 | 2019-05-28 15:00:35 -0700 | [diff] [blame] | 220 | func printVersion() { |
| 221 | fmt.Println("VOLTHA Read-Write Core") |
| 222 | fmt.Println(version.VersionInfo.String(" ")) |
| 223 | } |
| 224 | |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 225 | func main() { |
| 226 | start := time.Now() |
| 227 | |
| 228 | cf := config.NewRWCoreFlags() |
| 229 | cf.ParseCommandArguments() |
| 230 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 231 | // Set the instance ID as the hostname |
| 232 | var instanceId string |
| 233 | hostName := utils.GetHostName() |
| 234 | if len(hostName) > 0 { |
| 235 | instanceId = hostName |
| 236 | } else { |
| 237 | log.Fatal("HOSTNAME not set") |
| 238 | } |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 239 | |
| 240 | //Setup default logger - applies for packages that do not have specific logger set |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 241 | if _, err := log.SetDefaultLogger(log.JSON, cf.LogLevel, log.Fields{"instanceId": instanceId}); err != nil { |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 242 | log.With(log.Fields{"error": err}).Fatal("Cannot setup logging") |
| 243 | } |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 244 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 245 | // Update all loggers (provisioned via init) with a common field |
| 246 | if err := log.UpdateAllLoggers(log.Fields{"instanceId": instanceId}); err != nil { |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 247 | log.With(log.Fields{"error": err}).Fatal("Cannot setup logging") |
| 248 | } |
| 249 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 250 | // Update all loggers to log level specified as input parameter |
| 251 | log.SetAllLogLevel(cf.LogLevel) |
khenaidoo | 2c6a099 | 2019-04-29 13:46:56 -0400 | [diff] [blame] | 252 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 253 | //log.SetPackageLogLevel("github.com/opencord/voltha-go/rw_core/core", log.DebugLevel) |
khenaidoo | b920354 | 2018-09-17 22:56:37 -0400 | [diff] [blame] | 254 | |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 255 | defer log.CleanUp() |
| 256 | |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 257 | // Print version / build information and exit |
David K. Bainbridge | f430cd5 | 2019-05-28 15:00:35 -0700 | [diff] [blame] | 258 | if cf.DisplayVersionOnly { |
| 259 | printVersion() |
| 260 | return |
| 261 | } |
| 262 | |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 263 | // Print banner if specified |
| 264 | if cf.Banner { |
| 265 | printBanner() |
| 266 | } |
| 267 | |
| 268 | log.Infow("rw-core-config", log.Fields{"config": *cf}) |
| 269 | |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 270 | // Create the core |
| 271 | rw := newRWCore(cf) |
| 272 | |
| 273 | // Create a context adding the status update channel |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 274 | ctx, cancel := context.WithCancel(context.Background()) |
| 275 | defer cancel() |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 276 | |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 277 | /* |
| 278 | * Create and start the liveness and readiness container management probes. This |
| 279 | * is done in the main function so just in case the main starts multiple other |
| 280 | * objects there can be a single probe end point for the process. |
| 281 | */ |
| 282 | p := &probe.Probe{} |
Kent Hagerman | c461883 | 2019-10-07 12:24:36 -0400 | [diff] [blame] | 283 | go p.ListenAndServe(fmt.Sprintf("%s:%d", rw.config.ProbeHost, rw.config.ProbePort)) |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 284 | |
| 285 | // Add the probe to the context to pass to all the services started |
| 286 | probeCtx := context.WithValue(ctx, probe.ProbeContextKey, p) |
| 287 | |
| 288 | // Start the core |
| 289 | go rw.start(probeCtx, instanceId) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 290 | |
| 291 | code := waitForExit() |
khenaidoo | 5c11af7 | 2018-07-20 17:21:05 -0400 | [diff] [blame] | 292 | log.Infow("received-a-closing-signal", log.Fields{"code": code}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 293 | |
| 294 | // Cleanup before leaving |
David K. Bainbridge | b4a9ab0 | 2019-09-20 15:12:16 -0700 | [diff] [blame] | 295 | rw.stop(probeCtx) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 296 | |
| 297 | elapsed := time.Since(start) |
khenaidoo | 631fe54 | 2019-05-31 15:44:43 -0400 | [diff] [blame] | 298 | log.Infow("rw-core-run-time", log.Fields{"core": instanceId, "time": elapsed / time.Second}) |
khenaidoo | cfee5f4 | 2018-07-19 22:47:38 -0400 | [diff] [blame] | 299 | } |