blob: fc55683ae4e84bf0d72a4465911ac797f6ce6962 [file] [log] [blame]
Richard Jankowski215a3e22018-10-04 13:56:11 -04001/*
2 * Copyright 2018-present Open Networking Foundation
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18 * Two voltha cores receive the same request; each tries to acquire ownership of the request
19 * by writing its identifier (e.g. container name or pod name) to the transaction key named
Richard Jankowskie4d77662018-10-17 13:53:21 -040020 * after the serial number of the request. The core that loses the race for acquisition
21 * monitors the progress of the core actually serving the request by watching for changes
22 * in the value of the transaction key. Once the request is complete, the
23 * serving core closes the transaction by invoking the KVTransaction's Close method, which
Richard Jankowski215a3e22018-10-04 13:56:11 -040024 * replaces the value of the transaction (i.e. serial number) key with the string
25 * TRANSACTION_COMPLETE. The standby core observes this update, stops watching the transaction,
26 * and then deletes the transaction key.
27 *
Richard Jankowski215a3e22018-10-04 13:56:11 -040028 */
npujar1d86a522019-11-14 17:11:16 +053029
Richard Jankowski215a3e22018-10-04 13:56:11 -040030package core
31
32import (
npujar467fe752020-01-16 20:17:45 +053033 "context"
npujar1d86a522019-11-14 17:11:16 +053034 "time"
35
serkant.uluderya2ae470f2020-01-21 11:13:09 -080036 "github.com/opencord/voltha-lib-go/v3/pkg/db/kvstore"
37 "github.com/opencord/voltha-lib-go/v3/pkg/log"
Kent Hagerman46dcd9d2019-09-18 16:42:59 -040038 "google.golang.org/grpc/codes"
39 "google.golang.org/grpc/status"
Richard Jankowski215a3e22018-10-04 13:56:11 -040040)
41
42// Transaction acquisition results
43const (
khenaidoo89b0e942018-10-21 21:11:33 -040044 UNKNOWN = iota
npujar1d86a522019-11-14 17:11:16 +053045 SeizedBySelf
46 CompletedByOther
47 AbandonedByOther
48 AbandonedWatchBySelf
Richard Jankowski215a3e22018-10-04 13:56:11 -040049)
50
Kent Hagerman46dcd9d2019-09-18 16:42:59 -040051var errorTransactionNotAcquired = status.Error(codes.Canceled, "transaction-not-acquired")
52
npujar1d86a522019-11-14 17:11:16 +053053// Transaction constant
Richard Jankowski215a3e22018-10-04 13:56:11 -040054const (
npujar1d86a522019-11-14 17:11:16 +053055 TransactionComplete = "TRANSACTION-COMPLETE"
Richard Jankowski215a3e22018-10-04 13:56:11 -040056)
57
khenaidoo09771ef2019-10-11 14:25:02 -040058// Transaction constants used to guarantee the Core processing a request hold on to the transaction until
khenaidoof684e1b2019-10-28 19:00:37 -040059// it either completes it (either successfully or times out) or the Core itself crashes (
60// e.g. a server failure).
khenaidoo09771ef2019-10-11 14:25:02 -040061// If a request has a timeout of x seconds then the Core processing the request will renew the transaction lease
62// every x/NUM_TXN_RENEWAL_PER_REQUEST seconds. After the Core completes the request it stops renewing the
63// transaction and sets the transaction value to TRANSACTION_COMPLETE. If the processing Core crashes then it
64// will not renew the transaction causing the KV store to delete the transaction after its renewal period. The
65// Core watching the transaction will then take over.
66// Since the MIN_TXN_RENEWAL_INTERVAL_IN_SEC is 3 seconds then for any transaction that completes within 3 seconds
67// there won't be a transaction renewal done.
68const (
npujar1d86a522019-11-14 17:11:16 +053069 NumTxnRenewalPerRequest = 2
70 MinTxnRenewalIntervalInSec = 3
71 MinTxnReservationDurationInSec = 5
khenaidoo09771ef2019-10-11 14:25:02 -040072)
73
npujar1d86a522019-11-14 17:11:16 +053074// TransactionContext represent transaction context attributes
Richard Jankowski215a3e22018-10-04 13:56:11 -040075type TransactionContext struct {
khenaidoo09771ef2019-10-11 14:25:02 -040076 kvClient kvstore.Client
77 kvOperationTimeout int
78 owner string
79 txnPrefix string
Richard Jankowski215a3e22018-10-04 13:56:11 -040080}
khenaidoo89b0e942018-10-21 21:11:33 -040081
Richard Jankowski215a3e22018-10-04 13:56:11 -040082var ctx *TransactionContext
83
khenaidoo89b0e942018-10-21 21:11:33 -040084var txnState = []string{
85 "UNKNOWN",
86 "SEIZED-BY-SELF",
87 "COMPLETED-BY-OTHER",
88 "ABANDONED-BY-OTHER",
khenaidoo09771ef2019-10-11 14:25:02 -040089 "ABANDONED_WATCH_BY_SELF"}
Richard Jankowski215a3e22018-10-04 13:56:11 -040090
91func init() {
npujar1d86a522019-11-14 17:11:16 +053092 _, err := log.AddPackage(log.JSON, log.DebugLevel, nil)
93 if err != nil {
94 log.Errorw("unable-to-register-package-to-the-log-map", log.Fields{"error": err})
95 }
Richard Jankowski215a3e22018-10-04 13:56:11 -040096}
97
npujar1d86a522019-11-14 17:11:16 +053098// NewTransactionContext creates transaction context instance
Richard Jankowski215a3e22018-10-04 13:56:11 -040099func NewTransactionContext(
khenaidoo89b0e942018-10-21 21:11:33 -0400100 owner string,
101 txnPrefix string,
102 kvClient kvstore.Client,
khenaidoo09771ef2019-10-11 14:25:02 -0400103 kvOpTimeout int) *TransactionContext {
Richard Jankowski215a3e22018-10-04 13:56:11 -0400104
khenaidoo89b0e942018-10-21 21:11:33 -0400105 return &TransactionContext{
khenaidoo09771ef2019-10-11 14:25:02 -0400106 owner: owner,
107 txnPrefix: txnPrefix,
108 kvClient: kvClient,
109 kvOperationTimeout: kvOpTimeout}
Richard Jankowski215a3e22018-10-04 13:56:11 -0400110}
111
112/*
113 * Before instantiating a KVTransaction, a TransactionContext must be created.
npujar1d86a522019-11-14 17:11:16 +0530114 * The parameters stored in the context govern the behavior of all KVTransaction
Richard Jankowski215a3e22018-10-04 13:56:11 -0400115 * instances.
116 *
117 * :param owner: The owner (i.e. voltha core name) of a transaction
118 * :param txnPrefix: The key prefix under which all transaction IDs, or serial numbers,
119 * will be created (e.g. "service/voltha/transactions")
120 * :param kvClient: The client API used for all interactions with the KV store. Currently
121 * only the etcd client is supported.
Richard Jankowski199fd862019-03-18 14:49:51 -0400122 * :param: kvOpTimeout: The maximum time, in seconds, to be taken by any KV operation
123 * used by this package
Richard Jankowski215a3e22018-10-04 13:56:11 -0400124 */
npujar1d86a522019-11-14 17:11:16 +0530125
126// SetTransactionContext creates new transaction context
Richard Jankowski215a3e22018-10-04 13:56:11 -0400127func SetTransactionContext(owner string,
khenaidoo89b0e942018-10-21 21:11:33 -0400128 txnPrefix string,
129 kvClient kvstore.Client,
khenaidoo09771ef2019-10-11 14:25:02 -0400130 kvOpTimeout int) error {
Richard Jankowski215a3e22018-10-04 13:56:11 -0400131
khenaidoo09771ef2019-10-11 14:25:02 -0400132 ctx = NewTransactionContext(owner, txnPrefix, kvClient, kvOpTimeout)
khenaidoo89b0e942018-10-21 21:11:33 -0400133 return nil
Richard Jankowski215a3e22018-10-04 13:56:11 -0400134}
135
npujar1d86a522019-11-14 17:11:16 +0530136// KVTransaction represent KV transaction attributes
Richard Jankowskie4d77662018-10-17 13:53:21 -0400137type KVTransaction struct {
khenaidoo09771ef2019-10-11 14:25:02 -0400138 monitorCh chan int
npujar1d86a522019-11-14 17:11:16 +0530139 txnID string
khenaidoo09771ef2019-10-11 14:25:02 -0400140 txnKey string
Richard Jankowski215a3e22018-10-04 13:56:11 -0400141}
142
143/*
144 * A KVTransaction constructor
145 *
146 * :param txnId: The serial number of a voltha request.
Richard Jankowskie4d77662018-10-17 13:53:21 -0400147 * :return: A KVTransaction instance
Richard Jankowski215a3e22018-10-04 13:56:11 -0400148 */
npujar1d86a522019-11-14 17:11:16 +0530149
150// NewKVTransaction creates KV transaction instance
151func NewKVTransaction(txnID string) *KVTransaction {
khenaidoo89b0e942018-10-21 21:11:33 -0400152 return &KVTransaction{
npujar1d86a522019-11-14 17:11:16 +0530153 txnID: txnID,
154 txnKey: ctx.txnPrefix + txnID}
Richard Jankowski215a3e22018-10-04 13:56:11 -0400155}
156
157/*
khenaidoo09771ef2019-10-11 14:25:02 -0400158 * Acquired is invoked by a Core, upon reception of a request, to reserve the transaction key in the KV store. The
159 * request may be resource specific (i.e will include an ID for that resource) or may be generic (i.e. list a set of
160 * resources). If the request is resource specific then this function should be invoked with the ownedByMe flag to
161 * indicate whether this Core owns this resource. In the case where this Core owns this resource or it is a generic
162 * request then we will proceed to reserve the transaction key in the KV store for a minimum time specified by the
163 * minDuration param. If the reservation request fails (i.e. the other Core got the reservation before this one - this
164 * can happen only for generic request) then the Core will start to watch for changes to the key to determine
165 * whether the other Core completed the transaction successfully or the Core just died. If the Core does not own the
166 * resource then we will proceed to watch the transaction key.
Richard Jankowski215a3e22018-10-04 13:56:11 -0400167 *
khenaidoo09771ef2019-10-11 14:25:02 -0400168 * :param minDuration: minimum time to reserve the transaction key in the KV store
169 * :param ownedByMe: specify whether the request is about a resource owned or not. If it's absent then this is a
170 * generic request that has no specific resource ID (e.g. list)
171 *
172 * :return: A boolean specifying whether the resource was acquired. An error is return in case this function is invoked
173 * for a resource that is nonexistent.
Richard Jankowski215a3e22018-10-04 13:56:11 -0400174 */
npujar1d86a522019-11-14 17:11:16 +0530175
176// Acquired aquires transaction status
npujar467fe752020-01-16 20:17:45 +0530177func (c *KVTransaction) Acquired(ctx context.Context, minDuration int64, ownedByMe ...bool) (bool, error) {
khenaidoo89b0e942018-10-21 21:11:33 -0400178 var acquired bool
npujar1d86a522019-11-14 17:11:16 +0530179 var currOwner string
khenaidoo89b0e942018-10-21 21:11:33 -0400180 var res int
Richard Jankowski215a3e22018-10-04 13:56:11 -0400181
khenaidoo89b0e942018-10-21 21:11:33 -0400182 // Convert milliseconds to seconds, rounding up
183 // The reservation TTL is specified in seconds
khenaidoo09771ef2019-10-11 14:25:02 -0400184 durationInSecs := minDuration / 1000
185 if remainder := minDuration % 1000; remainder > 0 {
khenaidoo89b0e942018-10-21 21:11:33 -0400186 durationInSecs++
187 }
npujar1d86a522019-11-14 17:11:16 +0530188 if durationInSecs < int64(MinTxnReservationDurationInSec) {
189 durationInSecs = int64(MinTxnReservationDurationInSec)
khenaidoo09771ef2019-10-11 14:25:02 -0400190 }
191 genericRequest := true
192 resourceOwned := false
193 if len(ownedByMe) > 0 {
194 genericRequest = false
195 resourceOwned = ownedByMe[0]
196 }
197 if resourceOwned || genericRequest {
198 // Keep the reservation longer that the minDuration (which is really the request timeout) to ensure the
199 // transaction key stays in the KV store until after the Core finalize a request timeout condition (which is
200 // a success from a request completion perspective).
npujar467fe752020-01-16 20:17:45 +0530201 if err := c.tryToReserveTxn(ctx, durationInSecs*2); err == nil {
npujar1d86a522019-11-14 17:11:16 +0530202 res = SeizedBySelf
khenaidoo09771ef2019-10-11 14:25:02 -0400203 } else {
204 log.Debugw("watch-other-server",
npujar1d86a522019-11-14 17:11:16 +0530205 log.Fields{"transactionId": c.txnID, "owner": currOwner, "timeout": durationInSecs})
npujar467fe752020-01-16 20:17:45 +0530206 res = c.Watch(ctx, durationInSecs)
khenaidoo09771ef2019-10-11 14:25:02 -0400207 }
208 } else {
npujar467fe752020-01-16 20:17:45 +0530209 res = c.Watch(ctx, durationInSecs)
khenaidoo09771ef2019-10-11 14:25:02 -0400210 }
211 switch res {
npujar1d86a522019-11-14 17:11:16 +0530212 case SeizedBySelf, AbandonedByOther:
khenaidoo09771ef2019-10-11 14:25:02 -0400213 acquired = true
214 default:
215 acquired = false
216 }
npujar1d86a522019-11-14 17:11:16 +0530217 log.Debugw("acquire-transaction-status", log.Fields{"transactionId": c.txnID, "acquired": acquired, "result": txnState[res]})
khenaidoo09771ef2019-10-11 14:25:02 -0400218 return acquired, nil
219}
Richard Jankowski215a3e22018-10-04 13:56:11 -0400220
npujar467fe752020-01-16 20:17:45 +0530221func (c *KVTransaction) tryToReserveTxn(ctxt context.Context, durationInSecs int64) error {
npujar1d86a522019-11-14 17:11:16 +0530222 var currOwner string
khenaidoo09771ef2019-10-11 14:25:02 -0400223 var res int
npujar1d86a522019-11-14 17:11:16 +0530224 var err error
npujar467fe752020-01-16 20:17:45 +0530225 value, _ := ctx.kvClient.Reserve(ctxt, c.txnKey, ctx.owner, durationInSecs)
khenaidoo89b0e942018-10-21 21:11:33 -0400226 if value != nil {
khenaidoo09771ef2019-10-11 14:25:02 -0400227 if currOwner, err = kvstore.ToString(value); err != nil { // This should never happen
npujar1d86a522019-11-14 17:11:16 +0530228 log.Errorw("unexpected-owner-type", log.Fields{"transactionId": c.txnID, "error": err})
khenaidoo09771ef2019-10-11 14:25:02 -0400229 return err
230 }
231 if currOwner == ctx.owner {
npujar1d86a522019-11-14 17:11:16 +0530232 log.Debugw("acquired-transaction", log.Fields{"transactionId": c.txnID, "result": txnState[res]})
khenaidoo09771ef2019-10-11 14:25:02 -0400233 // Setup the monitoring channel
234 c.monitorCh = make(chan int)
npujar467fe752020-01-16 20:17:45 +0530235 go c.holdOnToTxnUntilProcessingCompleted(ctxt, c.txnKey, ctx.owner, durationInSecs)
khenaidoo09771ef2019-10-11 14:25:02 -0400236 return nil
khenaidoo89b0e942018-10-21 21:11:33 -0400237 }
238 }
khenaidoo09771ef2019-10-11 14:25:02 -0400239 return status.Error(codes.PermissionDenied, "reservation-denied")
Richard Jankowski215a3e22018-10-04 13:56:11 -0400240}
241
npujar1d86a522019-11-14 17:11:16 +0530242// Watch watches transaction
npujar467fe752020-01-16 20:17:45 +0530243func (c *KVTransaction) Watch(ctxt context.Context, durationInSecs int64) int {
Richard Jankowski199fd862019-03-18 14:49:51 -0400244 var res int
Scott Baker0e78ba22020-02-24 17:58:47 -0800245 events := ctx.kvClient.Watch(ctxt, c.txnKey, false)
A R Karthick43ba1fb2019-10-03 16:24:21 +0000246 defer ctx.kvClient.CloseWatch(c.txnKey, events)
Richard Jankowski199fd862019-03-18 14:49:51 -0400247
khenaidoo09771ef2019-10-11 14:25:02 -0400248 transactionWasAcquiredByOther := false
249
250 //Check whether the transaction was already completed by the other Core before we got here.
npujar467fe752020-01-16 20:17:45 +0530251 if kvp, _ := ctx.kvClient.Get(ctxt, c.txnKey); kvp != nil {
khenaidoo09771ef2019-10-11 14:25:02 -0400252 transactionWasAcquiredByOther = true
253 if val, err := kvstore.ToString(kvp.Value); err == nil {
npujar1d86a522019-11-14 17:11:16 +0530254 if val == TransactionComplete {
255 res = CompletedByOther
khenaidoo09771ef2019-10-11 14:25:02 -0400256 // Do an immediate delete of the transaction in the KV Store to free up KV Storage faster
npujar467fe752020-01-16 20:17:45 +0530257 err = c.Delete(ctxt)
npujar1d86a522019-11-14 17:11:16 +0530258 if err != nil {
259 log.Errorw("unable-to-delete-the-transaction", log.Fields{"error": err})
260 }
khenaidoo09771ef2019-10-11 14:25:02 -0400261 return res
262 }
263 } else {
264 // An unexpected value - let's get out of here as something did not go according to plan
npujar1d86a522019-11-14 17:11:16 +0530265 res = AbandonedWatchBySelf
266 log.Debugw("cannot-read-transaction-value", log.Fields{"txn": c.txnID, "error": err})
khenaidoo09771ef2019-10-11 14:25:02 -0400267 return res
268 }
269 }
270
A R Karthick919f6db2019-08-29 18:14:56 +0000271 for {
272 select {
A R Karthick919f6db2019-08-29 18:14:56 +0000273 case event := <-events:
khenaidoo09771ef2019-10-11 14:25:02 -0400274 transactionWasAcquiredByOther = true
npujar1d86a522019-11-14 17:11:16 +0530275 log.Debugw("received-event", log.Fields{"txn": c.txnID, "type": event.EventType})
A R Karthick919f6db2019-08-29 18:14:56 +0000276 if event.EventType == kvstore.DELETE {
277 // The other core failed to process the request
npujar1d86a522019-11-14 17:11:16 +0530278 res = AbandonedByOther
A R Karthick919f6db2019-08-29 18:14:56 +0000279 } else if event.EventType == kvstore.PUT {
280 key, e1 := kvstore.ToString(event.Key)
281 val, e2 := kvstore.ToString(event.Value)
khenaidoo09771ef2019-10-11 14:25:02 -0400282 if e1 == nil && e2 == nil && key == c.txnKey {
npujar1d86a522019-11-14 17:11:16 +0530283 if val == TransactionComplete {
284 res = CompletedByOther
khenaidoo09771ef2019-10-11 14:25:02 -0400285 // Successful request completion has been detected. Remove the transaction key
npujar467fe752020-01-16 20:17:45 +0530286 err := c.Delete(ctxt)
npujar1d86a522019-11-14 17:11:16 +0530287 if err != nil {
288 log.Errorw("unable-to-delete-the-transaction", log.Fields{"error": err})
289 }
A R Karthick919f6db2019-08-29 18:14:56 +0000290 } else {
khenaidoo09771ef2019-10-11 14:25:02 -0400291 log.Debugw("Ignoring-PUT-event", log.Fields{"val": val, "key": key})
A R Karthick919f6db2019-08-29 18:14:56 +0000292 continue
293 }
khenaidoo09771ef2019-10-11 14:25:02 -0400294 } else {
npujar1d86a522019-11-14 17:11:16 +0530295 log.Warnw("received-unexpected-PUT-event", log.Fields{"txn": c.txnID, "key": key, "ctxKey": c.txnKey})
A R Karthick919f6db2019-08-29 18:14:56 +0000296 }
Richard Jankowski199fd862019-03-18 14:49:51 -0400297 }
khenaidoo09771ef2019-10-11 14:25:02 -0400298 case <-time.After(time.Duration(durationInSecs) * time.Second):
299 // Corner case: In the case where the Core owning the device dies and before this Core takes ownership of
300 // this device there is a window where new requests will end up being watched instead of being processed.
301 // Grab the request if the other Core did not create the transaction in the KV store.
302 // TODO: Use a peer-monitoring probe to switch over (still relies on the probe frequency). This will
303 // guarantee that the peer is actually gone instead of limiting the time the peer can get hold of a
304 // request.
305 if !transactionWasAcquiredByOther {
npujar1d86a522019-11-14 17:11:16 +0530306 log.Debugw("timeout-no-peer", log.Fields{"txId": c.txnID})
307 res = AbandonedByOther
khenaidoo09771ef2019-10-11 14:25:02 -0400308 } else {
309 continue
310 }
Richard Jankowski199fd862019-03-18 14:49:51 -0400311 }
A R Karthick919f6db2019-08-29 18:14:56 +0000312 break
Richard Jankowski199fd862019-03-18 14:49:51 -0400313 }
314 return res
315}
316
npujar1d86a522019-11-14 17:11:16 +0530317// Close closes transaction
npujar467fe752020-01-16 20:17:45 +0530318func (c *KVTransaction) Close(ctxt context.Context) error {
npujar1d86a522019-11-14 17:11:16 +0530319 log.Debugw("close", log.Fields{"txn": c.txnID})
khenaidoo09771ef2019-10-11 14:25:02 -0400320 // Stop monitoring the key (applies only when there has been no transaction switch over)
321 if c.monitorCh != nil {
322 close(c.monitorCh)
npujar467fe752020-01-16 20:17:45 +0530323 err := ctx.kvClient.Put(ctxt, c.txnKey, TransactionComplete)
npujar1d86a522019-11-14 17:11:16 +0530324
325 if err != nil {
326 log.Errorw("unable-to-write-a-key-value-pair-to-the-KV-store", log.Fields{"error": err})
327 }
khenaidoo09771ef2019-10-11 14:25:02 -0400328 }
329 return nil
Richard Jankowski215a3e22018-10-04 13:56:11 -0400330}
331
npujar1d86a522019-11-14 17:11:16 +0530332// Delete deletes transaction
npujar467fe752020-01-16 20:17:45 +0530333func (c *KVTransaction) Delete(ctxt context.Context) error {
npujar1d86a522019-11-14 17:11:16 +0530334 log.Debugw("delete", log.Fields{"txn": c.txnID})
npujar467fe752020-01-16 20:17:45 +0530335 return ctx.kvClient.Delete(ctxt, c.txnKey)
khenaidoo09771ef2019-10-11 14:25:02 -0400336}
337
338// holdOnToTxnUntilProcessingCompleted renews the transaction lease until the transaction is complete. durationInSecs
339// is used to calculate the frequency at which the Core processing the transaction renews the lease. This function
340// exits only when the transaction is Closed, i.e completed.
npujar467fe752020-01-16 20:17:45 +0530341func (c *KVTransaction) holdOnToTxnUntilProcessingCompleted(ctxt context.Context, key string, owner string, durationInSecs int64) {
npujar1d86a522019-11-14 17:11:16 +0530342 log.Debugw("holdOnToTxnUntilProcessingCompleted", log.Fields{"txn": c.txnID})
343 renewInterval := durationInSecs / NumTxnRenewalPerRequest
344 if renewInterval < MinTxnRenewalIntervalInSec {
345 renewInterval = MinTxnRenewalIntervalInSec
khenaidoo09771ef2019-10-11 14:25:02 -0400346 }
347forLoop:
348 for {
349 select {
350 case <-c.monitorCh:
npujar1d86a522019-11-14 17:11:16 +0530351 log.Debugw("transaction-renewal-exits", log.Fields{"txn": c.txnID})
khenaidoo09771ef2019-10-11 14:25:02 -0400352 break forLoop
353 case <-time.After(time.Duration(renewInterval) * time.Second):
npujar467fe752020-01-16 20:17:45 +0530354 if err := ctx.kvClient.RenewReservation(ctxt, c.txnKey); err != nil {
khenaidoo09771ef2019-10-11 14:25:02 -0400355 // Log and continue.
356 log.Warnw("transaction-renewal-failed", log.Fields{"txnId": c.txnKey, "error": err})
357 }
358 }
359 }
Richard Jankowski215a3e22018-10-04 13:56:11 -0400360}