Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package model |
| 18 | |
| 19 | import ( |
| 20 | "bytes" |
| 21 | "compress/gzip" |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 22 | "github.com/golang/protobuf/proto" |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 23 | "github.com/opencord/voltha-go/common/log" |
| 24 | "github.com/opencord/voltha-go/db/kvstore" |
| 25 | "reflect" |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 26 | "strings" |
| 27 | "sync" |
| 28 | ) |
| 29 | |
| 30 | // PersistedRevision holds information of revision meant to be saved in a persistent storage |
| 31 | type PersistedRevision struct { |
| 32 | Revision |
| 33 | Compress bool |
| 34 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 35 | events chan *kvstore.Event |
| 36 | kvStore *Backend |
| 37 | mutex sync.RWMutex |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 38 | isStored bool |
| 39 | isWatched bool |
| 40 | } |
| 41 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 42 | type watchCache struct { |
| 43 | Cache sync.Map |
| 44 | } |
| 45 | |
| 46 | var watchCacheInstance *watchCache |
| 47 | var watchCacheOne sync.Once |
| 48 | |
| 49 | func Watches() *watchCache { |
| 50 | watchCacheOne.Do(func() { |
| 51 | watchCacheInstance = &watchCache{Cache: sync.Map{}} |
| 52 | }) |
| 53 | return watchCacheInstance |
| 54 | } |
| 55 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 56 | // NewPersistedRevision creates a new instance of a PersistentRevision structure |
| 57 | func NewPersistedRevision(branch *Branch, data interface{}, children map[string][]Revision) Revision { |
| 58 | pr := &PersistedRevision{} |
| 59 | pr.kvStore = branch.Node.GetRoot().KvStore |
| 60 | pr.Revision = NewNonPersistedRevision(nil, branch, data, children) |
| 61 | return pr |
| 62 | } |
| 63 | |
| 64 | // Finalize is responsible of saving the revision in the persistent storage |
| 65 | func (pr *PersistedRevision) Finalize(skipOnExist bool) { |
| 66 | pr.store(skipOnExist) |
| 67 | } |
| 68 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 69 | func (pr *PersistedRevision) store(skipOnExist bool) { |
| 70 | if pr.GetBranch().Txid != "" { |
| 71 | return |
| 72 | } |
| 73 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 74 | log.Debugw("ready-to-store-revision", log.Fields{"hash": pr.GetHash(), "name": pr.GetName(), "data": pr.GetData()}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 75 | |
| 76 | if blob, err := proto.Marshal(pr.GetConfig().Data.(proto.Message)); err != nil { |
| 77 | // TODO report error |
| 78 | } else { |
| 79 | if pr.Compress { |
| 80 | var b bytes.Buffer |
| 81 | w := gzip.NewWriter(&b) |
| 82 | w.Write(blob) |
| 83 | w.Close() |
| 84 | blob = b.Bytes() |
| 85 | } |
| 86 | |
| 87 | if err := pr.kvStore.Put(pr.GetName(), blob); err != nil { |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 88 | log.Warnw("problem-storing-revision", log.Fields{"error": err, "hash": pr.GetHash(), "name": pr.GetName(), "data": pr.GetConfig().Data}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 89 | } else { |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 90 | log.Debugw("storing-revision", log.Fields{"hash": pr.GetHash(), "name": pr.GetName(), "data": pr.GetConfig().Data}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 91 | pr.isStored = true |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | func (pr *PersistedRevision) SetupWatch(key string) { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 97 | if key == "" { |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 98 | log.Debugw("ignoring-watch", log.Fields{"key": key, "revision-hash": pr.GetHash()}) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 99 | return |
| 100 | } |
| 101 | |
| 102 | if _, exists := Watches().Cache.LoadOrStore(key+"-"+pr.GetHash(), struct{}{}); exists { |
| 103 | return |
| 104 | } |
| 105 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 106 | if pr.events == nil { |
| 107 | pr.events = make(chan *kvstore.Event) |
| 108 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 109 | log.Debugw("setting-watch-channel", log.Fields{"key": key, "revision-hash": pr.GetHash()}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 110 | |
| 111 | pr.SetName(key) |
| 112 | pr.events = pr.kvStore.CreateWatch(key) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 113 | } |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 114 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 115 | if !pr.isWatched { |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 116 | pr.isWatched = true |
| 117 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 118 | log.Debugw("setting-watch-routine", log.Fields{"key": key, "revision-hash": pr.GetHash()}) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 119 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 120 | // Start watching |
| 121 | go pr.startWatching() |
| 122 | } |
| 123 | } |
| 124 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 125 | func (pr *PersistedRevision) startWatching() { |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 126 | log.Debugw("starting-watch", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 127 | |
| 128 | StopWatchLoop: |
| 129 | for { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 130 | if pr.IsDiscarded() { |
| 131 | break StopWatchLoop |
| 132 | } |
| 133 | |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 134 | select { |
| 135 | case event, ok := <-pr.events: |
| 136 | if !ok { |
| 137 | log.Errorw("event-channel-failure: stopping watch loop", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
| 138 | break StopWatchLoop |
| 139 | } |
| 140 | |
| 141 | log.Debugw("received-event", log.Fields{"type": event.EventType, "watch": pr.GetName()}) |
| 142 | |
| 143 | switch event.EventType { |
| 144 | case kvstore.DELETE: |
| 145 | log.Debugw("delete-from-memory", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
| 146 | pr.Revision.Drop("", true) |
| 147 | break StopWatchLoop |
| 148 | |
| 149 | case kvstore.PUT: |
| 150 | log.Debugw("update-in-memory", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
| 151 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 152 | data := reflect.New(reflect.TypeOf(pr.GetData()).Elem()) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 153 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 154 | if err := proto.Unmarshal(event.Value.([]byte), data.Interface().(proto.Message)); err != nil { |
| 155 | log.Errorw("failed-to-unmarshal-watch-data", log.Fields{"key": pr.GetHash(), "watch": pr.GetName(), "error": err}) |
| 156 | } else { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 157 | log.Debugw("un-marshaled-watch-data", log.Fields{"key": pr.GetHash(), "watch": pr.GetName(), "data": data.Interface()}) |
| 158 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 159 | var pathLock string |
| 160 | var pac *proxyAccessControl |
| 161 | var blobs map[string]*kvstore.KVPair |
| 162 | |
| 163 | // The watch reported new persistence data. |
| 164 | // Construct an object that will be used to update the memory |
| 165 | blobs = make(map[string]*kvstore.KVPair) |
| 166 | key, _ := kvstore.ToString(event.Key) |
| 167 | blobs[key] = &kvstore.KVPair{ |
| 168 | Key: key, |
| 169 | Value: event.Value, |
| 170 | Session: "", |
| 171 | Lease: 0, |
| 172 | } |
| 173 | |
| 174 | if pr.GetNode().GetProxy() != nil { |
| 175 | // |
| 176 | // If a proxy exists for this revision, use it to lock access to the path |
| 177 | // and prevent simultaneous updates to the object in memory |
| 178 | // |
| 179 | pathLock, _ = pr.GetNode().GetProxy().parseForControlledPath(pr.GetNode().GetProxy().getFullPath()) |
| 180 | |
| 181 | //If the proxy already has a request in progress, then there is no need to process the watch |
| 182 | log.Debugw("checking-if-path-is-locked", log.Fields{"key": pr.GetHash(), "pathLock": pathLock}) |
| 183 | if PAC().IsReserved(pathLock) { |
| 184 | log.Debugw("operation-in-progress", log.Fields{ |
| 185 | "key": pr.GetHash(), |
| 186 | "path": pr.GetNode().GetProxy().getFullPath(), |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 187 | "operation": pr.GetNode().GetProxy().Operation.String(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 188 | }) |
| 189 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 190 | // Identify the operation type and determine if the watch event should be applied or not. |
| 191 | switch pr.GetNode().GetProxy().Operation { |
| 192 | case PROXY_REMOVE: |
| 193 | fallthrough |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 194 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 195 | case PROXY_ADD: |
| 196 | fallthrough |
| 197 | |
| 198 | case PROXY_UPDATE: |
| 199 | // We will need to reload once the operation completes. |
| 200 | // Therefore, the data of the current event is most likely out-dated |
| 201 | // and should be ignored |
| 202 | log.Debugw("ignore-watch-event", log.Fields{ |
| 203 | "key": pr.GetHash(), |
| 204 | "path": pr.GetNode().GetProxy().getFullPath(), |
| 205 | "operation": pr.GetNode().GetProxy().Operation.String(), |
| 206 | }) |
| 207 | |
| 208 | continue |
| 209 | |
| 210 | case PROXY_CREATE: |
| 211 | fallthrough |
| 212 | |
| 213 | case PROXY_LIST: |
| 214 | fallthrough |
| 215 | |
| 216 | case PROXY_GET: |
| 217 | fallthrough |
| 218 | |
| 219 | default: |
| 220 | log.Debugw("process-watch-event", log.Fields{ |
| 221 | "key": pr.GetHash(), |
| 222 | "path": pr.GetNode().GetProxy().getFullPath(), |
| 223 | "operation": pr.GetNode().GetProxy().Operation.String(), |
| 224 | }) |
| 225 | } |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 226 | } |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 227 | |
| 228 | // Reserve the path to prevent others to modify while we reload from persistence |
| 229 | log.Debugw("reserve-and-lock-path", log.Fields{"key": pr.GetHash(), "path": pathLock}) |
| 230 | pac = PAC().ReservePath(pr.GetNode().GetProxy().getFullPath(), pr.GetNode().GetProxy(), pathLock) |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 231 | pac.lock() |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 232 | pr.GetNode().GetProxy().Operation = PROXY_UPDATE |
| 233 | pac.SetProxy(pr.GetNode().GetProxy()) |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 234 | |
| 235 | // Load changes and apply to memory |
| 236 | pr.LoadFromPersistence(pr.GetName(), "", blobs) |
| 237 | |
| 238 | log.Debugw("release-and-unlock-path", log.Fields{"key": pr.GetHash(), "path": pathLock}) |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 239 | pac.getProxy().Operation = PROXY_GET |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 240 | pac.unlock() |
| 241 | PAC().ReleasePath(pathLock) |
| 242 | |
| 243 | } else { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 244 | // This block should be reached only if coming from a non-proxied request |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 245 | log.Debugw("revision-with-no-proxy", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
| 246 | |
| 247 | // Load changes and apply to memory |
| 248 | pr.LoadFromPersistence(pr.GetName(), "", blobs) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 249 | } |
| 250 | } |
| 251 | |
| 252 | default: |
| 253 | log.Debugw("unhandled-event", log.Fields{"key": pr.GetHash(), "watch": pr.GetName(), "type": event.EventType}) |
| 254 | } |
| 255 | } |
| 256 | } |
| 257 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 258 | Watches().Cache.Delete(pr.GetName() + "-" + pr.GetHash()) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 259 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 260 | log.Debugw("exiting-watch", log.Fields{"key": pr.GetHash(), "watch": pr.GetName()}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 261 | } |
| 262 | |
| 263 | // UpdateData modifies the information in the data model and saves it in the persistent storage |
| 264 | func (pr *PersistedRevision) UpdateData(data interface{}, branch *Branch) Revision { |
| 265 | log.Debugw("updating-persisted-data", log.Fields{"hash": pr.GetHash()}) |
| 266 | |
| 267 | newNPR := pr.Revision.UpdateData(data, branch) |
| 268 | |
| 269 | newPR := &PersistedRevision{ |
| 270 | Revision: newNPR, |
| 271 | Compress: pr.Compress, |
| 272 | kvStore: pr.kvStore, |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 273 | events: pr.events, |
| 274 | } |
| 275 | |
| 276 | if newPR.GetHash() != pr.GetHash() { |
| 277 | newPR.isWatched = false |
| 278 | newPR.isStored = false |
| 279 | pr.Drop(branch.Txid, false) |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 280 | pr.Drop(branch.Txid, false) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 281 | } else { |
| 282 | newPR.isWatched = true |
| 283 | newPR.isStored = true |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | return newPR |
| 287 | } |
| 288 | |
| 289 | // UpdateChildren modifies the children of a revision and of a specific component and saves it in the persistent storage |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 290 | func (pr *PersistedRevision) UpdateChildren(name string, children []Revision, |
| 291 | branch *Branch) Revision { |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 292 | log.Debugw("updating-persisted-children", log.Fields{"hash": pr.GetHash()}) |
| 293 | |
| 294 | newNPR := pr.Revision.UpdateChildren(name, children, branch) |
| 295 | |
| 296 | newPR := &PersistedRevision{ |
| 297 | Revision: newNPR, |
| 298 | Compress: pr.Compress, |
| 299 | kvStore: pr.kvStore, |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 300 | events: pr.events, |
| 301 | } |
| 302 | |
| 303 | if newPR.GetHash() != pr.GetHash() { |
| 304 | newPR.isWatched = false |
| 305 | newPR.isStored = false |
| 306 | pr.Drop(branch.Txid, false) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 307 | } else { |
| 308 | newPR.isWatched = true |
| 309 | newPR.isStored = true |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 310 | } |
| 311 | |
| 312 | return newPR |
| 313 | } |
| 314 | |
| 315 | // UpdateAllChildren modifies the children for all components of a revision and saves it in the peristent storage |
| 316 | func (pr *PersistedRevision) UpdateAllChildren(children map[string][]Revision, branch *Branch) Revision { |
| 317 | log.Debugw("updating-all-persisted-children", log.Fields{"hash": pr.GetHash()}) |
| 318 | |
| 319 | newNPR := pr.Revision.UpdateAllChildren(children, branch) |
| 320 | |
| 321 | newPR := &PersistedRevision{ |
| 322 | Revision: newNPR, |
| 323 | Compress: pr.Compress, |
| 324 | kvStore: pr.kvStore, |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 325 | events: pr.events, |
| 326 | } |
| 327 | |
| 328 | if newPR.GetHash() != pr.GetHash() { |
| 329 | newPR.isWatched = false |
| 330 | newPR.isStored = false |
| 331 | pr.Drop(branch.Txid, false) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 332 | } else { |
| 333 | newPR.isWatched = true |
| 334 | newPR.isStored = true |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 335 | } |
| 336 | |
| 337 | return newPR |
| 338 | } |
| 339 | |
| 340 | // Drop takes care of eliminating a revision hash that is no longer needed |
| 341 | // and its associated config when required |
| 342 | func (pr *PersistedRevision) Drop(txid string, includeConfig bool) { |
| 343 | pr.Revision.Drop(txid, includeConfig) |
| 344 | } |
| 345 | |
| 346 | // Drop takes care of eliminating a revision hash that is no longer needed |
| 347 | // and its associated config when required |
| 348 | func (pr *PersistedRevision) StorageDrop(txid string, includeConfig bool) { |
| 349 | log.Debugw("dropping-revision", |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 350 | log.Fields{"txid": txid, "hash": pr.GetHash(), "config-hash": pr.GetConfig().Hash}) |
Matt Jeanneret | cab955f | 2019-04-10 15:45:57 -0400 | [diff] [blame] | 351 | |
| 352 | pr.mutex.Lock() |
| 353 | defer pr.mutex.Unlock() |
| 354 | if pr.kvStore != nil && txid == "" { |
| 355 | if pr.isStored { |
| 356 | if pr.isWatched { |
| 357 | pr.kvStore.DeleteWatch(pr.GetName(), pr.events) |
| 358 | pr.isWatched = false |
| 359 | } |
| 360 | |
| 361 | if err := pr.kvStore.Delete(pr.GetName()); err != nil { |
| 362 | log.Errorw("failed-to-remove-revision", log.Fields{"hash": pr.GetHash(), "error": err.Error()}) |
| 363 | } else { |
| 364 | pr.isStored = false |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | } else { |
| 369 | if includeConfig { |
| 370 | log.Debugw("attempted-to-remove-transacted-revision-config", log.Fields{"hash": pr.GetConfig().Hash, "txid": txid}) |
| 371 | } |
| 372 | log.Debugw("attempted-to-remove-transacted-revision", log.Fields{"hash": pr.GetHash(), "txid": txid}) |
| 373 | } |
| 374 | |
| 375 | pr.Revision.Drop(txid, includeConfig) |
| 376 | } |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 377 | |
| 378 | // verifyPersistedEntry validates if the provided data is available or not in memory and applies updates as required |
| 379 | func (pr *PersistedRevision) verifyPersistedEntry(data interface{}, typeName string, keyName string, keyValue string, txid string) (response Revision) { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 380 | // Parent which holds the current node entry |
| 381 | parent := pr.GetBranch().Node.Root |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 382 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 383 | // Get a copy of the parent's children |
| 384 | children := make([]Revision, len(parent.GetBranch(NONE).Latest.GetChildren(typeName))) |
| 385 | copy(children, parent.GetBranch(NONE).Latest.GetChildren(typeName)) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 386 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 387 | // Verify if a child with the provided key value can be found |
| 388 | if childIdx, childRev := pr.GetNode().findRevByKey(children, keyName, keyValue); childRev != nil { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 389 | // A child matching the provided key exists in memory |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 390 | // Verify if the data differs from what was retrieved from persistence |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 391 | if childRev.GetData().(proto.Message).String() != data.(proto.Message).String() { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 392 | log.Debugw("revision-data-is-different", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 393 | "key": childRev.GetHash(), |
| 394 | "name": childRev.GetName(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 395 | "data": childRev.GetData(), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 396 | }) |
| 397 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 398 | // |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 399 | // Data has changed; replace the child entry and update the parent revision |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 400 | // |
| 401 | |
| 402 | // BEGIN Lock child -- prevent any incoming changes |
| 403 | childRev.GetBranch().LatestLock.Lock() |
| 404 | |
| 405 | // Update child |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 406 | updatedChildRev := childRev.UpdateData(data, childRev.GetBranch()) |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 407 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 408 | updatedChildRev.GetNode().SetProxy(childRev.GetNode().GetProxy()) |
| 409 | updatedChildRev.SetupWatch(updatedChildRev.GetName()) |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 410 | childRev.Drop(txid, false) |
| 411 | updatedChildRev.SetLastUpdate() |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 412 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 413 | // Update cache |
| 414 | GetRevCache().Cache.Store(updatedChildRev.GetName(), updatedChildRev) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 415 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 416 | childRev.GetBranch().LatestLock.Unlock() |
| 417 | // END lock child |
| 418 | |
| 419 | // Update child entry |
| 420 | children[childIdx] = updatedChildRev |
| 421 | |
| 422 | // BEGIN lock parent -- Update parent |
| 423 | parent.GetBranch(NONE).LatestLock.Lock() |
| 424 | |
| 425 | updatedRev := parent.GetBranch(NONE).Latest.UpdateChildren(typeName, children, parent.GetBranch(NONE)) |
| 426 | parent.GetBranch(NONE).Node.makeLatest(parent.GetBranch(NONE), updatedRev, nil) |
| 427 | |
| 428 | parent.GetBranch(NONE).LatestLock.Unlock() |
| 429 | // END lock parent |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 430 | |
| 431 | // Drop the previous child revision |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 432 | parent.GetBranch(NONE).Latest.ChildDrop(typeName, childRev.GetHash()) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 433 | |
| 434 | if updatedChildRev != nil { |
| 435 | log.Debugw("verify-persisted-entry--adding-child", log.Fields{ |
| 436 | "key": updatedChildRev.GetHash(), |
| 437 | "name": updatedChildRev.GetName(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 438 | "data": updatedChildRev.GetData(), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 439 | }) |
| 440 | response = updatedChildRev |
| 441 | } |
| 442 | } else { |
| 443 | // Data is the same. Continue to the next entry |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 444 | log.Debugw("same-revision-data", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 445 | "key": childRev.GetHash(), |
| 446 | "name": childRev.GetName(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 447 | "data": childRev.GetData(), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 448 | }) |
| 449 | if childRev != nil { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 450 | log.Debugw("keeping-same-revision-data", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 451 | "key": childRev.GetHash(), |
| 452 | "name": childRev.GetName(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 453 | "data": childRev.GetData(), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 454 | }) |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 455 | |
| 456 | // Update timestamp to reflect when it was last read and to reset tracked timeout |
| 457 | childRev.SetLastUpdate() |
| 458 | GetRevCache().Cache.Store(childRev.GetName(), childRev) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 459 | response = childRev |
| 460 | } |
| 461 | } |
| 462 | } else { |
| 463 | // There is no available child with that key value. |
| 464 | // Create a new child and update the parent revision. |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 465 | log.Debugw("no-such-revision-entry", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 466 | "key": keyValue, |
| 467 | "name": typeName, |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 468 | "data": data, |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 469 | }) |
| 470 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 471 | // BEGIN child lock |
| 472 | pr.GetBranch().LatestLock.Lock() |
| 473 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 474 | // Construct a new child node with the retrieved persistence data |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 475 | childRev = pr.GetBranch().Node.MakeNode(data, txid).Latest(txid) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 476 | |
| 477 | // We need to start watching this entry for future changes |
| 478 | childRev.SetName(typeName + "/" + keyValue) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 479 | childRev.SetupWatch(childRev.GetName()) |
| 480 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 481 | pr.GetBranch().Node.makeLatest(pr.GetBranch(), childRev, nil) |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 482 | pr.GetBranch().LatestLock.Unlock() |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 483 | // END child lock |
| 484 | |
| 485 | // |
| 486 | // Add the child to the parent revision |
| 487 | // |
| 488 | |
| 489 | // BEGIN parent lock |
| 490 | parent.GetBranch(NONE).LatestLock.Lock() |
| 491 | children = append(children, childRev) |
| 492 | updatedRev := parent.GetBranch(NONE).Latest.UpdateChildren(typeName, children, parent.GetBranch(NONE)) |
| 493 | updatedRev.GetNode().SetProxy(parent.GetBranch(NONE).Node.GetProxy()) |
| 494 | |
| 495 | parent.GetBranch(NONE).Node.makeLatest(parent.GetBranch(NONE), updatedRev, nil) |
| 496 | parent.GetBranch(NONE).LatestLock.Unlock() |
| 497 | // END parent lock |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 498 | |
| 499 | // Child entry is valid and can be included in the response object |
| 500 | if childRev != nil { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 501 | log.Debugw("adding-revision-to-response", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 502 | "key": childRev.GetHash(), |
| 503 | "name": childRev.GetName(), |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 504 | "data": childRev.GetData(), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 505 | }) |
| 506 | response = childRev |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | return response |
| 511 | } |
| 512 | |
| 513 | // LoadFromPersistence retrieves data from kv store at the specified location and refreshes the memory |
| 514 | // by adding missing entries, updating changed entries and ignoring unchanged ones |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 515 | func (pr *PersistedRevision) LoadFromPersistence( |
| 516 | path string, txid string, blobs map[string]*kvstore.KVPair) []Revision { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 517 | pr.mutex.Lock() |
| 518 | defer pr.mutex.Unlock() |
| 519 | |
| 520 | log.Debugw("loading-from-persistence", log.Fields{"path": path, "txid": txid}) |
| 521 | |
| 522 | var response []Revision |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 523 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 524 | for strings.HasPrefix(path, "/") { |
| 525 | path = path[1:] |
| 526 | } |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 527 | |
| 528 | if pr.kvStore != nil && path != "" { |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 529 | if blobs == nil || len(blobs) == 0 { |
| 530 | log.Debugw("retrieve-from-kv", log.Fields{"path": path, "txid": txid}) |
| 531 | blobs, _ = pr.kvStore.List(path) |
| 532 | } |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 533 | |
| 534 | partition := strings.SplitN(path, "/", 2) |
| 535 | name := partition[0] |
| 536 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 537 | var nodeType interface{} |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 538 | if len(partition) < 2 { |
| 539 | path = "" |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 540 | nodeType = pr.GetBranch().Node.Type |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 541 | } else { |
| 542 | path = partition[1] |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 543 | nodeType = pr.GetBranch().Node.Root.Type |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 544 | } |
| 545 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 546 | field := ChildrenFields(nodeType)[name] |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 547 | |
| 548 | if field != nil && field.IsContainer { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 549 | log.Debugw("parsing-data-blobs", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 550 | "path": path, |
| 551 | "name": name, |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 552 | "size": len(blobs), |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 553 | }) |
| 554 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 555 | for _, blob := range blobs { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 556 | output := blob.Value.([]byte) |
| 557 | |
| 558 | data := reflect.New(field.ClassType.Elem()) |
| 559 | |
| 560 | if err := proto.Unmarshal(output, data.Interface().(proto.Message)); err != nil { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 561 | log.Errorw("failed-to-unmarshal", log.Fields{ |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 562 | "path": path, |
| 563 | "txid": txid, |
| 564 | "error": err, |
| 565 | }) |
| 566 | } else if path == "" { |
| 567 | if field.Key != "" { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 568 | log.Debugw("no-path-with-container-key", log.Fields{ |
| 569 | "path": path, |
| 570 | "txid": txid, |
| 571 | "data": data.Interface(), |
| 572 | }) |
| 573 | |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 574 | // Retrieve the key identifier value from the data structure |
| 575 | // based on the field's key attribute |
| 576 | _, key := GetAttributeValue(data.Interface(), field.Key, 0) |
| 577 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 578 | if entry := pr.verifyPersistedEntry(data.Interface(), name, field.Key, key.String(), |
| 579 | txid); entry != nil { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 580 | response = append(response, entry) |
| 581 | } |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 582 | } else { |
| 583 | log.Debugw("path-with-no-container-key", log.Fields{ |
| 584 | "path": path, |
| 585 | "txid": txid, |
| 586 | "data": data.Interface(), |
| 587 | }) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 588 | } |
| 589 | |
| 590 | } else if field.Key != "" { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 591 | log.Debugw("path-with-container-key", log.Fields{ |
| 592 | "path": path, |
| 593 | "txid": txid, |
| 594 | "data": data.Interface(), |
| 595 | }) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 596 | // The request is for a specific entry/id |
| 597 | partition := strings.SplitN(path, "/", 2) |
| 598 | key := partition[0] |
| 599 | if len(partition) < 2 { |
| 600 | path = "" |
| 601 | } else { |
| 602 | path = partition[1] |
| 603 | } |
| 604 | keyValue := field.KeyFromStr(key) |
| 605 | |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 606 | if entry := pr.verifyPersistedEntry(data.Interface(), name, field.Key, keyValue.(string), |
| 607 | txid); entry != nil { |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 608 | response = append(response, entry) |
| 609 | } |
| 610 | } |
| 611 | } |
| 612 | |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 613 | log.Debugw("no-more-data-blobs", log.Fields{"path": path, "name": name}) |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 614 | } else { |
Chaitrashree G S | be6ab94 | 2019-05-24 06:42:49 -0400 | [diff] [blame] | 615 | log.Debugw("cannot-process-field", log.Fields{ |
manikkaraj k | 9eb6cac | 2019-05-09 12:32:03 -0400 | [diff] [blame] | 616 | "type": pr.GetBranch().Node.Type, |
Matt Jeanneret | 384d8c9 | 2019-05-06 14:27:31 -0400 | [diff] [blame] | 617 | "name": name, |
| 618 | }) |
| 619 | } |
| 620 | } |
| 621 | |
| 622 | return response |
| 623 | } |