SEBA-438 Create multithread-save analysis tool
Change-Id: I0753bb3cac033e3e60f2d3e6e3f7c6a79be75ff2
diff --git a/analysis/README.md b/analysis/README.md
new file mode 100644
index 0000000..1ddb42b
--- /dev/null
+++ b/analysis/README.md
@@ -0,0 +1,10 @@
+# Analysis Tools
+
+This directory contains tools useful for analyzing the behavior of CORD and
+its components.
+
+## multithread-save
+
+This tool helps identify potential race conditions in XOS caused by the same field
+being saved from multiple tasks, where a task is a sync step, event step, pull
+step, or model policy.
\ No newline at end of file
diff --git a/analysis/multithread-save/README.md b/analysis/multithread-save/README.md
new file mode 100644
index 0000000..5ec10bf
--- /dev/null
+++ b/analysis/multithread-save/README.md
@@ -0,0 +1,59 @@
+# Multithread Save
+
+With loglevel DEBUG, the XOS synchronizers will write to the logs when a
+particular model is saved. This tool pulls the information about object
+saves from the logs and analyzes it. The goal is to identify potential race
+conditions caused by the same field being saved from multiple tasks, where a
+task is a sync step, event step, pull step, or model policy.
+
+For each model, the tool will print a warning if `save()` was called on a
+model that hadn't changed (it's better to use `save_if_changed()`). For fields
+in the model that were saved from code in two or more files, it will print out
+the file paths. The idea is that these files should be scrutinized to make
+sure the object is being updated in a safe way.
+
+## Usage
+
+```bash
+./analyze_logs.sh
+```
+
+## Sample Output
+```text
+Inspecting model: RCORDSubscriber
+
+Inspecting model: OLTDevice
+ [WARNING] Object being saved with no changes
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ [WARNING] Field saved from multiple tasks: oper_status
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ /opt/xos/synchronizers/volt/steps/sync_olt_device.py
+ [WARNING] Field saved from multiple tasks: of_id
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ /opt/xos/synchronizers/volt/steps/sync_olt_device.py
+ [WARNING] Field saved from multiple tasks: serial_number
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ /opt/xos/synchronizers/volt/steps/sync_olt_device.py
+ [WARNING] Field saved from multiple tasks: backend_status
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ /opt/xos/synchronizers/volt/steps/sync_olt_device.py
+ [WARNING] Field saved from multiple tasks: device_id
+ /opt/xos/synchronizers/volt/pull_steps/pull_olts.py
+ /opt/xos/synchronizers/volt/steps/sync_olt_device.py
+
+Inspecting model: VOLTServiceInstance
+ [WARNING] Object being saved with no changes
+ /opt/xos/synchronizers/volt/steps/sync_volt_service_instance.py
+ [WARNING] Field saved from multiple tasks: onu_device_id
+ /opt/xos/synchronizers/volt/steps/sync_volt_service_instance.py
+ /opt/xos/synchronizers/volt/model_policies/model_policy_voltserviceinstance.py
+ [WARNING] Field saved from multiple tasks: subscribed_links_ids
+ /opt/xos/synchronizers/volt/steps/sync_volt_service_instance.py
+ /opt/xos/synchronizers/volt/model_policies/model_policy_voltserviceinstance.py
+
+Inspecting model: ONUDevice
+
+Inspecting model: AttWorkflowDriverWhiteListEntry
+ [WARNING] Object being saved with no changes
+ /opt/xos/synchronizers/att-workflow-driver/model_policies/model_policy_att_workflow_driver_whitelistentry.py
+```
diff --git a/analysis/multithread-save/analyze_logs.sh b/analysis/multithread-save/analyze_logs.sh
new file mode 100755
index 0000000..19f44b8
--- /dev/null
+++ b/analysis/multithread-save/analyze_logs.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# copy-ssh-keys.sh - Adds ssh keys to nodes given as parameters to the script,
+# after removing them from the ~/.ssh/known_hosts file on the local system.
+#
+# This script should be run interactively as it will prompt for input, and only
+# invoked once, so as not to add multiple copies of the SSH key to the remote
+# system.
+
+AGGLOGS=/tmp/logs.out
+
+kubectl logs -l release=att-workflow > $AGGLOGS
+kubectl logs -l release=seba-services --all-containers >> $AGGLOGS
+grep "save()" $AGGLOGS | ./logparse.py
diff --git a/analysis/multithread-save/logparse.py b/analysis/multithread-save/logparse.py
new file mode 100755
index 0000000..47e1c76
--- /dev/null
+++ b/analysis/multithread-save/logparse.py
@@ -0,0 +1,92 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/python
+
+from pyparsing import Combine, Group, Optional, ParseException, Regex, Word, ZeroOrMore, alphas, nums
+import fileinput
+import pprint
+
+Written = {}
+
+Date = Combine(Word(nums)+"-"+Word(nums)+"-"+Word(nums))
+Time = Combine(Word(nums)+":"+Word(nums)+":"+Word(nums)+"."+Word(nums))
+Msg = Regex("is new|no changes|updated")
+QuotedWord = "'"+Word(alphas+"_")+"'"+Optional(", ")
+Array = "["+ZeroOrMore(QuotedWord)+"]"
+Key = Word(alphas+"_")
+Value = Array | Word(alphas+"_-/.")
+KeyValue = Combine(Key("key")+"="+Value("value"))
+Logmsg = Date+"T"+Time+"Z [debug ] save(): "+Msg+KeyValue*(2,3)
+
+entries = {}
+
+def add_syncstep(model, field, syncstep):
+ if model not in entries:
+ entries[model] = {}
+ if field in entries[model]:
+ if syncstep not in entries[model][field]:
+ entries[model][field].append(syncstep)
+ else:
+ entries[model][field] = [syncstep]
+
+
+for line in fileinput.input():
+ entry = {}
+ try:
+ logline = Logmsg.parseString(line)
+ #print logline
+ entry["status"] = logline[4]
+ for field in range(5, len(logline)):
+ key, value = logline[field].split("=")
+ if key == "changed_fields":
+ entry[key] = eval(value)
+ else:
+ entry[key] = value
+
+ if entry["syncstep"] == "None":
+ continue
+
+ if "changed_fields" in entry:
+ for field in entry["changed_fields"]:
+ add_syncstep(entry["classname"], field, entry["syncstep"])
+ else:
+ if entry["status"] == "no changes":
+ add_syncstep(entry["classname"], "all", entry["syncstep"])
+
+
+ except ParseException, err:
+ print err.line
+ print " "*(err.column-1) + "^"
+ print err
+
+pp = pprint.PrettyPrinter()
+
+for obj in entries:
+ all_syncstep = []
+ print "\nInspecting model: ", obj
+ entry = entries[obj]
+ if "all" in entry:
+ print " [WARNING] Object being saved with no changes"
+ all_syncstep = entry["all"]
+ for path in all_syncstep:
+ print " ", path
+ for field in entry:
+ if field == "all":
+ continue
+ syncstep = list(set(entry[field] + all_syncstep))
+ if len(syncstep) > 1:
+ print " [WARNING] Field saved from multiple tasks: ", field
+ for path in syncstep:
+ print " ", path