VOL-2341 Fix protoc merging separate messages together
Change-Id: I8702faad7ed3c1bf8bfb113aea0dc57f681f0bd2
diff --git a/voltha/voltha-kafka-dump/Dockerfile b/voltha/voltha-kafka-dump/Dockerfile
index 6ddf464..dfce9f0 100755
--- a/voltha/voltha-kafka-dump/Dockerfile
+++ b/voltha/voltha-kafka-dump/Dockerfile
@@ -21,7 +21,7 @@
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && \
- apt-get install -y unzip=6.0-21+deb9u2 curl=7.52.1-5+deb9u9 git=1:2.11.0-3+deb9u4 libatomic1=6.3.0-18+deb9u1 ca-certificates=20161130+nmu1+deb9u1 --no-install-recommends && \
+ apt-get install -y unzip=6.0-21+deb9u2 curl=7.52.1-5+deb9u9 git=1:2.11.0-3+deb9u4 libatomic1=6.3.0-18+deb9u1 ca-certificates=20161130+nmu1+deb9u1 python3.5=3.5.3-1+deb9u1 --no-install-recommends && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
@@ -41,4 +41,4 @@
# Install voltha-protos
RUN git clone -b $VOLTHA_PROTOS_VERSION --single-branch https://github.com/opencord/voltha-protos.git
-COPY voltha-dump-events.sh ./
+COPY callprotoc.py voltha-dump-events.sh ./
diff --git a/voltha/voltha-kafka-dump/callprotoc.py b/voltha/voltha-kafka-dump/callprotoc.py
new file mode 100644
index 0000000..453d97a
--- /dev/null
+++ b/voltha/voltha-kafka-dump/callprotoc.py
@@ -0,0 +1,64 @@
+# Copyright 2019-present the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Separate the incoming stream of messages from kafkacat and call
+# protoc on each message.
+
+from __future__ import print_function
+
+import subprocess
+import sys
+
+
+def call_protoc(buf, msgName, protoFileName, includeDir, first):
+ process = subprocess.Popen(["protoc", "--decode="+msgName, protoFileName, "-I", includeDir], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ process.stdin.write(buf)
+ if not first:
+ print(",")
+ print(process.communicate()[0].decode("utf-8"))
+ process.stdin.close()
+
+
+def main():
+ if len(sys.argv) != 4:
+ print(sys.stderr, "syntax: callprotoc.py <msgname> <protofilename> <includedir>", file=sys.stderr)
+ sys.exit(-1)
+
+ msgName = sys.argv[1]
+ protoFileName = sys.argv[2]
+ includeDir = sys.argv[3]
+
+ print ("[")
+
+ buf = b""
+ first = True
+ in_bytes = sys.stdin.buffer.read(1)
+ while in_bytes:
+ buf = buf + in_bytes
+ while b"===VOLTHA-DELIM===" in buf:
+ (part, buf) = buf.split(b"===VOLTHA-DELIM===", 1)
+ if first:
+ first = False
+ call_protoc(part, msgName, protoFileName, includeDir, first)
+ in_bytes = sys.stdin.buffer.read(1)
+
+ # there is likely one trailing message still to print
+ if buf:
+ call_protoc(buf, msgName, protoFileName, includeDir, first)
+
+ print ("]")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/voltha/voltha-kafka-dump/voltha-dump-events.sh b/voltha/voltha-kafka-dump/voltha-dump-events.sh
index 906d4bb..b0eae94 100755
--- a/voltha/voltha-kafka-dump/voltha-dump-events.sh
+++ b/voltha/voltha-kafka-dump/voltha-dump-events.sh
@@ -20,5 +20,6 @@
if [[ $* == *--binary* ]]; then
kafkacat -u -C -b voltha-kafka.voltha -t voltha.events -D "" -o beginning -e
else
- kafkacat -u -C -b voltha-kafka.voltha -t voltha.events -D "" -o beginning -e | protoc --decode=voltha.Event /opt/voltha-kafka-dump/voltha-protos/protos/voltha_protos/events.proto -I /opt/voltha-kafka-dump/voltha-protos/protos
+ kafkacat -u -C -b voltha-kafka.voltha -t voltha.events -D "===VOLTHA-DELIM===" -o beginning -e | \
+ python3.5 ./callprotoc.py voltha.Event /opt/voltha-kafka-dump/voltha-protos/protos/voltha_protos/events.proto /opt/voltha-kafka-dump/voltha-protos/protos
fi