This commit includes:
1) Make descriptiuon fields mandatory
2) Make some minor changes to comply to IETF
3) Add a few test cases (more to come)
Change-Id: Id871b2015ddb72c72a03c9fd5d0e08461fe67398
diff --git a/experiments/proto2Yang/addressbook.proto b/experiments/proto2yang/addressbook.proto
similarity index 95%
rename from experiments/proto2Yang/addressbook.proto
rename to experiments/proto2yang/addressbook.proto
index bfdceea..fc1a10f 100644
--- a/experiments/proto2Yang/addressbook.proto
+++ b/experiments/proto2yang/addressbook.proto
@@ -25,6 +25,7 @@
}
repeated PhoneNumber phones = 4;
+ repeated string khen = 5;
}
// Our address book file is just one of these.
diff --git a/experiments/proto2Yang/descriptor.desc b/experiments/proto2yang/descriptor.desc
similarity index 100%
rename from experiments/proto2Yang/descriptor.desc
rename to experiments/proto2yang/descriptor.desc
Binary files differ
diff --git a/experiments/proto2Yang/descriptor_parser.py b/experiments/proto2yang/descriptor_parser.py
similarity index 100%
rename from experiments/proto2Yang/descriptor_parser.py
rename to experiments/proto2yang/descriptor_parser.py
diff --git a/experiments/proto2Yang/proto2yang.py b/experiments/proto2yang/proto2yang.py
similarity index 90%
rename from experiments/proto2Yang/proto2yang.py
rename to experiments/proto2yang/proto2yang.py
index 88ce1a7..8d9a40b 100755
--- a/experiments/proto2Yang/proto2yang.py
+++ b/experiments/proto2yang/proto2yang.py
@@ -26,7 +26,7 @@
$ python -m grpc.tools.protoc -I.
--plugin=protoc-gen-custom=./proto2yang.py --custom_out=. <proto file>.proto
- - the above will produce a <proto file>.yang file formatted for yang
+ - the above will produce a ietf-<proto file>.yang file formatted for yang
- two examples of proto that can be used in the same directory are
yang.proto and addressbook.proto
@@ -42,37 +42,34 @@
from google.protobuf.descriptor import FieldDescriptor
template_yang = Template("""
-module {{ module.name }} {
-
- namespace "https://gerrit.opencord.org/voltha/{{ module.package }}";
+module ietf-{{ module.name }} {
yang-version 1.1;
-
+ namespace "urn:ietf:params:xml:ns:yang:ietf-{{ module.name }}";
prefix "voltha";
- revision 2016-11-15 {{ module.revision }} {
- {% if module.description %}
- /* {{ message.description }} */
- {% else %}
+ organization "CORD";
+ contact
+ " Any name";
+
+ description
+ "{{ module.description }}";
+
+ revision "2016-11-15" {
description "Initial revision.";
- {% endif %}
+ reference "reference";
}
{% for enum in module.enums %}
- {% if enum.description %}
- /* {{ enum.description }} */
- {% endif %}
typedef {{ enum.name }} {
type enumeration {
{% for v in enum.value %}
- {% if v.description %}
enum {{ v.name }} {
description "{{ v.description }}";
}
- {% else %}
- enum {{ v.name }} ;
- {% endif %}
{% endfor %}
}
+ description
+ "{{ enum.description }}";
}
{% endfor %}
@@ -82,14 +79,10 @@
{% else %}
container {{ message.name }} {
{% endif %}
- {% if message.description %}
- /* {{ message.description }} */
- {% endif %}
+ description
+ "{{ message.description }}";
{% for field in message.fields %}
{% if field.type_ref %}
- {% if field.description %}
- /* {{ field.description }} */
- {% endif %}
{% for dict_item in module.referred_messages_with_keys %}
{% if dict_item.name == field.type %}
list {{ field.name }} {
@@ -98,6 +91,8 @@
max-elements 1;
{% endif %}
uses {{ field.type }};
+ description
+ "{{ field.description }}";
}
{% endif %}
{% endfor %}
@@ -110,13 +105,13 @@
fraction-digits 5;
}
{% else %}
- type {{ field.type }} ;
+ type {{ field.type }};
{% endif %}
- {% if field.description %}
description
- "{{ field.description }}" ;
- {% endif %}
+ "{{ field.description }}";
}
+ description
+ "{{ field.description }}";
}
{% else %}
leaf {{ field.name }} {
@@ -125,32 +120,25 @@
fraction-digits 5;
}
{% else %}
- type {{ field.type }} ;
+ type {{ field.type }};
{% endif %}
- {% if field.description %}
description
- "{{ field.description }}" ;
- {% endif %}
+ "{{ field.description }}";
}
{% endif %}
{% endfor %}
{% for enum_type in message.enums %}
- {% if enum_type.description %}
- /* {{ enum_type.description }} */
- {% endif %}
typedef {{ enum_type.name }} {
type enumeration {
{% for v in enum_type.value %}
- {% if v.description %}
enum {{ v.name }} {
description "{{ v.description }}";
}
- {% else %}
- enum {{ v.name }} ;
- {% endif %}
{% endfor %}
}
+ description
+ "{{ enum_type.description }}";
}
{% endfor %}
@@ -165,17 +153,16 @@
/* {{ service.description }}" */
{% endif %}
{% for method in service.methods %}
- {% if method.description %}
- /* {{ method.description }} */
- {% endif %}
rpc {{ service.service }}-{{ method.method }} {
+ description
+ "{{ method.description }}";
{% if method.input %}
input {
{% if method.input_ref %}
- uses {{ method.input }} ;
+ uses {{ method.input }};
{% else %}
leaf {{ method.input }} {
- type {{ method.input }} ;
+ type {{ method.input }};
}
{% endif %}
}
@@ -183,10 +170,10 @@
{% if method.output %}
output {
{% if method.output_ref %}
- uses {{ method.output }} ;
+ uses {{ method.output }};
{% else %}
leaf {{ method.output }} {
- type {{ method.output }} ;
+ type {{ method.output }};
}
{% endif %}
}
@@ -418,7 +405,8 @@
# TODO: We should have a separate file for each output. There is an
# issue reusing the same filename with an incremental suffix. Using
# a different file name works but not the actual proto file name
- f.name = proto_file.name.replace('.proto', '.yang')
+ f.name = '{}-{}'.format('ietf', proto_file.name.replace('.proto',
+ '.yang'))
# f.name = '{}_{}{}'.format(_rchop(proto_file.name, '.proto'), idx,
# '.yang')
# idx += 1
diff --git a/experiments/proto2Yang/yang.proto b/experiments/proto2yang/yang.proto
similarity index 81%
rename from experiments/proto2Yang/yang.proto
rename to experiments/proto2yang/yang.proto
index 94959a3..718951c 100644
--- a/experiments/proto2Yang/yang.proto
+++ b/experiments/proto2yang/yang.proto
@@ -2,8 +2,6 @@
package experiment;
-//import "google/protobuf/empty.proto";
-
message AsyncEvent {
int32 seq = 1;
enum EventType {
@@ -15,27 +13,26 @@
string details = 3;
}
-enum khenType {
- BIG_KHEN = 0;
- SMALL_KHEN = 1;
- NO_KHEN = 2;
+enum SimpleEnum {
+ APPLE = 0;
+ BANANA = 1;
+ ORANGE = 2;
}
-
message Packet {
int32 source = 1;
bytes content = 2;
- message Result {
+ message InnerPacket {
string url = 1;
string title = 2;
repeated string snippets = 3;
- message Success {
+ message InnerInnerPacket {
string input = 1;
string desc = 2;
}
- repeated Success success = 4;
+ repeated InnerInnerPacket inner_inner_packet = 4;
}
- repeated Result results = 3;
+ repeated InnerPacket inner_packets = 3;
}
message Echo {
diff --git a/tests/utests/netconf/proto2yang/__init__.py b/tests/utests/netconf/proto2yang/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/utests/netconf/proto2yang/__init__.py
diff --git a/tests/utests/netconf/proto2yang/protobuf_to_yang_test.py b/tests/utests/netconf/proto2yang/protobuf_to_yang_test.py
new file mode 100644
index 0000000..dacaaf5
--- /dev/null
+++ b/tests/utests/netconf/proto2yang/protobuf_to_yang_test.py
@@ -0,0 +1,558 @@
+#
+# Copyright 2016 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+import os
+from unittest import TestCase
+import time
+from tests.itests.docutests.test_utils import run_command_to_completion_with_raw_stdout
+from tests.utests.chameleon.protoc_plugins.test_utils import load_file, \
+ unindent, save_file
+
+proto_to_yang_cmd='python -m grpc.tools.protoc -I{} ' \
+ '--plugin=protoc-gen-custom=/voltha/experiments' \
+ '/proto2yang/proto2yang.py --custom_out={} {}'
+
+yang_validate_cmd="pyang -f tree --ietf {}"
+
+TEMP_PATH="/tmp/proto2yang"
+TEMP_INPUT_PROTO_FILE="test.proto"
+TEMP_OUTPUT_YANG_FILE="ietf-test.yang"
+TEMP_PROTO_PATH='{}/{}'.format(TEMP_PATH, TEMP_INPUT_PROTO_FILE)
+TEMP_YANG_PATH='{}/{}'.format(TEMP_PATH, TEMP_OUTPUT_YANG_FILE)
+
+
+class ProtoToYang(TestCase):
+
+ def setup(self):
+ if not os.path.exists(TEMP_PATH):
+ os.makedirs(TEMP_PATH)
+
+
+ def _compare_file(self, response, expected_response):
+ # compare two files and strip empty lines, blanks, etc
+ def _filter(x):
+ x.strip()
+ return x is not None
+
+ response = filter(_filter,response.split())
+ expected_response = filter(_filter,expected_response.split())
+ print response
+ print expected_response
+
+ self.assertEqual(set(response), set(expected_response))
+
+ def _gen_yang(self, proto):
+ try:
+ save_file(os.path.join(TEMP_PATH, TEMP_INPUT_PROTO_FILE), proto)
+ cmd = proto_to_yang_cmd.format(TEMP_PATH, TEMP_PATH, TEMP_PROTO_PATH
+ )
+ print 'running command: {}'.format(cmd)
+ response, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ return load_file(TEMP_YANG_PATH)
+ except Exception as e:
+ print('Failure to generate yang file {}'.format(e))
+
+
+ def test_01_empty_proto(self):
+ print "Test_01_empty_proto_Start:------------------"
+ t0 = time.time()
+
+ proto = unindent("""
+ syntax = "proto3";
+ package test;
+ """)
+
+ expected_response = """
+ module ietf-test {
+ yang-version 1.1;
+ namespace "urn:ietf:params:xml:ns:yang:ietf-test";
+ prefix "voltha";
+
+ organization "CORD";
+ contact
+ " Any name";
+
+ description
+ "";
+
+ revision "2016-11-15" {
+ description "Initial revision.";
+ reference "reference";
+ }
+ }
+ """
+
+ try:
+ yang = self._gen_yang(proto)
+ self._compare_file(yang, expected_response)
+ finally:
+ print "Test_01_empty_proto_End:------------------ took {} " \
+ "secs\n\n".format(time.time() - t0)
+
+
+ def test_02_empty_message_with_service(self):
+ print "Test_02_empty_message_with_service_Start:------------------"
+ t0 = time.time()
+
+ proto = unindent("""
+ syntax = "proto3";
+ package test;
+ message Null {}
+ service TestService {
+ rpc Get(Null) returns(Null);
+ }
+ """)
+
+ expected_response = """
+ module ietf-test {
+ yang-version 1.1;
+ namespace "urn:ietf:params:xml:ns:yang:ietf-test";
+ prefix "voltha";
+
+ organization "CORD";
+ contact
+ " Any name";
+
+ description
+ "";
+
+ revision "2016-11-15" {
+ description "Initial revision.";
+ reference "reference";
+ }
+
+ grouping Null {
+ description
+ "";
+ }
+
+ rpc TestService-Get {
+ description
+ "";
+ input {
+ uses Null;
+ }
+ output {
+ uses Null;
+ }
+ }
+ }
+ """
+
+ try:
+ yang = self._gen_yang(proto)
+ self._compare_file(yang, expected_response)
+ finally:
+ print "Test_02_empty_message_with_service_End:------------------ took {} " \
+ "secs\n\n".format(time.time() - t0)
+
+
+ def test_03_simple_message_with_service(self):
+ print "Test__03_simple_message_with_service_Start:------------------"
+ t0 = time.time()
+
+ proto = unindent("""
+ syntax = "proto3";
+ package test;
+
+ // Simple Message
+ message Simple {
+ string str = 1; // a string attribute
+ int32 int = 2; // an int32 attribute
+ }
+
+ // Service to get things done
+ service TestService {
+
+ /* Get simple answer
+ *
+ * Returns the true answer to all of life's persistent questions.
+ */
+ rpc Get(Simple) returns(Simple);
+ }
+ """)
+
+ expected_response = """
+ module ietf-test {
+ yang-version 1.1;
+ namespace "urn:ietf:params:xml:ns:yang:ietf-test";
+ prefix "voltha";
+
+ organization "CORD";
+ contact
+ " Any name";
+
+ description
+ "";
+
+ revision "2016-11-15" {
+ description "Initial revision.";
+ reference "reference";
+ }
+
+
+ grouping Simple {
+ description
+ "Simple Message";
+ leaf str {
+ type string;
+ description
+ "a string attribute";
+ }
+
+ leaf int {
+ type int32;
+ description
+ "an int32 attribute";
+ }
+
+ }
+
+ /* Service to get things done" */
+ rpc TestService-Get {
+ description
+ "Get simple answer
+
+ Returns the true answer to all of life's persistent questions.";
+ input {
+ uses Simple;
+ }
+ output {
+ uses Simple;
+ }
+ }
+ }
+ """
+
+ try:
+ yang = self._gen_yang(proto)
+ self._compare_file(yang, expected_response)
+ finally:
+ print "Test_03_simple_message_with_service_End" \
+ ":------------------ took {} secs\n\n".format(time.time() - t0)
+
+
+ def test_04_mix_types(self):
+ print "Test__04_mix_types_Start:------------------"
+ t0 = time.time()
+
+ proto = unindent("""
+ syntax = "proto3";
+
+ package experiment;
+
+ message AsyncEvent {
+ int32 seq = 1;
+ enum EventType {
+ BIG_BANG = 0; // just a big bang
+ SMALL_BANG = 1; // so small bang
+ NO_BANG = 2;
+ }
+ EventType type = 2;
+ string details = 3;
+ }
+
+ enum SimpleEnum {
+ APPLE = 0;
+ BANANA = 1;
+ ORANGE = 2;
+ }
+
+ message Packet {
+ int32 source = 1;
+ bytes content = 2;
+ message InnerPacket {
+ string url = 1;
+ string title = 2;
+ repeated string snippets = 3;
+ message InnerInnerPacket {
+ string input = 1;
+ string desc = 2;
+ }
+ repeated InnerInnerPacket inner_inner_packet = 4;
+ }
+ repeated InnerPacket inner_packets = 3;
+ }
+
+ message Echo {
+ string msg = 1;
+ float delay = 2;
+ }
+
+ message testMessage{
+ string test2 = 1;
+ int32 test3 = 2;
+ }
+
+ service ExperimentalService {
+
+ rpc GetEcho(Echo) returns(Echo);
+
+ // For server to send async stream to client
+ rpc ReceiveStreamedEvents(Packet)
+ returns(stream AsyncEvent);
+
+ // For server to send async packets to client
+ rpc ReceivePackets(Echo) returns(stream Packet);
+
+ // For client to send async packets to server
+ rpc SendPackets(stream Packet) returns(Echo);
+
+ }
+ """)
+
+ expected_response = """
+ module ietf-test {
+ yang-version 1.1;
+ namespace "urn:ietf:params:xml:ns:yang:ietf-test";
+ prefix "voltha";
+
+ organization "CORD";
+ contact
+ " Any name";
+
+ description
+ "";
+
+ revision "2016-11-15" {
+ description "Initial revision.";
+ reference "reference";
+ }
+
+ typedef SimpleEnum {
+ type enumeration {
+ enum APPLE {
+ description "";
+ }
+ enum BANANA {
+ description "";
+ }
+ enum ORANGE {
+ description "";
+ }
+ }
+ description
+ "";
+ }
+
+ grouping AsyncEvent {
+ description
+ "";
+ leaf seq {
+ type int32;
+ description
+ "";
+ }
+
+ leaf type {
+ type EventType;
+ description
+ "";
+ }
+
+ leaf details {
+ type string;
+ description
+ "";
+ }
+
+ typedef EventType {
+ type enumeration {
+ enum BIG_BANG {
+ description "";
+ }
+ enum SMALL_BANG {
+ description "";
+ }
+ enum NO_BANG {
+ description "";
+ }
+ }
+ description
+ "";
+ }
+
+ }
+
+ grouping Packet {
+ description
+ "";
+ leaf source {
+ type int32;
+ description
+ "";
+ }
+
+ leaf content {
+ type binary;
+ description
+ "";
+ }
+
+ list inner_packets {
+ key "url";
+ uses InnerPacket;
+ description
+ "";
+ }
+
+ grouping InnerPacket {
+ description
+ "";
+ leaf url {
+ type string;
+ description
+ "";
+ }
+
+ leaf title {
+ type string;
+ description
+ "";
+ }
+
+ list snippets {
+ key "snippets";
+ leaf snippets {
+ type string;
+ description
+ "";
+ }
+ description
+ "";
+ }
+
+ list inner_inner_packet {
+ key "input";
+ uses InnerInnerPacket;
+ description
+ "";
+ }
+
+ grouping InnerInnerPacket {
+ description
+ "";
+ leaf input {
+ type string;
+ description
+ "";
+ }
+
+ leaf desc {
+ type string;
+ description
+ "";
+ }
+
+ }
+
+ }
+
+ }
+
+ grouping Echo {
+ description
+ "";
+ leaf msg {
+ type string;
+ description
+ "";
+ }
+
+ leaf delay {
+ type decimal64 {
+ fraction-digits 5;
+ }
+ description
+ "";
+ }
+
+ }
+
+ container testMessage {
+ description
+ "";
+ leaf test2 {
+ type string;
+ description
+ "";
+ }
+
+ leaf test3 {
+ type int32;
+ description
+ "";
+ }
+
+ }
+
+ rpc ExperimentalService-GetEcho {
+ description
+ "";
+ input {
+ uses Echo;
+ }
+ output {
+ uses Echo;
+ }
+ }
+
+ rpc ExperimentalService-ReceiveStreamedEvents {
+ description
+ "For server to send async stream to client";
+ input {
+ uses Packet;
+ }
+ output {
+ uses AsyncEvent;
+ }
+ }
+
+ rpc ExperimentalService-ReceivePackets {
+ description
+ "For server to send async packets to client";
+ input {
+ uses Echo;
+ }
+ output {
+ uses Packet;
+ }
+ }
+
+ rpc ExperimentalService-SendPackets {
+ description
+ "For client to send async packets to server";
+ input {
+ uses Packet;
+ }
+ output {
+ uses Echo;
+ }
+ }
+
+
+ }
+ """
+
+ try:
+ yang = self._gen_yang(proto)
+ self._compare_file(yang, expected_response)
+ finally:
+ print "Test_04_mix_types_End" \
+ ":------------------ took {} secs\n\n".format(time.time() - t0)
+