[SEBA-496]
Update pip packages used in XOS containers, flatten containers
Fixed invalid YAML generated by xosgenx modeldefs.xtarget
Change-Id: Ic051170f3f9ef5a5d0dbba61aa6843458a435d49
diff --git a/lib/xos-api/requirements.txt b/lib/xos-api/requirements.txt
index f990247..1fb843d 100644
--- a/lib/xos-api/requirements.txt
+++ b/lib/xos-api/requirements.txt
@@ -1,11 +1,9 @@
-PyYAML~=3.12
-Twisted~=16.6.0
-googleapis-common-protos~=1.5.6
-grpcio-tools~=1.12.0
-grpcio~=1.12.0
+Twisted~=18.9.0
+googleapis-common-protos~=1.5.8
+grpcio-tools~=1.19.0
+grpcio~=1.19.0
multistructlog~=2.1.0
-pykwalify~=1.6.0
python-consul~=1.1.0
simplejson~=3.16.0
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-config/Makefile b/lib/xos-config/Makefile
deleted file mode 100644
index 650eb5f..0000000
--- a/lib/xos-config/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosconfig"
\ No newline at end of file
diff --git a/lib/xos-config/requirements.txt b/lib/xos-config/requirements.txt
index 5528415..1ed683f 100644
--- a/lib/xos-config/requirements.txt
+++ b/lib/xos-config/requirements.txt
@@ -1,2 +1,2 @@
-PyYAML~=3.12
-pykwalify~=1.6.0
+PyYAML~=5.1
+pykwalify~=1.7.0
diff --git a/lib/xos-genx/Makefile b/lib/xos-genx/Makefile
deleted file mode 100644
index feee40e..0000000
--- a/lib/xos-genx/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosgenx"
-
diff --git a/lib/xos-genx/requirements.txt b/lib/xos-genx/requirements.txt
index b210781..924fbb6 100644
--- a/lib/xos-genx/requirements.txt
+++ b/lib/xos-genx/requirements.txt
@@ -1,8 +1,8 @@
Jinja2~=2.10
-PyYAML~=3.12
-astunparse~=1.5.0
+PyYAML~=5.1
+astunparse~=1.6.2
colorama~=0.4.1
-inflect~=1.0.1
+inflect~=2.1.0
plyxproto~=4.0.0
ply~=3.11
six~=1.12.0
diff --git a/lib/xos-genx/xos-genx-tests/test_translator.py b/lib/xos-genx/xos-genx-tests/test_translator.py
index f98894b..ae4289a 100644
--- a/lib/xos-genx/xos-genx-tests/test_translator.py
+++ b/lib/xos-genx/xos-genx-tests/test_translator.py
@@ -125,8 +125,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
-
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 4)
def test_gui_hidden_models(self):
@@ -147,7 +146,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 1)
self.assertIn("Bar", output)
self.assertNotIn("Foo", output)
@@ -165,7 +164,7 @@
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- yaml_ir = yaml.load(output)
+ yaml_ir = yaml.safe_load(output)
self.assertEqual(len(yaml_ir["items"]), 1)
self.assertIn("name", output)
self.assertNotIn("secret", output)
@@ -336,7 +335,7 @@
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
- read_only = [s for s in output.splitlines() if "read_only: True" in s]
+ read_only = [s for s in output.splitlines() if "read_only: true" in s]
self.assertEqual(len(read_only), 3) # readonly is 1 for ParentFoo and 2 for Foo
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 07dbc35..a48d95a 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -146,7 +146,7 @@
elif l["link_type"] == "onetomany":
on_field = l["dst_port"]
outlist.append(
- "- {model: %s, type: %s, on_field: %s}\n"
+ "{model: %s, type: %s, on_field: %s}"
% (l["peer"]["name"], l["link_type"], on_field)
)
seen.append(l["peer"])
diff --git a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
index ec7e5f2..94b4136 100644
--- a/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
+++ b/lib/xos-genx/xosgenx/targets/modeldefs.xtarget
@@ -1,53 +1,57 @@
+---
+# Created by modeldefs.xtarget
items:
{%- for m in proto.messages | sort(attribute='name') %}
-{%- if m.name != 'XOSBase' and xproto_unquote(xproto_first_non_empty([m.options.gui_hidden, 'False'])) != 'True' %}
-- app: {{ xproto_unquote(xproto_first_non_empty([m.options.name, m.options.app_label, options.name, context.app_label])) }}
- {%- if m.options.description %}
- description: "{{ xproto_unquote(m.options.description) }}"
- {%- endif %}
- {%- if m.options.verbose_name %}
- verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
- {%- endif %}
- fields:
- {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
- {% for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') -%}
- {% if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') -%}
- - hint: {% if f.options.help_text %}{{ xproto_unquote(f.options.help_text) }}{% else %}''{% endif %}
- {% if not f.link -%}
- name: {{ f.name }}
- {%- else -%}
- name: {{ f.name }}_id
- relation: {model: {{ f.options.model }}, type: {{ f.options.link_type }}}
- {% endif %}
- {%- if f.options.default %}
- default: "{{ xproto_unquote(xproto_default_to_gui(f.options.default)) }}"
- {%- endif %}
- {%- if f.options.choices %}
- options:
- {% for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
- - {{ xproto_dict_to_sorted_string(o) }}
- {% endfor %}
- {%- endif %}
- type: {{ xproto_type_to_ui_type(f) }}
- read_only: {{ xproto_is_true(f.options.feedback_state) }}
- {% set validators = xproto_validators(f) -%}
- {% if validators -%}
- validators:
- {% for v in validators | sort(attribute='name',reverse=True) -%}
- - {{ v | yaml }}
- {% endfor %}
- {% else -%}
- validators: []
- {% endif %}
- {% endif -%}
- {% endfor %}
- name: {{ m.name }}
- {%- set goodlinks = xproto_links_to_modeldef_relations( xproto_base_links(m, proto.message_table) + m.links ) + xproto_links_to_modeldef_relations( xproto_base_rlinks(m, proto.message_table) + m.rlinks )%}
- {% if goodlinks %}
- relations:
- {{ goodlinks | join('\n') | indent(width=2)}}
- {%- else %}
- relations: []
- {%- endif %}
+{% if m.name != 'XOSBase' and xproto_unquote(xproto_first_non_empty([m.options.gui_hidden, 'False'])) != 'True' %}
+ - name: {{ m.name }}
+ app: {{ xproto_unquote(xproto_first_non_empty([m.options.name, m.options.app_label, options.name, context.app_label])) }}
+ {%- if m.options.verbose_name %}
+ verbose_name: "{{ xproto_unquote(m.options.verbose_name) }}"
+ {%- endif %}
+ {%- if m.options.description %}
+ description: "{{ xproto_unquote(m.options.description) }}"
+ {%- endif %}
+ {%- set goodlinks = xproto_links_to_modeldef_relations( xproto_base_links(m, proto.message_table) + m.links ) + xproto_links_to_modeldef_relations( xproto_base_rlinks(m, proto.message_table) + m.rlinks )%}
+ {%- if goodlinks %}
+ relations:
+ {%- for gl in goodlinks %}
+ - {{ gl }}
+ {%- endfor %}
+ {%- else %}
+ relations: []
+ {%- endif %}
+ fields:
+ {%- set id_field = {'type':'int32', 'name':'id', 'options':{}} %}
+ {%- for f in (xproto_base_fields(m, proto.message_table) + m.fields + [id_field]) | sort(attribute='name') %}
+ {%- if xproto_unquote(xproto_first_non_empty([f.options.gui_hidden, 'False'])) != 'True' and (not f.link or f.options.link_type != 'manytomany') %}
+ {%- if not f.link %}
+ - name: {{ f.name }}
+ {%- else %}
+ - name: {{ f.name }}_id
+ relation: {model: {{ f.options.model }}, type: {{ f.options.link_type }}}
+ {%- endif %}
+ hint: {% if f.options.help_text %}"{{ xproto_unquote(f.options.help_text) }}"{% else %}""{% endif %}
+ {%- if f.options.default %}
+ default: "{{ xproto_unquote(xproto_default_to_gui(f.options.default)) }}"
+ {%- endif %}
+ {%- if f.options.choices %}
+ options:
+ {%- for o in xproto_options_choices_to_dict(xproto_unquote(f.options.choices)) %}
+ - {{ xproto_dict_to_sorted_string(o) }}
+ {%- endfor %}
+ {%- endif %}
+ type: {{ xproto_type_to_ui_type(f) }}
+ read_only: {{ xproto_is_true(f.options.feedback_state) | lower }}
+ {%- set validators = xproto_validators(f) %}
+ {%- if validators %}
+ validators:
+ {%- for v in validators | sort(attribute='name', reverse=True) %}
+ - {{ v | yaml | indent(width=12)}}
+ {%- endfor %}
+ {%- else %}
+ validators: []
+ {%- endif %}
+ {%- endif %}
+ {%- endfor %}
{%- endif %}
-{% endfor -%}
+{%- endfor %}
diff --git a/lib/xos-kafka/requirements.txt b/lib/xos-kafka/requirements.txt
index 1888a03..dee0cea 100644
--- a/lib/xos-kafka/requirements.txt
+++ b/lib/xos-kafka/requirements.txt
@@ -1,3 +1,3 @@
-confluent-kafka~=0.11.5
+confluent-kafka==0.11.5
multistructlog~=2.1.0
-xosconfig~=3.0.0
+xosconfig~=3.0.1
diff --git a/lib/xos-migrate/requirements.txt b/lib/xos-migrate/requirements.txt
index 0ea31a5..4643655 100644
--- a/lib/xos-migrate/requirements.txt
+++ b/lib/xos-migrate/requirements.txt
@@ -1,7 +1,6 @@
-PyYAML~=3.12
+Django~=1.11.20
django-extensions~=2.1.6
djangorestframework~=3.9.1
-django~=1.11.20
multistructlog~=2.1.0
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-synchronizer/requirements.txt b/lib/xos-synchronizer/requirements.txt
index 616f5fb..c539816 100644
--- a/lib/xos-synchronizer/requirements.txt
+++ b/lib/xos-synchronizer/requirements.txt
@@ -1,8 +1,10 @@
Jinja2~=2.10
-astunparse~=1.5.0
+ansible~=2.7.9
+astunparse~=1.6.2
confluent-kafka==0.11.5
-inflect~=1.0.1
+inflect~=2.1.0
multistructlog~=2.1.0
-networkx~=1.11
-xosconfig~=3.0.0
-xosgenx~=3.0.0
+networkx~=2.2
+six~=1.12.0
+xosconfig~=3.0.1
+xosgenx~=3.0.1
diff --git a/lib/xos-synchronizer/xossynchronizer/event_loop.py b/lib/xos-synchronizer/xossynchronizer/event_loop.py
index 15f005c..0c69694 100644
--- a/lib/xos-synchronizer/xossynchronizer/event_loop.py
+++ b/lib/xos-synchronizer/xossynchronizer/event_loop.py
@@ -136,7 +136,7 @@
"dst_accessor": dst_accessor,
}
model_dependency_graph.add_edge(
- src_model, dst_model, edge_label
+ src_model, dst_model, **edge_label
)
model_dependency_graph_rev = model_dependency_graph.reverse(copy=True)
@@ -635,14 +635,14 @@
edge_type = oG[i1][i0]["type"]
if edge_type == PROXY_EDGE:
oG.remove_edge(i1, i0)
- oG.add_edge(i0, i1, {"type": edge_type})
+ oG.add_edge(i0, i1, type=edge_type)
except KeyError:
- oG.add_edge(i0, i1, {"type": edge_type})
+ oG.add_edge(i0, i1, type=edge_type)
except KeyError:
pass
components = weakly_connected_component_subgraphs(oG)
- cohort_indexes = [reversed(topological_sort(g)) for g in components]
+ cohort_indexes = [reversed(list(topological_sort(g))) for g in components]
cohorts = [
[objects[i] for i in cohort_index] for cohort_index in cohort_indexes
]
diff --git a/lib/xos-util/Makefile b/lib/xos-util/Makefile
deleted file mode 100644
index 4143dba..0000000
--- a/lib/xos-util/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-test:
- nosetests -s -v --with-id --with-coverage --cover-html --cover-erase --cover-xml --cover-package="xosutil"