CORD-2247: Write basic static checker for xproto
Change-Id: I63a96972e16fd8dd62d4dec840eede66cbb26368
diff --git a/lib/xos-genx/xosgenx/checkers/model_policy.xchecker b/lib/xos-genx/xosgenx/checkers/model_policy.xchecker
new file mode 100644
index 0000000..abbdd0d
--- /dev/null
+++ b/lib/xos-genx/xosgenx/checkers/model_policy.xchecker
@@ -0,0 +1,5 @@
+{% for m in proto.messages %}
+{% if not m.options.no_policy %}
+{{ xproto_check_policy(m) }}
+{% endif %}
+{% endfor %}
diff --git a/lib/xos-genx/xosgenx/checkers/service_model_base.xchecker b/lib/xos-genx/xosgenx/checkers/service_model_base.xchecker
new file mode 100644
index 0000000..85881f0
--- /dev/null
+++ b/lib/xos-genx/xosgenx/checkers/service_model_base.xchecker
@@ -0,0 +1,32 @@
+{# Using regular variables has scoping issues. #}
+{# See: https://stackoverflow.com/questions/7537439/how-to-increment-a-variable-on-a-for-loop-in-jinja-template/7537466 #}
+{% set counter = {
+ 'num_service_models': 0,
+ 'num_service_instance_models': 0,
+ 'num_orphaned_models': 0,
+ } %}
+{% macro increment(key) %}
+ {% if counter.update({key: counter[key] + 1}) %} {% endif %}
+{% endmacro %}
+{% for m in proto.messages %}
+{% set matched = False %}
+{% set base_names = m.bases | map(attribute='name') | list %}
+{% if 'Service' in base_names %}
+{{ increment('num_service_models') }}
+{% set matched = True %}
+{% endif %}
+{% if not matched and 'ServiceInstance' in base_names or 'Tenant' in base_names or 'TenantWithContainer' in base_names %}
+{{ increment('num_service_instance_models') }}
+{% set matched = True %}
+{% endif %}
+{% if not matched and 'XOSBase' not in base_names %}
+501 Model does not have a parent - {{ m.name }}
+{% endif %}
+{% endfor %}
+{% if counter.num_service_models !=1 %}
+502 {{ counter.num_service_models }} Service models instead of 1
+{% elif counter.num_service_instance_models !=1 %}
+503 {{ counter.num_service_instance_models }} ServiceInstance models instead of 1
+{% else %}
+200 OK
+{% endif %}
diff --git a/lib/xos-genx/xosgenx/checkers/service_options.xchecker b/lib/xos-genx/xosgenx/checkers/service_options.xchecker
new file mode 100644
index 0000000..383cdbd
--- /dev/null
+++ b/lib/xos-genx/xosgenx/checkers/service_options.xchecker
@@ -0,0 +1,12 @@
+{% set required_options = ['name','verbose_name','app_label'] %}
+
+{% for m in proto.messages %}
+{% for o in required_options %}
+{% if not options[o] and not m.options[o] %}
+504 Required option {{ o }} is missing from model {{ m.name }}
+{% endif %}
+
+{% set required_field_options = ['tosca_key'] %}
+
+{% endfor %}
+{% endfor %}
diff --git a/lib/xos-genx/xosgenx/checkers/syncstep.xchecker b/lib/xos-genx/xosgenx/checkers/syncstep.xchecker
new file mode 100644
index 0000000..836a2c2
--- /dev/null
+++ b/lib/xos-genx/xosgenx/checkers/syncstep.xchecker
@@ -0,0 +1,5 @@
+{% for m in proto.messages %}
+{% if not m.options.no_sync %}
+{{ xproto_check_synchronizer(m) }}
+{% endif %}
+{% endfor %}
diff --git a/lib/xos-genx/xosgenx/generator.py b/lib/xos-genx/xosgenx/generator.py
index 650e688..33e96b1 100755
--- a/lib/xos-genx/xosgenx/generator.py
+++ b/lib/xos-genx/xosgenx/generator.py
@@ -26,7 +26,7 @@
loader = jinja2.PackageLoader(__name__, 'templates')
env = jinja2.Environment(loader=loader)
-class XOSGenerator:
+class XOSProcessor:
@staticmethod
def _read_input_from_files(files):
@@ -78,8 +78,8 @@
@staticmethod
def _load_jinja2_extensions(os_template_env, attic):
- os_template_env.globals['include_file'] = XOSGenerator._include_file(attic) # Generates a function
- os_template_env.globals['file_exists'] = XOSGenerator._file_exists(attic) # Generates a function
+ os_template_env.globals['include_file'] = XOSProcessor._include_file(attic) # Generates a function
+ os_template_env.globals['file_exists'] = XOSProcessor._file_exists(attic) # Generates a function
os_template_env.filters['yaml'] = yaml.dump
for f in dir(jinja2_extensions):
@@ -168,8 +168,7 @@
return ne_pointer
@staticmethod
- def generate(args):
-
+ def process(args, operator = None):
# Setting defaults
if not hasattr(args, 'attic'):
args.attic = None
@@ -196,19 +195,25 @@
raise Exception("[XosGenX] The output dir must be a directory!")
if hasattr(args, 'files'):
- inputs = XOSGenerator._read_input_from_files(args.files)
+ inputs = XOSProcessor._read_input_from_files(args.files)
elif hasattr(args, 'inputs'):
inputs = args.inputs
else:
raise Exception("[XosGenX] No inputs provided!")
- template_path = XOSGenerator._get_template(args.target)
+ if not operator:
+ operator = args.target
+ template_path = XOSProcessor._get_template(operator)
+ else:
+ template_path = operator
+
+
[template_folder, template_name] = os.path.split(template_path)
os_template_loader = jinja2.FileSystemLoader(searchpath=[template_folder])
os_template_env = jinja2.Environment(loader=os_template_loader)
- os_template_env = XOSGenerator._load_jinja2_extensions(os_template_env, args.attic)
+ os_template_env = XOSProcessor._load_jinja2_extensions(os_template_env, args.attic)
template = os_template_env.get_template(template_name)
- context = XOSGenerator._add_context(args)
+ context = XOSProcessor._add_context(args)
parser = plyxproto.ProtobufAnalyzer()
try:
@@ -216,7 +221,7 @@
except plyxproto.ParsingError, e:
line, start, end = e.error_range
- ptr = XOSGenerator._find_last_nonempty_line(inputs, start)
+ ptr = XOSProcessor._find_last_nonempty_line(inputs, start)
if start == 0:
beginning = ''
@@ -236,14 +241,14 @@
exit(1)
- v = XOSGenerator._attach_parser(ast, args)
+ v = XOSProcessor._attach_parser(ast, args)
if args.output is not None and args.write_to_file == "model":
rendered = {}
for i, model in enumerate(v.models):
models = {}
models[model] = v.models[model]
- messages = [XOSGenerator._find_message_by_model_name(v.messages, model)]
+ messages = [XOSProcessor._find_message_by_model_name(v.messages, model)]
rendered[model] = template.render(
{"proto":
@@ -257,7 +262,7 @@
"options": v.options
}
)
- XOSGenerator._write_file_per_model(rendered, args.output, args.dest_extension, args.quiet)
+ XOSProcessor._write_file_per_model(rendered, args.output, args.dest_extension, args.quiet)
else:
rendered = template.render(
{"proto":
@@ -272,8 +277,8 @@
}
)
if args.output is not None and args.write_to_file == "target":
- XOSGenerator._write_split_target(rendered, args.output, args.quiet)
+ XOSProcessor._write_split_target(rendered, args.output, args.quiet)
elif args.output is not None and args.write_to_file == "single":
- XOSGenerator._write_single_file(rendered, args.output, args.dest_file, args.quiet)
+ XOSProcessor._write_single_file(rendered, args.output, args.dest_file, args.quiet)
return rendered
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
index a81dcb6..859594c 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
@@ -19,3 +19,4 @@
from .fol2 import *
from .gui import *
from .tosca import *
+from .checklib import *
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/base.py b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
index 5943e66..f8224b2 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/base.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
@@ -271,4 +271,4 @@
return list
else:
- return False
\ No newline at end of file
+ return False
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
new file mode 100644
index 0000000..a34fd7f
--- /dev/null
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
@@ -0,0 +1,61 @@
+import ast
+
+def xproto_check_synchronizer(m):
+ try:
+ sync_step_path = 'synchronizer/steps/sync_%s.py'%m['name'].lower()
+ sync_step = open(sync_step_path).read()
+ except IOError:
+ return '510 Model needs a sync step %s'%sync_step_path
+
+ try:
+ sync_step_ast = ast.parse(sync_step)
+ except SyntaxError:
+ return '511 Could not parse sync step %s'%sync_step_path
+
+ classes = filter(lambda x:isinstance(x, ast.ClassDef), sync_step_ast.body)
+ found_sync_step_class = False
+
+ for c in classes:
+ base_names = [v.id for v in c.bases]
+ if 'SyncStep' in base_names or 'SyncInstanceUsingAnsible' in base_names:
+ attributes = filter(lambda x:isinstance(x, ast.Assign), c.body)
+ for a in attributes:
+ target_names = [t.id for t in a.targets]
+ values = a.value.elts if isinstance(a.value, ast.List) else [a.value]
+ value_names = [v.id for v in values]
+
+ if 'observes' in target_names and m['name'] in value_names:
+ found_sync_step_class = True
+ break
+
+ if not found_sync_step_class:
+ return '512 Synchronizer needs a sync step class with an observes field containing %s'%m['name']
+ else:
+ return '200 OK'
+
+
+def xproto_check_policy(m):
+ try:
+ model_policy_path = 'synchronizer/model_policies/model_policy_%s.py'%m['name'].lower()
+ model_policy = open(model_policy_path).read()
+ except IOError:
+ return '510 Model needs a model policy %s'%model_policy_path
+
+ try:
+ model_policy_ast = ast.parse(model_policy)
+ except SyntaxError:
+ return '511 Could not parse sync step %s'%model_policy_path
+
+ classes = filter(lambda x:isinstance(x, ast.ClassDef), model_policy_ast.body)
+ found_model_policy_class = False
+ for c in classes:
+ base_names = [v.id for v in c.bases]
+ if 'Policy' in base_names or 'TenantWithContainerPolicy' in base_names:
+ found_model_policy_class = True
+ break
+
+ if not found_model_policy_class:
+ return '513 Synchronizer needs a model policy class'
+ else:
+ return '200 OK'
+
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 7c65388..50bcf0e 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -124,4 +124,4 @@
outlist.append('- {model: %s, type: %s, on_field: %s}\n' % (l['peer']['name'], l['link_type'], on_field))
seen.append(l['peer'])
- return outlist
\ No newline at end of file
+ return outlist
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
index 51cd2bb..9e3b4a8 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
@@ -62,4 +62,4 @@
keys.append(of)
- return keys
\ No newline at end of file
+ return keys
diff --git a/lib/xos-genx/xosgenx/xosgen.py b/lib/xos-genx/xosgenx/xosgen.py
index 7893f22..fa734b5 100755
--- a/lib/xos-genx/xosgenx/xosgen.py
+++ b/lib/xos-genx/xosgenx/xosgen.py
@@ -21,7 +21,6 @@
parse = argparse.ArgumentParser(description='XOS Generative Toolchain')
parse.add_argument('--rev', dest='rev', action='store_true',default=False, help='Convert proto to xproto')
-parse.add_argument('--target', dest='target', action='store',default=None, help='Output format, corresponding to <output>.yaml file', required=True)
parse.add_argument('--output', dest='output', action='store',default=None, help='Destination dir')
parse.add_argument('--attic', dest='attic', action='store',default=None, help='The location at which static files are stored')
parse.add_argument('--kvpairs', dest='kv', action='store',default=None, help='Key value pairs to make available to the target')
@@ -31,41 +30,67 @@
group.add_argument('--dest-file', dest='dest_file', action='store',default=None, help='Output file name (if write-to-file is set to single)')
group.add_argument('--dest-extension', dest='dest_extension', action='store',default=None, help='Output file extension (if write-to-file is set to single)')
+group = parse.add_mutually_exclusive_group(required=True)
+group.add_argument('--target', dest='target', action='store',default=None, help='Output format, corresponding to <output>.yaml file')
+group.add_argument('--checkers', dest='checkers', action='store', default=None, help='Comma-separated list of static checkers')
+
parse.add_argument('files', metavar='<input file>', nargs='+', action='store', help='xproto files to compile')
+CHECK = 1
+GEN = 2
+
class XosGen:
@staticmethod
def init(args=None):
-
if not args:
args = parse.parse_args()
args.quiet = False
- # convert output to absolute path
- if args.output is not None and not os.path.isabs(args.output):
- args.output = os.path.abspath(os.getcwd() + '/' + args.output)
- if not '/' in args.target:
- # if the target is not a path, it refer to a library included one
- args.target = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/targets/" + args.target)
- if not os.path.isabs(args.target):
- args.target = os.path.abspath(os.getcwd() + '/' + args.target)
+ if args.target:
+ op = GEN
+ subdir = '/targets/'
+ elif args.checkers:
+ op = CHECK
+ subdir = '/checkers/'
+ else:
+ parse.error("At least one of --target and --checkers is required")
- # check if there's a line that starts with +++ in the target
- # if so, then the output file names are left to the target to decide
- # also, if dest-file or dest-extension are supplied, then an error is generated.
- plusplusplus = reduce(lambda acc, line: True if line.startswith('+++') else acc, open(args.target).read().splitlines(), False)
+ operators = args.checkers.split(',') if hasattr(args, 'checkers') and args.checkers else [args.target]
- if plusplusplus and args.write_to_file != 'target':
- parse.error('%s chooses the names of the files that it generates, you must set --write-to-file to "target"' % args.target)
+ for i in xrange(len(operators)):
+ if not '/' in operators[i]:
+ # if the target is not a path, it refer to a library included one
+ operators[i] = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + subdir + operators[i])
+ if not os.path.isabs(operators[i]):
+ operators[i] = os.path.abspath(os.getcwd() + '/' + operators[i])
- if args.write_to_file != 'single' and (args.dest_file):
- parse.error('--dest-file requires --write-to-file to be set to "single"')
+ if op == GEN:
+ # convert output to absolute path
+ if args.output is not None and not os.path.isabs(args.output):
+ args.output = os.path.abspath(os.getcwd() + '/' + args.output)
- if args.write_to_file != 'model' and (args.dest_extension):
- parse.error('--dest-extension requires --write-to-file to be set to "model"')
+ operator = operators[0]
+
+ # check if there's a line that starts with +++ in the target
+ # if so, then the output file names are left to the target to decide
+ # also, if dest-file or dest-extension are supplied, then an error is generated.
+ plusplusplus = reduce(lambda acc, line: True if line.startswith('+++') else acc, open(operator).read().splitlines(), False)
+
+ if plusplusplus and args.write_to_file != 'target':
+ parse.error('%s chooses the names of the files that it generates, you must set --write-to-file to "target"' % operator)
+
+ if args.write_to_file != 'single' and (args.dest_file):
+ parse.error('--dest-file requires --write-to-file to be set to "single"')
+
+ if args.write_to_file != 'model' and (args.dest_extension):
+ parse.error('--dest-extension requires --write-to-file to be set to "model"')
+
+ else:
+ if args.write_to_file or args.dest_extension:
+ parse.error('Checkers cannot write to files')
inputs = []
@@ -74,9 +99,28 @@
inputs.append(os.path.abspath(os.getcwd() + '/' + fname))
else:
inputs.append(fname)
+
args.files = inputs
- generated = XOSGenerator.generate(args)
+ if op==GEN:
+ generated = XOSProcessor.process(args, operators[0])
+ if not args.output and not args.write_to_file:
+ print generated
+ elif op==CHECK:
+ for o in operators:
+ verdict_str = XOSProcessor.process(args, o)
+ vlst = verdict_str.split('\n')
- if not args.output and not args.write_to_file:
- print generated
\ No newline at end of file
+ try:
+ verdict = next(v for v in vlst if v.strip())
+ status_code, status_string = verdict.split(' ', 1)
+ status_code = int(status_code)
+ except:
+ print "Checker %s returned mangled output" % o
+ exit(1)
+
+ if status_code != 200:
+ print '%s: %s - %s' % (o, status_code, status_string)
+ exit(1)
+ else:
+ print '%s: OK'%o