[CORD-1569]
Autogenerate documentation of build system variables
Change-Id: I839f46d681e8f6954316f0ea4e9a79395501459f
diff --git a/docs/scripts/defaults.md.j2 b/docs/scripts/defaults.md.j2
new file mode 100644
index 0000000..7d7405d
--- /dev/null
+++ b/docs/scripts/defaults.md.j2
@@ -0,0 +1,22 @@
+# Build System Variable Glossary
+
+{{ def_docs['frontmatter']['description'] }}
+
+{% for key, val in def_docs|dictsort %}
+### {{ key }}
+
+{{ val['description'] }}
+
+Default value:
+```
+{{ val['defval_pp'] }}
+```
+
+Used in:
+
+{% for file in val['reflist']|sort(attribute='path') -%}
+ - [{{ file.path }}]({{ file.link }})
+{% endfor -%}
+
+{% endfor %}
+
diff --git a/docs/scripts/defaultsdoc.py b/docs/scripts/defaultsdoc.py
new file mode 100644
index 0000000..f75fa85
--- /dev/null
+++ b/docs/scripts/defaultsdoc.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# defaultsdoc.py - documentation for ansible default vaules
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import fnmatch
+import jinja2
+import logging
+import os
+import pprint
+import re
+import sys
+import xml.etree.ElementTree as ET
+import yaml
+import markedyaml
+
+# logging setup
+sh = logging.StreamHandler(sys.stderr)
+sh.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+
+LOG = logging.getLogger("defaultsdoc.py")
+LOG.addHandler(sh)
+
+# parse args
+parser = argparse.ArgumentParser()
+
+parser.add_argument('-p', '--playbook_dir', default='../platform-install/',
+ action='append', required=False,
+ help="path to base playbook directory")
+
+parser.add_argument('-d', '--descriptions', default='scripts/descriptions.md',
+ action='store', required=False,
+ help="markdown file with descriptions")
+
+parser.add_argument('-t', '--template', default='scripts/defaults.md.j2',
+ action='store', required=False,
+ help="jinja2 template to fill with defaults")
+
+parser.add_argument('-o', '--output', default='defaults.md',
+ action='store', required=False,
+ help="output file")
+
+args = parser.parse_args()
+
+# find the branch we're on via the repo manifest
+manifest_path = os.path.abspath("../../.repo/manifest.xml")
+try:
+ tree = ET.parse(manifest_path)
+ manifest_xml = tree.getroot()
+ repo_default = manifest_xml.find('default')
+ repo_branch = repo_default.attrib['revision']
+except Exception:
+ LOG.exception("Error loading repo manifest")
+ sys.exit(1)
+
+role_defs = []
+profile_defs = []
+group_defs = []
+
+# frontmatter section is any text at the top of the descriptions.md file, and
+# comes before all other sections
+def_docs = {'frontmatter':{'description':''}}
+
+# find all the files to be processed
+for dirpath, dirnames, filenames in os.walk(args.playbook_dir):
+ basepath = re.sub(args.playbook_dir, '', dirpath)
+ for filename in filenames :
+ filepath = os.path.join(basepath, filename)
+
+ if fnmatch.fnmatch(filepath, "roles/*/defaults/*.yml"):
+ role_defs.append(filepath)
+
+ if fnmatch.fnmatch(filepath, "profile_manifests/*.yml"):
+ profile_defs.append(filepath)
+
+ if fnmatch.fnmatch(filepath, "group_vars/*.yml"):
+ group_defs.append(filepath)
+
+
+
+for rd in role_defs:
+ rd_vars = {}
+ # trim slash so basename grabs the final directory name
+ rd_basedir = os.path.basename(args.playbook_dir[:-1])
+ try:
+ rd_fullpath = os.path.abspath(os.path.join(args.playbook_dir, rd))
+ rd_partialpath = os.path.join(rd_basedir, rd)
+
+ # partial URL, without line nums
+ rd_url = "https://github.com/opencord/platform-install/tree/%s/%s" % (repo_branch, rd)
+
+
+ rd_fh= open(rd_fullpath, 'r')
+
+ # markedloader is for line #'s
+ loader = markedyaml.MarkedLoader(rd_fh.read())
+ marked_vars = loader.get_data()
+
+ rd_fh.seek(0) # go to front of file
+
+ # yaml.safe_load is for vars in a better format
+ rd_vars = yaml.safe_load(rd_fh)
+
+ rd_fh.close()
+
+ except yaml.YAMLError:
+ LOG.exception("Problem loading file: %s" % rd)
+ sys.exit(1)
+
+ if rd_vars:
+
+ for key, val in rd_vars.iteritems():
+
+ # build full URL to lines. Lines numbered from zero, so +1 on them to match github
+ if marked_vars[key].start_mark.line == marked_vars[key].end_mark.line:
+ full_url = "%s#L%d" % (rd_url, marked_vars[key].start_mark.line+1)
+ else:
+ full_url = "%s#L%d-L%d" % (rd_url, marked_vars[key].start_mark.line, marked_vars[key].end_mark.line)
+
+ if key in def_docs:
+ if def_docs[key]['defval'] == val:
+ def_docs[key]['reflist'].append({'path':rd_partialpath, 'link':full_url})
+ else:
+ LOG.error(" %s has different default > %s : %s" % (rd, key, val))
+ else:
+ to_print = { str(key): val }
+ pp = yaml.dump(to_print, indent=4, allow_unicode=False, default_flow_style=False)
+
+ def_docs[key] = {
+ 'defval': val,
+ 'defval_pp': pp,
+ 'description': "",
+ 'reflist': [{'path':rd_partialpath, 'link':full_url}],
+ }
+
+# read in descriptions file
+descriptions = {}
+with open(args.descriptions, 'r') as descfile:
+ desc_name = 'frontmatter'
+ desc_lines = ''
+
+ for d_l in descfile:
+ # see if this is a header line at beginning of docs
+ desc_header = re.match(r"##\s+([\w_]+)", d_l)
+
+ if desc_header:
+ # add previous description to dict
+ descriptions[desc_name] = desc_lines
+
+ # set this as the next name, wipe out lines
+ desc_name = desc_header.group(1)
+ desc_lines = ''
+ else:
+ desc_lines += d_l
+
+ descriptions[desc_name] = desc_lines
+
+# add descriptions to def_docs
+for d_name, d_text in descriptions.iteritems():
+ if d_name in def_docs:
+ def_docs[d_name]['description'] = d_text
+ else:
+ LOG.error("Description exists for '%s' but doesn't exist in defaults" % d_name)
+
+# check for missing descriptions
+for key in sorted(def_docs):
+ if not def_docs[key]['description']:
+ LOG.error("No description found for '%s'" % key)
+
+# Add to template and write to output file
+j2env = jinja2.Environment(
+ loader = jinja2.FileSystemLoader('.')
+)
+
+template = j2env.get_template(args.template)
+
+with open(args.output, 'w') as f:
+ f.write(template.render(def_docs=def_docs))
diff --git a/docs/scripts/descriptions.md b/docs/scripts/descriptions.md
new file mode 100644
index 0000000..6a15590
--- /dev/null
+++ b/docs/scripts/descriptions.md
@@ -0,0 +1,660 @@
+This documents every variable available in the build system. The repos these
+variables are used in are:
+
+ - [cord](https://github.com/opencord/cord) (aka "build" when checked out)
+ - [maas](https://github.com/opencord/maas)
+ - [platform-install](https://github.com/opencord/platform-install)
+
+## apt_cacher_name
+
+DNS name for the apt-cacher-ng server used by compute nodes and LXC containers
+on head node.
+
+## apt_cacher_port
+
+Port number used for apt-cacher-ng.
+
+## apt_ssl_sites
+
+APT package sources are on HTTPS servers. These bypass apt-cacher-ng, which
+can't proxy SSL connections.
+
+## build_cord_dir
+
+Directory on the `build` node that the `cord` directory (root directory checked
+out by `repo`) is copied to.
+
+## build_docker_tag
+
+Tag applied to all built (and downloaded standalone) Docker images.
+
+## ca_digest
+
+Name of the digest (aka "hash") algorithm used when creating CA SSL
+certificates.
+
+## ca_im_days
+
+Length of time in days that an Intermediate Certificate Authority cert will be
+valid for.
+
+## ca_im_phrase
+
+The passphrase (password) used to encrypt the Intermediate Certificate
+Authority's private key.
+
+## ca_im_subj
+
+The subject (name in the certificate) of the Intermediate Certificate
+Authority.
+
+## ca_root_days
+
+Length of time in days that the Root Certificate Authority cert will be valid
+for.
+
+## ca_root_phrase
+
+The passphrase (password) used to encrypt the Root Certificate Authority's
+private key. Default is for this to be autogenerated by the password lookup in
+ansible and stored in [credentials_dir](#credentials_dir).
+
+## ca_root_subj
+
+The subject (name in the certificate) of the Root Certificate Authority.
+
+## ca_size
+
+Size of the keys used in generating the CA certificates, in bits.
+
+## cert_days
+
+Length of times that a standard server/client certificate will be valid for
+
+## cert_digest
+
+Name of the digest (aka "hash") algorithm used when creating SSL certificates.
+
+## cert_size
+
+Size of the keys used in generating the server/client certificates, in bits.
+
+## charm_versions
+
+List of Juju charms and the versions used.
+
+## client_certs
+
+List of client SSL certificates to generate
+
+## cloudlab_extrafs
+
+Filesystem device to use for extra space when on CloudLab
+
+## cloudlab_links
+
+Symbolic links to create to use the extra space that is mounted when using
+CloudLab
+
+## compute_external_interfaces
+
+List of possible VTN external interfaces on the compute node, for setting up
+OpenStack with the VTN ML2 plugin.
+
+## config_cord_dir
+
+Location of the `cord` directory on the config node. See also
+[build_cord_dir](#build_cord_dir).
+
+## config_cord_profile_dir
+
+Location of the `cord_profile` directory on the config node.
+
+## cord_config_app_version
+
+Version of the CORD config ONOS app to use
+
+## cord_in_a_box
+
+Used to determine if this is a Cord-in-a-Box virtual pod installation
+
+## cord_vtn_app_version
+
+Version of the CORD VTN ONOS app to use
+
+## credentials_dir
+
+The location of the `credentials_dir` directory on the head node
+
+## delete_cord_profile_dir
+
+Boolean value, whether or not to delete the `cord_profile` directory when
+tearing down XOS on a pod
+
+## deploy_docker_registry
+
+DNS name or IP address of the Docker Registry
+
+## deploy_docker_tag
+
+Tag used to identify which docker images to use when performing a deployment.
+
+## deployment_flavors
+
+Names of OpenStack "flavors" of VM's that can be deployed.
+
+## deployment_type
+
+Deployment type, used in XOS to identify the type of deployment.
+
+## dhcpd_subnets
+
+Used to configure the DHCP server used in OpenCloud and other non-MaaS
+deployments.
+
+## dns_check_domain
+
+Domaing to check when performing the prerequisite check.
+
+## dns_check_ipv4
+
+IP address of [dns_check_domain](#dns_check_domain) for DNS resolution
+prerequisite check.
+
+## dns_search
+
+Which domain suffixes to search for hosts in (non-MaaS)
+
+## dns_servers
+
+IP addresses of DNS servers
+
+## dns_ttl
+
+Time-to-live for DNS entries when using NSD (non-MaaS)
+
+## docker_apt_repo
+
+Name of the Docker APT repo to install Docker from
+
+## docker_opts
+
+Options to provide to Docker to configure the
+
+## dpdk_lcore_mask
+
+DPDK option to set which CPU cores to use. More documentation at:
+[http://docs.openvswitch.org/en/latest/intro/install/dpdk/#setup-ovs](http://docs.openvswitch.org/en/latest/intro/install/dpdk/#setup-ovs)
+
+## dpdk_socket_memory
+
+DPDK option concerning memory allocation.
+
+## enabled_gui_extensions
+
+List of GUI extensions enabled and loaded into the Web UI
+
+## fabric_interfaces
+
+External VTN interface connected to the fabric switches.
+
+## fabric_network_cfg_json
+
+Filename of the JSON file used to configure the Fabric ONOS.
+
+## frontend_only
+
+`frontend_only` suppresses starting synchronzier containers as a part of the
+XOS container set. It is used in testing scenarios where synchronizers aren't
+needed.
+
+## gerrit_changesets
+
+List of gerrit
+
+## gui_api_endpoint
+
+Partial URI to the API endpoint used by the GUI to contact the XOS API
+
+## gui_background
+
+Backgrund image used behind login screen when logging into XOS.
+
+## gui_favicon
+
+Favicon used in URL bar for XOS web UI.
+
+## gui_logo
+
+Logo used in XOS web UI.
+
+## gui_payoff
+
+Text below the Logo in the XOS web UI.
+
+## gui_project_name
+
+Name of the type of POD being deployed, shown in XOS UI.
+
+## gui_routes
+
+Links given in the top-level of the XOS web UI to specific objects, to feature
+them in the sidebar.
+
+## gui_websocket
+
+URI path used by XOS web UI for the websocket connection.
+
+## gw_port
+
+Port on the XOS ws container for XOS UI connections
+
+## head_cord_dir
+
+Location on the headnode where the `cord` directory is copied.
+
+## head_cord_profile_dir
+
+Location on the headnode where the `cord_profile` directory is copied.
+
+## head_credentials_dir
+
+Location on the headnode where the `credentials` diretory is copied.
+
+## head_lxd_list
+
+List of LXD containers (for Juju/OpenStack) to create.
+
+## head_mavenrepo_dir
+
+Location on the headnode to create the `mavenrepo` directory, which contains
+the docker-compose.yml file for the Maven repo docker container that serves up
+ONOS Apps to the ONOS instances on the headnode.
+
+## head_onos_cord_dir
+
+Location on the headnode to create the `onos_cord` directory, which contains
+configuration and the docker-compose.yml file for starting the ONOS instance
+that runs the VTN app.
+
+## head_onos_fabric_dir
+
+Location on the headnode to create the `onos_fabric` directory, which contains
+configuration and the docker-compose.yml file for starting the ONOS instance
+that runs the Fabric app.
+
+## headnode
+
+Name of the headnode on the system, used to configure NSD DNS aliases.
+
+## hugepages
+
+DPDK setting to control memory allocation.
+
+## hwaddr_prefix
+
+MAC address prefix used when creating LXD containers, to assign them DHCP addresses.
+
+## image_dir
+
+Directory to download OpenStack glance images into on the head node.
+
+## juju_config_path
+
+Path to Juju configuration file.
+
+## keystone_admin_password
+
+Password for OpenStack Keystone `admin` user.
+
+## kvm_check
+
+Whether or not to perform a check for processor virtualization features
+required for the KVM hypervisor to work.
+
+## log4j_port
+
+Port used by ONOS containers for sending log4j logging messages to ElasticStack.
+
+## logging_host
+
+Hostname (or IP) for the ElasticStack logging host machine.
+
+## maas_xos_admin_pass
+
+Contains the XOS admin password, used for loading TOSCA with up MaaS. Can't
+use the standard [xos_admin_pass](#xos_admin_pass) as these playbooks are run
+from the MaaS provisioner container.
+
+## management_hosts_net_cidr
+
+CIDR for the management_hosts VTN network.
+
+## management_hosts_net_range_xos_high
+
+Last IP address to assign as a part of the management_hosts VTN network.
+
+## management_hosts_net_range_xos_low
+
+First IP address to assign as a part of the management_hosts VTN network.
+
+## management_network_cidr
+
+CIDR of the head node management network that connects between the OpenStack
+LXC containers and compute nodes.
+
+## mgmt_interface
+
+Physical management network interface on head node.
+
+## mgmt_ipv4_first_octets
+
+First 3 octets of the IP address of the management network.
+
+## mgmt_name_reverse_unbound
+
+The same value as [mgmt_ipv4_first_octets](#mgmt_ipv4_first_octets) but
+formatted for Unbound for use as a reverse DNS lookup zone.
+
+## mgmtbr_ext_interface
+
+Network interface on head node to add to the `mgmtbr` bridge.
+
+## mgmtbr_nat_interface
+
+Network interface connected to the internet that NAT is performed on for
+nodes that use the `mgmtbr` bridge.
+
+## min_memtotal_mb
+
+Minimum amount of memory to allow for a full virtual POD to be built with.
+
+## min_processor_vcpus
+
+Minimum number of CPU's to allow for a full virtual POD to be built with.
+
+## nsd_conf
+
+Path to the `nsd.conf` file for configuring the NSD authoritative nameserver.
+
+## nsd_group
+
+Group used by the NSD nameserver.
+
+## nsd_ip
+
+IP address of the NSD nameserver. Usually this is set to the loopback address,
+as Unbound runs on the external interfaces.
+
+## nsd_zones
+
+Configuration of DNS Zones that NSD provides authoritative DNS lookups for.
+
+## nsd_zonesdir
+
+Directory where DNS Zone files are kept for NSD.
+
+## onos_cord_port
+
+Port used for SSH connections to the `ONOS CORD` instance.
+
+## onos_debug_appnames
+
+Names of ONOS Apps loaded to change the logging level on for debugging purposes.
+
+## onos_debug_level
+
+The logging level (`INFO`, `DEBUG`, `TRACE`, etc.) to set ONOS Apps listed in
+[onos_debug_appnames](#onos_debug_appnames).
+
+## onos_docker_image
+
+Name of the docker image used to bring up ONOS containers.
+
+## onos_log_level
+
+Default logging level ONOS should log at.
+
+## physical_node_list
+
+List of physical nodes to set up in DNS.
+
+## pki_dir
+
+Location where SSL certificates are generated on the `config` node. Contains
+subdirectories for root and intermediate CA certificates.
+
+## pmd_cpu_mask
+
+DPDK setting for CPU pinning.
+
+## pod_sshkey_name
+
+Name of the SSH key generated to be used by the pod, specifically for logging
+into instance VM's that are brought up.
+
+## profile_library
+
+The name of the profile-specific onboarding TOSCA file.
+
+## pull_docker_registry
+
+DNS Name or IP of the Docker Registry to pull images from.
+
+## pull_docker_tag
+
+Tag for pulling Docker images.
+
+## repo_checksum
+
+Checksum of the [repo](https://code.google.com/archive/p/git-repo/) download.
+
+## repo_dl_url
+
+URL of `repo` to download.
+
+## repo_manifest_url
+
+URL of Gerrit manifest repository that `repo` fetches it's list of git
+repositories from.
+
+## requests_ca_bundle
+
+When using python's requests module, name of the CA certificate bundle file to
+use to validate SSL certificates.
+
+## run_dist_upgrade
+
+Whether or not to run `apt-get dist-upgrrade` on a system in the course of
+setting it up.
+
+## server_certs
+
+List of SSL certificates to generate for server use.
+
+## site_humanname
+
+Human readable name to use for the CORD site.
+
+## site_name
+
+Machine readable name to use for the CORD site. This should be one word, without spaces.
+
+## site_suffix
+
+The DNS suffix applied to all machines created for this site. Must be a valid DNS name.
+
+## ssh_ca_phrase
+
+The passphrase used to encrypt the Root CA key when creating a SSL hierarchy.
+
+## ssh_client_genkeys
+
+Names of SSH Client keys to generate and sign by the SSH CA.
+
+## ssh_host_genkeys
+
+Names of SSH Host keys to generatte and sign by the SSH CA.
+
+## ssh_keysize
+
+Size in bits of SSH keys to generate
+
+## ssh_keytype
+
+The key type of the SSH keys. `rsa` is used currently, may change this as
+support for newer key algorithms is added to the underlying platform.
+
+## ssh_pki_dir
+
+Directory where SSH keys are generated.
+
+## ssl_cert_subj_prefix
+
+SSL certificate prefix substring to use when generating certificates.
+
+## trust_store_pw
+
+Java KeyStore password used for encrypting SSL certificates. This currently
+doesn't contain any secure certificates, just the generated CA
+root/intermediate certificates for validation of SSL connections.
+
+## unbound_conf
+
+Path for the Unbound recursive DNS resolver configuration file.
+
+## unbound_group
+
+Group name used by Unbound server.
+
+## unbound_interfaces
+
+List of network interfaces that Unbound should listen on.
+
+## unbound_listen_all
+
+Whether Unbound should listen on all available network interfaces.
+
+## unbound_listen_on_default
+
+Whether Unbound should listen on the default gateway interface (as known to Ansible).
+
+## use_apt_cache
+
+Enables the use of `apt-cacher-ng` to cache APT packages on Head/LXC/Compute nodes.
+
+## use_dpdk
+
+Enable DPDK in OpenStack Nova and Neutron
+
+## use_fabric
+
+Start and use ONOS in a container to manage fabric switches
+
+## use_maas
+
+Use MaaS to manage compute nodes and switches.
+
+## use_management_hosts
+
+Whether the management_hosts network type in VTN should be enabled.
+
+## use_openstack
+
+Bring up and use OpenStack to manage VM's.
+
+## use_redis
+
+Use redis as a message bus inside XOS.
+
+## use_vtn
+
+Use the ONOS VTN app to manage networks for virtual instances.
+
+## vcpu_pin_set
+
+DPDK setting to specify CPU pinning.
+
+## vtn_management_host_net_interface
+
+Network interface to use on the head/compute nodes for the management_host network.
+
+## xos_admin_first
+
+First name of the XOS Admin user
+
+## xos_admin_last
+
+Last tname of the XOS Admin user
+
+## xos_admin_pass
+
+Password of the XOS Admin user (autogenerated by default)
+
+## xos_admin_user
+
+Username (email) of the XOS Admin user
+
+## xos_bootstrap_ui_port
+
+Port to connect to to bootstrap the XOS interface.
+
+## xos_chameleon_port
+
+Port used by Chameleon in XOS.
+
+## xos_db_name
+
+XOS Postgres database name
+
+## xos_db_password
+
+XOS Postgres database password.
+
+## xos_db_username
+
+XOS Postgres database username.
+
+## xos_dir
+
+Path of XOS directory within Docker containers.
+
+## xos_docker_networks
+
+Name of networks created in Docker for XOS containers.
+
+## xos_grpc_insecure_port
+
+Insecure (non-SSL) port used for GRPC connections to the XOS API.
+
+## xos_grpc_secure_port
+
+Secure (SSL) port used for GRPC connections to the XOS API.
+
+## xos_images
+
+List of OpenStack Glance images in QCOW2 format that are downloaded.
+
+## xos_other_templates
+
+List of templates to generate when creating the `cord_profile` directory.
+
+## xos_services
+
+List of XOS services to load, including the name, path and whether SSH keypairs
+should be included for the services.
+
+## xos_tosca_config_templates
+
+List of XOS tosca templates to load that make up the service graph of a
+profile.
+
+## xos_ui_port
+
+XOS Web UI port to use for API access.
+
+## xos_users
+
+List of additional users to create in XOS, in addition to
+[xos_admin_user](#xos_admin_user).
+
diff --git a/docs/scripts/markedyaml.py b/docs/scripts/markedyaml.py
new file mode 100644
index 0000000..f7c1484
--- /dev/null
+++ b/docs/scripts/markedyaml.py
@@ -0,0 +1,118 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# markedyaml.py
+# generates nodes with start/end line and column values
+# start line seems off with single-line items, correct with multiline
+#
+# Original code from here: https://gist.github.com/dagss/5008118
+# Request for licensing clarification made on 2017-09-19
+# Contains improvements to support more types (bool/int/etc.)
+
+import yaml
+from yaml.composer import Composer
+from yaml.reader import Reader
+from yaml.scanner import Scanner
+from yaml.composer import Composer
+from yaml.resolver import Resolver
+from yaml.parser import Parser
+from yaml.constructor import Constructor, BaseConstructor, SafeConstructor
+
+def create_node_class(cls):
+ class node_class(cls):
+ def __init__(self, x, start_mark, end_mark):
+ cls.__init__(self, x)
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+ def __new__(self, x, start_mark, end_mark):
+ return cls.__new__(self, x)
+ node_class.__name__ = '%s_node' % cls.__name__
+ return node_class
+
+dict_node = create_node_class(dict)
+list_node = create_node_class(list)
+unicode_node = create_node_class(unicode)
+int_node = create_node_class(int)
+float_node = create_node_class(float)
+
+class NodeConstructor(SafeConstructor):
+ # To support lazy loading, the original constructors first yield
+ # an empty object, then fill them in when iterated. Due to
+ # laziness we omit this behaviour (and will only do "deep
+ # construction") by first exhausting iterators, then yielding
+ # copies.
+ def construct_yaml_map(self, node):
+ obj, = SafeConstructor.construct_yaml_map(self, node)
+ return dict_node(obj, node.start_mark, node.end_mark)
+
+ def construct_yaml_seq(self, node):
+ obj, = SafeConstructor.construct_yaml_seq(self, node)
+ return list_node(obj, node.start_mark, node.end_mark)
+
+ def construct_yaml_str(self, node):
+ obj = SafeConstructor.construct_scalar(self, node)
+ assert isinstance(obj, unicode)
+ return unicode_node(obj, node.start_mark, node.end_mark)
+
+ def construct_yaml_bool(self, node):
+ obj = SafeConstructor.construct_yaml_bool(self, node)
+ return int_node(obj, node.start_mark, node.end_mark)
+
+ def construct_yaml_int(self, node):
+ obj = SafeConstructor.construct_scalar(self, node)
+ return int_node(obj, node.start_mark, node.end_mark)
+
+ def construct_yaml_float(self, node):
+ obj = SafeConstructor.construct_scalar(self, node)
+ return float_node(obj, node.start_mark, node.end_mark)
+
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:map',
+ NodeConstructor.construct_yaml_map)
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:seq',
+ NodeConstructor.construct_yaml_seq)
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:str',
+ NodeConstructor.construct_yaml_str)
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:bool',
+ NodeConstructor.construct_yaml_bool)
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:int',
+ NodeConstructor.construct_yaml_int)
+
+NodeConstructor.add_constructor(
+ u'tag:yaml.org,2002:float',
+ NodeConstructor.construct_yaml_float)
+
+
+class MarkedLoader(Reader, Scanner, Parser, Composer, NodeConstructor, Resolver):
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ NodeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+def get_data(stream):
+ return MarkedLoader(stream).get_data()
+