Apply non-core changes in CORD-912 to master
remove vestigial templates
create admin-openrc.sh in cord_profile_dir and home dir

Change-Id: I52a7cef1ea9e0dc7a37d9888fcfdc093434777ef
diff --git a/roles/dhcpd/defaults/main.yml b/roles/dhcpd/defaults/main.yml
new file mode 100644
index 0000000..b1fd226
--- /dev/null
+++ b/roles/dhcpd/defaults/main.yml
@@ -0,0 +1,27 @@
+---
+# dhcpd/defaults/main.yml
+
+dns_search: []
+dns_servers: []
+
+# http://serverfault.com/questions/40712/what-range-of-mac-addresses-can-i-safely-use-for-my-virtual-machines
+hwaddr_prefix: "c2a4"
+
+dhcpd_subnets: []
+
+# example dhcpd_subnets:
+#
+# dhcpd_subnets:
+#   - interface: eth1
+#     cidr: 192.168.200.1/24
+#     dhcp_first: 129
+#     dhcp_last: 254
+#     tftp_server: 192.168.200.1
+#     static_nodes:
+#       - name: node1
+#         ipv4_last_octet: 30
+#         filename: boot.tftp
+#     other_static:
+#       - head_lxd_list
+
+
diff --git a/roles/dhcpd/handlers/main.yml b/roles/dhcpd/handlers/main.yml
new file mode 100644
index 0000000..38a25c5
--- /dev/null
+++ b/roles/dhcpd/handlers/main.yml
@@ -0,0 +1,8 @@
+---
+# dhcpd/handlers/main.yml
+
+- name: restart-dhcpd
+  service:
+    name: isc-dhcp-server
+    state: restarted
+
diff --git a/roles/dhcpd/tasks/main.yml b/roles/dhcpd/tasks/main.yml
new file mode 100644
index 0000000..8c103de
--- /dev/null
+++ b/roles/dhcpd/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+# dhcpd/tasks/main.yml
+
+- name: Install dhcpd
+  apt:
+    name: "{{ item }}"
+    update_cache: yes
+    cache_valid_time: 3600
+  with_items:
+    - isc-dhcp-server
+
+- name: Create /etc/dhcp/dhcpd.conf from template
+  template:
+    src: dhcpd.conf.j2
+    dest: /etc/dhcp/dhcpd.conf
+    mode: "0644"
+    owner: root
+    group: root
+    #    validate: 'dhcpd -t -cf %s'
+  notify:
+    - restart-dhcpd
+
+- name: Set interfaces for dhcpd to listen on
+  template:
+    src: isc-dhcp-server.j2
+    dest: /etc/default/isc-dhcp-server
+    mode: "0644"
+    owner: root
+    group: root
+  notify:
+    - restart-dhcpd
+
diff --git a/roles/dhcpd/templates/dhcpd.conf.j2 b/roles/dhcpd/templates/dhcpd.conf.j2
new file mode 100644
index 0000000..bbf9d8b
--- /dev/null
+++ b/roles/dhcpd/templates/dhcpd.conf.j2
@@ -0,0 +1,65 @@
+# dhcpd.conf
+# Managed by Ansible!
+
+{% if dns_search %}
+option domain-name "{{ dns_search[0] }}";
+option domain-search "{{ dns_search | join('", "') }}";
+{% endif %}
+
+{% if dns_servers %}
+option domain-name-servers {{ dns_servers | join(", ") }};
+{% endif %}
+
+{% for subnet in dhcpd_subnets %}
+subnet {{ subnet.cidr | ipaddr('network') }} netmask {{ subnet.cidr | ipaddr('netmask') }} {
+  option routers {{ subnet.router | default(subnet.cidr | ipaddr('1') | ipaddr('address')) }};
+  range {{ subnet.cidr | ipaddr(subnet.dhcp_first | default("129")) | ipaddr('address') }} {{ subnet.cidr | ipaddr(subnet.dhcp_last | default("254")) | ipaddr('address') }};
+{% if subnet.dns_search is defined %}
+  option domain-name {{ subnet.dns_search [0] }};
+  option domain-search {{ subnet.dns_search| join(", ") }};
+{% endif %}
+{% if subnet.dns_servers is defined %}
+  option domain-name-servers {{ subnet.dns_servers | join(", ") }};
+{% endif %}
+  default-lease-time {{ subnet.lease_time | default("240") }};
+  max-lease-time {{ subnet.max_lease_time | default("480") }};
+{% if subnet.pxe_filename is defined %}
+  filename "{{ subnet.pxe_filename }}";
+  next-server {{ subnet.tftp_server | default(subnet.cidr | ipaddr('1') | ipaddr('address')) }};
+{% endif %}
+{% if subnet.static_nodes is defined %}
+  # hosts from list: static_nodes
+{% for node in subnet.static_nodes %}
+  host {{ node.name }} {
+    option host-name "{{ node.name }}";
+{% set host_ipaddr = (subnet.cidr | ipaddr(node.ipv4_last_octet) | ipaddr('address')) %}
+    fixed-address {{ host_ipaddr }};
+    hardware ethernet {{ node.hwaddr | default(hwaddr_prefix ~ (host_ipaddr | ip4_hex)) | hwaddr('unix') }};
+{% if node.pxe_filename is defined %}
+    filename "{{ node.pxe_filename }}";
+    next-server {{ subnet.tftp_server | default(subnet.cidr | ipaddr('1') | ipaddr('address')) }};
+{% endif %}
+  }
+{% endfor %}
+{% endif %}
+{% if subnet.other_static is defined %}
+{% for hostlist in subnet.other_static %}
+  # hosts from list: {{ hostlist }}
+{% set nodes = vars[hostlist] %}
+{% for node in nodes %}
+  host {{ node.name }} {
+    option host-name "{{ node.name }}";
+{% set host_ipaddr = (subnet.cidr | ipaddr(node.ipv4_last_octet) | ipaddr('address')) %}
+    fixed-address {{ host_ipaddr }};
+    hardware ethernet {{ node.hwaddr | default(hwaddr_prefix ~ (host_ipaddr | ip4_hex)) | hwaddr('unix') }};
+{% if node.pxe_filename is defined %}
+    filename "{{ subnet.pxe_filename }}";
+    next-server {{ subnet.tftp_server | default(subnet.cidr | ipaddr('1') | ipaddr('address')) }};
+{% endif %}
+  }
+{% endfor %}
+{% endfor %}
+{% endif %}
+}
+
+{% endfor %}
diff --git a/roles/dhcpd/templates/isc-dhcp-server.j2 b/roles/dhcpd/templates/isc-dhcp-server.j2
new file mode 100644
index 0000000..98efb1d
--- /dev/null
+++ b/roles/dhcpd/templates/isc-dhcp-server.j2
@@ -0,0 +1,22 @@
+# Defaults for isc-dhcp-server initscript
+# sourced by /etc/init.d/isc-dhcp-server
+# installed at /etc/default/isc-dhcp-server by the maintainer scripts
+
+#
+# This is a POSIX shell fragment
+#
+
+# Path to dhcpd's config file (default: /etc/dhcp/dhcpd.conf).
+#DHCPD_CONF=/etc/dhcp/dhcpd.conf
+
+# Path to dhcpd's PID file (default: /var/run/dhcpd.pid).
+#DHCPD_PID=/var/run/dhcpd.pid
+
+# Additional options to start dhcpd with.
+#       Don't use options -cf or -pf here; use DHCPD_CONF/ DHCPD_PID instead
+#OPTIONS=""
+
+# On what interfaces should the DHCP server (dhcpd) serve DHCP requests?
+#       Separate multiple interfaces with spaces, e.g. "eth0 eth1".
+INTERFACES="{{ dhcpd_subnets | map(attribute='interface') | join(' ') }}"
+
diff --git a/roles/dns-configure/defaults/main.yml b/roles/dns-configure/defaults/main.yml
index defbf98..0101eb7 100644
--- a/roles/dns-configure/defaults/main.yml
+++ b/roles/dns-configure/defaults/main.yml
@@ -9,3 +9,5 @@
 # Set this to search domain suffixes
 # dns_search: {}
 
+unbound_listen_on_default: False
+
diff --git a/roles/dns-configure/tasks/main.yml b/roles/dns-configure/tasks/main.yml
index 07b0d5d..792748e 100644
--- a/roles/dns-configure/tasks/main.yml
+++ b/roles/dns-configure/tasks/main.yml
@@ -1,15 +1,34 @@
 ---
 # roles/dns-configure/tasks.yml
 
-- name: Configure resolv.conf to use nameservers
+- name: Make sure resolvconf is doing DNS resolver mangling
+  apt:
+    name: resolvconf
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Create resolvconf configuration files
   template:
-    src="resolv.conf.j2"
-    dest="/etc/resolv.conf"
-    mode=0644 owner=root group=root
+    src: "{{ item }}.j2"
+    dest: "/etc/resolvconf/resolv.conf.d/{{ item }}"
+    mode: 0644
+    owner: root
+    group: root
+  with_items:
+    - base
+    - head
+  register: resolvconf_configured
+
+- name: Tell resolvconf to refresh /etc/resolv.conf file if changed
+  become: yes
+  command: resolvconf -u
+  when: resolvconf_configured.changed
+  tags:
+   - skip_ansible_lint # needs to run before the DNS check which happens next, so can't be a handler
 
 - name: Check that VM's can be found in DNS
   shell: "dig +short {{ item.name }}.{{ site_suffix }} | grep {{ item.ipv4_last_octet }}"
-  with_items: "{{ head_vm_list }}"
+  with_items: "{{ head_lxd_list }}"
   tags:
    - skip_ansible_lint # purely a way to pass/fail config done so far. Ansible needs a "dns_query" module
 
diff --git a/roles/dns-configure/templates/base.j2 b/roles/dns-configure/templates/base.j2
new file mode 100644
index 0000000..7eadcf1
--- /dev/null
+++ b/roles/dns-configure/templates/base.j2
@@ -0,0 +1,3 @@
+{% if dns_search is defined %}
+search{% for searchdom in dns_search %} {{ searchdom }}{% endfor %}
+{% endif %}
diff --git a/roles/dns-configure/templates/head.j2 b/roles/dns-configure/templates/head.j2
new file mode 100644
index 0000000..f19e8cc
--- /dev/null
+++ b/roles/dns-configure/templates/head.j2
@@ -0,0 +1,14 @@
+# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
+# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
+# Make changes to  /etc/resolvconf/resolv.conf.d instead
+# Modified by Ansible
+{% if unbound_listen_on_default %}
+{% for host in groups['head'] %}
+nameserver {{ hostvars[host].ansible_default_ipv4.address }}
+{% endfor %}
+{% endif %}
+{% if dns_servers is defined %}
+{% for ns in dns_servers %}
+nameserver {{ ns }}
+{% endfor %}
+{% endif %}
diff --git a/roles/dns-nsd/templates/nsd.conf.j2 b/roles/dns-nsd/templates/nsd.conf.j2
index 29ba41a..46813bb 100644
--- a/roles/dns-nsd/templates/nsd.conf.j2
+++ b/roles/dns-nsd/templates/nsd.conf.j2
@@ -12,7 +12,7 @@
   zonesdir: {{ nsd_zonesdir }}
 
 remote-control:
-  control-enable: yes
+  control-enable: no
 
 # zones to load
 {% for zone in nsd_zones %}
diff --git a/roles/dns-nsd/templates/zone.forward.j2 b/roles/dns-nsd/templates/zone.forward.j2
index 895d8a3..129fa7c 100644
--- a/roles/dns-nsd/templates/zone.forward.j2
+++ b/roles/dns-nsd/templates/zone.forward.j2
@@ -1,5 +1,6 @@
 ;## NSD authoritative only DNS
 ;## FORWARD Zone
+;# created by ansible
 
 $ORIGIN {{ item.name }}. ; default zone domain
 $TTL {{ item.ttl | default(dns_ttl) }} ; default time to live
@@ -27,8 +28,9 @@
 {% endfor %}
 {% endif %}
 
-; Set from list of nodes
-{% set nodes = vars[item.nodelist] %}
+{% for nodelist in item.nodelists %}
+; Created from nodelist: {{ nodelist }}
+{% set nodes = vars[nodelist] %}
 {% for node in nodes %}
 {{ node.name }}    IN    A    {{ item.ipv4_first_octets ~ "." ~ node.ipv4_last_octet }}
 {% if node.aliases is defined %}
@@ -37,4 +39,5 @@
 {% endfor %}
 {% endif %}
 {% endfor %}
+{% endfor %}
 
diff --git a/roles/dns-nsd/templates/zone.reverse.j2 b/roles/dns-nsd/templates/zone.reverse.j2
index f327d4b..1ebdcd3 100644
--- a/roles/dns-nsd/templates/zone.reverse.j2
+++ b/roles/dns-nsd/templates/zone.reverse.j2
@@ -1,10 +1,10 @@
 ;## NSD authoritative only DNS
 ;## REVERSE Zone for {{ item.name }}
+;# created by ansible
 
 $ORIGIN {{ item.name }}. ; default zone domain
 $TTL {{ item.ttl | default(dns_ttl) }} ; default time to live
 
-
 {{ (item.ipv4_first_octets ~ ".0") | ipaddr('revdns') | regex_replace('^0\.','') }} IN SOA {{ item.soa }}.{{ item.name }}. admin.{{ item.name }}. (
          {{ item.serial | default(ansible_date_time.epoch) }}   ; Serial, must be incremented every time you change this file
          3600        ; Refresh [1hr]
@@ -13,9 +13,13 @@
          60          ; Min TTL [1m]
          )
 
-{% set nodes = vars[item.nodelist] %}
+; -- PTR records --
 
-;PTR records
+{% for nodelist in item.nodelists %}
+; Created from nodelist: {{ nodelist }}
+{% set nodes = vars[nodelist] %}
 {% for node in nodes %}
 {{ (item.ipv4_first_octets ~ "." ~ node.ipv4_last_octet) | ipaddr('revdns') }} IN PTR {{ node.name }}
 {% endfor %}
+{% endfor %}
+
diff --git a/roles/dns-unbound/defaults/main.yml b/roles/dns-unbound/defaults/main.yml
index d0553b1..c462ba0 100644
--- a/roles/dns-unbound/defaults/main.yml
+++ b/roles/dns-unbound/defaults/main.yml
@@ -1,5 +1,11 @@
 ---
+# dns-unbound/defaults/main.yml
 
 unbound_conf: "/var/unbound/etc/unbound.conf"
 unbound_group: "wheel"
 
+unbound_listen_on_default: False
+
+unbound_listen_all: False
+
+unbound_interfaces: []
diff --git a/roles/dns-unbound/templates/unbound.conf.j2 b/roles/dns-unbound/templates/unbound.conf.j2
index ff5ccbd..d82a45f 100644
--- a/roles/dns-unbound/templates/unbound.conf.j2
+++ b/roles/dns-unbound/templates/unbound.conf.j2
@@ -18,6 +18,11 @@
   # allow from localhost
   access-control: 127.0.0.0/24 allow
 
+{% if unbound_listen_all %}
+  # allow from everywhere
+  access-control: 0.0.0.0/0 allow
+{% endif %}
+
 {% if unbound_listen_on_default %}
   # allow from default interfaces
   access-control: {{ ansible_default_ipv4.address }}/{{ (ansible_default_ipv4.address ~ "/" ~ ansible_default_ipv4.netmask) | ipaddr('prefix') }} allow
@@ -26,7 +31,7 @@
 {% if unbound_interfaces is defined %}
   # allow from local networks
 {% for cidr_ipv4 in unbound_interfaces %}
-  access-control: {{ cidr_ipv4 }} allow
+  access-control: {{ cidr_ipv4 | ipaddr('0') }} allow
 {% endfor %}
 {% endif %}
 
diff --git a/roles/interface-config/defaults/main.yml b/roles/interface-config/defaults/main.yml
new file mode 100644
index 0000000..f9056b0
--- /dev/null
+++ b/roles/interface-config/defaults/main.yml
@@ -0,0 +1,9 @@
+---
+# interface-config/defaults/main.yml
+
+mgmt_interface: eth1
+
+mgmt_ipv4_first_octets: "192.168.200"
+
+physical_node_list: []
+
diff --git a/roles/interface-config/tasks/main.yml b/roles/interface-config/tasks/main.yml
new file mode 100644
index 0000000..3954696
--- /dev/null
+++ b/roles/interface-config/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+# interface-config/tasks/main.yml
+
+- name: Create network interface for management network
+  template:
+    src: eth.cfg.j2
+    dest: "/etc/network/interfaces.d/{{ mgmt_interface }}.cfg"
+    owner: root
+    group: root
+    mode: 0644
+  register: mgmtint_config
+
+- name: Bring up management network if reconfigured
+  when: mgmtint_config.changed
+  command: "ifup {{ mgmt_interface }}"
+  tags:
+    - skip_ansible_lint # needs to be run before next steps
+
diff --git a/roles/interface-config/templates/eth.cfg.j2 b/roles/interface-config/templates/eth.cfg.j2
new file mode 100644
index 0000000..2376335
--- /dev/null
+++ b/roles/interface-config/templates/eth.cfg.j2
@@ -0,0 +1,12 @@
+{% for node in physical_node_list if node.name == ansible_hostname %}
+auto {{ node.interface | default(mgmt_interface) }}
+
+iface {{ node.interface | default(mgmt_interface) }} inet static
+    address {{ mgmt_ipv4_first_octets }}.{{ node.ipv4_last_octet }}
+    network {{ mgmt_ipv4_first_octets }}.0
+    netmask 255.255.255.0
+    broadcast {{ mgmt_ipv4_first_octets }}.255
+{% if node.gateway_enabled is defined and node.gateway_enabled %}
+    gateway {{ mgmt_ipv4_first_octets }}.1
+{% endif %}
+{% endfor %}
diff --git a/roles/juju-compute-setup/tasks/main.yml b/roles/juju-compute-setup/tasks/main.yml
index d061697..f4e3918 100644
--- a/roles/juju-compute-setup/tasks/main.yml
+++ b/roles/juju-compute-setup/tasks/main.yml
@@ -9,12 +9,6 @@
 # list of active juju_machines names: juju_machines.keys()
 # list of active juju_services names: juju_services.keys()
 
-# FIXME: Need to add firewall rules to head node or compute machines won't be
-# able to talk to head node VM's.  iptables cmd's look like this:
-#
-# iptables -A FORWARD -i eth0 -o mgmtbr -s <extnet> -d <vmnet> -j ACCEPT
-# iptables -A FORWARD -i mgmtbr -o eth0 -s <vmnet> -d <extnet> -j ACCEPT
-
 - name: Add machines to Juju
   when: "{{ groups['compute'] | difference( juju_machines.keys() ) | length }}"
   command: "juju add-machine ssh:{{ item }}"
@@ -56,20 +50,20 @@
     prompt="Waiting for Juju..."
     seconds=20
 
-# 160*15s = 2400s = 40m max wait
+# 100*30s = 3000s = 50m max wait
 - name: Wait for nova-compute nodes to come online
   juju_facts:
   until: item in juju_compute_nodes.keys() and juju_compute_nodes[item]['workload-status']['message'] == "Unit is ready"
-  retries: 160
-  delay: 15
+  retries: 100
+  delay: 30
   with_items: "{{ groups['compute'] }}"
 
 - name: verify that the nodes appear in nova
   action: shell bash -c "source ~/admin-openrc.sh; nova hypervisor-list | grep '{{ item }}'"
   register: result
   until: result | success
-  retries: 10
-  delay: 10
+  retries: 20
+  delay: 15
   with_items: "{{ groups['compute'] }}"
   tags:
    - skip_ansible_lint # this really should be the os_server module, but ansible doesn't know about juju created openstack
diff --git a/roles/juju-finish/tasks/main.yml b/roles/juju-finish/tasks/main.yml
index e173e0c..6a8c8a0 100644
--- a/roles/juju-finish/tasks/main.yml
+++ b/roles/juju-finish/tasks/main.yml
@@ -29,10 +29,3 @@
   tags:
    - skip_ansible_lint # checking/waiting on a system to be up
 
-- name: Create admin-openrc.sh credentials file
-  template:
-   src=admin-openrc.sh.j2
-   dest="{{ item }}/admin-openrc.sh"
-  with_items:
-    - "{{ ansible_user_dir }}"
-    - "{{ cord_profile_dir }}"
diff --git a/roles/juju-setup/tasks/main.yml b/roles/juju-setup/tasks/main.yml
index 69b13bf..159bd1b 100644
--- a/roles/juju-setup/tasks/main.yml
+++ b/roles/juju-setup/tasks/main.yml
@@ -41,7 +41,7 @@
 
 - name: Check that Juju is actually ready
   juju_facts:
-  until: juju_machines["juju.cord.lab"] is defined and juju_machines["juju.cord.lab"]["agent_state"] == "started"
+  until: 'juju_machines["juju.{{ site_suffix }}"] is defined and juju_machines["juju.{{ site_suffix }}"]["agent_state"] == "started"'
   retries: 40
   delay: 15
 
@@ -66,12 +66,12 @@
   retries: 3
   delay: 15
 
-- name: Deploy services that are hosted in their own VM
+- name: Deploy services that are hosted in their own LXD container
   when: "{{ lxd_service_list | difference( juju_services.keys() ) | length }}"
   command: "juju deploy {{ charm_versions[item] | default(item) }} --to {{ juju_machines[item~'.'~site_suffix]['machine_id'] }} --config={{ juju_config_path }}"
   with_items: "{{ lxd_service_list | difference( juju_services.keys() ) }}"
 
-- name: Deploy services that don't have their own VM
+- name: Deploy services that don't have their own container
   when: "{{ standalone_service_list | difference( juju_services.keys() ) | length }}"
   command: "juju deploy {{ charm_versions[item] | default(item) }} --config={{ juju_config_path }}"
   with_items: "{{ standalone_service_list | difference( juju_services.keys() ) }}"
@@ -85,3 +85,12 @@
     - relations
   tags:
    - skip_ansible_lint # benign to do this more than once, hard to check for
+
+- name: Create admin-openrc.sh OpenStack credentials file
+  template:
+    src: admin-openrc.sh.j2
+    dest: "{{ item }}/admin-openrc.sh"
+  with_items:
+    - "{{ ansible_user_dir }}"
+    - "{{ cord_profile_dir }}"
+
diff --git a/roles/juju-finish/templates/admin-openrc.sh.j2 b/roles/juju-setup/templates/admin-openrc.sh.j2
similarity index 100%
rename from roles/juju-finish/templates/admin-openrc.sh.j2
rename to roles/juju-setup/templates/admin-openrc.sh.j2
diff --git a/roles/pki-cert/defaults/main.yml b/roles/pki-cert/defaults/main.yml
index bbd9c5f..4d55149 100644
--- a/roles/pki-cert/defaults/main.yml
+++ b/roles/pki-cert/defaults/main.yml
@@ -5,6 +5,7 @@
 cert_digest: "sha256"
 cert_days: 180
 
-# list of server certificates to create
+# lists of certificates to create
 server_certs: []
+client_certs: []
 
diff --git a/roles/pki-cert/tasks/main.yml b/roles/pki-cert/tasks/main.yml
index f162f2f..b7cbdd3 100644
--- a/roles/pki-cert/tasks/main.yml
+++ b/roles/pki-cert/tasks/main.yml
@@ -43,12 +43,12 @@
   with_items: "{{ server_certs }}"
   tags:
      - skip_ansible_lint # diagnostic command
-  register: chain_verify
+  register: server_chain_verify
 
 - name: Assert that verify of cert succeeded
   assert:
     that: "'OK' in '{{ item.stdout }}'"
-  with_items: "{{ chain_verify.results }}"
+  with_items: "{{ server_chain_verify.results }}"
 
 - name: Get the intermediate cert into im_cert var
   command: >
@@ -57,7 +57,7 @@
   tags:
      - skip_ansible_lint # concat of files
 
-- name: Get the cert into server_cert var
+- name: Get the certs into server_certs var
   command: >
     openssl x509 -in {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
   with_items: "{{ server_certs }}"
@@ -65,9 +65,72 @@
      - skip_ansible_lint # concat of files
   register: server_certs_raw
 
-- name: Create chained server cert
+- name: Create chained server certs
   copy:
     dest: "{{ pki_dir }}/intermediate_ca/certs/{{ item.item.cn }}_cert_chain.pem"
     content: "{{ item.stdout }}\n{{ im_cert.stdout }}"
   with_items: "{{ server_certs_raw.results }}"
 
+- name: Generate client private key (no pw)
+  command: >
+    openssl genrsa
+      -out {{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem"
+  with_items: "{{ client_certs }}"
+
+- name: Generate client CSR
+  command: >
+    openssl req -config {{ pki_dir }}/intermediate_ca/openssl.cnf
+      -key {{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem
+      -new -sha256 -subj "{{ item.subj }}"
+      -out {{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem"
+  environment:
+    KEY_ALTNAMES: "{{ item.altnames | join(', ') }}"
+  with_items: "{{ client_certs }}"
+
+- name: Sign client cert
+  command: >
+    openssl ca -config {{ pki_dir }}/intermediate_ca/openssl.cnf -batch
+      -passin file:{{ pki_dir }}/intermediate_ca/private/ca_im_phrase
+      -extensions user_cert
+      -days {{ cert_days }} -md {{ cert_digest }}
+      -in {{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem
+      -out {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem"
+  environment:
+    KEY_ALTNAMES: "{{ item.altnames | join(', ') }}"
+  with_items: "{{ client_certs }}"
+
+- name: Verify cert against root + im chain
+  command: >
+    openssl verify -purpose sslclient
+      -CAfile {{ pki_dir }}/intermediate_ca/certs/im_cert_chain.pem
+      {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  with_items: "{{ client_certs }}"
+  tags:
+     - skip_ansible_lint # diagnostic command
+  register: client_chain_verify
+
+- name: Assert that verify of cert succeeded
+  assert:
+    that: "'OK' in '{{ item.stdout }}'"
+  with_items: "{{ client_chain_verify.results }}"
+
+- name: Get the certs into client_certs var
+  command: >
+    openssl x509 -in {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  with_items: "{{ client_certs }}"
+  tags:
+     - skip_ansible_lint # concat of files
+  register: client_certs_raw
+
+- name: Create chained client cert
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/certs/{{ item.item.cn }}_cert_chain.pem"
+    content: "{{ item.stdout }}\n{{ im_cert.stdout }}"
+  with_items: "{{ client_certs_raw.results }}"
+
diff --git a/roles/pki-intermediate-ca/defaults/main.yml b/roles/pki-intermediate-ca/defaults/main.yml
index feecca8..c8ec9c9 100644
--- a/roles/pki-intermediate-ca/defaults/main.yml
+++ b/roles/pki-intermediate-ca/defaults/main.yml
@@ -10,7 +10,7 @@
 ca_im_days: 730
 
 # passphrases for the certificate
-ca_im_phrase: "{{ lookup('password', credentials_dir+'/ca_im_phrase length=64') }}"
+ca_im_phrase: "{{ lookup('password', credentials_dir ~ '/ca_im_phrase length=64') }}"
 
 # noninteractive csr subject
 ca_im_subj: "/C=US/ST=California/L=Menlo Park/O=ON.Lab/OU=Test Deployment/CN=CORD Test Deployment Intermediate CA"
diff --git a/roles/pki-root-ca/defaults/main.yml b/roles/pki-root-ca/defaults/main.yml
index 8f6888c..9fc4952 100644
--- a/roles/pki-root-ca/defaults/main.yml
+++ b/roles/pki-root-ca/defaults/main.yml
@@ -10,7 +10,7 @@
 ca_root_days: 3650
 
 # passphrases for the key
-ca_root_phrase: "{{ lookup('password', credentials_dir+'/ca_root_phrase length=64') }}"
+ca_root_phrase: "{{ lookup('password', credentials_dir ~ '/ca_root_phrase length=64') }}"
 
 # noninteractive csr subject
 ca_root_subj: "/C=US/ST=California/L=Menlo Park/O=ON.Lab/OU=Test Deployment/CN=CORD Test Deployment Root CA"
diff --git a/roles/ssh-pki/defaults/main.yml b/roles/ssh-pki/defaults/main.yml
new file mode 100644
index 0000000..1e8574e
--- /dev/null
+++ b/roles/ssh-pki/defaults/main.yml
@@ -0,0 +1,20 @@
+---
+# ssh-pki/tasks/main.yml
+
+pki_dir: "/opt/pki"
+ssh_pki_dir: "/opt/ssh_pki"
+credentials_dir: "/opt/credentials"
+
+# password on SSH CA
+ssh_ca_phrase: "{{ lookup('password', credentials_dir ~ '/ssh_ca_phrase length=64') }}"
+
+# ssh-keygen parameters
+ssh_keytype: rsa
+ssh_keysize: 4096
+
+# lists of keys to generate
+ssh_client_genkeys:
+  - name: headnode
+
+ssh_host_genkeys: []
+
diff --git a/roles/ssh-pki/tasks/main.yml b/roles/ssh-pki/tasks/main.yml
new file mode 100644
index 0000000..44dbe64
--- /dev/null
+++ b/roles/ssh-pki/tasks/main.yml
@@ -0,0 +1,76 @@
+---
+# ssh-pki/tasks/main.yml
+
+- name: Create SSH CA Directory
+  file:
+    dest: "{{ item }}"
+    state: directory
+    owner: "{{ ansible_user_id }}"
+    mode: 0700
+  with_items:
+    - "{{ ssh_pki_dir }}"
+    - "{{ ssh_pki_dir }}/ca"
+    - "{{ ssh_pki_dir }}/client_certs"
+    - "{{ ssh_pki_dir }}/host_certs"
+
+- name: Generate SSH CA Cert
+  command: >
+    ssh-keygen
+      -q -N "{{ ssh_ca_phrase }}"
+      -t {{ ssh_keytype }}
+      -b {{ ssh_keysize }}
+      -C "CORD SSH CA"
+      -f {{ ssh_pki_dir }}/ca/cord_ssh_ca_cert
+  args:
+    creates: "{{ ssh_pki_dir }}/ca/cord_ssh_ca_cert.pub"
+
+- name: Generate SSH Client Certs
+  command: >
+    ssh-keygen
+      -q -N ""
+      -t {{ item.keytype | default(ssh_keytype) }}
+      -b {{ item.keysize | default(ssh_keysize) }}
+      -C "CORD SSH client key for {{ item.name }}"
+      -f {{ ssh_pki_dir }}/client_certs/{{ item.name }}_sshkey
+  args:
+    creates: "{{ ssh_pki_dir }}/client_certs/{{ item.name }}_sshkey.pub"
+  with_items: "{{ ssh_client_genkeys }}"
+  register: client_ssh_key_generated
+
+- name: Sign SSH Client Certs with SSH CA
+  command: >
+    ssh-keygen
+      -q -P "{{ ssh_ca_phrase }}"
+      -I "{{ item.name }}"
+      -n "{{ item.name }}"
+      -s {{ ssh_pki_dir }}/ca/cord_ssh_ca_cert
+      {{ ssh_pki_dir }}/client_certs/{{ item.name }}_sshkey.pub
+  args:
+    creates: "{{ ssh_pki_dir }}/client_certs/{{ item.name }}_sshkey-cert.pub"
+  with_items: "{{ ssh_client_genkeys }}"
+
+- name: Generate SSH Host Certs
+  command: >
+    ssh-keygen
+      -q -N ""
+      -t {{ item.keytype | default(ssh_keytype) }}
+      -b {{ item.keysize | default(ssh_keysize) }}
+      -C "CORD SSH host key for {{ item.name }}"
+      -f {{ ssh_pki_dir }}/host_certs/{{ item.name }}_sshkey
+  args:
+    creates: "{{ ssh_pki_dir }}/host_certs/{{ item.name }}_sshkey.pub"
+  with_items: "{{ ssh_host_genkeys }}"
+  register: host_ssh_keys_generated
+
+- name: Generate SSH Host Certs
+  command: >
+    ssh-keygen
+      -q -P "{{ ssh_ca_phrase }}" -h
+      -I "{{ item.name }}"
+      -n "{{ item.name }},{{ item.name }}.{{ site_suffix }}"
+      -s {{ ssh_pki_dir }}/ca/cord_ssh_ca_cert
+      {{ ssh_pki_dir }}/host_certs/{{ item.name }}_sshkey.pub
+  args:
+    creates: "{{ ssh_pki_dir }}/host_certs/{{ item.name }}_sshkey-cert.pub"
+  with_items: "{{ ssh_host_genkeys }}"
+