pki work, and keystone cert generated
ignore retry files
load variables for localhost as wel
split root/intermediate generation
use array for creating server certs
configure openstack with certs from server via lookup('file',...
move root CA cert to old location, testing
indent ssl info
more places where the CA cert is used
don't have juju self-manage certs
juju requires certs be base64 encoded (not documented)
install both root/intermediate CA certs, as juju/trusty apache is too old to support chaining
provide ca/im chain to juju keystone config
yaml error
updated name for onos source per jono
fixed the onos-fabric-install role
whitespace
copy CA certs to compute node
stop wasting time
diagnostically print contents of /usr/local/share/ca-certificates/ dir

Change-Id: Idbd4891736b07690a260bf3d117c547de1ae7424
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.retry
diff --git a/cord-deploy-openstack.yml b/cord-deploy-openstack.yml
index 3a0e596..97a4efc 100644
--- a/cord-deploy-openstack.yml
+++ b/cord-deploy-openstack.yml
@@ -2,7 +2,7 @@
 # Deploys OpenStack in LXD containers on the CORD head node
 
 - name: Include vars
-  hosts: all
+  hosts: all, localhost
   tasks:
     - name: Include variables
       include_vars: "{{ item }}"
@@ -17,6 +17,19 @@
     - { role: head-prep, become: yes }
     - create-lxd
 
+- name: Create SSL root/intermediate CAs and server certificates
+  connection: local
+  hosts: localhost
+  roles:
+    - pki-root-ca
+    - pki-intermediate-ca
+    - pki-cert
+
+- name: Install CA certificates
+  hosts: head
+  roles:
+    - pki-install
+
 - name: Start OpenStack install
   hosts: head
   roles:
@@ -29,3 +42,4 @@
   hosts: head
   roles:
     - juju-finish
+
diff --git a/credentials/.gitignore b/credentials/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/credentials/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/credentials/README.md b/credentials/README.md
new file mode 100644
index 0000000..7c8e469
--- /dev/null
+++ b/credentials/README.md
@@ -0,0 +1,5 @@
+# credentials
+
+This directory contains credentials autogenerated by ansible during playbook
+runs, when they aren't already defined by some other means.
+
diff --git a/pki-setup.yml b/pki-setup.yml
new file mode 100644
index 0000000..f8d24c6
--- /dev/null
+++ b/pki-setup.yml
@@ -0,0 +1,21 @@
+---
+# Create a SSL CA for CORD pod use
+
+- name: Include vars
+  hosts: localhost
+  connection: local
+  tasks:
+    - name: Include variables
+      include_vars: "{{ item }}"
+      with_items:
+        - "vars/{{ cord_profile }}.yml"
+        - vars/local_vars.yml
+
+- name: Create Root CA, Intermediate CA, Server certs
+  hosts: localhost
+  connection: local
+  roles:
+    - pki-root-ca
+    - pki-intermediate-ca
+    - pki-cert
+
diff --git a/pki/.gitignore b/pki/.gitignore
new file mode 100644
index 0000000..e58b508
--- /dev/null
+++ b/pki/.gitignore
@@ -0,0 +1,4 @@
+openssl.cfg
+intermediate_ca
+root_ca
+
diff --git a/pki/README.md b/pki/README.md
new file mode 100644
index 0000000..2abc67e
--- /dev/null
+++ b/pki/README.md
@@ -0,0 +1,5 @@
+# pki
+
+This folder contains files uses to manage the certificate authority (CA), so
+handle with care.
+
diff --git a/roles/compute-prep/handlers/main.yml b/roles/compute-prep/handlers/main.yml
index eee1556..d7e4f7b 100644
--- a/roles/compute-prep/handlers/main.yml
+++ b/roles/compute-prep/handlers/main.yml
@@ -3,3 +3,7 @@
 
 - name: run rc.local
   command: /etc/rc.local
+
+- name: update-ca-certifictes on compute node
+  command: update-ca-certificates
+
diff --git a/roles/compute-prep/tasks/main.yml b/roles/compute-prep/tasks/main.yml
index 76dffe2..1607e5f 100644
--- a/roles/compute-prep/tasks/main.yml
+++ b/roles/compute-prep/tasks/main.yml
@@ -15,6 +15,22 @@
     name: ubuntu
     groups: adm
 
+- name: Copy over CA certs
+  synchronize:
+    src: "/usr/local/share/ca-certificates/"
+    dest: "/usr/local/share/ca-certificates/"
+  notify:
+    - update-ca-certifictes on compute node
+
+- name: List certs in /usr/local/share/ca-certificates/
+  command: "ls -la /usr/local/share/ca-certificates/"
+  register: certs_on_compute
+  tags:
+    - skip_ansible_lint # diagnostics
+
+- name: Output from listing certs
+  debug: var=certs_on_compute
+
 - name: Add head node ubuntu user key
   authorized_key:
     user: ubuntu
diff --git a/roles/docker-compose/tasks/main.yml b/roles/docker-compose/tasks/main.yml
index 85b8c14..1c1b0d0 100644
--- a/roles/docker-compose/tasks/main.yml
+++ b/roles/docker-compose/tasks/main.yml
@@ -19,7 +19,7 @@
 #- name: Copy SSL Certs to ONOS so docker-compose can find it
 #  when: "{{ head_vm_list | map(attribute='name') | list | intersect(['onos-cord-1']) | list | length }}"
 #  command: ansible onos-cord-1 -u ubuntu -m copy \
-#    -a "src=/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt dest=~/cord/xos-certs.crt"
+#    -a "src=/usr/local/share/ca-certificates/cord_ca_cert.pem dest=~/cord/xos-certs.crt"
 
 #- name: Build ONOS image with docker-compose
 #  when: "{{ head_vm_list | map(attribute='name') | list | intersect(['onos-cord-1']) | list | length }}"
diff --git a/roles/juju-finish/handlers/main.yml b/roles/juju-finish/handlers/main.yml
deleted file mode 100644
index 695b2cb..0000000
--- a/roles/juju-finish/handlers/main.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-- name: update-ca-certificates
-  become: yes
-  command: update-ca-certificates
-
-- name: Move cert to all service VMs
-  command: ansible services -b -u ubuntu -m copy -a "src={{ ansible_user_dir }}/keystone_juju_ca_cert.crt dest=/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt owner=root group=root mode=0644"
-
-- name: update-ca-certificates in service VMs
-  command: ansible services -b -u ubuntu -m command -a "update-ca-certificates"
diff --git a/roles/juju-finish/tasks/main.yml b/roles/juju-finish/tasks/main.yml
index be58be3..32dfd85 100644
--- a/roles/juju-finish/tasks/main.yml
+++ b/roles/juju-finish/tasks/main.yml
@@ -1,3 +1,6 @@
+---
+# juju-finish/tasks/main.yml
+
 # run another time, so services will be in juju_services list
 - name: Obtain Juju Facts after service creation
   juju_facts:
@@ -31,22 +34,3 @@
    src=admin-openrc.sh.j2
    dest={{ ansible_user_dir }}/admin-openrc.sh
 
-- name: Copy nova-cloud-controller CA certificate to head
-  command: juju scp {{ juju_services['nova-cloud-controller']['units'].keys()[0] }}:/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt {{ ansible_user_dir }}
-  register: result
-  until: result | success
-  retries: 40
-  delay: 15
-  tags:
-   - skip_ansible_lint # checking/waiting on file availibilty
-
-- name: Copy cert to system location
-  become: yes
-  copy:
-    src: "{{ ansible_user_dir }}/keystone_juju_ca_cert.crt"
-    dest: "/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt"
-    remote_src: true
-  notify:
-    - update-ca-certificates
-    - Move cert to all service VMs
-    - update-ca-certificates in service VMs
diff --git a/roles/juju-setup/defaults/main.yml b/roles/juju-setup/defaults/main.yml
index c6ad753..74d7eee 100644
--- a/roles/juju-setup/defaults/main.yml
+++ b/roles/juju-setup/defaults/main.yml
@@ -8,3 +8,4 @@
 
 juju_config_path: /usr/local/src/juju_config.yml
 charm_versions: {}
+
diff --git a/roles/juju-setup/templates/cord_juju_config.yml.j2 b/roles/juju-setup/templates/cord_juju_config.yml.j2
index 0ee33a2..0b07590 100644
--- a/roles/juju-setup/templates/cord_juju_config.yml.j2
+++ b/roles/juju-setup/templates/cord_juju_config.yml.j2
@@ -14,9 +14,11 @@
   admin-password: "{{ keystone_admin_password }}"
   os-public-hostname: "keystone.{{ site_suffix }}"
   ha-mcastport: 5403
-  https-service-endpoints: "True"
   openstack-origin: "cloud:trusty-kilo"
   use-https: "yes"
+  ssl_key: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/private/keystone.{{ site_suffix }}_key.pem') | b64encode }}
+  ssl_cert: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/certs/keystone.{{ site_suffix }}_cert.pem') | b64encode }}
+  ssl_ca: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/certs/im_cert_chain.pem') | b64encode }}
 
 mongodb: {}
 
@@ -57,4 +59,3 @@
 rabbitmq-server:
   ssl: "on"
 
-
diff --git a/roles/juju-setup/templates/opencloud_juju_config.yml.j2 b/roles/juju-setup/templates/opencloud_juju_config.yml.j2
index 4345b19..564f28f 100644
--- a/roles/juju-setup/templates/opencloud_juju_config.yml.j2
+++ b/roles/juju-setup/templates/opencloud_juju_config.yml.j2
@@ -12,8 +12,10 @@
   admin-password: "{{ keystone_admin_password }}"
   os-public-hostname: "keystone.{{ site_suffix }}"
   use-https: "yes"
-  https-service-endpoints: "True"
   openstack-origin: "cloud:trusty-kilo"
+  ssl_key: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/private/keystone.{{ site_suffix }}_key.pem') | b64encode }}
+  ssl_cert: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/certs/keystone.{{ site_suffix }}_cert_chain.pem') | b64encode }}
+  ssl_ca: {{ lookup('file', '{{ playbook_dir }}/pki/intermediate_ca/certs/im_cert_chain.pem') | b64encode }}
 
 mongodb: {}
 
diff --git a/roles/onos-cord-install/tasks/main.yml b/roles/onos-cord-install/tasks/main.yml
index 1393570..eb1f64e 100644
--- a/roles/onos-cord-install/tasks/main.yml
+++ b/roles/onos-cord-install/tasks/main.yml
@@ -38,10 +38,13 @@
 
 - name: Copy SSL Certs to ONOS so docker-compose can find it
   copy:
-    src: "/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt"
-    dest: "{{ onos_cord_dest }}/xos-certs.crt"
+    src: "/usr/local/share/ca-certificates/{{ item }}"
+    dest: "{{ onos_cord_dest }}/{{ item }}"
     owner: "{{ ansible_user_id }}"
     remote_src: True
+  with_items:
+    - "cord_root_ca.crt"
+    - "cord_intermediate_ca.crt"
 
 - name: Build onos image
   command: docker-compose build chdir={{ onos_cord_dest }}
diff --git a/roles/onos-cord-install/templates/Dockerfile.j2 b/roles/onos-cord-install/templates/Dockerfile.j2
index a9973be..e79f87b 100644
--- a/roles/onos-cord-install/templates/Dockerfile.j2
+++ b/roles/onos-cord-install/templates/Dockerfile.j2
@@ -4,15 +4,21 @@
 MAINTAINER Zack Williams <zdw@cs.arizona.edu>
 
 # Add SSL certs
-COPY xos-certs.crt /usr/local/share/ca-certificates/xos-certs.crt
+COPY cord_root_ca.crt /usr/local/share/ca-certificates/cord_root_ca.crt
+COPY cord_intermediate_ca.crt /usr/local/share/ca-certificates/cord_intermediate_ca.crt
 RUN update-ca-certificates
 
 # Create Java KeyStore from certs
-RUN openssl x509 -in /usr/local/share/ca-certificates/xos-certs.crt \
-      -outform der -out /usr/local/share/ca-certificates/xos-certs.der && \
-    keytool -import -noprompt -storepass {{ trust_store_pw }} -alias xos-certs \
-      -file /usr/local/share/ca-certificates/xos-certs.der \
-      -keystore /usr/local/share/ca-certificates/xos-certs.jks
+RUN openssl x509 -in /usr/local/share/ca-certificates/cord_root_ca.crt \
+      -outform der -out /usr/local/share/ca-certificates/cord_root_ca.der && \
+    openssl x509 -in /usr/local/share/ca-certificates/cord_intermediate_ca.crt \
+      -outform der -out /usr/local/share/ca-certificates/cord_intermediate_ca.der && \
+    keytool -import -noprompt -storepass {{ trust_store_pw }} -alias cord_root_ca \
+      -file /usr/local/share/ca-certificates/cord_root_ca.der \
+      -keystore /usr/local/share/ca-certificates/cord_ca_certs.jks && \
+    keytool -import -noprompt -storepass {{ trust_store_pw }} -alias cord_intermediate_ca \
+      -file /usr/local/share/ca-certificates/cord_intermediate_ca.der \
+      -keystore /usr/local/share/ca-certificates/cord_ca_certs.jks
 
 # Updated onos-service to use the jks
 COPY onos-service /root/onos/bin/onos-service
diff --git a/roles/onos-cord-install/templates/onos-service.j2 b/roles/onos-cord-install/templates/onos-service.j2
index 7eef6f5..00a337e 100644
--- a/roles/onos-cord-install/templates/onos-service.j2
+++ b/roles/onos-cord-install/templates/onos-service.j2
@@ -10,7 +10,7 @@
 # Do modify the keystore location/password and truststore location/password accordingly
 #export JAVA_OPTS="${JAVA_OPTS:--DenableNettyTLS=true -Djavax.net.ssl.keyStore=/home/ubuntu/onos.jks -Djavax.net.ssl.keyStorePassword=222222 -Djavax.net.ssl.trustStore=/home/ubuntu/onos.jks -Djavax.net.ssl.trustStorePassword=222222}"
 
-export JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/local/share/ca-certificates/xos-certs.jks -Djavax.net.ssl.trustStorePassword={{ trust_store_pw }}" 
+export JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/local/share/ca-certificates/cord_ca_certs.jks -Djavax.net.ssl.trustStorePassword={{ trust_store_pw }}"
 
 set -e  # exit on error
 set -u  # exit on undefined variable
diff --git a/roles/pki-cert/defaults/main.yml b/roles/pki-cert/defaults/main.yml
new file mode 100644
index 0000000..bbd9c5f
--- /dev/null
+++ b/roles/pki-cert/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+# pki-cert/defaults/main.yml
+
+cert_size: 2048
+cert_digest: "sha256"
+cert_days: 180
+
+# list of server certificates to create
+server_certs: []
+
diff --git a/roles/pki-cert/tasks/main.yml b/roles/pki-cert/tasks/main.yml
new file mode 100644
index 0000000..f162f2f
--- /dev/null
+++ b/roles/pki-cert/tasks/main.yml
@@ -0,0 +1,73 @@
+---
+# pki-cert/tasks/main.yml
+
+- name: Generate server private key (no pw)
+  command: >
+    openssl genrsa
+      -out {{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem"
+  with_items: "{{ server_certs }}"
+
+- name: Generate server CSR
+  command: >
+    openssl req -config {{ pki_dir }}/intermediate_ca/openssl.cnf
+      -key {{ pki_dir }}/intermediate_ca/private/{{ item.cn }}_key.pem
+      -new -sha256 -subj "{{ item.subj }}"
+      -out {{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem"
+  environment:
+    KEY_ALTNAMES: "{{ item.altnames | join(', ') }}"
+  with_items: "{{ server_certs }}"
+
+- name: Sign server cert
+  command: >
+    openssl ca -config {{ pki_dir }}/intermediate_ca/openssl.cnf -batch
+      -passin file:{{ pki_dir }}/intermediate_ca/private/ca_im_phrase
+      -extensions server_cert
+      -days {{ cert_days }} -md {{ cert_digest }}
+      -in {{ pki_dir }}/intermediate_ca/csr/{{ item.cn }}_csr.pem
+      -out {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem"
+  environment:
+    KEY_ALTNAMES: "{{ item.altnames | join(', ') }}"
+  with_items: "{{ server_certs }}"
+
+- name: Verify cert against root + im chain
+  command: >
+    openssl verify -purpose sslserver
+      -CAfile {{ pki_dir }}/intermediate_ca/certs/im_cert_chain.pem
+      {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  with_items: "{{ server_certs }}"
+  tags:
+     - skip_ansible_lint # diagnostic command
+  register: chain_verify
+
+- name: Assert that verify of cert succeeded
+  assert:
+    that: "'OK' in '{{ item.stdout }}'"
+  with_items: "{{ chain_verify.results }}"
+
+- name: Get the intermediate cert into im_cert var
+  command: >
+    openssl x509 -in {{ pki_dir }}/intermediate_ca/certs/im_cert.pem
+  register: im_cert
+  tags:
+     - skip_ansible_lint # concat of files
+
+- name: Get the cert into server_cert var
+  command: >
+    openssl x509 -in {{ pki_dir }}/intermediate_ca/certs/{{ item.cn }}_cert.pem
+  with_items: "{{ server_certs }}"
+  tags:
+     - skip_ansible_lint # concat of files
+  register: server_certs_raw
+
+- name: Create chained server cert
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/certs/{{ item.item.cn }}_cert_chain.pem"
+    content: "{{ item.stdout }}\n{{ im_cert.stdout }}"
+  with_items: "{{ server_certs_raw.results }}"
+
diff --git a/roles/pki-install/handlers/main.yml b/roles/pki-install/handlers/main.yml
new file mode 100644
index 0000000..409ab0f
--- /dev/null
+++ b/roles/pki-install/handlers/main.yml
@@ -0,0 +1,16 @@
+---
+# pki-install/handlers/main.yml
+
+- name: Run update-ca-certificates on head node
+  become: yes
+  command: update-ca-certificates
+
+- name: Copy root CA cert to all service VMs
+  command: ansible services -b -u ubuntu -m copy -a "src=/usr/local/share/ca-certificates/cord_root_ca.crt dest=/usr/local/share/ca-certificates/cord_root_ca.crt owner=root group=root mode=0644"
+
+- name: Copy intermediate CA cert to all service VMs
+  command: ansible services -b -u ubuntu -m copy -a "src=/usr/local/share/ca-certificates/cord_intermediate_ca.crt dest=/usr/local/share/ca-certificates/cord_intermediate_ca.crt owner=root group=root mode=0644"
+
+- name: update-ca-certificates in service VMs
+  command: ansible services -b -u ubuntu -m command -a "update-ca-certificates"
+
diff --git a/roles/pki-install/tasks/main.yml b/roles/pki-install/tasks/main.yml
new file mode 100644
index 0000000..136b8c7
--- /dev/null
+++ b/roles/pki-install/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+# pki-install/tasks/main.yml
+
+- name: Copy CA certificates to head node
+  become: yes
+  copy:
+    src: "{{ playbook_dir }}/pki/{{ item.src }}"
+    dest: "/usr/local/share/ca-certificates/{{ item.dest }}"
+  with_items:
+    - src: "root_ca/certs/ca_cert.pem"
+      dest: "cord_root_ca.crt"
+    - src: "intermediate_ca/certs/im_cert.pem"
+      dest: "cord_intermediate_ca.crt"
+  notify:
+    - Run update-ca-certificates on head node
+    - Copy root CA cert to all service VMs
+    - Copy intermediate CA cert to all service VMs
+    - update-ca-certificates in service VMs
diff --git a/roles/pki-intermediate-ca/defaults/main.yml b/roles/pki-intermediate-ca/defaults/main.yml
new file mode 100644
index 0000000..24801d3
--- /dev/null
+++ b/roles/pki-intermediate-ca/defaults/main.yml
@@ -0,0 +1,16 @@
+---
+# pki-intermediate-ca/defaults/main.yml
+
+pki_dir: "{{ playbook_dir }}/pki"
+
+# crypto parameters
+ca_digest: "sha256"
+ca_size: 4096
+ca_im_days: 730
+
+# passphrases for the certificate
+ca_im_phrase: "{{ lookup('password', 'credentials/ca_im_phrase length=64') }}"
+
+# noninteractive csr subject
+ca_im_subj: "/C=US/ST=California/L=Menlo Park/O=ON.Lab/OU=Test Deployment/CN=CORD Test Deployment Intermediate CA"
+
diff --git a/roles/pki-intermediate-ca/tasks/main.yml b/roles/pki-intermediate-ca/tasks/main.yml
new file mode 100644
index 0000000..8485dc2
--- /dev/null
+++ b/roles/pki-intermediate-ca/tasks/main.yml
@@ -0,0 +1,117 @@
+---
+# pki-ca/tasks/main.yml
+
+- name: Create intermediate CA directory
+  file:
+    dest: "{{ pki_dir }}/intermediate_ca"
+    state: directory
+
+- name: Create intermediate CA openssl.cnf from template
+  template:
+    src: openssl_im.cnf.j2
+    dest: "{{ pki_dir }}/intermediate_ca/openssl.cnf"
+    force: no
+
+- name: Create subdirs for intermediate CA
+  file:
+    dest: "{{ pki_dir }}/intermediate_ca/{{ item }}"
+    state: directory
+  with_items:
+    - certs
+    - crl
+    - csr
+    - newcerts
+
+- name: Create private CA directory
+  file:
+    dest: "{{ pki_dir }}/intermediate_ca/private"
+    state: directory
+    mode: 0700
+
+- name: Create serial file
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/serial"
+    content: "01"
+    force: no
+
+- name: Create empty index file if it doesn't exist
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/index.txt"
+    content: ""
+    force: no
+
+- name: Save intermediate passphrase to intermediate_ca/private/ca_im_phrase
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/private/ca_im_phrase"
+    content: "{{ ca_im_phrase }}"
+    mode: 0400
+
+- name: Generate intermediate key
+  command: >
+    openssl genrsa -aes256
+      -out {{ pki_dir }}/intermediate_ca/private/im_key.pem
+      -passout file:{{ pki_dir }}/intermediate_ca/private/ca_im_phrase
+      {{ ca_size }}
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/private/im_key.pem"
+
+- name: Set permissions on intermediate key
+  file:
+    dest: "{{ pki_dir }}/intermediate_ca/private/im_key.pem"
+    mode: 0400
+
+- name: Create intermediate CSR
+  command: >
+    openssl req -config {{ pki_dir }}/intermediate_ca/openssl.cnf
+      -key {{ pki_dir }}/intermediate_ca/private/im_key.pem
+      -passin file:{{ pki_dir }}/intermediate_ca/private/ca_im_phrase
+      -new -sha256 -subj "{{ ca_im_subj }}"
+      -out {{ pki_dir }}/intermediate_ca/csr/intermediate_ca_csr.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/certs/intermediate_ca_csr.pem"
+  environment:
+    KEY_ALTNAMES: ""
+
+- name: Create intermediate cert from CSR with root CA
+  command: >
+    openssl ca -config {{ pki_dir }}/root_ca/openssl.cnf -batch
+      -extensions v3_intermediate_ca
+      -passin file:{{ pki_dir }}/root_ca/private/ca_root_phrase
+      -days {{ ca_im_days }} -md {{ ca_digest }}
+      -in {{ pki_dir }}/intermediate_ca/csr/intermediate_ca_csr.pem
+      -out {{ pki_dir }}/intermediate_ca/certs/im_cert.pem
+  args:
+    creates: "{{ pki_dir }}/intermediate_ca/certs/im_cert.pem"
+
+- name: Verify intemediate cert
+  command: >
+    openssl verify
+      -CAfile {{ pki_dir }}/root_ca/certs/ca_cert.pem
+      {{ pki_dir }}/intermediate_ca/certs/im_cert.pem
+  register: im_verify
+  tags:
+     - skip_ansible_lint # diagnostic command
+
+- name: Assert that verify of intermediate cert succeeded
+  assert:
+    that: "'OK' in '{{ im_verify.stdout }}'"
+
+- name: Get the root cert into ca_cert var
+  command: >
+    openssl x509 -in {{ pki_dir }}/root_ca/certs/ca_cert.pem
+  register: ca_cert
+  tags:
+     - skip_ansible_lint # concat of files
+
+- name: Get the intermediate cert into im_cert var
+  command: >
+    openssl x509 -in {{ pki_dir }}/intermediate_ca/certs/im_cert.pem
+  register: im_cert
+  tags:
+     - skip_ansible_lint # concat of files
+
+- name: Create intermediate cert chain
+  copy:
+    dest: "{{ pki_dir }}/intermediate_ca/certs/im_cert_chain.pem"
+    content: "{{ im_cert.stdout }}\n{{ ca_cert.stdout }}"
+
diff --git a/roles/pki-intermediate-ca/templates/openssl_im.cnf.j2 b/roles/pki-intermediate-ca/templates/openssl_im.cnf.j2
new file mode 100644
index 0000000..6647d83
--- /dev/null
+++ b/roles/pki-intermediate-ca/templates/openssl_im.cnf.j2
@@ -0,0 +1,107 @@
+# Created by openssl_im.cnf.j2, configured by ansible
+
+[ ca ]
+default_ca  = CA_default
+
+[ CA_default ]
+dir               = {{ pki_dir }}/intermediate_ca
+certs             = $dir/certs
+crl_dir           = $dir/crl
+new_certs_dir     = $dir/newcerts
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/private/.randfile
+
+private_key       = $dir/private/im_key.pem
+certificate       = $dir/certs/im_cert.pem
+
+crlnumber         = $dir/crl/crlnumber
+crl               = $dir/crl/im_crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# Make new requests easier to sign - allow two subjects with same name
+# (Or revoke the old certificate first.)
+unique_subject    = no
+
+default_md        = {{ ca_digest }}
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = {{ ca_im_days }}
+preserve          = no
+
+# for CA that signs client certs
+policy            = policy_loose
+
+[ policy_loose ]
+# Allow the intermediate CA to sign more types of certs
+countryName             = optional
+stateOrProvinceName     = optional
+localityName            = optional
+organizationName        = optional
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+default_bits         = {{ ca_size }}
+default_md           = {{ ca_digest }}
+distinguished_name   = req_distinguished_name
+string_mask          = utf8only
+x509_extensions      = v3_intermediate_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = Country Name (2 letter code)
+stateOrProvinceName             = State or Province Name
+localityName                    = Locality Name
+0.organizationName              = Organization Name
+organizationalUnitName          = Organizational Unit Name
+commonName                      = Common Name
+emailAddress                    = Email Address
+
+# Some defaults
+countryName_default             = US
+stateOrProvinceName_default     = California
+localityName_default            = Menlo Park
+0.organizationName_default      = ON.Lab
+organizationalUnitName_default  = Test Deployment
+emailAddress_default            = privateca@opencord.org
+
+[ v3_intermediate_ca ]
+# Extensions for a typical intermediate CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:TRUE, pathlen:0
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ server_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = CA:FALSE
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = ${ENV::KEY_ALTNAMES}
+
+[ user_cert ]
+# Extensions for client certificates (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = CA:FALSE
+keyUsage = critical, digitalSignature, keyEncipherment, nonRepudiation
+extendedKeyUsage = clientAuth, emailProtection
+
+[ crl_ext ]
+# Extension for CRLs (`man x509v3_config`).
+authorityKeyIdentifier=keyid:always
+
+[ ocsp ]
+# Extension for OCSP signing certificates (`man ocsp`).
+basicConstraints = CA:FALSE
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+keyUsage = critical, digitalSignature
+extendedKeyUsage = critical, OCSPSigning
+
diff --git a/roles/pki-root-ca/defaults/main.yml b/roles/pki-root-ca/defaults/main.yml
new file mode 100644
index 0000000..ebfd5ed
--- /dev/null
+++ b/roles/pki-root-ca/defaults/main.yml
@@ -0,0 +1,14 @@
+---
+# pki-root-ca/defaults/main.yml
+pki_dir: "{{ playbook_dir }}/pki"
+
+# ca parameters
+ca_digest: "sha256"
+ca_size: 4096
+ca_root_days: 3650
+
+# passphrases for the key
+ca_root_phrase: "{{ lookup('password', 'credentials/ca_root_phrase length=64') }}"
+
+# noninteractive csr subject
+ca_root_subj: "/C=US/ST=California/L=Menlo Park/O=ON.Lab/OU=Test Deployment/CN=CORD Test Deployment Root CA"
diff --git a/roles/pki-root-ca/tasks/main.yml b/roles/pki-root-ca/tasks/main.yml
new file mode 100644
index 0000000..eb23d09
--- /dev/null
+++ b/roles/pki-root-ca/tasks/main.yml
@@ -0,0 +1,73 @@
+---
+# pki-root-ca/tasks/main.yml
+
+- name: Create root CA directory
+  file:
+    dest: "{{ pki_dir }}/root_ca"
+    state: directory
+
+- name: Create root CA openssl.cnf from template
+  template:
+    src: openssl_root.cnf.j2
+    dest: "{{ pki_dir }}/root_ca/openssl.cnf"
+    force: no
+
+- name: Create subdirs for root CA
+  file:
+    dest: "{{ pki_dir }}/root_ca/{{ item }}"
+    state: directory
+  with_items:
+    - certs
+    - crl
+    - newcerts
+
+- name: Create private CA directory
+  file:
+    dest: "{{ pki_dir }}/root_ca/private"
+    state: directory
+    mode: 0700
+
+- name: Create serial file
+  copy:
+    dest: "{{ pki_dir }}/root_ca/serial"
+    content: "1000"
+    force: no
+
+- name: Create empty index file if it doesn't exist
+  copy:
+    dest: "{{ pki_dir }}/root_ca/index.txt"
+    content: ""
+    force: no
+
+- name: Save root passphrase to root_ca/private/ca_root_phrase
+  copy:
+    dest: "{{ pki_dir }}/root_ca/private/ca_root_phrase"
+    content: "{{ ca_root_phrase }}"
+    mode: 0400
+
+- name: Generate root key
+  command: >
+    openssl genrsa -aes256
+      -out {{ pki_dir }}/root_ca/private/ca_key.pem
+      -passout file:{{ pki_dir }}/root_ca/private/ca_root_phrase
+      {{ ca_size }}
+  args:
+    creates: "{{ pki_dir }}/root_ca/private/ca_key.pem"
+
+- name: Set permissions on root key
+  file:
+    dest: "{{ pki_dir }}/root_ca/private/ca_key.pem"
+    mode: 0400
+
+- name: Create root certificate
+  command: >
+    openssl req -config {{ pki_dir }}/root_ca/openssl.cnf
+      -key {{ pki_dir }}/root_ca/private/ca_key.pem
+      -passin file:{{ pki_dir }}/root_ca/private/ca_root_phrase
+      -new -x509 -days {{ ca_root_days }}
+      -sha256 -extensions v3_ca
+      -subj "{{ ca_root_subj }}"
+      -out {{ pki_dir }}/root_ca/certs/ca_cert.pem
+  args:
+    creates: "{{ pki_dir }}/root_ca/certs/ca_cert.pem"
+
diff --git a/roles/pki-root-ca/templates/openssl_root.cnf.j2 b/roles/pki-root-ca/templates/openssl_root.cnf.j2
new file mode 100644
index 0000000..a7ffced
--- /dev/null
+++ b/roles/pki-root-ca/templates/openssl_root.cnf.j2
@@ -0,0 +1,95 @@
+# Created by openssl_root.cnf.j2, configured by ansible
+
+[ ca ]
+default_ca  = CA_default
+
+[ CA_default ]
+dir               = {{ pki_dir }}/root_ca
+certs             = $dir/certs
+crl_dir           = $dir/crl
+new_certs_dir     = $dir/newcerts
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/private/.randfile
+
+private_key       = $dir/private/ca_key.pem
+certificate       = $dir/certs/ca_cert.pem
+
+crlnumber         = $dir/crl/crlnumber
+crl               = $dir/crl/ca_crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# Make new requests easier to sign - allow two subjects with same name
+# (Or revoke the old certificate first.)
+unique_subject    = no
+
+default_md        = {{ ca_digest }}
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = {{ ca_root_days }}
+preserve          = no
+
+# for CA that only signs intermediate CA certs
+policy            = policy_strict
+
+[ policy_strict ]
+# Used by root CA to sign intermediate CA's, should match
+countryName             = match
+stateOrProvinceName     = match
+organizationName        = match
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+default_bits         = {{ ca_size }}
+default_md           = {{ ca_digest }}
+distinguished_name   = req_distinguished_name
+string_mask          = utf8only
+x509_extensions      = v3_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = Country Name (2 letter code)
+stateOrProvinceName             = State or Province Name
+localityName                    = Locality Name
+0.organizationName              = Organization Name
+organizationalUnitName          = Organizational Unit Name
+commonName                      = Common Name
+emailAddress                    = Email Address
+
+# Some defaults
+countryName_default             = US
+stateOrProvinceName_default     = California
+localityName_default            = Menlo Park
+0.organizationName_default      = ON.Lab
+organizationalUnitName_default  = Test Deployment
+emailAddress_default            = privateca@opencord.org
+
+[ v3_ca ]
+# Extensions for a typical CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:TRUE
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ v3_intermediate_ca ]
+# Extensions for a typical intermediate CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:TRUE, pathlen:0
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ crl_ext ]
+# Extension for CRLs (`man x509v3_config`).
+authorityKeyIdentifier=keyid:always
+
+[ ocsp ]
+# Extension for OCSP signing certificates (`man ocsp`).
+basicConstraints = CA:FALSE
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+keyUsage = critical, digitalSignature
+extendedKeyUsage = critical, OCSPSigning
+
diff --git a/roles/xos-install/tasks/main.yml b/roles/xos-install/tasks/main.yml
index ffa4f9d..58d2e50 100644
--- a/roles/xos-install/tasks/main.yml
+++ b/roles/xos-install/tasks/main.yml
@@ -16,18 +16,10 @@
    - python-keystoneclient
    - python-glanceclient
 
-# ---- copy repos from the dev machine to the head node ----
-
-- name: Check to see if orchestration directory exists
-  local_action: stat path="{{ playbook_dir }}/../../orchestration"
-  register: orchestration
-
 - name: Copy repositories to the head node
   synchronize:
       src: "{{ playbook_dir }}/../../orchestration/{{ item }}"
       dest: "{{ ansible_user_dir }}/"
-  when:
-      False # orchestration.stat.exists == True  # XXX
   with_items:
       - service-profile
       - xos
@@ -45,22 +37,6 @@
   with_items:
       - vtn
       - olt
-  when:
-      False # (orchestration.stat.exists == True) and (onos_apps.stat.exists == True)   # XXX
-
-
-# ----  alternatively, check out repos from Internet ---
-
-- name: Clone service-profile repo
-  git:
-    repo={{ service_profile_repo_url }}
-    dest={{ service_profile_repo_dest }}
-    version={{ service_profile_repo_branch }}
-    force=yes
-  when:
-    True # orchestration.stat.exists == False # XXX
-
-# ----  install keys ----
 
 - name: Copy over SSH keys
   command: cp ~/.ssh/{{ item }} {{ service_profile_repo_dest }}/{{ xos_configuration }}/
diff --git a/vars/cord.yml b/vars/cord.yml
index 0a41fd1..c1d8798 100644
--- a/vars/cord.yml
+++ b/vars/cord.yml
@@ -23,13 +23,21 @@
 # site domain suffix
 site_suffix: cord.lab
 
+# SSL server certificate generation
+server_certs:
+  - cn: "keystone.{{ site_suffix }}"
+    subj: "/C=US/ST=California/L=Menlo Park/O=ON.Lab/OU=Test Deployment/CN=keystone.{{ site_suffix }}"
+    altnames:
+      - "DNS:keystone.{{ site_suffix }}"
+      - "DNS:{{ site_suffix }}"
+
 # resolv.conf settings
 dns_search:
-  - cord.lab
+  - "{{ site_suffix }}"
 
 # NSD/Unbound settings
 nsd_zones:
-  - name: cord.lab
+  - name: "{{ site_suffix }}"
     ipv4_first_octets: 192.168.122
     name_reverse_unbound: "168.192.in-addr.arpa"
     soa: ns1
@@ -55,4 +63,4 @@
 data_plane_ip: 10.168.0.253/24
 
 # CORD ONOS app version
-cord_app_version: 1.2-SNAPSHOT
\ No newline at end of file
+cord_app_version: 1.2-SNAPSHOT
diff --git a/vars/local_vars.yml b/vars/local_vars.yml
new file mode 100644
index 0000000..9153e0a
--- /dev/null
+++ b/vars/local_vars.yml
@@ -0,0 +1,3 @@
+# local_custom_vars.yaml
+# Put any local customizations to variables in this file.
+