Merge "Update Jenkins executors to Ubuntu 18.04"
diff --git a/global-jjb b/global-jjb
index 5dc3432..5a6965f 160000
--- a/global-jjb
+++ b/global-jjb
@@ -1 +1 @@
-Subproject commit 5dc3432cae2f13d9e5151a00a76a78ce73d92d70
+Subproject commit 5a6965f43f220bf6002bfde60897afd216f52ab0
diff --git a/jjb/ci-management/packer.yaml b/jjb/ci-management/packer.yaml
new file mode 100644
index 0000000..83b6d18
--- /dev/null
+++ b/jjb/ci-management/packer.yaml
@@ -0,0 +1,20 @@
+- project:
+    archive-artifacts: '**/*.log'
+    branch: 'master'
+    build-timeout: '60'
+    build-node: 'ubuntu18.04-basebuild-1c-1g'
+
+    jobs:
+      - '{project-name}-packer-jobs':
+          platforms:
+            - 'ubuntu-18.04'
+          templates:
+            - basebuild_1804
+
+    name: packer-jobs
+    project: 'ci-management'
+    project-name: 'ci-management-ami'
+
+    jjb-version: 3.2.0
+    packer-version: 1.6.5
+    packer-builder: aws
diff --git a/packer/common-packer b/packer/common-packer
index 4a5b0cd..e96974f 160000
--- a/packer/common-packer
+++ b/packer/common-packer
@@ -1 +1 @@
-Subproject commit 4a5b0cd9032938194c4813fe36663ddee4f9e60e
+Subproject commit e96974fa52edb7d8e7f27d314155bd088255e07d
diff --git a/packer/provision/baseline.yaml b/packer/provision/baseline.yaml
new file mode 100644
index 0000000..1aad172
--- /dev/null
+++ b/packer/provision/baseline.yaml
@@ -0,0 +1,88 @@
+---
+- hosts: all
+  become_user: root
+  become_method: sudo
+
+  pre_tasks:
+    - include_role: name=lfit.system-update
+
+    - name: Install base packages
+      include_tasks: "{{item}}"
+      with_first_found:
+        - "install-base-pkgs-{{ansible_distribution}}.yaml"
+        - "install-base-pkgs-{{ansible_os_family}}.yaml"
+
+    - name: Allow jenkins user sudo access
+      copy:
+        dest: /etc/sudoers.d/89-jenkins-user-defaults
+        content: |
+          Defaults:jenkins !requiretty
+          jenkins ALL=(ALL) NOPASSWD:ALL
+        validate: /usr/sbin/visudo -cf %s
+      become: yes
+
+  roles:
+    - lfit.lf-recommended-tools
+    - lfit.lf-dev-libs
+    - lfit.haveged-install
+    - lfit.java-install
+    - lfit.python-install
+    - lfit.shellcheck-install
+    - lfit.sysstat-install
+
+  post_tasks:
+    - name: Update /etc/nss-switch.conf to map hostname with IP
+      # Update /etc/nss-switch.conf to map hostname with IP instead of using `localhost`
+      # from /etc/hosts which is required by some of the Java API's to avoid
+      # Java UnknownHostException: "Name or service not known" error.
+      replace:
+        path: /etc/nsswitch.conf
+        regexp: '^hosts:(\s+.*)?$'
+        replace: 'hosts:\1 myhostname'
+        backup: yes
+      become: yes
+
+    - name: Disable periodic updates
+      block:
+        - name: Set all periodic update options to 0
+          replace:
+            path: /etc/apt/apt.conf.d/10periodic
+            regexp: "1"
+            replace: "0"
+        - name: Set all auto update options to 0
+          replace:
+            path: /etc/apt/apt.conf.d/20auto-upgrades
+            regexp: "1"
+            replace: "0"
+        - name: Disable unattended upgrades
+          lineinfile:
+            path: /etc/apt/apt.conf.d/10periodic
+            regexp: "^APT::Periodic::Unattended-Upgrade"
+            line: 'APT::Periodic::Unattended-Upgrade "0";'
+            create: yes
+        - name: Uninstall unattended upgrades
+          apt:
+            name: unattended-upgrades
+            state: absent
+        - name: Prevent unattended upgrades from being installed
+          dpkg_selections:
+            name: unattended-upgrades
+            selection: hold
+        - name: Disable apt-daily.* systemd services
+          systemd:
+            name: "{{service}}"
+            enabled: no
+            masked: yes
+          with_items:
+            - apt-daily.service
+            - apt-daily.timer
+            - apt-daily-upgrade.service
+            - apt-daily-upgrade.timer
+          loop_control:
+            loop_var: service
+      when: ansible_distribution == 'Ubuntu'
+      become: yes
+
+    - name: System Reseal
+      script: system-reseal.sh
+      become: true
diff --git a/packer/provision/install-base-pkgs-Ubuntu.yaml b/packer/provision/install-base-pkgs-Ubuntu.yaml
new file mode 100644
index 0000000..2ab6308
--- /dev/null
+++ b/packer/provision/install-base-pkgs-Ubuntu.yaml
@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2020 Open Networking Foundation <info@opennetworking.org>
+#
+# SPDX-License-Identifier: LicenseRef-ONF-Member-Only-1.0
+
+---
+- name: Install base packages
+  apt:
+    name:
+      - build-essential
+      - devscripts
+      - dh-systemd
+      - equivs
+      - gdebi
+    state: latest
+  become: yes
diff --git a/packer/provision/local-builder.yaml b/packer/provision/local-builder.yaml
new file mode 100644
index 0000000..8315fa0
--- /dev/null
+++ b/packer/provision/local-builder.yaml
@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2020 Open Networking Foundation <info@opennetworking.org>
+#
+# SPDX-License-Identifier: LicenseRef-ONF-Member-Only-1.0
+
+---
+- import_playbook: ../provision/baseline.yaml
diff --git a/packer/provision/onf-playbook.yaml b/packer/provision/onf-playbook.yaml
new file mode 100644
index 0000000..206fec8
--- /dev/null
+++ b/packer/provision/onf-playbook.yaml
@@ -0,0 +1,231 @@
+---
+# SPDX-FileCopyrightText: 2020 Open Networking Foundation <info@opennetworking.org>
+# SPDX-License-Identifier: LicenseRef-ONF-Member-Only-1.0
+
+- name: "Provision Packages for Jenkins image"
+  hosts: default
+  become: true
+
+  tasks:
+    - name: Install apt packages
+      apt:
+        name:
+          - "facter"
+          - "libxml2-utils"
+          - "bzip2"
+          - "curl"
+          - "ebtables"
+          - "enchant"
+          - "ethtool"
+          - "git"
+          - "graphviz"
+          - "jq"
+          - "kafkacat"
+          - "less"
+          - "libpcap-dev"
+          - "libxml2-utils"
+          - "maven"
+          - "ruby"
+          - "screen"
+          - "socat"
+          - "ssh"
+          - "sshpass"
+          - "zip"
+          # below four packages are required by npm
+          - "nodejs"
+          - "libssl1.0-dev"
+          - "nodejs-dev"
+          - "node-gyp"
+          - "npm"
+        state: "present"
+        update_cache: true
+        cache_valid_time: 3600
+
+    - name: Download repo launcher
+      get_url:
+        url: "https://gerrit.googlesource.com/git-repo/+/refs/tags/v2.12.2/repo?format=TEXT"
+        checksum: "sha256:f5afffcc9afae128efd2b325ff19544a0b78acceb33a2edf368ce2de94e8c33e"
+        dest: /tmp/repo.b64
+
+    - name: Decode and make repo launcher executable
+      shell:
+        cmd: |
+          base64 --decode /tmp/repo.b64 > /usr/local/bin/repo;
+          chmod 755 /usr/local/bin/repo
+        creates: /usr/local/bin/repo
+
+    - name: Download helm archive
+      get_url:
+        url: "https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz"
+        checksum: "sha256:01b317c506f8b6ad60b11b1dc3f093276bb703281cb1ae01132752253ec706a2"
+        dest: "/tmp/helm.tgz"
+
+    - name: Unarchive helm
+      unarchive:
+        src: "/tmp/helm.tgz"
+        dest: "/tmp"
+        remote_src: yes
+
+    - name: Install helm binary
+      copy:
+        src: /tmp/linux-amd64/helm
+        dest: /usr/local/bin/helm
+        mode: "0755"
+        remote_src: yes
+
+    - name: Download/install kubectl binary
+      get_url:
+        url: "https://storage.googleapis.com/kubernetes-release/release/v1.18.15/bin/linux/amd64/kubectl"
+        checksum: "sha256:eb5a5dd0a72795942ab81d1e4331625e80a90002c8bb39b2cb15aa707a3812c6"
+        dest: /usr/local/bin/kubectl
+        mode: "0755"
+
+    - name: load /etc/docker/daemon.json from file
+      slurp:
+        src: /etc/docker/daemon.json
+      register: imported_var
+
+    - name: append more key/values
+      set_fact:
+        imported_var: "{{ imported_var.content|b64decode|from_json | default([]) | combine({ 'registry-mirrors': ['https://mirror.registry.opennetworking.org'] }) }}"
+
+    - name: write var to file
+      copy:
+        content: "{{ imported_var | to_nice_json }}"
+        dest: /etc/docker/daemon.json
+
+    - name: restart Docker service
+      systemd:
+        name: docker
+        state: restarted
+        daemon_reload: true
+
+    - name: Install multi python3 packages with version specifiers
+      pip:
+        name:
+          - ansible
+          - ansible-lint
+          - docker
+          - docker-compose
+          - git-review
+          - httpie
+          - netaddr
+          - pylint
+          - tox
+          - twine
+          - virtualenv
+          - yamllint
+        executable: pip3
+    - name: Install multi python2 packages with version specifiers
+      pip:
+        name:
+        - Jinja2
+        - coverage
+        - certifi
+        - cryptography
+        - git+https://github.com/linkchecker/linkchecker.git@v9.4.0
+        - graphviz
+        - isort
+        - more-itertools==5.0.0
+        - mock>2.0.0<2.1.0
+        - ndg-httpsclient
+        - nose2>0.9.0<0.10.0
+        - pyopenssl
+        - pexpect
+        - pyyaml>3.10.0<3.11.0
+        - requests>2.14.0<2.15.0
+        - robotframework
+        - robotframework-httplibrary
+        - robotframework-kafkalibrary
+        - robotframework-lint
+        - robotframework-requests
+        - robotframework-sshlibrary
+        - six
+        - urllib3
+
+    - name: Install multi ruby packages with version specifiers
+      gem:
+        name: mdl
+        version: 0.5.0
+
+    - name: Install gitbook-cli npm package with version specifiers
+      npm:
+        name: gitbook-cli
+        global: true
+
+    - name: Install markdownlint npm package with version specifiers
+      npm:
+        name: markdownlint
+        global: true
+
+    - name: Install typings npm package with version specifiers
+      npm:
+        name: typings
+        global: true
+
+    - name: Download minikube
+      get_url:
+        url: "https://storage.googleapis.com/minikube/releases/latest/minikube_1.18.0-0_amd64.deb"
+        checksum: "sha256:6e3918b601704014f3d0b0a09e3116f1ea528ac255525743a800b5f0b5856622"
+        dest: /tmp/minikube.deb
+
+    - name: Install minikube deb
+      apt:
+        deb: /tmp/minikube.deb
+
+    - name: Download protobuf
+      get_url:
+        url: "https://github.com/google/protobuf/releases/download/v3.7.0/protoc-3.7.0-linux-x86_64.zip"
+        checksum: "sha256:a1b8ed22d6dc53c5b8680a6f1760a305b33ef471bece482e92728f00ba2a2969"
+        dest: /tmp/protobuf.zip
+
+    - name: Unarchive protobuf
+      unarchive:
+        src: "/tmp/protobuf.zip"
+        dest: "/usr/local"
+        remote_src: yes
+
+    - name: Download pandoc
+      get_url:
+        url: "https://github.com/jgm/pandoc/releases/download/2.10.1/pandoc-2.10.1-1-amd64.deb"
+        checksum: "sha256:4515d6fe2bf8b82765d8dfa1e1b63ccb0ff3332d60389f948672eaa37932e936"
+        dest: /tmp/pandoc.deb
+
+    - name: Install pandoc deb
+      apt:
+        deb: /tmp/pandoc.deb
+
+    - name: Download yq
+      get_url:
+        url: "https://github.com/mikefarah/yq/releases/download/3.4.0/yq_linux_amd64"
+        checksum: "sha256:f6bd1536a743ab170b35c94ed4c7c4479763356bd543af5d391122f4af852460"
+        dest: /usr/local/bin/yq
+
+    - name: Change yq Permission
+      file:
+        path: /usr/local/bin/yq
+        mode: 0755
+
+    - name: Download hadolint
+      get_url:
+        url: "https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-Linux-x86_64"
+        checksum: "sha256:f9bc9de12438b463ca84e77fde70b07b155d4da07ca21bc3f4354a62c6199db4"
+        dest: /usr/local/bin/hadolint
+
+    - name: Change hadolint Permission
+      file:
+        path: /usr/local/bin/hadolint
+        mode: 0755
+
+    - name: Recursively remove download files and folders
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /tmp/linux-amd64
+        - /tmp/helm.tgz
+        - /tmp/minikube.deb
+        - /tmp/protobuf.zip
+        - /tmp/pandoc.deb
+        - /tmp/repo.b64
+        - /tmp/golang_1.16.3.tar.gz
diff --git a/packer/provision/system-reseal.sh b/packer/provision/system-reseal.sh
new file mode 100644
index 0000000..002edf5
--- /dev/null
+++ b/packer/provision/system-reseal.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2016 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# Cleans up VM as preparation for image snapshotting
+
+# vim: sw=2 ts=2 sts=2 et :
+
+rm -rf ~/.viminfo \
+    /etc/Pegasus/*.cnf \
+    /etc/Pegasus/*.crt \
+    /etc/Pegasus/*.csr \
+    /etc/Pegasus/*.pem \
+    /etc/Pegasus/*.srl \
+    /etc/ssh/ssh*key* \
+    /root/.ssh/* \
+    /root/anaconda-ks.cfg \
+    /root/anaconda-post.log \
+    /root/initial-setup-ks.cfg \
+    /root/install.log \
+    /root/install.log.syslog \
+    /tmp/packer \
+    /var/cache/fontconfig/* \
+    /var/cache/gdm/* \
+    /var/cache/man/* \
+    /var/lib/AccountService/users/* \
+    /var/lib/cloud/* \
+    /var/lib/fprint/* \
+    /var/lib/logrotate.status \
+    /var/log/*.log* \
+    /var/log/BackupPC/LOG \
+    /var/log/ConsoleKit/* \
+    /var/log/anaconda.syslog \
+    /var/log/anaconda/* \
+    /var/log/apache2/*_log \
+    /var/log/apache2/*_log-* \
+    /var/log/apt/* \
+    /var/log/aptitude* \
+    /var/log/audit/* \
+    /var/log/btmp* \
+    /var/log/ceph/*.log \
+    /var/log/chrony/*.log \
+    /var/log/cron* \
+    /var/log/cups/*_log \
+    /var/log/debug* \
+    /var/log/dmesg* \
+    /var/log/exim4/* \
+    /var/log/faillog* \
+    /var/log/gdm/* \
+    /var/log/glusterfs/*glusterd.vol.log \
+    /var/log/glusterfs/glusterfs.log \
+    /var/log/httpd/*log \
+    /var/log/installer/* \
+    /var/log/jetty/jetty-console.log \
+    /var/log/journal/* \
+    /var/log/lastlog* \
+    /var/log/libvirt/libvirtd.log \
+    /var/log/libvirt/lxc/*.log \
+    /var/log/libvirt/qemu/*.log \
+    /var/log/libvirt/uml/*.log \
+    /var/log/lightdm/* \
+    /var/log/mail/* \
+    /var/log/maillog* \
+    /var/log/messages* \
+    /var/log/ntp \
+    /var/log/ntpstats/* \
+    /var/log/ppp/connect-errors \
+    /var/log/rhsm/* \
+    /var/log/sa/* \
+    /var/log/secure* \
+    /var/log/setroubleshoot/*.log \
+    /var/log/spooler* \
+    /var/log/squid/*.log \
+    /var/log/syslog* \
+    /var/log/tallylog* \
+    /var/log/tuned/tuned.log \
+    /var/log/wtmp* \
+    /var/named/data/named.run
+
+# Force a system sync and sleep to get around any SSD issues
+echo "Forcing sync and sleep for 10sec"
+sync
+sleep 10
diff --git a/packer/templates/basebuild_1804.json b/packer/templates/basebuild_1804.json
new file mode 100644
index 0000000..668b282
--- /dev/null
+++ b/packer/templates/basebuild_1804.json
@@ -0,0 +1,88 @@
+{
+    "variables": {
+        "ansible_roles_path": ".galaxy",
+        "aws_access_key": null,
+        "aws_security_key": null,
+        "arch": "x86_64",
+        "cloud_user_data": null,
+        "distro": null,
+        "instance_type": "t2.medium",
+        "security_group_id": null,
+        "source_ami_filter_name": null,
+        "source_ami_filter_owner": null,
+        "source_ami_filter_product_code": "",
+        "subnet_id": null,
+        "ssh_user": null
+    },
+    "builders": [
+        {
+            "name": "aws",
+            "access_key": "{{user `aws_access_key`}}",
+            "ami_name": "{{user `distro`}} - basebuild - {{user `arch`}} - {{isotime \"20060102-1504\"}}",
+            "instance_type": "{{user `instance_type`}}",
+            "region": "us-west-2",
+            "secret_key": "{{user `aws_security_key`}}",
+            "security_group_id": "{{user `security_group_id`}}",
+            "source_ami_filter": {
+                "filters": {
+                    "name": "{{user `source_ami_filter_name`}}",
+                    "product-code": "{{user `source_ami_filter_product_code`}}",
+                    "architecture": "{{user `arch`}}",
+                    "root-device-type": "ebs",
+                    "virtualization-type": "hvm"
+                },
+                "most_recent": true,
+                "owners": ["{{user `source_ami_filter_owner`}}"]
+            },
+            "ssh_username": "{{user `ssh_user`}}",
+            "subnet_id": "{{user `subnet_id`}}",
+            "type": "amazon-ebs",
+            "user_data_file": "{{user `cloud_user_data`}}"
+        }
+    ],
+    "provisioners": [
+      {
+          "type": "shell-local",
+          "command": "./common-packer/ansible-galaxy.sh {{user `ansible_roles_path`}}"
+      },
+      {
+          "type": "ansible",
+          "user": "{{user `ssh_user`}}",
+          "playbook_file": "provision/local-builder.yaml",
+          "ansible_env_vars": [
+              "ANSIBLE_NOCOWS=1",
+              "ANSIBLE_PIPELINING=False",
+              "ANSIBLE_HOST_KEY_CHECKING=False",
+              "ANSIBLE_ROLES_PATH={{user `ansible_roles_path`}}",
+              "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+              "ANSIBLE_STDOUT_CALLBACK=debug"
+          ]
+      },
+      {
+          "type": "ansible",
+          "user": "{{user `ssh_user`}}",
+          "playbook_file": "./common-packer/provision/local-docker.yaml",
+          "ansible_env_vars": [
+              "ANSIBLE_NOCOWS=1",
+              "ANSIBLE_PIPELINING=False",
+              "ANSIBLE_HOST_KEY_CHECKING=False",
+              "ANSIBLE_ROLES_PATH={{user `ansible_roles_path`}}",
+              "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+              "ANSIBLE_STDOUT_CALLBACK=debug"
+          ]
+      },
+      {
+          "type": "ansible",
+          "user": "{{user `ssh_user`}}",
+          "playbook_file": "provision/onf-playbook.yaml",
+          "ansible_env_vars": [
+              "ANSIBLE_NOCOWS=1",
+              "ANSIBLE_PIPELINING=False",
+              "ANSIBLE_HOST_KEY_CHECKING=False",
+              "ANSIBLE_ROLES_PATH={{user `ansible_roles_path`}}",
+              "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+              "ANSIBLE_STDOUT_CALLBACK=debug"
+          ]
+      }
+  ]
+}
diff --git a/packer/vars/ubuntu-18.04.json b/packer/vars/ubuntu-18.04.json
new file mode 100644
index 0000000..a1ee888
--- /dev/null
+++ b/packer/vars/ubuntu-18.04.json
@@ -0,0 +1,11 @@
+{
+  "source_ami_filter_name": "*ubuntu*18.04*",
+  "source_ami_filter_owner": "aws-marketplace",
+  "source_ami_filter_product_code": "3iplms73etrdhxdepv72l6ywj",
+
+  "ssh_user": "ubuntu",
+
+  "distro": "Ubuntu 18.04",
+  "arch": "x86_64",
+  "cloud_user_data": null
+}