Add CI job for molecule tests

Adds the following zuul job:

kayobe-tox-molecule

It runs ansible role tests using the molecule framework.

Note that we are currently running tox as root to work around issues
with applying docker group membership in the Zuul CI environment.

The ubuntu-1604 platform has been removed because the job takes a long
time to run even on one platform.

Change-Id: I8be24be828c0e124d822d7b39a02169c92b81eb3
Story: #2001637
Task: #6646

ansible/roles/kolla-openstack/molecule/default/molecule.yml

@@ -8,8 +8,6 @@ lint:
 platforms:
   - name: centos-7
     image: centos:7
-  - name: ubuntu-1604
-    image: ubuntu:16.04
 provisioner:
   name: ansible
   lint:

ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml

@@ -8,8 +8,6 @@ lint:
 platforms:
   - name: centos-7
     image: centos:7
-  - name: ubuntu-1604
-    image: ubuntu:16.04
 provisioner:
   name: ansible
   inventory:

ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml

@@ -13,11 +13,15 @@
         - "{{ kolla_inspector_ipa_kernel_path }}"
         - "{{ kolla_inspector_ipa_ramdisk_path }}"
 
+    # NOTE(mgoddard): Previously we were creating empty files for the kernel
+    # and ramdisk, but this was found to cause ansible to hang on recent
+    # versions of docker. Using non-empty files seems to resolve the issue.
+    # See https://github.com/ansible/ansible/issues/36725.
     - name: Ensure ironic inspector kernel and ramdisk images exist
       local_action:
-        module: file
-        path: "{{ item }}"
-        state: touch
+        module: copy
+        content: fake image
+        dest: "{{ item }}"
       with_items:
         - "{{ kolla_inspector_ipa_kernel_path }}"
         - "{{ kolla_inspector_ipa_ramdisk_path }}"

playbooks/kayobe-tox-molecule/pre.yml

@@ -0,0 +1,4 @@
+---
+- hosts: all
+  roles:
+    - role: docker-engine

roles/docker-engine/tasks/main.yml

@@ -0,0 +1,79 @@
+---
+# Docker community edition installation procedure for Ubuntu taken from
+# https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce-1.
+
+# TODO: Configure docker to use the registry mirror provided by OpenStack
+# infra. Kolla-ansible has code referencing
+# http://{{ zuul_site_mirror_fqdn }}:8081/registry-1.docker/, but this did not
+# seem to work.
+
+- name: Fail if OS family is not supported
+  fail:
+    msg: "OS family {{ ansible_os_family }} is not supported"
+  when: ansible_os_family not in ["Debian"]
+
+- name: Update apt package cache
+  apt:
+    update_cache: true
+  become: true
+
+- name: Ensure dependencies are installed
+  apt:
+    name: "{{ item }}"
+    state: installed
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+    - curl
+    - software-properties-common
+  become: true
+
+- name: Ensure docker apt gpg key is installed
+  apt_key:
+    url: "https://download.docker.com/linux/ubuntu/gpg"
+    id: "0EBFCD88"
+    state: present
+  become: true
+
+- name: Ensure docker CE stable repository is present
+  apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable"
+    state: present
+  become: true
+
+- name: Update apt package cache
+  apt:
+    update_cache: true
+  become: true
+
+- name: Ensure docker-ce is installed
+  apt:
+    name: docker-ce
+    state: installed
+  become: true
+
+- name: Ensure docker group exists
+  group:
+    name: docker
+  become: true
+
+- name: Add user to docker group
+  user:
+    name: "{{ ansible_user_id }}"
+    append: yes
+    groups: docker
+  register: group_result
+  become: true
+
+# NOTE(mgoddard): We need to reset the SSH connection to pick up the new group
+# membership. For Ansible 2.3+, ideally we'd use a meta task with the
+# reset_connection option but due to
+# https://github.com/ansible/ansible/issues/27520 this does not work (checked
+# in Ansible 2.3.2.0). Various methods have been attempted, but none have been
+# found to work. Instead, we use sudo when executing tox.
+# Attempted solutions:
+# * reset_connection - see above.
+# * kill local ssh/remove control socket - Zuul blocks running local commands.
+# * kill remote sshd - causes the task to fail, error cannot be ignored.
+# * pause for SSH connection timeout - fails due to
+#   https://github.com/ansible/ansible/issues/31694.

tox.ini

@@ -47,12 +47,19 @@ commands =
     {toxinidir}/tools/test-ansible.sh {posargs}
 
 [testenv:molecule]
+whitelist_externals =
+    bash
+    sudo
 commands =
     # Install ansible role dependencies from Galaxy.
     ansible-galaxy install \
         -r {toxinidir}/requirements.yml \
         -p {toxinidir}/ansible/roles
-    {toxinidir}/tools/test-molecule.sh {posargs}
+    # NOTE(mgoddard): We are executing the molecule command using sudo. This is
+    # to work around the inability to reset the SSH connection in order to pick
+    # up the Linux group membership change for the docker group. See the
+    # docker-engine role for further details.
+    sudo bash -c "source {envdir}/bin/activate && {toxinidir}/tools/test-molecule.sh {posargs}"
 
 [testenv:alint]
 commands = ansible-lint ansible/*.yaml

zuul.d/jobs.yaml

@@ -1,19 +1,33 @@
 ---
-# Tox job that checks Ansible playbook syntax.
 - job:
     name: kayobe-tox-ansible-syntax
+    description: |
+      Tox job that checks Ansible playbook syntax.
     parent: openstack-tox
     vars:
       tox_envlist: ansible-syntax
 
-# Tox job that runs native Ansible role tests.
 - job:
     name: kayobe-tox-ansible
+    description: |
+      Tox job that runs native Ansible role tests.
     parent: openstack-tox-with-sudo
     vars:
       tox_envlist: ansible
 
-# Base job for testing overcloud deployment.
+- job:
+    name: kayobe-tox-molecule
+    description: |
+      Tox job that runs molecule-based Ansible role tests.
+    parent: openstack-tox-with-sudo
+    pre-run: playbooks/kayobe-tox-molecule/pre.yml
+    timeout: 3600
+    required-projects:
+      - name: openstack/requirements
+        override-checkout: stable/pike
+    vars:
+      tox_envlist: molecule
+
 - job:
     name: kayobe-overcloud-base
     description: |

zuul.d/project.yaml

@@ -9,6 +9,7 @@
         - build-openstack-sphinx-docs
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
+        - kayobe-tox-molecule
         - kayobe-overcloud-centos
         - kayobe-seed-centos
 
@@ -22,5 +23,6 @@
         - build-openstack-sphinx-docs
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
+        - kayobe-tox-molecule
         - kayobe-overcloud-centos
         - kayobe-seed-centos