Implement overlayfs-backed containers

Since overlayfs is now part of the mainstream kernel, it makes sense
to take advantage of it to speed up the container creation process.

This patch implements the ability to set 'overlayfs' as the backing
store for containers. This is an optional feature which must
explicitly be set by the deployer.

As this only works with Ansible > 2.0 a pre-requisite check has been
implemented to protect against its use with prior versions. The
inventory and key prep playbook have been adjusted to work with
Ansible versions <2 and >2.

The logging for the veth cleanup has been removed as overlayfs
containers do not have access to the /dev/log device. This causes
the container setup to be incomplete.

Depends-On: I20123b83af10c0890f4d5945b9fb230acd664213
Change-Id: If5b11ef6d94fe863a75efd174957ad43b9d2e030

defaults/main.yml

@@ -29,9 +29,16 @@ lxc_container_rootfs_directory: "{{ lxc_container_directory }}/{{ container_name
 lxc_container_fs_size: 5G
 lxc_container_fs_type: ext4
 
-# Default store is lvm, however will fall back to dir if the
-# container_volume_group is not found.
-lxc_container_backing_store: lvm
+# The container backing store can be set to 'overlayfs' to use overlayfs
+# This should only be done for production use with a linux kernel > 3.14
+# which is when overlayfs was merged into the mainline kernel.
+# lxc_container_backing_store: overlayfs
+# Other store options are "dir" and "lvm".
+
+# If the container backing store is LVM, the automatic detection will
+# require the presence of the lxc_container_vg_name volume group. If
+# this is not found then the automatic detection will fail back to
+# using the directory backing.
 lxc_container_vg_name: lxc
 
 lxc_container_default_mtu: "1500"

releasenotes/notes/container-create-overlayfs-46f3c4c0ecacaadf.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - The container creation process now allows ``overlayfs`` to be set as the
+    ``lxc_container_backing_store``. When this is set it will use a snapshot
+    of the base container to build the containers. The ``overlayfs``
+    backing store is not recommended to be used for production unless the host
+    kernel version is 3.18 or higher.

tasks/container_create.yml

@@ -19,13 +19,14 @@
   failed_when: false
   changed_when: vg_result.rc != 0
   delegate_to: "{{ physical_host }}"
+  when: lxc_container_backing_store is not defined or lxc_container_backing_store == "lvm"
   tags:
     - lxc-container-vg-detect
 
-- name: Set container backend "dir" if "lvm" not found
+- name: Set container backend to "dir" or "lvm" based on whether the lxc VG was found
   set_fact:
-     lxc_container_backing_store: dir
-  when: vg_result.rc != 0
+     lxc_container_backing_store: "{{ (vg_result.rc != 0) | ternary('dir', 'lvm') }}"
+  when: vg_result.rc is defined
   tags:
     - lxc-container-vg-detect
 
@@ -62,7 +63,7 @@
     state: started
     backing_store: "{{ lxc_container_backing_store }}"
     directory: "{{ lxc_container_rootfs_directory }}"
-    fs_size: "{{ lxc_container_fs_size }}"
+    fs_size: "{{ properties.container_fs_size | default(lxc_container_fs_size) }}"
     fs_type: "{{ lxc_container_fs_type }}"
     vg_name: "{{ lxc_container_vg_name }}"
     template_options: "{{ lxc_container_download_template_options }}"
@@ -74,9 +75,62 @@
       - "lxc.hook.autodev=/var/lib/lxc/{{ inventory_hostname }}/autodev"
       - "lxc.mount.entry=/openstack/backup/{{ inventory_hostname }} var/backup none defaults,bind,rw 0 0"
   delegate_to: "{{ physical_host }}"
+  when:
+    - lxc_container_backing_store != "overlayfs"
   tags:
     - lxc-container-create
 
+# Due to https://github.com/ansible/ansible-modules-extras/issues/2577 the
+# next two tasks do not use the lxc_container module.
+# TODO(odyssey4me): Revisit this once a fix has merged
+- name: Check if container exists (overlayfs)
+  command: "lxc-info -n {{ inventory_hostname }}"
+  failed_when: false
+  delegate_to: "{{ physical_host }}"
+  register: lxc_container_info
+  when: lxc_container_backing_store == "overlayfs"
+
+# Due to https://bugs.launchpad.net/ubuntu/+source/lxc/+bug/1591510
+# the '-B' option is used, instead of the more understandable
+# '--backingstore'.
+# TODO(odyssey4me): Revisit this once a fix has merged
+- name: Create container (overlayfs)
+  command: >
+    lxc-copy --snapshot -B overlayfs
+    --name {{ lxc_container_map.distro }}-{{ lxc_container_map.release }}-{{ lxc_container_map.arch }}
+    --newname {{ inventory_hostname }}
+    -L {{ properties.container_fs_size | default(lxc_container_fs_size) }}
+  delegate_to: "{{ physical_host }}"
+  when:
+    - lxc_container_backing_store == "overlayfs"
+    - lxc_container_info.rc != 0
+  tags:
+    - lxc-container-create
+
+# Note that the first four container_config flags are normally applied through
+# the lxc_container module's 'config' option, but the module doesn't support
+# applying that option after the container has been created. This is a workaround
+# until either that is fixed, or a better alternative is identified.
+# TODO(odyssey4me): revisit this for better options some time
+- name: Execute container commands and add container config (overlayfs)
+  lxc_container:
+    name: "{{ inventory_hostname }}"
+    container_command: "{{ lxc_container_map.prep_commands }}"
+    container_config:
+      - "lxc.start.auto=1"
+      - "lxc.start.delay=15"
+      - "lxc.group=onboot"
+      - "lxc.group=openstack"
+      - "lxc.autodev=1"
+      - "lxc.pts=1024"
+      - "lxc.kmsg=0"
+      - "lxc.hook.autodev=/var/lib/lxc/{{ inventory_hostname }}/autodev"
+      - "lxc.mount.entry=/openstack/backup/{{ inventory_hostname }} var/backup none defaults,bind,rw 0 0"
+  delegate_to: "{{ physical_host }}"
+  when: lxc_container_backing_store == "overlayfs"
+  tags:
+    - lxc-container-config
+
 - name: Container network interfaces
   lxc_container:
     name: "{{ inventory_hostname }}"

tasks/main.yml

@@ -13,6 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Ansible version and LXC backing store check
+  fail:
+    msg: "Using overlayfs is not supported when using Ansible version < 2"
+  when:
+    - lxc_container_backing_store is defined
+    - lxc_container_backing_store == "overlayfs"
+    - ansible_version.major < 2
+
+- name: Kernel version and LXC backing store check
+  debug:
+    msg: "Using overlayfs is not recommended when using Kernel version < 3.18"
+  when:
+    - lxc_container_backing_store is defined
+    - lxc_container_backing_store == "overlayfs"
+    - hostvars[physical_host]['ansible_kernel'] | version_compare('3.18.0-0-generic', '<')
+
 - name: Gather variables for each operating system
   include_vars: "{{ item }}"
   with_first_found:

templates/veth-cleanup.sh.j2

@@ -4,12 +4,10 @@ export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 # LXC eth0 is considered special and not managed by the base container_networks
 #  data structure. This is being added outside of the loop for this reason.
 ip link del {{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}_eth0 || true
-logger "LXC container {{ inventory_hostname }} removing veth {{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}_eth0"
 
 # Veth cleanup for items in the container_networks data structure
 {% for key, value in container_networks.items() %}
-    {% if value.type is not defined or value.type == 'veth' %}
+{% if value.type is not defined or value.type == 'veth' %}
 ip link del {{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}_{{ value.interface }} || true
-logger "LXC container {{ inventory_hostname }} removing veth {{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}_{{ value.interface }}"
-    {% endif %}
+{% endif %}
 {% endfor %}