Configurable lxc cache and rootfs path

The rootfs of the containers use large disk space and is defaulted to
/var/lib/lxc. It's desirable to configure an alternative rootfs directory on a
partition with large enough free space to hold the rootfs.

The rootfs of a container is hard-linked to the lxc cache when created by lxc
templates. Therefore both have to reside on the same partition and the lxc
cache shall be configurable as well as the rootfs directory.

The lxc-ubuntu template is patched by lxc-host role to use the configured lxc
cache folder. This patch should be removed once the updated upstream lxc with
support of configuring the cache path goes mainstream.

To confgiure these parameters, set the following variables in global_overrides
of openstack_user_config.yml:
    - lxc_container_directory, defaults to /var/lib/lxc
    - lxc_container_cache_path, defaults to /var/cache/lxc

Closes-Bug: #1463905
Change-Id: I9ccf0c45e0a1875644750e0d70ce613265040176

playbooks/lxc-containers-destroy.yml

@@ -35,6 +35,7 @@
         - "/openstack/backup/{{ container_name }}"
         - "/openstack/log/{{ container_name }}"
         - "/var/lib/lxc/{{ container_name }}"
+        - "{{ lxc_container_directory|default('/var/lib/lxc') }}/{{ container_name }}"
       delegate_to: "{{ physical_host }}"
       tags:
         - container-directories

playbooks/roles/lxc_container_create/defaults/main.yml

@@ -18,6 +18,10 @@ lxc_container_config: /etc/lxc/lxc-openstack.conf
 # Default container template to build from
 lxc_container_template: ubuntu
 
+# lxc container rootfs directory and cache path
+lxc_container_directory: "/var/lib/lxc"
+lxc_container_cache_path: "/var/cache/lxc"
+
 # container_fs.* is only used with building on an LVM backend
 lxc_container_fs_size: 5G
 lxc_container_fs_type: ext4

playbooks/roles/lxc_container_create/tasks/container_create.yml

@@ -57,6 +57,7 @@
     template: "{{ properties.container_template|default(lxc_container_template) }}"
     state: started
     backing_store: "{{ properties.container_backing_store|default(lxc_container_backing_store) }}"
+    directory: "{{ lxc_container_directory }}/{{ container_name }}"
     fs_size: "{{ properties.container_fs_size|default(lxc_container_fs_size) }}"
     fs_type: "{{ properties.container_fs_type|default(lxc_container_fs_type) }}"
     vg_name: "{{ properties.container_vg_name|default(lxc_container_vg_name) }}"

playbooks/roles/lxc_container_destroy/tasks/main.yml

@@ -30,6 +30,7 @@
     - "/openstack/backup/{{ container_name }}"
     - "/openstack/log/{{ container_name }}"
     - "/var/lib/lxc/{{ container_name }}"
+    - "{{ lxc_container_directory|default('/var/lib/lxc') }}/{{ container_name }}"
   delegate_to: "{{ physical_host }}"
   tags:
     - container-directories

playbooks/roles/lxc_hosts/defaults/main.yml

@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# lxc container rootfs directory and cache path
+lxc_container_directory: "/var/lib/lxc"
+lxc_container_cache_path: "/var/cache/lxc"
+
 # lxc container net network
 lxc_net_bridge: lxcbr0
 lxc_net_bridge_port: none

playbooks/roles/lxc_hosts/tasks/lxc_cache.yml

@@ -29,7 +29,7 @@
 - name: Move lxc cached image into place
   unarchive:
     src: "/var/cache/lxc_{{ item.name }}"
-    dest: "/var/cache/lxc/"
+    dest: "{{ lxc_container_cache_path }}/"
     copy: "no"
   with_items: lxc_container_caches
   when: cache_download|changed

playbooks/roles/lxc_hosts/tasks/lxc_install.yml

@@ -26,3 +26,16 @@
   with_items: lxc_apt_packages
   tags:
     - lxc-packages
+
+# The functionality with changing the container cache has been added into the
+# upstream LXC templates with patch [ https://github.com/lxc/lxc/pull/558 ]
+# TODO: remove the below patch and pass lxc_container_cache_path to lxc
+# templates as appropriate once the lxc update goes mainstream
+- name: Patch lxc-ubuntu cache path
+  replace:
+    dest: /usr/share/lxc/templates/lxc-ubuntu
+    regexp: '\$LOCALSTATEDIR/cache/lxc'
+    replace: "{{ lxc_container_cache_path }}"
+    backup: yes
+  tags:
+    - lxc-cache-path

playbooks/roles/lxc_hosts/tasks/lxc_pre_install.yml

@@ -27,6 +27,8 @@
     - /usr/share/lxc/templates
     - /openstack
     - /openstack/backup
+    - "{{ lxc_container_directory }}"
+    - "{{ lxc_container_cache_path }}"
   tags:
     - lxc-directories
 
@@ -72,14 +74,14 @@
     - lxc-irqbalance
 
 - name: Drop lxc-openstack app armor profile
-  copy:
+  template:
     src: "{{ item.src }}"
     dest: "{{ item.dest }}"
     owner: "{{ item.owner|default('root') }}"
     group: "{{ item.group|default('root') }}"
     mode: "{{ item.mode|default('0644') }}"
   with_items:
-    - { src: lxc-openstack, dest: "/etc/apparmor.d/lxc/lxc-openstack" }
+    - { src: lxc-openstack.apparmor.j2, dest: "/etc/apparmor.d/lxc/lxc-openstack" }
   notify:
     - Load lxc-openstack apparmor profile
     - Restart apparmor

playbooks/roles/lxc_hosts/templates/lxc-openstack.apparmor.j2

@@ -17,10 +17,10 @@ profile lxc-openstack flags=(attach_disconnected,mediate_deleted) {
   mount fstype=nbd* -> /**,
   mount fstype=nfs* -> /**,
   mount fstype=devpts,
-  
+
 # allow System access.
   mount fstype=cgroup -> /sys/fs/cgroup/**,
-  mount fstype=proc -> /var/cache/lxc/**,
-  mount fstype=sysfs -> /var/cache/lxc/**,
-  mount options=(rw,bind) /var/cache/lxc/**/dev/shm/ -> /var/cache/lxc/**/run/shm/,
+  mount fstype=proc -> {{ lxc_container_cache_path }}/**,
+  mount fstype=sysfs -> {{ lxc_container_cache_path }}/**,
+  mount options=(rw,bind) {{ lxc_container_cache_path }}/**/dev/shm/ -> {{ lxc_container_cache_path }}/**/run/shm/,
 }