Implement container bind mount for all logs

All of the log directories for all containers will now be bind mounted to the
host. This change ensures that containers are not running into an issue with
Full file systems due to logs which is common when a container is backed by
a blocked device (lvm, zfs, btrfs).

Closes-Bug: #1588051
Change-Id: I25a481c0409f1a45494a8668f00c5393672e853c
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/common-tasks/os-lxc-container-setup.yml

@@ -27,39 +27,97 @@
 #  If extra container configurations are desirable set the
 #  "extra_container_config" list to strings containing the options needed.
 
+- name: Set default bind mounts
+  set_fact:
+    lxc_default_bind_mounts:
+      - bind_dir_path: "/var/log"
+        mount_path: "/openstack/log/{{ inventory_hostname }}"
+  when: lxc_default_bind_mounts is undefined
+
 - name: Ensure mount directories exists
   file:
     path: "{{ item['mount_path'] }}"
     state: "directory"
-  with_items: "{{ list_of_bind_mounts | default([]) }}"
+  with_items:
+    - "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
   when:
     - list_of_bind_mounts is defined
     - not is_metal | bool
 
-- name: LXC Directory bind mount
+- name: LXC bind mount directories
   lxc_container:
     name: "{{ inventory_hostname }}"
     container_command: |
       [[ ! -d "{{ item['bind_dir_path'] }}" ]] && mkdir -p "{{ item['bind_dir_path'] }}"
-    container_config:
-      - "lxc.mount.entry={{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
-  with_items: "{{ list_of_bind_mounts | default([]) }}"
+  with_items:
+    - "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
   register: _bm
   when:
     - list_of_bind_mounts is defined
     - not is_metal | bool
 
-- name: Extra lxc config
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_config: "{{ extra_container_config }}"
+- name: Add bind mount configuration to container
+  lineinfile:
+    dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
+    line: "lxc.mount.entry = {{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
+    backup: "true"
+  with_items:
+    - "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
+  when:
+    - list_of_bind_mounts is defined
+    - not is_metal | bool
+  register: _mc
+
+- name: Extra lxc config
+  lineinfile:
+    dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
+    line: "{{ item.split('=')[0] }} = {{ item.split('=', 1)[1] }}"
+    insertafter: "^{{ item.split('=')[0] }}"
+    backup: "true"
+  with_items: "{{ extra_container_config }}"
+  delegate_to: "{{ physical_host }}"
+  register: _ec
   when:
     - extra_container_config is defined
     - not is_metal | bool
-  register: _ec
+
+# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
+# this uses the LXC CLI tools to ensure that we get logging.
+# TODO(odyssey4me): revisit this once the bug is fixed and released
+- name: Lxc container restart
+  command: >
+    lxc-stop --name {{ inventory_hostname }}
+    --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
+    --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
+  delegate_to: "{{ physical_host }}"
+  register: container_stop
+  until: container_stop | success
+  retries: 3
+  when:
+    - not is_metal | bool
+    - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
+
+# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
+# this uses the LXC CLI tools to ensure that we get logging.
+# TODO(odyssey4me): revisit this once the bug is fixed and released
+- name: Start Container
+  command: >
+    lxc-start --daemon --name {{ inventory_hostname }}
+    --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
+    --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
+  delegate_to: "{{ physical_host }}"
+  register: container_start
+  until: container_start | success
+  retries: 3
+  when:
+    - not is_metal | bool
+    - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
 
 - name: Wait for container ssh
   wait_for:
@@ -68,11 +126,9 @@
     search_regex: "OpenSSH"
     host: "{{ ansible_ssh_host }}"
   delegate_to: "{{ physical_host }}"
-  when:
-    - >
-      (_bm is defined and _bm | changed) or
-      (_ec is defined and _ec | changed)
-    - not is_metal | bool
   register: ssh_wait_check
   until: ssh_wait_check | success
   retries: 3
+  when:
+    - (_bm is defined and _bm | changed) or (_ec is defined and _ec | changed)
+    - not is_metal | bool

playbooks/galera-install.yml

@@ -19,6 +19,11 @@
   gather_facts: "{{ gather_facts | default(True) }}"
   user: root
   tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-mysql_logs"
+            dest: "/var/log/mysql_logs"
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
         list_of_bind_mounts:

playbooks/inventory/group_vars/repo_all.yml

@@ -17,11 +17,6 @@
 repo_server_package_state: "{{ package_state }}"
 repo_build_package_state: "{{ package_state }}"
 
-# The default bind mount to hold the repo data
-repo_all_lxc_container_bind_mounts:
-  - mount_path: "/openstack/{{ inventory_hostname }}"
-    bind_dir_path: "/var/www"
-
 # Optionally set this variable to the location on the deployment
 # host where a set of git clones may be sourced to stage the repo
 # server.

playbooks/repo-server.yml

@@ -29,7 +29,9 @@
 
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
-        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}"
+        list_of_bind_mounts:
+          - mount_path: "/openstack/{{ inventory_hostname }}"
+            bind_dir_path: "/var/www"
       when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists
 
     - include: common-tasks/os-lxc-container-setup.yml
@@ -37,11 +39,14 @@
         repo_build_git_cache_bind_mount:
           - mount_path: "{{ repo_build_git_cache }}"
             bind_dir_path: "{{ repo_build_git_cache }}"
-        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}"
+        list_of_bind_mounts:
+          - mount_path: "/openstack/{{ inventory_hostname }}"
+            bind_dir_path: "/var/www"
+          - mount_path: "{{ repo_build_git_cache }}"
+            bind_dir_path: "{{ repo_build_git_cache }}"
       when:
         - repo_build_git_cache is defined
         - _local_git_cache.stat.exists
-
   roles:
     - { role: "repo_server", tags: [ "repo-server" ] }
     - role: "rsyslog_client"

playbooks/rsyslog-install.yml

@@ -22,7 +22,7 @@
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
         list_of_bind_mounts:
-          - bind_dir_path: "{{ storage_directory }}"
+          - bind_dir_path: "{{ rsyslog_server_storage_directory }}"
             mount_path: "/openstack/{{ inventory_hostname }}/log-storage"
     - include: common-tasks/package-cache-proxy.yml
   roles:
@@ -31,5 +31,4 @@
       tags:
         - "system-crontab-coordination"
   vars:
-    storage_directory: "{{ rsyslog_server_storage_directory }}"
     is_metal: "{{ properties.is_metal|default(false) }}"

releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml

@@ -0,0 +1,25 @@
+---
+features:
+  - Containers will now bind mount all logs to the physical host
+    machine in the "/openstack/log/{{ inventory_hostname }}"
+    location. This change will ensure containers using a block
+    backed file system (lvm, zfs, bfrfs) do not run into issues
+    with full file systems due to logging.
+upgrade:
+  - When upgrading deployers will need to ensure they have a
+    backup of all logging from within the container prior to
+    running the playbooks. If the logging node is present within
+    the deployment all logs should already be sync'd with the
+    logging server and no action is required. As a pre-step it's
+    recommended that deployers clean up logging directories from
+    within containers prior to running the playbooks. After the
+    playbooks have run the bind mount will be in effect at
+    "/var/log" which will mount over all previous log files and
+    directories.
+  - Due to a new bind mount at "/var/log" all containers will be
+    restarted. This is a required restart. It is recommended that
+    deployers run the container restarts in serial to not impact
+    production workloads.
+fixes:
+  - Logging within the container has been bind mounted to the hosts
+    this reslves issue `1588051 <https://bugs.launchpad.net/openstack-ansible/+bug/1588051>_`

scripts/gate-check-commit.sh

@@ -79,9 +79,6 @@ popd
 # Implement the log directory
 mkdir -p /openstack/log
 
-# Implement the log directory link for openstack-infra log publishing
-ln -sf /openstack/log "$(dirname "${0}")/../logs"
-
 pushd "$(dirname "${0}")/../playbooks"
   # Disable Ansible color output
   export ANSIBLE_NOCOLOR=1

scripts/scripts-library.sh

@@ -132,7 +132,15 @@ function exit_fail {
 }
 
 function gate_job_exit_tasks {
-  [[ -d "/openstack/log" ]] && chmod -R 0777 /openstack/log
+  # If this is a gate node from OpenStack-Infra Store all logs into the
+  #  execution directory after gate run.
+  if [[ -d "/etc/nodepool" ]];then
+    GATE_LOG_DIR="$(dirname "${0}")/../logs"
+    mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
+    rsync -av --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true
+    rsync -av --ignore-errors /openstack/log/ "${GATE_LOG_DIR}/openstack" || true
+    chmod -R 0777 "${GATE_LOG_DIR}"
+  fi
 }
 
 function print_info {