From 992e616046a737357183394808ea5a0c48a49065 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 4 Aug 2016 08:35:41 -0500 Subject: [PATCH] Implement container bind mount for all logs All of the log directories for all containers will now be bind mounted to the host. This change ensures that containers are not running into an issue with Full file systems due to logs which is common when a container is backed by a blocked device (lvm, zfs, btrfs). Closes-Bug: #1588051 Change-Id: I25a481c0409f1a45494a8668f00c5393672e853c Signed-off-by: Kevin Carter --- .../common-tasks/os-lxc-container-setup.yml | 86 +++++++++++++++---- playbooks/galera-install.yml | 5 ++ playbooks/inventory/group_vars/repo_all.yml | 5 -- playbooks/repo-server.yml | 11 ++- playbooks/rsyslog-install.yml | 3 +- .../bindmount-logs-3c23aab5b5ed3440.yaml | 25 ++++++ scripts/gate-check-commit.sh | 3 - scripts/scripts-library.sh | 10 ++- 8 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml diff --git a/playbooks/common-tasks/os-lxc-container-setup.yml b/playbooks/common-tasks/os-lxc-container-setup.yml index 1c64fc5737..2e3523840a 100644 --- a/playbooks/common-tasks/os-lxc-container-setup.yml +++ b/playbooks/common-tasks/os-lxc-container-setup.yml @@ -27,39 +27,97 @@ # If extra container configurations are desirable set the # "extra_container_config" list to strings containing the options needed. +- name: Set default bind mounts + set_fact: + lxc_default_bind_mounts: + - bind_dir_path: "/var/log" + mount_path: "/openstack/log/{{ inventory_hostname }}" + when: lxc_default_bind_mounts is undefined + - name: Ensure mount directories exists file: path: "{{ item['mount_path'] }}" state: "directory" - with_items: "{{ list_of_bind_mounts | default([]) }}" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" when: - list_of_bind_mounts is defined - not is_metal | bool -- name: LXC Directory bind mount +- name: LXC bind mount directories lxc_container: name: "{{ inventory_hostname }}" container_command: | [[ ! -d "{{ item['bind_dir_path'] }}" ]] && mkdir -p "{{ item['bind_dir_path'] }}" - container_config: - - "lxc.mount.entry={{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0" - with_items: "{{ list_of_bind_mounts | default([]) }}" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" register: _bm when: - list_of_bind_mounts is defined - not is_metal | bool -- name: Extra lxc config - lxc_container: - name: "{{ inventory_hostname }}" - container_config: "{{ extra_container_config }}" +- name: Add bind mount configuration to container + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + line: "lxc.mount.entry = {{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0" + backup: "true" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" + when: + - list_of_bind_mounts is defined + - not is_metal | bool + register: _mc + +- name: Extra lxc config + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + line: "{{ item.split('=')[0] }} = {{ item.split('=', 1)[1] }}" + insertafter: "^{{ item.split('=')[0] }}" + backup: "true" + with_items: "{{ extra_container_config }}" + delegate_to: "{{ physical_host }}" + register: _ec when: - extra_container_config is defined - not is_metal | bool - register: _ec + +# Due to https://github.com/ansible/ansible-modules-extras/issues/2691 +# this uses the LXC CLI tools to ensure that we get logging. +# TODO(odyssey4me): revisit this once the bug is fixed and released +- name: Lxc container restart + command: > + lxc-stop --name {{ inventory_hostname }} + --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log + --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }} + delegate_to: "{{ physical_host }}" + register: container_stop + until: container_stop | success + retries: 3 + when: + - not is_metal | bool + - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed) + +# Due to https://github.com/ansible/ansible-modules-extras/issues/2691 +# this uses the LXC CLI tools to ensure that we get logging. +# TODO(odyssey4me): revisit this once the bug is fixed and released +- name: Start Container + command: > + lxc-start --daemon --name {{ inventory_hostname }} + --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log + --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }} + delegate_to: "{{ physical_host }}" + register: container_start + until: container_start | success + retries: 3 + when: + - not is_metal | bool + - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed) - name: Wait for container ssh wait_for: @@ -68,11 +126,9 @@ search_regex: "OpenSSH" host: "{{ ansible_ssh_host }}" delegate_to: "{{ physical_host }}" - when: - - > - (_bm is defined and _bm | changed) or - (_ec is defined and _ec | changed) - - not is_metal | bool register: ssh_wait_check until: ssh_wait_check | success retries: 3 + when: + - (_bm is defined and _bm | changed) or (_ec is defined and _ec | changed) + - not is_metal | bool diff --git a/playbooks/galera-install.yml b/playbooks/galera-install.yml index c8abff840e..5811c30992 100644 --- a/playbooks/galera-install.yml +++ b/playbooks/galera-install.yml @@ -19,6 +19,11 @@ gather_facts: "{{ gather_facts | default(True) }}" user: root tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-mysql_logs" + dest: "/var/log/mysql_logs" - include: common-tasks/os-lxc-container-setup.yml vars: list_of_bind_mounts: diff --git a/playbooks/inventory/group_vars/repo_all.yml b/playbooks/inventory/group_vars/repo_all.yml index b13fc265ff..2ad20fce5d 100644 --- a/playbooks/inventory/group_vars/repo_all.yml +++ b/playbooks/inventory/group_vars/repo_all.yml @@ -17,11 +17,6 @@ repo_server_package_state: "{{ package_state }}" repo_build_package_state: "{{ package_state }}" -# The default bind mount to hold the repo data -repo_all_lxc_container_bind_mounts: - - mount_path: "/openstack/{{ inventory_hostname }}" - bind_dir_path: "/var/www" - # Optionally set this variable to the location on the deployment # host where a set of git clones may be sourced to stage the repo # server. diff --git a/playbooks/repo-server.yml b/playbooks/repo-server.yml index e65c7c6c91..e6c8d09dd0 100644 --- a/playbooks/repo-server.yml +++ b/playbooks/repo-server.yml @@ -29,7 +29,9 @@ - include: common-tasks/os-lxc-container-setup.yml vars: - list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}" + list_of_bind_mounts: + - mount_path: "/openstack/{{ inventory_hostname }}" + bind_dir_path: "/var/www" when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists - include: common-tasks/os-lxc-container-setup.yml @@ -37,11 +39,14 @@ repo_build_git_cache_bind_mount: - mount_path: "{{ repo_build_git_cache }}" bind_dir_path: "{{ repo_build_git_cache }}" - list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}" + list_of_bind_mounts: + - mount_path: "/openstack/{{ inventory_hostname }}" + bind_dir_path: "/var/www" + - mount_path: "{{ repo_build_git_cache }}" + bind_dir_path: "{{ repo_build_git_cache }}" when: - repo_build_git_cache is defined - _local_git_cache.stat.exists - roles: - { role: "repo_server", tags: [ "repo-server" ] } - role: "rsyslog_client" diff --git a/playbooks/rsyslog-install.yml b/playbooks/rsyslog-install.yml index e0a9ee1e80..dd9b1e9662 100644 --- a/playbooks/rsyslog-install.yml +++ b/playbooks/rsyslog-install.yml @@ -22,7 +22,7 @@ - include: common-tasks/os-lxc-container-setup.yml vars: list_of_bind_mounts: - - bind_dir_path: "{{ storage_directory }}" + - bind_dir_path: "{{ rsyslog_server_storage_directory }}" mount_path: "/openstack/{{ inventory_hostname }}/log-storage" - include: common-tasks/package-cache-proxy.yml roles: @@ -31,5 +31,4 @@ tags: - "system-crontab-coordination" vars: - storage_directory: "{{ rsyslog_server_storage_directory }}" is_metal: "{{ properties.is_metal|default(false) }}" diff --git a/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml b/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml new file mode 100644 index 0000000000..1ceea65fa1 --- /dev/null +++ b/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml @@ -0,0 +1,25 @@ +--- +features: + - Containers will now bind mount all logs to the physical host + machine in the "/openstack/log/{{ inventory_hostname }}" + location. This change will ensure containers using a block + backed file system (lvm, zfs, bfrfs) do not run into issues + with full file systems due to logging. +upgrade: + - When upgrading deployers will need to ensure they have a + backup of all logging from within the container prior to + running the playbooks. If the logging node is present within + the deployment all logs should already be sync'd with the + logging server and no action is required. As a pre-step it's + recommended that deployers clean up logging directories from + within containers prior to running the playbooks. After the + playbooks have run the bind mount will be in effect at + "/var/log" which will mount over all previous log files and + directories. + - Due to a new bind mount at "/var/log" all containers will be + restarted. This is a required restart. It is recommended that + deployers run the container restarts in serial to not impact + production workloads. +fixes: + - Logging within the container has been bind mounted to the hosts + this reslves issue `1588051 _` diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh index db820e3437..8a62d2b448 100755 --- a/scripts/gate-check-commit.sh +++ b/scripts/gate-check-commit.sh @@ -79,9 +79,6 @@ popd # Implement the log directory mkdir -p /openstack/log -# Implement the log directory link for openstack-infra log publishing -ln -sf /openstack/log "$(dirname "${0}")/../logs" - pushd "$(dirname "${0}")/../playbooks" # Disable Ansible color output export ANSIBLE_NOCOLOR=1 diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index aa0fd22b6a..b39c09c4f7 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -132,7 +132,15 @@ function exit_fail { } function gate_job_exit_tasks { - [[ -d "/openstack/log" ]] && chmod -R 0777 /openstack/log + # If this is a gate node from OpenStack-Infra Store all logs into the + # execution directory after gate run. + if [[ -d "/etc/nodepool" ]];then + GATE_LOG_DIR="$(dirname "${0}")/../logs" + mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack" + rsync -av --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true + rsync -av --ignore-errors /openstack/log/ "${GATE_LOG_DIR}/openstack" || true + chmod -R 0777 "${GATE_LOG_DIR}" + fi } function print_info {