Merge "Implement container bind mount for all logs"

This commit is contained in:
Jenkins 2016-08-30 13:52:12 +00:00 committed by Gerrit Code Review
commit fa9c114e51
8 changed files with 119 additions and 29 deletions

View File

@ -27,39 +27,97 @@
# If extra container configurations are desirable set the
# "extra_container_config" list to strings containing the options needed.
- name: Set default bind mounts
set_fact:
lxc_default_bind_mounts:
- bind_dir_path: "/var/log"
mount_path: "/openstack/log/{{ inventory_hostname }}"
when: lxc_default_bind_mounts is undefined
- name: Ensure mount directories exists
file:
path: "{{ item['mount_path'] }}"
state: "directory"
with_items: "{{ list_of_bind_mounts | default([]) }}"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
- "{{ lxc_default_bind_mounts }}"
delegate_to: "{{ physical_host }}"
when:
- list_of_bind_mounts is defined
- not is_metal | bool
- name: LXC Directory bind mount
- name: LXC bind mount directories
lxc_container:
name: "{{ inventory_hostname }}"
container_command: |
[[ ! -d "{{ item['bind_dir_path'] }}" ]] && mkdir -p "{{ item['bind_dir_path'] }}"
container_config:
- "lxc.mount.entry={{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
with_items: "{{ list_of_bind_mounts | default([]) }}"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
- "{{ lxc_default_bind_mounts }}"
delegate_to: "{{ physical_host }}"
register: _bm
when:
- list_of_bind_mounts is defined
- not is_metal | bool
- name: Extra lxc config
lxc_container:
name: "{{ inventory_hostname }}"
container_config: "{{ extra_container_config }}"
- name: Add bind mount configuration to container
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
line: "lxc.mount.entry = {{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
backup: "true"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
- "{{ lxc_default_bind_mounts }}"
delegate_to: "{{ physical_host }}"
when:
- list_of_bind_mounts is defined
- not is_metal | bool
register: _mc
- name: Extra lxc config
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
line: "{{ item.split('=')[0] }} = {{ item.split('=', 1)[1] }}"
insertafter: "^{{ item.split('=')[0] }}"
backup: "true"
with_items: "{{ extra_container_config }}"
delegate_to: "{{ physical_host }}"
register: _ec
when:
- extra_container_config is defined
- not is_metal | bool
register: _ec
# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
# this uses the LXC CLI tools to ensure that we get logging.
# TODO(odyssey4me): revisit this once the bug is fixed and released
- name: Lxc container restart
command: >
lxc-stop --name {{ inventory_hostname }}
--logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
--logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
delegate_to: "{{ physical_host }}"
register: container_stop
until: container_stop | success
retries: 3
when:
- not is_metal | bool
- (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
# this uses the LXC CLI tools to ensure that we get logging.
# TODO(odyssey4me): revisit this once the bug is fixed and released
- name: Start Container
command: >
lxc-start --daemon --name {{ inventory_hostname }}
--logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
--logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
delegate_to: "{{ physical_host }}"
register: container_start
until: container_start | success
retries: 3
when:
- not is_metal | bool
- (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
- name: Wait for container ssh
wait_for:
@ -68,11 +126,9 @@
search_regex: "OpenSSH"
host: "{{ ansible_ssh_host }}"
delegate_to: "{{ physical_host }}"
when:
- >
(_bm is defined and _bm | changed) or
(_ec is defined and _ec | changed)
- not is_metal | bool
register: ssh_wait_check
until: ssh_wait_check | success
retries: 3
when:
- (_bm is defined and _bm | changed) or (_ec is defined and _ec | changed)
- not is_metal | bool

View File

@ -19,6 +19,11 @@
gather_facts: "{{ gather_facts | default(True) }}"
user: root
tasks:
- include: common-tasks/os-log-dir-setup.yml
vars:
log_dirs:
- src: "/openstack/log/{{ inventory_hostname }}-mysql_logs"
dest: "/var/log/mysql_logs"
- include: common-tasks/os-lxc-container-setup.yml
vars:
list_of_bind_mounts:

View File

@ -17,11 +17,6 @@
repo_server_package_state: "{{ package_state }}"
repo_build_package_state: "{{ package_state }}"
# The default bind mount to hold the repo data
repo_all_lxc_container_bind_mounts:
- mount_path: "/openstack/{{ inventory_hostname }}"
bind_dir_path: "/var/www"
# Optionally set this variable to the location on the deployment
# host where a set of git clones may be sourced to stage the repo
# server.

View File

@ -29,7 +29,9 @@
- include: common-tasks/os-lxc-container-setup.yml
vars:
list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}"
list_of_bind_mounts:
- mount_path: "/openstack/{{ inventory_hostname }}"
bind_dir_path: "/var/www"
when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists
- include: common-tasks/os-lxc-container-setup.yml
@ -37,11 +39,14 @@
repo_build_git_cache_bind_mount:
- mount_path: "{{ repo_build_git_cache }}"
bind_dir_path: "{{ repo_build_git_cache }}"
list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}"
list_of_bind_mounts:
- mount_path: "/openstack/{{ inventory_hostname }}"
bind_dir_path: "/var/www"
- mount_path: "{{ repo_build_git_cache }}"
bind_dir_path: "{{ repo_build_git_cache }}"
when:
- repo_build_git_cache is defined
- _local_git_cache.stat.exists
roles:
- { role: "repo_server", tags: [ "repo-server" ] }
- role: "rsyslog_client"

View File

@ -22,7 +22,7 @@
- include: common-tasks/os-lxc-container-setup.yml
vars:
list_of_bind_mounts:
- bind_dir_path: "{{ storage_directory }}"
- bind_dir_path: "{{ rsyslog_server_storage_directory }}"
mount_path: "/openstack/{{ inventory_hostname }}/log-storage"
- include: common-tasks/package-cache-proxy.yml
roles:
@ -31,5 +31,4 @@
tags:
- "system-crontab-coordination"
vars:
storage_directory: "{{ rsyslog_server_storage_directory }}"
is_metal: "{{ properties.is_metal|default(false) }}"

View File

@ -0,0 +1,25 @@
---
features:
- Containers will now bind mount all logs to the physical host
machine in the "/openstack/log/{{ inventory_hostname }}"
location. This change will ensure containers using a block
backed file system (lvm, zfs, bfrfs) do not run into issues
with full file systems due to logging.
upgrade:
- When upgrading deployers will need to ensure they have a
backup of all logging from within the container prior to
running the playbooks. If the logging node is present within
the deployment all logs should already be sync'd with the
logging server and no action is required. As a pre-step it's
recommended that deployers clean up logging directories from
within containers prior to running the playbooks. After the
playbooks have run the bind mount will be in effect at
"/var/log" which will mount over all previous log files and
directories.
- Due to a new bind mount at "/var/log" all containers will be
restarted. This is a required restart. It is recommended that
deployers run the container restarts in serial to not impact
production workloads.
fixes:
- Logging within the container has been bind mounted to the hosts
this reslves issue `1588051 <https://bugs.launchpad.net/openstack-ansible/+bug/1588051>_`

View File

@ -79,9 +79,6 @@ popd
# Implement the log directory
mkdir -p /openstack/log
# Implement the log directory link for openstack-infra log publishing
ln -sf /openstack/log "$(dirname "${0}")/../logs"
pushd "$(dirname "${0}")/../playbooks"
# Disable Ansible color output
export ANSIBLE_NOCOLOR=1

View File

@ -132,7 +132,15 @@ function exit_fail {
}
function gate_job_exit_tasks {
[[ -d "/openstack/log" ]] && chmod -R 0777 /openstack/log
# If this is a gate node from OpenStack-Infra Store all logs into the
# execution directory after gate run.
if [[ -d "/etc/nodepool" ]];then
GATE_LOG_DIR="$(dirname "${0}")/../logs"
mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
rsync -av --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true
rsync -av --ignore-errors /openstack/log/ "${GATE_LOG_DIR}/openstack" || true
chmod -R 0777 "${GATE_LOG_DIR}"
fi
}
function print_info {