From 14c668c2e8738ee96adb4e3a49a80a98c7017db3 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 13 Nov 2014 11:24:08 +0000 Subject: [PATCH] Remove rsyslog containers from swift-hosts * Setup rsyslog configuration on the storage nodes * Removes the requirement for an 'lxc' VG on swift nodes * Adjust environment to not start rsyslog containers * Fix log names for error logs to ensure they are configured * Fix the container setup plays to allow an empty list of containers for a host Fixes: #522 --- etc/rpc_deploy/rpc_environment.yml | 1 - etc/rpc_deploy/rpc_user_config.yml | 2 +- .../playbooks/openstack/swift-storage.yml | 1 + .../playbooks/setup/build-containers.yml | 2 +- .../playbooks/setup/containers-setup.yml | 2 +- .../playbooks/setup/restart-containers.yml | 2 +- .../tasks/container_create.yml | 58 ++++++++ .../roles/container_create/tasks/main.yml | 45 +------ .../tasks/container_restart.yml | 43 ++++++ .../roles/container_restart/tasks/main.yml | 30 +---- .../container_setup/tasks/container_setup.yml | 124 ++++++++++++++++++ .../roles/container_setup/tasks/main.yml | 111 +--------------- .../templates/swift-rsyslog.conf.j2 | 8 +- 13 files changed, 240 insertions(+), 189 deletions(-) create mode 100644 rpc_deployment/roles/container_create/tasks/container_create.yml create mode 100644 rpc_deployment/roles/container_restart/tasks/container_restart.yml create mode 100644 rpc_deployment/roles/container_setup/tasks/container_setup.yml diff --git a/etc/rpc_deploy/rpc_environment.yml b/etc/rpc_deploy/rpc_environment.yml index b31f6a5d5b..4f93bff5e8 100644 --- a/etc/rpc_deploy/rpc_environment.yml +++ b/etc/rpc_deploy/rpc_environment.yml @@ -262,7 +262,6 @@ container_skel: - storage_containers - log_containers - network_containers - - swift_containers contains: - rsyslog utility_container: diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index f9517c8d35..90acb186c0 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -15,7 +15,7 @@ # This is the md5 of the environment file # this will ensure consistency when deploying. -environment_version: dc4cd5fe9c07eee223e8bbb1c5bbaad5 +environment_version: 3511a43b8e4cc39af4beaaa852b5f917 # User defined CIDR used for containers # Global cidr/s used for everything. diff --git a/rpc_deployment/playbooks/openstack/swift-storage.yml b/rpc_deployment/playbooks/openstack/swift-storage.yml index 229d06bdae..6ac30656f3 100644 --- a/rpc_deployment/playbooks/openstack/swift-storage.yml +++ b/rpc_deployment/playbooks/openstack/swift-storage.yml @@ -18,6 +18,7 @@ roles: - swift_common - swift_storage_setup + - rsyslog_config vars_files: - inventory/group_vars/swift_all.yml diff --git a/rpc_deployment/playbooks/setup/build-containers.yml b/rpc_deployment/playbooks/setup/build-containers.yml index d89d3e4fe5..571e60f4e5 100644 --- a/rpc_deployment/playbooks/setup/build-containers.yml +++ b/rpc_deployment/playbooks/setup/build-containers.yml @@ -19,6 +19,6 @@ - container_create vars: default_container_groups: "{{ hostvars[inventory_hostname]['container_types'] }}" - container_groups: "{{ groups[container_group|default(default_container_groups)] }}" + container_groups: "{{ groups[container_group|default(default_container_groups)] | default('') }}" - include: containers-setup.yml diff --git a/rpc_deployment/playbooks/setup/containers-setup.yml b/rpc_deployment/playbooks/setup/containers-setup.yml index 5faee6110c..f3c48ae89f 100644 --- a/rpc_deployment/playbooks/setup/containers-setup.yml +++ b/rpc_deployment/playbooks/setup/containers-setup.yml @@ -21,7 +21,7 @@ - vars/config_vars/container_interfaces.yml vars: default_container_groups: "{{ hostvars[inventory_hostname]['container_types'] }}" - container_groups: "{{ groups[container_group|default(default_container_groups)] }}" + container_groups: "{{ groups[container_group|default(default_container_groups)] | default('') }}" required_container_config_options: - "lxc.mount.entry=/openstack/log/{{ hostvars[item]['container_name'] }} var/log/{{ hostvars[item]['service_name'] }} none defaults,bind,rw 0 0" - "lxc.mount.entry=/openstack/backup/{{ hostvars[item]['container_name'] }} var/backup none defaults,bind,rw 0 0" diff --git a/rpc_deployment/playbooks/setup/restart-containers.yml b/rpc_deployment/playbooks/setup/restart-containers.yml index a6ea07a259..f9a373ec47 100644 --- a/rpc_deployment/playbooks/setup/restart-containers.yml +++ b/rpc_deployment/playbooks/setup/restart-containers.yml @@ -20,4 +20,4 @@ - container_restart vars: default_container_groups: "{{ hostvars[inventory_hostname]['container_types'] }}" - container_groups: "{{ groups[container_group|default(default_container_groups)] }}" + container_groups: "{{ groups[container_group|default(default_container_groups)] | default('') }}" diff --git a/rpc_deployment/roles/container_create/tasks/container_create.yml b/rpc_deployment/roles/container_create/tasks/container_create.yml new file mode 100644 index 0000000000..6dd0257269 --- /dev/null +++ b/rpc_deployment/roles/container_create/tasks/container_create.yml @@ -0,0 +1,58 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create an LXC container. + +- name: Check for lxc volume group + shell: > + (which vgs > /dev/null && vgs | grep -o "lxc") || false + register: vg_result + ignore_errors: True + +- name: Create Container Local + lxc: > + name={{ hostvars[item]['container_name'] }} + template={{ container_template }} + config={{ hostvars[item]['container_config'] }} + command=create + state=running + template_options=" + --release {{ container_release }} + " + when: vg_result.rc == 1 + with_items: container_groups + +- name: Check Container Bridge exists + file: > + state=file + path="/sys/class/net/{{ management_bridge }}/bridge/bridge_id" + +- name: Create Container LVM + lxc: > + name={{ hostvars[item]['container_name'] }} + template={{ container_template }} + config={{ hostvars[item]['container_config'] }} + command=create + state=running + bdev=lvm + lvname={{ hostvars[item]['container_name'] }} + vgname=lxc + fstype={{ hostvars[item]['container_lvm_fstype'] }} + fssize={{ hostvars[item]['container_lvm_fssize'] }} + template_options=" + --release {{ container_release }} + " + when: vg_result.rc == 0 + with_items: container_groups diff --git a/rpc_deployment/roles/container_create/tasks/main.yml b/rpc_deployment/roles/container_create/tasks/main.yml index 6dd0257269..096b24edcc 100644 --- a/rpc_deployment/roles/container_create/tasks/main.yml +++ b/rpc_deployment/roles/container_create/tasks/main.yml @@ -13,46 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Create an LXC container. - -- name: Check for lxc volume group - shell: > - (which vgs > /dev/null && vgs | grep -o "lxc") || false - register: vg_result - ignore_errors: True - -- name: Create Container Local - lxc: > - name={{ hostvars[item]['container_name'] }} - template={{ container_template }} - config={{ hostvars[item]['container_config'] }} - command=create - state=running - template_options=" - --release {{ container_release }} - " - when: vg_result.rc == 1 - with_items: container_groups - -- name: Check Container Bridge exists - file: > - state=file - path="/sys/class/net/{{ management_bridge }}/bridge/bridge_id" - -- name: Create Container LVM - lxc: > - name={{ hostvars[item]['container_name'] }} - template={{ container_template }} - config={{ hostvars[item]['container_config'] }} - command=create - state=running - bdev=lvm - lvname={{ hostvars[item]['container_name'] }} - vgname=lxc - fstype={{ hostvars[item]['container_lvm_fstype'] }} - fssize={{ hostvars[item]['container_lvm_fssize'] }} - template_options=" - --release {{ container_release }} - " - when: vg_result.rc == 0 - with_items: container_groups +- include: container_create.yml + when: container_groups|length > 0 diff --git a/rpc_deployment/roles/container_restart/tasks/container_restart.yml b/rpc_deployment/roles/container_restart/tasks/container_restart.yml new file mode 100644 index 0000000000..ddcabba15f --- /dev/null +++ b/rpc_deployment/roles/container_restart/tasks/container_restart.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Test Container Networking + wait_for: > + port=22 + timeout=20 + search_regex=OpenSSH + host={{ hostvars[item]['container_address'] }} + with_items: container_groups + delegate_to: localhost + register: network_check + ignore_errors: yes + +- name: Restart containers + lxc: > + name={{ hostvars[item]['container_name'] }} + command=restart + with_items: container_groups + when: network_check|failed + +- name: Check Networking After Restart + wait_for: > + port=22 + timeout={{ container_start_timeout }} + search_regex=OpenSSH + host={{ hostvars[item]['container_address'] }} + with_items: container_groups + delegate_to: localhost + when: network_check|failed + diff --git a/rpc_deployment/roles/container_restart/tasks/main.yml b/rpc_deployment/roles/container_restart/tasks/main.yml index ddcabba15f..becbd9e8d1 100644 --- a/rpc_deployment/roles/container_restart/tasks/main.yml +++ b/rpc_deployment/roles/container_restart/tasks/main.yml @@ -13,31 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Test Container Networking - wait_for: > - port=22 - timeout=20 - search_regex=OpenSSH - host={{ hostvars[item]['container_address'] }} - with_items: container_groups - delegate_to: localhost - register: network_check - ignore_errors: yes - -- name: Restart containers - lxc: > - name={{ hostvars[item]['container_name'] }} - command=restart - with_items: container_groups - when: network_check|failed - -- name: Check Networking After Restart - wait_for: > - port=22 - timeout={{ container_start_timeout }} - search_regex=OpenSSH - host={{ hostvars[item]['container_address'] }} - with_items: container_groups - delegate_to: localhost - when: network_check|failed - +- include: container_restart.yml + when: container_groups|length > 0 diff --git a/rpc_deployment/roles/container_setup/tasks/container_setup.yml b/rpc_deployment/roles/container_setup/tasks/container_setup.yml new file mode 100644 index 0000000000..252f5ef950 --- /dev/null +++ b/rpc_deployment/roles/container_setup/tasks/container_setup.yml @@ -0,0 +1,124 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Obtain the Systems SSH-Key + set_fact: + container_ssh_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" + +- name: Fail when empty or non-existent SSH pub key + fail: > + msg="Failing - ~/.ssh/id_rsa.pub file doesn't exist or is empty" + when: container_ssh_key == "" + +- name: Set the SSH key in place + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command=" + mkdir -p ~/.ssh/; + if [ ! -f \"~/.ssh/authorized_keys\" ];then + touch ~/.ssh/authorized_keys; + fi; + grep '{{ container_ssh_key }}' ~/.ssh/authorized_keys || + echo '{{ container_ssh_key }}' | tee -a ~/.ssh/authorized_keys; + " + with_items: container_groups + +- name: Set base network interface + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command=" + echo -e '{{ container_interface }}' | tee /etc/network/interfaces; + " + with_items: container_groups + +- name: Set management network interface + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command=" + echo -e '{{ management_interface }}' | tee /etc/network/interfaces.d/management.cfg; + " + with_items: container_groups + +- name: Ensure SSH is avail at boot + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command="update-rc.d ssh defaults" + with_items: container_groups + +- name: Ensure SSH is available for root + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command="sed -i 's/PermitRootLogin.*/PermitRootLogin\ yes/g' /etc/ssh/sshd_config" + with_items: container_groups + +- name: Ensure SSH started + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command="service ssh restart" + with_items: container_groups + +- name: Ensure required inner directories + lxc: > + name={{ hostvars[item.1]['container_name'] }} + command=attach + container_command="mkdir -p {{ item.0 }}" + with_nested: + - [ "/monitoring", "/etc/network/interfaces.d", "/var/backup" ] + - container_groups + +- name: Create Required local log directories + file: > + path="{{ item.0 }}/{{ hostvars[item.1]['container_name'] }}" + state=directory + with_nested: + - [ "/openstack/backup", "/openstack/log" ] + - container_groups + +- name: Create Required local monitoring directories + file: > + path={{ item }} + state=directory + with_items: + - "/openstack/monitoring" + +- name: Ensure required inner service directories + lxc: > + name={{ hostvars[item.1]['container_name'] }} + command=attach + container_command="mkdir -p {{ item.0 }}/{{ hostvars[item.1]['service_name'] }}" + with_nested: + - [ "/etc", "/var/log" ] + - container_groups + +- name: Ensure python2.7 installed + lxc: > + name={{ hostvars[item]['container_name'] }} + command=attach + container_command="apt-get -y install python2.7; rm /usr/bin/python; ln -s /usr/bin/python2.7 /usr/bin/python" + with_items: container_groups + +- name: Ensure Required container config options + lxc: > + name={{ hostvars[item]['container_name'] }} + command=config + options="{{ required_container_config_options }}" + state=running + with_items: container_groups diff --git a/rpc_deployment/roles/container_setup/tasks/main.yml b/rpc_deployment/roles/container_setup/tasks/main.yml index 252f5ef950..b330a5fb08 100644 --- a/rpc_deployment/roles/container_setup/tasks/main.yml +++ b/rpc_deployment/roles/container_setup/tasks/main.yml @@ -13,112 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Obtain the Systems SSH-Key - set_fact: - container_ssh_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" - -- name: Fail when empty or non-existent SSH pub key - fail: > - msg="Failing - ~/.ssh/id_rsa.pub file doesn't exist or is empty" - when: container_ssh_key == "" - -- name: Set the SSH key in place - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command=" - mkdir -p ~/.ssh/; - if [ ! -f \"~/.ssh/authorized_keys\" ];then - touch ~/.ssh/authorized_keys; - fi; - grep '{{ container_ssh_key }}' ~/.ssh/authorized_keys || - echo '{{ container_ssh_key }}' | tee -a ~/.ssh/authorized_keys; - " - with_items: container_groups - -- name: Set base network interface - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command=" - echo -e '{{ container_interface }}' | tee /etc/network/interfaces; - " - with_items: container_groups - -- name: Set management network interface - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command=" - echo -e '{{ management_interface }}' | tee /etc/network/interfaces.d/management.cfg; - " - with_items: container_groups - -- name: Ensure SSH is avail at boot - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command="update-rc.d ssh defaults" - with_items: container_groups - -- name: Ensure SSH is available for root - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command="sed -i 's/PermitRootLogin.*/PermitRootLogin\ yes/g' /etc/ssh/sshd_config" - with_items: container_groups - -- name: Ensure SSH started - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command="service ssh restart" - with_items: container_groups - -- name: Ensure required inner directories - lxc: > - name={{ hostvars[item.1]['container_name'] }} - command=attach - container_command="mkdir -p {{ item.0 }}" - with_nested: - - [ "/monitoring", "/etc/network/interfaces.d", "/var/backup" ] - - container_groups - -- name: Create Required local log directories - file: > - path="{{ item.0 }}/{{ hostvars[item.1]['container_name'] }}" - state=directory - with_nested: - - [ "/openstack/backup", "/openstack/log" ] - - container_groups - -- name: Create Required local monitoring directories - file: > - path={{ item }} - state=directory - with_items: - - "/openstack/monitoring" - -- name: Ensure required inner service directories - lxc: > - name={{ hostvars[item.1]['container_name'] }} - command=attach - container_command="mkdir -p {{ item.0 }}/{{ hostvars[item.1]['service_name'] }}" - with_nested: - - [ "/etc", "/var/log" ] - - container_groups - -- name: Ensure python2.7 installed - lxc: > - name={{ hostvars[item]['container_name'] }} - command=attach - container_command="apt-get -y install python2.7; rm /usr/bin/python; ln -s /usr/bin/python2.7 /usr/bin/python" - with_items: container_groups - -- name: Ensure Required container config options - lxc: > - name={{ hostvars[item]['container_name'] }} - command=config - options="{{ required_container_config_options }}" - state=running - with_items: container_groups +- include: container_setup.yml + when: container_groups|length > 0 diff --git a/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 index 3402fc2fe7..6551d38c61 100644 --- a/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 +++ b/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 @@ -6,17 +6,17 @@ #local1.*;local1.!notice ?HourlyProxyLog local1.*;local1.!notice /var/log/swift/proxy.log -local1.notice /var/log/swift/proxy.error +local1.notice /var/log/swift/proxy-error.log local1.* ~ local2.*;local2.!notice /openstack/log/{{ inventory_hostname }}/account.log -local2.notice /openstack/log/{{ inventory_hostname }}/account.error +local2.notice /openstack/log/{{ inventory_hostname }}/account-error.log local2.* ~ local3.*;local3.!notice /openstack/log/{{ inventory_hostname }}/container.log -local3.notice /openstack/log/{{ inventory_hostname }}/container.error +local3.notice /openstack/log/{{ inventory_hostname }}/container-error.log local3.* ~ local4.*;local4.!notice /openstack/log/{{ inventory_hostname }}/object.log -local4.notice /openstack/log/{{ inventory_hostname }}/object.error +local4.notice /openstack/log/{{ inventory_hostname }}/object-error.log local4.*