From 53027de8c8e89b9611d1f61102fbe73f6eb1c100 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 6 May 2016 11:24:44 +0100 Subject: [PATCH] Setup rsync module per object server drive This PR sets up rsync module per drive for swift object servers. This improves resource distribution, and also allows us to utilise rsync module changes to prevent disk writes to specific drives when they begin filling up. Enable this option by setting "swift_rsync_module_per_drive: True", this will still default to False to match upstream defaults. Additionally the rsync max connections has been increased to match upstream defaults. Additionally we include rsync.d/*.conf files by default and set this directory up, so that we can add individual configuration to disable specific drives. Change-Id: I2019cade5bf5f2878497d30ce738dff07786fa64 --- defaults/main.yml | 5 ++- ...ync-module-per-drive-79b05af8276e7d6e.yaml | 12 ++++++ tasks/swift_pre_install.yml | 1 + templates/object-server-replicator.conf.j2 | 3 ++ templates/object-server.conf.j2 | 3 ++ templates/rsyncd.conf.j2 | 38 +++++++++++++++++++ 6 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/swift-rsync-module-per-drive-79b05af8276e7d6e.yaml diff --git a/defaults/main.yml b/defaults/main.yml index 1cb10dda..4bd4fec4 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -217,11 +217,14 @@ swift: {} # default: True # Set rsync max_connections vars -swift_max_rsync_connections: 2 +swift_max_rsync_connections: 4 swift_account_max_rsync_connections: "{{ swift_max_rsync_connections }}" swift_container_max_rsync_connections: "{{ swift_max_rsync_connections }}" swift_object_max_rsync_connections: "{{ swift_max_rsync_connections }}" +# Set Swift to use rsync module per object server drive +swift_rsync_module_per_drive: False + # Set the managed regions as a list of swift regions to manage # Use for global clusters, default when not set is all regions. # swift_managed_regions: diff --git a/releasenotes/notes/swift-rsync-module-per-drive-79b05af8276e7d6e.yaml b/releasenotes/notes/swift-rsync-module-per-drive-79b05af8276e7d6e.yaml new file mode 100644 index 00000000..53990462 --- /dev/null +++ b/releasenotes/notes/swift-rsync-module-per-drive-79b05af8276e7d6e.yaml @@ -0,0 +1,12 @@ +--- +features: + - Enable rsync module per object server drive by setting + the ``swift_rsync_module_per_drive`` setting to ``True``. + Set this to configure rsync and swift to utilise individual + configuration per drive. This is required when disabling + rsyncs to individual disks. For example, in a disk full + scenario. +upgrade: + - The ``swift_max_rsync_connections`` default value has + changed from 2 to 4 in order to match the OpenStack swift + documented value. diff --git a/tasks/swift_pre_install.yml b/tasks/swift_pre_install.yml index ade9dcff..0f5bf3a9 100644 --- a/tasks/swift_pre_install.yml +++ b/tasks/swift_pre_install.yml @@ -69,6 +69,7 @@ - { path: "/openstack/log/{{ inventory_hostname }}", owner: "{{ swift_syslog_user_name }}", group: "{{ swift_syslog_group_name }}" } - { path: "/var/cache/swift" } - { path: "{{ swift_system_home_folder }}" } + - { path: "/etc/rsync.d", owner: "root", group: "root" } tags: - swift-dirs diff --git a/templates/object-server-replicator.conf.j2 b/templates/object-server-replicator.conf.j2 index 0f66605f..ed31aef2 100644 --- a/templates/object-server-replicator.conf.j2 +++ b/templates/object-server-replicator.conf.j2 @@ -23,3 +23,6 @@ replication_server = True log_facility = LOG_LOCAL4 concurrency = 6 reclaim_age = {{ reclaim_age | default(604800) }} +{% if swift_rsync_module_per_drive %} +rsync_module = {replication_ip}::object_{device} +{% endif %} diff --git a/templates/object-server.conf.j2 b/templates/object-server.conf.j2 index 6a29cb90..f29eda01 100644 --- a/templates/object-server.conf.j2 +++ b/templates/object-server.conf.j2 @@ -39,6 +39,9 @@ recon_lock_path = /var/lock log_facility = LOG_LOCAL4 concurrency = 6 reclaim_age = {{ reclaim_age | default(604800) }} +{% if swift_rsync_module_per_drive %} +rsync_module = {replication_ip}::object_{device} +{% endif %} {% endif %} [object-updater] diff --git a/templates/rsyncd.conf.j2 b/templates/rsyncd.conf.j2 index fbfae80e..5ea2e46b 100644 --- a/templates/rsyncd.conf.j2 +++ b/templates/rsyncd.conf.j2 @@ -18,8 +18,46 @@ path = {{ swift_vars.mount_point | default(swift.mount_point) }} read only = false lock file = /var/lock/container.lock +{% if swift_rsync_module_per_drive %} +{### Set up per drive rsync modules #} +{### Lets get the default groups for drives and find the default storage_policy #} +{% set def_groups = [ 'account', 'container' ] %} +{% for policy in swift.storage_policies %} +{% if policy.policy.default is defined and policy.policy.default == True %} +{% set _update = def_groups.append(policy.policy.name) %} +{% endif %} +{% endfor %} +{### swift_vars needs to exist for swift_vars[setting] checks to work #} +{% if swift_vars is not defined %} +{% set swift_vars = {} %} +{% endif %} +{% set drives = swift_vars.drives | default(swift.drives | default([])) %} +{### Loop through the drives to determine if they are in the object group #} +{% for drive in drives %} +{% set devAdded = False %} +{### Check if groups is defined per host or drive #} +{% set groups = drives.groups | default(swift_vars.groups | default(swift.groups | default(def_groups))) %} +{% for group in groups %} +{% if group not in ['account', 'container'] %} +{### We only want to add a device once so check if it was added already #} +{% if not devAdded | bool %} +[object_{{ drive.name }}] +max connections = {{ swift_object_max_rsync_connections }} +path = {{ swift_vars.mount_point | default(swift.mount_point) }} +read only = false +lock file = /var/lock/object_{{ drive.name }}.lock + +{% endif %} +{% set devAdded = True %} +{% endif %} +{% endfor %} +{% endfor %} +{% else %} [object] max connections = {{ swift_object_max_rsync_connections }} path = {{ swift_vars.mount_point | default(swift.mount_point) }} read only = false lock file = /var/lock/object.lock + +{% endif %} +&include /etc/rsync.d