From a37533cff90d73cb18ce87343e72dd2ae5dae478 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 12 Nov 2014 16:42:37 +0000 Subject: [PATCH] Add support for dedicated replication network * Allow 2 servers to run (one on the replication network) * Split out the replication configuration * Set a specific replication_network * Fix service names Partial-fixes: #516 --- .../inventory/group_vars/swift_all.yml | 4 + .../roles/swift_account/tasks/main.yml | 13 +++- .../account-server-replicator.conf.j2 | 45 +++++++++++ .../templates/account-server.conf.j2 | 2 + .../roles/swift_container/tasks/main.yml | 11 ++- .../container-server-replicator.conf.j2 | 37 +++++++++ .../templates/container-server.conf.j2 | 2 + .../swift_init_scripts/templates/init-config | 16 +++- .../roles/swift_object/tasks/main.yml | 11 ++- .../object-server-replicator.conf.j2 | 77 +++++++++++++++++++ .../templates/object-server.conf.j2 | 2 + .../roles/swift_proxy/tasks/main.yml | 2 +- .../openstack_service_vars/swift_account.yml | 3 +- .../swift_container.yml | 6 +- .../openstack_service_vars/swift_object.yml | 5 +- .../openstack_service_vars/swift_proxy.yml | 2 +- 16 files changed, 224 insertions(+), 14 deletions(-) create mode 100644 rpc_deployment/roles/swift_account/templates/account-server-replicator.conf.j2 create mode 100644 rpc_deployment/roles/swift_container/templates/container-server-replicator.conf.j2 create mode 100644 rpc_deployment/roles/swift_object/templates/object-server-replicator.conf.j2 diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 91c6c9265d..4fed62a95d 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -68,3 +68,7 @@ container_directories: - /var/lock/swift - /etc/swift - /etc/swift/rings/ + - /etc/swift/object-server + - /etc/swift/container-server + - /etc/swift/account-server + - /etc/swift/proxy-server diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml index 6c6011fd0e..8233f2dec9 100644 --- a/rpc_deployment/roles/swift_account/tasks/main.yml +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -13,10 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: "swift account server configuration" +- name: "swift account server config with replication" template: > src="account-server.conf.j2" - dest="/etc/swift/account-server.conf" + dest="/etc/swift/account-server/account-server.conf" owner={{ system_user }} mode=0644 notify: Restart swift service + +- name: "swift account server replication config" + template: > + src="account-server-replicator.conf.j2" + dest="/etc/swift/account-server/account-server-replicator.conf" + owner={{ system_user }} + mode=0644 + notify: Restart swift service + when: swift.replication_network is defined diff --git a/rpc_deployment/roles/swift_account/templates/account-server-replicator.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server-replicator.conf.j2 new file mode 100644 index 0000000000..527f40eb63 --- /dev/null +++ b/rpc_deployment/roles/swift_account/templates/account-server-replicator.conf.j2 @@ -0,0 +1,45 @@ +[DEFAULT] +{% set repl_bridge = 'ansible_' + swift.replication_network|replace('-', '_') %} +bind_ip = {{ hostvars[inventory_hostname][repl_bridge]['ipv4']['address'] }} +bind_port = {{ swift_account_port }} +workers = 2 + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:swift#account +replication_server = True + +[account-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = account-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +per_diff = 10000 +# max_diffs = 100 +# concurrency = 8 +# interval = 30 +# +# How long without an error before a node's error count is reset. This will +# also be how long before a node is reenabled after suppression is triggered. +# error_suppression_interval = 60 +# +# How many errors can accumulate before a node is temporarily ignored. +# error_suppression_limit = 10 +# +# node_timeout = 10 +# conn_timeout = 0.5 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# Time in seconds to wait between replication passes +# Note: if the parameter 'interval' is defined then it will be used in place +# of run_pause. +# run_pause = 30 +# +# recon_cache_path = /var/cache/swift diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 index 9631ded201..a8fd8b2494 100644 --- a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -86,6 +86,7 @@ log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift recon_lock_path = /var/lock/swift +{% if swift.replication_network is not defined %} [account-replicator] # You can override the default log routing for this app here (don't use set!): # log_name = account-replicator @@ -118,6 +119,7 @@ per_diff = 10000 # run_pause = 30 # # recon_cache_path = /var/cache/swift +{% endif %} [account-auditor] # You can override the default log routing for this app here (don't use set!): diff --git a/rpc_deployment/roles/swift_container/tasks/main.yml b/rpc_deployment/roles/swift_container/tasks/main.yml index 9974043c75..08dc5dcff5 100644 --- a/rpc_deployment/roles/swift_container/tasks/main.yml +++ b/rpc_deployment/roles/swift_container/tasks/main.yml @@ -16,7 +16,16 @@ - name: "swift container server configuration" template: > src="container-server.conf.j2" - dest="/etc/swift/container-server.conf" + dest="/etc/swift/container-server/container-server.conf" owner={{ system_user }} mode=0644 notify: Restart swift service + +- name: "swift container server replication config" + template: > + src="container-server-replicator.conf.j2" + dest="/etc/swift/container-server/container-server-replicator.conf" + owner={{ system_user }} + mode=0644 + notify: Restart swift service + when: swift.replication_network is defined diff --git a/rpc_deployment/roles/swift_container/templates/container-server-replicator.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server-replicator.conf.j2 new file mode 100644 index 0000000000..369cbdd863 --- /dev/null +++ b/rpc_deployment/roles/swift_container/templates/container-server-replicator.conf.j2 @@ -0,0 +1,37 @@ +[DEFAULT] +{% set repl_bridge = 'ansible_' + swift.replication_network|replace('-', '_') %} +bind_ip = {{ hostvars[inventory_hostname][repl_bridge]['ipv4']['address'] }} +bind_port = {{ swift_container_port }} +workers = 2 + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:swift#container +replication_server = True + +[container-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = container-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +# per_diff = 1000 +# max_diffs = 100 +# concurrency = 8 +# interval = 30 +# node_timeout = 10 +# conn_timeout = 0.5 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# Time in seconds to wait between replication passes +# Note: if the parameter 'interval' is defined then it will be used in place +# of run_pause. +# run_pause = 30 +# +# recon_cache_path = /var/cache/swift diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 index aa9ea78459..915b4b205e 100644 --- a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -95,6 +95,7 @@ log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift recon_lock_path = /var/lock/swift +{% if swift.replication_network is not defined %} [container-replicator] # You can override the default log routing for this app here (don't use set!): # log_name = container-replicator @@ -119,6 +120,7 @@ log_facility = LOG_LOCAL2 # run_pause = 30 # # recon_cache_path = /var/cache/swift +{% endif %} [container-updater] # You can override the default log routing for this app here (don't use set!): diff --git a/rpc_deployment/roles/swift_init_scripts/templates/init-config b/rpc_deployment/roles/swift_init_scripts/templates/init-config index e259ac0815..1ec8b38414 100644 --- a/rpc_deployment/roles/swift_init_scripts/templates/init-config +++ b/rpc_deployment/roles/swift_init_scripts/templates/init-config @@ -9,6 +9,16 @@ stop on runlelvel [016] respawn +# If this is a replicator service and we have a repl network adjust the conf +{% if swift.replication_network is defined and 'replicator' in item %} +{% set repl_conf = '-replicator' %} +{% endif %} +# If this is a server-replicator and we have a repl network adjust the name/conf +{% if swift.replication_network is defined and 'server-replicator' in item %} +{% set repl_conf = '-replicator' %} +{% set item = 'swift-' + service_conf_loc %} +{% endif %} + # Set the RUNBIN environment variable env RUNBIN="/usr/local/bin/{{ item }}" @@ -26,14 +36,14 @@ end script # Post stop actions post-stop script - rm "/var/run/{{ item }}/{{ item }}.pid" + rm "/var/run/{{ item }}/{{ item }}{{ repl_conf | default('') }}.pid" end script # Run the start up job exec start-stop-daemon --start \ --chuid {{ system_user }} \ --make-pidfile \ - --pidfile /var/run/{{ item }}/{{ item }}.pid \ + --pidfile /var/run/{{ item }}/{{ item }}{{ repl_conf | default('') }}.pid \ --exec "{{ program_override|default('$RUNBIN') }}" \ - /etc/{{ service_name }}/{{ service_conf }} \ + /etc/{{ service_name }}/{{ service_conf_loc }}/{{ service_conf_loc }}{{ repl_conf | default('') }}.conf \ -- {{ program_config_options|default('') }} diff --git a/rpc_deployment/roles/swift_object/tasks/main.yml b/rpc_deployment/roles/swift_object/tasks/main.yml index e0e221c969..4351023132 100644 --- a/rpc_deployment/roles/swift_object/tasks/main.yml +++ b/rpc_deployment/roles/swift_object/tasks/main.yml @@ -16,7 +16,16 @@ - name: "swift object server configuration" template: > src="object-server.conf.j2" - dest="/etc/swift/object-server.conf" + dest="/etc/swift/object-server/object-server.conf" owner={{ system_user }} mode=0644 notify: Restart swift service + +- name: "swift object server replication config" + template: > + src="object-server-replicator.conf.j2" + dest="/etc/swift/object-server/object-server-replicator.conf" + owner={{ system_user }} + mode=0644 + notify: Restart swift service + when: swift.replication_network is defined diff --git a/rpc_deployment/roles/swift_object/templates/object-server-replicator.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server-replicator.conf.j2 new file mode 100644 index 0000000000..23a04a113a --- /dev/null +++ b/rpc_deployment/roles/swift_object/templates/object-server-replicator.conf.j2 @@ -0,0 +1,77 @@ +[DEFAULT] +{% set repl_bridge = 'ansible_' + swift.replication_network|replace('-', '_') %} +bind_ip = {{ hostvars[inventory_hostname][repl_bridge]['ipv4']['address'] }} +bind_port = {{ swift_object_port }} +workers = 2 + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:swift#object +replication_server = True + +[object-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = object-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +# daemonize = on +# run_pause = 30 +concurrency = 6 +# stats_interval = 300 +# +# The sync method to use; default is rsync but you can use ssync to try the +# EXPERIMENTAL all-swift-code-no-rsync-callouts method. Once ssync is verified +# as having performance comparable to, or better than, rsync, we plan to +# deprecate rsync so we can move on with more features for replication. +# sync_method = rsync +# +# max duration of a partition rsync +# rsync_timeout = 900 +# +# bandwidth limit for rsync in kB/s. 0 means unlimited +# rsync_bwlimit = 0 +# +# passed to rsync for io op timeout +# rsync_io_timeout = 30 +# +# node_timeout = +# max duration of an http request; this is for REPLICATE finalization calls and +# so should be longer than node_timeout +# http_timeout = 60 +# +# attempts to kill all workers if nothing replicates for lockup_timeout seconds +# lockup_timeout = 1800 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# ring_check_interval = 15 +# recon_cache_path = /var/cache/swift +# +# limits how long rsync error log lines are +# 0 means to log the entire line +# rsync_error_log_line_length = 0 +# +# handoffs_first and handoff_delete are options for a special case +# such as disk full in the cluster. These two options SHOULD NOT BE +# CHANGED, except for such an extreme situations. (e.g. disks filled up +# or are about to fill up. Anyway, DO NOT let your drives fill up) +# handoffs_first is the flag to replicate handoffs prior to canonical +# partitions. It allows to force syncing and deleting handoffs quickly. +# If set to a True value(e.g. "True" or "1"), partitions +# that are not supposed to be on the node will be replicated first. +# handoffs_first = False +# +# handoff_delete is the number of replicas which are ensured in swift. +# If the number less than the number of replicas is set, object-replicator +# could delete local handoffs even if all replicas are not ensured in the +# cluster. Object-replicator would remove local handoff partition directories +# after syncing partition when the number of successful responses is greater +# than or equal to this number. By default(auto), handoff partitions will be +# removed when it has successfully replicated to all the canonical nodes. +# handoff_delete = auto diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 index 8fa1d6af30..f033e3bd60 100644 --- a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -142,6 +142,7 @@ log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift recon_lock_path = /var/lock/swift +{% if swift.replication_network is not defined %} [object-replicator] # You can override the default log routing for this app here (don't use set!): # log_name = object-replicator @@ -206,6 +207,7 @@ concurrency = 6 # than or equal to this number. By default(auto), handoff partitions will be # removed when it has successfully replicated to all the canonical nodes. # handoff_delete = auto +{% endif %} [object-updater] # You can override the default log routing for this app here (don't use set!): diff --git a/rpc_deployment/roles/swift_proxy/tasks/main.yml b/rpc_deployment/roles/swift_proxy/tasks/main.yml index 0b754bd3d8..f247f74ce3 100644 --- a/rpc_deployment/roles/swift_proxy/tasks/main.yml +++ b/rpc_deployment/roles/swift_proxy/tasks/main.yml @@ -16,7 +16,7 @@ - name: swift proxy server configuration template: > src="proxy-server.conf.j2" - dest="/etc/swift/proxy-server.conf" + dest="/etc/swift/proxy-server/proxy-server.conf" owner={{ service_admin_username }} mode=0644 notify: Restart swift service diff --git a/rpc_deployment/vars/openstack_service_vars/swift_account.yml b/rpc_deployment/vars/openstack_service_vars/swift_account.yml index 54e22c6583..a90f718fdc 100644 --- a/rpc_deployment/vars/openstack_service_vars/swift_account.yml +++ b/rpc_deployment/vars/openstack_service_vars/swift_account.yml @@ -21,5 +21,6 @@ program_names: - swift-account-auditor - swift-account-replicator - swift-account-reaper + - swift-account-server-replicator -service_conf: account-server.conf +service_conf_loc: account-server diff --git a/rpc_deployment/vars/openstack_service_vars/swift_container.yml b/rpc_deployment/vars/openstack_service_vars/swift_container.yml index 9014f680f3..7188769fd9 100644 --- a/rpc_deployment/vars/openstack_service_vars/swift_container.yml +++ b/rpc_deployment/vars/openstack_service_vars/swift_container.yml @@ -20,6 +20,8 @@ program_names: - swift-container-server - swift-container-auditor - swift-container-replicator - - swift-container-reaper + - swift-container-sync + - swift-container-updater + - swift-container-server-replicator -service_conf: container-server.conf +service_conf_loc: container-server diff --git a/rpc_deployment/vars/openstack_service_vars/swift_object.yml b/rpc_deployment/vars/openstack_service_vars/swift_object.yml index 97c349de52..857bf4daed 100644 --- a/rpc_deployment/vars/openstack_service_vars/swift_object.yml +++ b/rpc_deployment/vars/openstack_service_vars/swift_object.yml @@ -20,6 +20,7 @@ program_names: - swift-object-server - swift-object-auditor - swift-object-replicator - - swift-object-reaper + - swift-object-updater + - swift-object-server-replicator -service_conf: object-server.conf +service_conf_loc: object-server diff --git a/rpc_deployment/vars/openstack_service_vars/swift_proxy.yml b/rpc_deployment/vars/openstack_service_vars/swift_proxy.yml index 6c5b1ba6ba..0a0de5321a 100644 --- a/rpc_deployment/vars/openstack_service_vars/swift_proxy.yml +++ b/rpc_deployment/vars/openstack_service_vars/swift_proxy.yml @@ -19,4 +19,4 @@ program_names: - swift-proxy-server -service_conf: proxy-server.conf +service_conf_loc: proxy-server