openstack/openstack-ansible
Code Issues Proposed changes

Merge pull request #604 from andymcc/ring_refactor

Browse Source 
Refactor swift_rings.py ring builder
This commit is contained in:
Andy McCrae 2014-11-24 14:53:21 +00:00
commit 6074510e1a
4 changed files with 157 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
etc/rpc_deploy/conf.d/swift.yml
View File

@ -66,7 +66,7 @@
## all swift settings are set under swift_vars.
##
## The storage_ip and repl_ip represent the IP that will go in the ring for storage and replication.
## E.g. for swift-node1 the IP string added to the ring would be 198.51.100.4:(service_port)R203.0.113.4:(service_port)
## These will be pulled from the server's interface (specified by storage_network & replication_network), but can be overriden by specifying them at the node or drive level.
## If only the storage_ip is specified then the repl_ip will default to the storage_ip
## If only the repl_ip is specified then the storage_ip will default to the host ip above.
## If neither are specified both will default to the host ip above.
@ -83,29 +83,21 @@
# ip: 192.0.2.4
# container_vars:
# swift_vars:
# storage_ip: 198.51.100.4
# repl_ip: 203.0.113.4
# zone: 0
# swift-node2:
# ip: 192.0.2.5
# container_vars:
# swift_vars:
# storage_ip: 198.51.100.5
# repl_ip: 203.0.113.5
# zone: 1
# swift-node3:
# ip: 192.0.2.6
# container_vars:
# swift_vars:
# storage_ip: 198.51.100.6
# repl_ip: 203.0.113.6
# zone: 2
# swift-node4:
# ip: 192.0.2.7
# container_vars:
# swift_vars:
# storage_ip: 198.51.100.7
# repl_ip: 203.0.113.7
# zone: 3
# swift-node5:
# ip: 192.0.2.8

30
rpc_deployment/roles/swift_ring_builder/tasks/main.yml
View File

@ -32,7 +32,33 @@
dest="/etc/swift/scripts/swift_rings.py"
mode=0700
- name: "build rings"
command: /usr/bin/python /etc/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_inventory.json
- name: "Build ring-contents files"
template: >
src=ring.contents.j2
dest="/etc/swift/scripts/{{ item.type }}.contents"
with_items:
- { item: "{{ swift.account }}", port: "{{ swift_account_port }}", type: "account" }
- { item: "{{ swift.container }}", port: "{{ swift_container_port}}", type: "container" }
- name: "Build ring-contents files for storage policies"
template: >
src=ring.contents.j2
dest="/etc/swift/scripts/object-{{ item[0].policy.index }}.contents"
with_nested:
- "{{ swift.storage_policies }}"
- [{ type: 'object', port: "{{ swift_object_port }}" }]
- name: "build rings for account/container from contents files"
command: /usr/bin/python /etc/swift/scripts/swift_rings.py -f /etc/swift/scripts/{{ item }}.contents
with_items:
- account
- container
args:
chdir: /etc/swift/rings/
- name: "build rings for storage policies from contents files"
command: /usr/bin/python /etc/swift/scripts/swift_rings.py -f /etc/swift/scripts/object-{{ item.policy.index }}.contents
with_items:
- "{{ swift.storage_policies }}"
args:
chdir: /etc/swift/rings/

89
rpc_deployment/roles/swift_ring_builder/templates/ring.contents.j2 Normal file
View File

@ -0,0 +1,89 @@
{### Check if this is an object storage policy #}
{% if item[1] is defined %}
{% set port = item[1]['port'] %}
{% set type = item[1]['type'] %}
{% set item = item[0]['policy'] %}
{### If the index is 0 then it needs to be object without index #}
{% if item.index == 0 %}
{% set builder_file = type %}
{% else %}
{% set builder_file = type + '-' + item.index|string %}
{% endif %}
{% set name = item.name %}
{### Otherwise this should be account or container rings #}
{### Make the port/type/item/builder_file/name vals uniform across rings #}
{% elif item.port is defined %}
{% set port = item.port %}
{% set type = item.type %}
{% set item = item.item %}
{% set builder_file = type %}
{% set name = type %}
{% endif %}
{### Lets get the min_part_hours, part_power and repl_number vals #}
{% set min_part_hours = item.min_part_hours | default(swift.min_part_hours | default(swift_default_min_part_hours)) %}
{% set part_power = item.part_power | default(swift.part_power) %}
{% set repl_number = item.repl_number | default(swift.repl_number | default(swift_default_replication_number)) %}
{### Create the builder dict #}
{% set builder = {} %}
{### This is a hacky way of updating the builder dict #}
{% set _update = builder.update({'min_part_hours':min_part_hours}) %}
{% set _update = builder.update({'repl_number':repl_number}) %}
{% set _update = builder.update({'part_power':part_power}) %}
{% set _update = builder.update({'builder_file':builder_file}) %}
{### Now we need to add the drives #}
{### Create an update the builder dict to have drives as an empty list #}
{% set _update = builder.update({'drives':[]}) %}
{### Lets get the default groups for drives and find the default storage_policy #}
{% set def_groups = [ 'account', 'container' ] %}
{% for policy in swift.storage_policies %}
{% if policy.policy.default is defined and policy.policy.default == True %}
{% set _update = def_groups.append(policy.policy.name) %}
{% endif %}
{% endfor %}
{### Loop through the swift_hosts #}
{% for host in groups['swift_hosts'] %}
{### Set the default storage_ip #}
{% set def_storage_ip = hostvars[host]['container_address'] %}
{% if swift.storage_network is defined %}
{% set storage_bridge = 'ansible_' + swift.storage_network|replace('-', '_') %}
{% set def_storage_ip = hostvars[host][storage_bridge]['ipv4']['address'] | default(hostvars[host]['container_address']) %}
{% endif %}
{### Set the default replication_ip #}
{% set def_repl_ip = def_storage_ip %}
{% if swift.replication_network is defined %}
{% set repl_bridge = 'ansible_' + swift.replication_network|replace('-', '_') %}
{% set def_repl_ip = hostvars[host][repl_bridge]['ipv4']['address'] | default(def_storage_ip) %}
{% endif %}
{### Get the drives use swift global as default #}
{% set drives = hostvars[host]['swift_vars']['drives'] | default(swift.drives | default([])) %}
{### Loop through the drives #}
{% for drive in drives %}
{### Check if groups is defined per host or drive #}
{% set groups = drive.groups | default(hostvars[host]['swift_vars']['groups'] | default(swift.groups | default(def_groups))) %}
{### Only build the device if it is part of the group we're building the ring for #}
{% if name in groups %}
{### Build an empty device which we'll update with the appropriate details #}
{% set device = {} %}
{% set weight = drive.weight | default(hostvars[host]['swift_vars']['weight'] | default(swift.weight | default(swift_default_drive_weight))) %}
{% set region = drive.region | default(hostvars[host]['swift_vars']['region'] | default(swift.region | default(1))) %}
{% set zone = drive.zone | default(hostvars[host]['swift_vars']['zone'] | default(swift.zone | default(swift_default_host_zone))) %}
{% set repl_ip = drive.repl_ip | default( hostvars[host]['swift_vars']['repl_ip'] | default(def_repl_ip)) %}
{% set repl_port = drive.repl_port | default((hostvars[host]['swift_vars']['repl_port']) | default(port)) %}
{% set storage_ip = drive.storage_ip | default(hostvars[host]['swift_vars']['storage_ip'] | default(def_storage_ip)) %}
{% set storage_port = drive.storage_port | default((hostvars[host]['swift_vars']['storage_port']) | default(port)) %}
{### Update the device with the appropriate values #}
{% set _update = device.update({'device':drive.name}) %}
{% set _update = device.update({'weight': weight}) %}
{% set _update = device.update({'region': region}) %}
{% set _update = device.update({'zone': zone}) %}
{% set _update = device.update({'repl_ip': repl_ip}) %}
{% set _update = device.update({'repl_port': repl_port|int}) %}
{% set _update = device.update({'ip': storage_ip}) %}
{% set _update = device.update({'port': storage_port|int}) %}
{### Append the device to the drives list of the builder dict #}
{% set _update = builder.drives.append(device) %}
{% endif %}
{% endfor %}
{% endfor %}
{### Output the builder file #}
{{ builder | to_nice_json }}

204
rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py
View File

@ -24,40 +24,9 @@ import threading
import json
import copy
USAGE = "usage: %prog -s <rpc_inventory.json>"
USAGE = "usage: %prog -f <swift_ring.contents>"
DEV_KEY = "%(ip)s:%(port)d/%(device)s"
DEFAULT_REPL = {{ swift_default_replication_number }}
DEFAULT_MIN_PART_HOURS = {{ swift_default_min_part_hours }}
DEFAULT_HOST_ZONE = {{ swift_default_host_zone }}
DEFAULT_HOST_WEIGHT = {{ swift_default_drive_weight }}
DEFAULT_ACCOUNT_PORT = {{ swift_account_port }}
DEFAULT_CONTAINER_PORT = {{ swift_container_port }}
DEFAULT_OBJECT_PORT = {{ swift_object_port }}
DEFAULT_SECTION_PORT = {
'account': DEFAULT_ACCOUNT_PORT,
'container': DEFAULT_CONTAINER_PORT,
'object': DEFAULT_OBJECT_PORT,
}
DEFAULT_GROUP_MAP = {
'account': 'account',
{% for policy in swift.storage_policies %}
{% if policy.policy.index == 0 %}
'object': '{{ policy.policy.name }}',
{% else %}
'object-{{ policy.policy.index}}': '{{ policy.policy.name }}',
{% endif %}
{% endfor %}
'container': 'container'
}
DEFAULT_GROUPS= [
'account',
{% for policy in swift.storage_policies %}
'{{ policy.policy.name }}',
{% endfor %}
'container'
]
DEVICE_KEY = "%(ip)s:%(port)d/%(device)s"
class RingValidationError(Exception):
pass
@ -98,32 +67,32 @@ def remove_host_from_ring(build_file, host):
def update_host_in_ring(build_file, new_host, old_host, validate=False):
if new_host.get('zone', 0) != old_host['zone']:
devstr = DEV_KEY % new_host
devstr = DEVICE_KEY % new_host
raise RingValidationError('Cannot update zone on %s, this can only be '
'done when the drive is added' % (devstr))
if new_host.get('region', 1) != old_host['region']:
devstr = DEV_KEY % new_host
devstr = DEVICE_KEY % new_host
raise RingValidationError('Cannot update region on %s, this can only '
'be done when the drive is added' % (devstr))
try:
r_ip = new_host.get('repl_ip', new_host['ip'])
r_port = new_host.get('repl_port', new_host['port'])
weight = new_host.get('weight', DEFAULT_HOST_WEIGHT)
weight = new_host.get('weight')
if r_ip != old_host['replication_ip'] or \
r_port != old_host['replication_port']:
host_d = {'r_ip': r_ip, 'r_port': r_port}
host_d.update(new_host)
host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(name)s" % host_d
host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(device)s" % host_d
if not validate:
run_and_wait(rb_main, ["swift-ring-builder", build_file,
"set_info", DEV_KEY % new_host,
"set_info", DEVICE_KEY % new_host,
host_str])
except Exception as ex:
raise RingValidationError(ex)
if weight != old_host['weight'] and not validate:
change_host_weight(build_file, DEV_KEY % new_host, weight)
change_host_weight(build_file, DEVICE_KEY % new_host, weight)
def add_host_to_ring(build_file, host, validate=False):
@ -131,7 +100,7 @@ def add_host_to_ring(build_file, host, validate=False):
try:
if host.get('region') is not None:
host_str += 'r%(region)d' % host
host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE))
host_str += "z%d" % (host.get('zone'))
host_str += "-%(ip)s:%(port)d" % host
if host.get('repl_ip'):
r_ip = host['repl_ip']
@ -141,8 +110,8 @@ def add_host_to_ring(build_file, host, validate=False):
r_ip = host.get('repl_ip', host['ip'])
r_port = host['repl_port']
host_str += "R%s:%d" % (r_ip, r_port)
host_str += "/%(name)s" % host
weight = host.get('weight', DEFAULT_HOST_WEIGHT)
host_str += "/%(device)s" % host
weight = host.get('weight')
except Exception as ex:
raise RingValidationError(ex)
if not validate:
@ -178,14 +147,11 @@ def get_build_file_data(build_file):
return build_file_data
def build_ring(section, conf, part_power, hosts, validate=False):
def build_ring(build_name, repl, min_part_hours, part_power, hosts, validate=False):
# Create the build file
build_file = "%s.builder" % (section)
build_file = "%s.builder" % (build_name)
build_file_data = get_build_file_data(build_file)
repl = conf.get('repl_number', DEFAULT_REPL)
min_part_hours = conf.get('min_part_hours',
DEFAULT_MIN_PART_HOURS)
update = build_file_data is not None
create_buildfile(build_file, part_power, repl, min_part_hours, update,
data=build_file_data, validate=validate)
@ -194,23 +160,16 @@ def build_ring(section, conf, part_power, hosts, validate=False):
if update:
for i, dev in enumerate(build_file_data['devs']):
if dev is not None:
old_hosts[DEV_KEY % dev] = i
section_key = section.split('-')[0]
service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key])
old_hosts[DEVICE_KEY % dev] = i
for host in hosts:
host_vars = hosts[host]
host_vars['device'] = host_vars['name']
host_vars['port'] = service_port
host_vars['groups'] = host_vars.get('groups', DEFAULT_GROUPS)
if DEFAULT_GROUP_MAP[section] in host_vars['groups']:
host_key = DEV_KEY % host_vars
if host_key in old_hosts:
old_host = build_file_data['devs'][old_hosts[host_key]]
update_host_in_ring(build_file, host_vars, old_host,
validate=validate)
old_hosts.pop(host_key)
else:
add_host_to_ring(build_file, host_vars, validate=validate)
host_key = DEVICE_KEY % host
if host_key in old_hosts:
old_host = build_file_data['devs'][old_hosts[host_key]]
update_host_in_ring(build_file, host, old_host,
validate=validate)
old_hosts.pop(host_key)
else:
add_host_to_ring(build_file, host, validate=validate)
if old_hosts and not validate:
# There are still old hosts, these hosts must've been removed
@ -225,122 +184,37 @@ def main(setup):
# load the json file
try:
with open(setup) as json_stream:
_inventory = json.load(json_stream)
_contents_file = json.load(json_stream)
except Exception as ex:
print("Failed to load json string %s" % (ex))
return 1
_hosts = {}
# Get the swift specific global vars
global_vars = _inventory['all']['vars']
check_section(global_vars, 'swift')
swift_vars = global_vars['swift']
if _inventory.get("swift_hosts"):
for host in _inventory['swift_hosts']['hosts']:
host_config = _inventory['_meta']['hostvars'][host]
host_vars = host_config['swift_vars']
host_ip = host_vars.get('storage_ip', host_config['container_address'])
if not host_vars.get('drives'):
if not swift_vars.get('drives'):
continue
else:
host_vars['drives'] = copy.deepcopy(swift_vars.get('drives'))
host_drives = host_vars.get('drives')
for host_drive in host_drives:
host_drive['ip'] = host_drive.get('storage_ip', host_ip)
if host_vars.get('groups'):
host_drive['groups'] = \
host_drive.get('groups', host_vars['groups'])
if host_vars.get('repl_ip'):
host_drive['repl_ip'] = \
host_drive.get('repl_ip', host_vars['repl_ip'])
if host_vars.get('repl_port'):
host_drive['repl_port'] = \
host_drive.get('repl_port', host_vars['repl_port'])
if host_vars.get('weight'):
host_drive['weight'] = \
host_drive.get('weight', host_vars['weight'])
key = "%s/%s" % (host_drive['ip'], host_drive['name'])
if key in _hosts:
print("%s already definined - duplicate device" % key)
return 1
_hosts[key] = host_drive
hosts = _contents_file['drives']
kargs = {'validate': True, 'hosts': hosts}
ring_call = [ _contents_file['builder_file'],
_contents_file['repl_number'],
_contents_file['min_part_hours'],
_contents_file['part_power']]
global_vars = _inventory['all']['vars']
check_section(global_vars, 'swift')
swift_vars = global_vars['swift']
if not swift_vars.get('part_power'):
print('No part_power specified - please set a part_power value')
return 1
part_power = swift_vars.get('part_power')
# If the repl_number or min_part hours are set on a "global" level in the
# conf lets set them here - otherwise use the overall default.
default_repl_num = swift_vars.get('repl_number', DEFAULT_REPL)
default_min_part_hours = swift_vars.get('min_part_hours',
DEFAULT_MIN_PART_HOURS)
ring_calls = []
# Create account ring - if the section is empty create an empty dict
# so defaults are used
if not has_section(swift_vars, 'account'):
swift_vars['account'] = {'repl_number': default_repl_num,
'min_part_hours': default_min_part_hours}
ring_calls.append(('account', swift_vars['account'], part_power))
# Create container ring - if the section is empty create an empty dict
# so defaults are used
if not has_section(swift_vars, 'container'):
swift_vars['container'] = {'repl_number': default_repl_num,
'min_part_hours': default_min_part_hours}
ring_calls.append(('container', swift_vars['container'], part_power))
# Create object rings (storage policies)
check_section(swift_vars, 'storage_policies')
indexes = set()
for policy in swift_vars['storage_policies']:
policy = policy['policy']
if policy['index'] in indexes:
print("Storage Policy index %d already in use" % (policy['index']))
return 4
if policy['index'] == 0:
buildfilename = 'object'
else:
buildfilename = 'object-%d' % (policy['index'])
indexes.add(policy['index'])
# Set default port/min_part_hours/repl_number
if 'min_part_hours' not in policy:
policy['min_part_hours'] = default_min_part_hours
if 'repl_number' not in policy:
policy['repl_number'] = default_repl_num
if 'port' not in policy:
policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT)
ring_calls.append((buildfilename, policy, part_power))
# Now that we have gathered all the options for building/update the rings
# lets validate them
kargs = {'validate': True, 'hosts': _hosts}
for ring_call in ring_calls:
try:
build_ring(*ring_call, **kargs)
except RingValidationError as ex:
print(ex)
return 2
try:
build_ring(*ring_call, **kargs)
except RingValidationError as ex:
print(ex)
return 2
# If the validation passes lets go ahead and build the rings.
kargs.pop('validate')
for ring_call in ring_calls:
build_ring(*ring_call, **kargs)
build_ring(*ring_call, **kargs)
if __name__ == "__main__":
parser = OptionParser(USAGE)
parser.add_option("-s", "--setup", dest="setup",
help="Specify the swift setup file.", metavar="FILE",
default="/etc/rpc_deploy/rpc_inventory.json")
parser.add_option("-f", "--file", dest="setup",
help="Specify the swift ring contents file.", metavar="FILE")
options, args = parser.parse_args(sys.argv[1:])
if options.setup and not exists(options.setup):
print("Swift setup file not found or doesn't exist")
print("Swift ring contents file not found or doesn't exist")
parser.print_help()
sys.exit(1)