diff --git a/etc/rpc_deploy/conf.d/swift.yml b/etc/rpc_deploy/conf.d/swift.yml index 767d90cc74..4610f1e570 100644 --- a/etc/rpc_deploy/conf.d/swift.yml +++ b/etc/rpc_deploy/conf.d/swift.yml @@ -66,7 +66,7 @@ ## all swift settings are set under swift_vars. ## ## The storage_ip and repl_ip represent the IP that will go in the ring for storage and replication. -## E.g. for swift-node1 the IP string added to the ring would be 198.51.100.4:(service_port)R203.0.113.4:(service_port) +## These will be pulled from the server's interface (specified by storage_network & replication_network), but can be overriden by specifying them at the node or drive level. ## If only the storage_ip is specified then the repl_ip will default to the storage_ip ## If only the repl_ip is specified then the storage_ip will default to the host ip above. ## If neither are specified both will default to the host ip above. @@ -83,29 +83,21 @@ # ip: 192.0.2.4 # container_vars: # swift_vars: -# storage_ip: 198.51.100.4 -# repl_ip: 203.0.113.4 # zone: 0 # swift-node2: # ip: 192.0.2.5 # container_vars: # swift_vars: -# storage_ip: 198.51.100.5 -# repl_ip: 203.0.113.5 # zone: 1 # swift-node3: # ip: 192.0.2.6 # container_vars: # swift_vars: -# storage_ip: 198.51.100.6 -# repl_ip: 203.0.113.6 # zone: 2 # swift-node4: # ip: 192.0.2.7 # container_vars: # swift_vars: -# storage_ip: 198.51.100.7 -# repl_ip: 203.0.113.7 # zone: 3 # swift-node5: # ip: 192.0.2.8 diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 87bf1c6046..231a9c40c3 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -32,7 +32,33 @@ dest="/etc/swift/scripts/swift_rings.py" mode=0700 -- name: "build rings" - command: /usr/bin/python /etc/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_inventory.json +- name: "Build ring-contents files" + template: > + src=ring.contents.j2 + dest="/etc/swift/scripts/{{ item.type }}.contents" + with_items: + - { item: "{{ swift.account }}", port: "{{ swift_account_port }}", type: "account" } + - { item: "{{ swift.container }}", port: "{{ swift_container_port}}", type: "container" } + +- name: "Build ring-contents files for storage policies" + template: > + src=ring.contents.j2 + dest="/etc/swift/scripts/object-{{ item[0].policy.index }}.contents" + with_nested: + - "{{ swift.storage_policies }}" + - [{ type: 'object', port: "{{ swift_object_port }}" }] + +- name: "build rings for account/container from contents files" + command: /usr/bin/python /etc/swift/scripts/swift_rings.py -f /etc/swift/scripts/{{ item }}.contents + with_items: + - account + - container + args: + chdir: /etc/swift/rings/ + +- name: "build rings for storage policies from contents files" + command: /usr/bin/python /etc/swift/scripts/swift_rings.py -f /etc/swift/scripts/object-{{ item.policy.index }}.contents + with_items: + - "{{ swift.storage_policies }}" args: chdir: /etc/swift/rings/ diff --git a/rpc_deployment/roles/swift_ring_builder/templates/ring.contents.j2 b/rpc_deployment/roles/swift_ring_builder/templates/ring.contents.j2 new file mode 100644 index 0000000000..a4723739e6 --- /dev/null +++ b/rpc_deployment/roles/swift_ring_builder/templates/ring.contents.j2 @@ -0,0 +1,89 @@ +{### Check if this is an object storage policy #} +{% if item[1] is defined %} +{% set port = item[1]['port'] %} +{% set type = item[1]['type'] %} +{% set item = item[0]['policy'] %} +{### If the index is 0 then it needs to be object without index #} +{% if item.index == 0 %} +{% set builder_file = type %} +{% else %} +{% set builder_file = type + '-' + item.index|string %} +{% endif %} +{% set name = item.name %} +{### Otherwise this should be account or container rings #} +{### Make the port/type/item/builder_file/name vals uniform across rings #} +{% elif item.port is defined %} +{% set port = item.port %} +{% set type = item.type %} +{% set item = item.item %} +{% set builder_file = type %} +{% set name = type %} +{% endif %} +{### Lets get the min_part_hours, part_power and repl_number vals #} +{% set min_part_hours = item.min_part_hours | default(swift.min_part_hours | default(swift_default_min_part_hours)) %} +{% set part_power = item.part_power | default(swift.part_power) %} +{% set repl_number = item.repl_number | default(swift.repl_number | default(swift_default_replication_number)) %} +{### Create the builder dict #} +{% set builder = {} %} +{### This is a hacky way of updating the builder dict #} +{% set _update = builder.update({'min_part_hours':min_part_hours}) %} +{% set _update = builder.update({'repl_number':repl_number}) %} +{% set _update = builder.update({'part_power':part_power}) %} +{% set _update = builder.update({'builder_file':builder_file}) %} +{### Now we need to add the drives #} +{### Create an update the builder dict to have drives as an empty list #} +{% set _update = builder.update({'drives':[]}) %} +{### Lets get the default groups for drives and find the default storage_policy #} +{% set def_groups = [ 'account', 'container' ] %} +{% for policy in swift.storage_policies %} +{% if policy.policy.default is defined and policy.policy.default == True %} +{% set _update = def_groups.append(policy.policy.name) %} +{% endif %} +{% endfor %} +{### Loop through the swift_hosts #} +{% for host in groups['swift_hosts'] %} +{### Set the default storage_ip #} +{% set def_storage_ip = hostvars[host]['container_address'] %} +{% if swift.storage_network is defined %} +{% set storage_bridge = 'ansible_' + swift.storage_network|replace('-', '_') %} +{% set def_storage_ip = hostvars[host][storage_bridge]['ipv4']['address'] | default(hostvars[host]['container_address']) %} +{% endif %} +{### Set the default replication_ip #} +{% set def_repl_ip = def_storage_ip %} +{% if swift.replication_network is defined %} +{% set repl_bridge = 'ansible_' + swift.replication_network|replace('-', '_') %} +{% set def_repl_ip = hostvars[host][repl_bridge]['ipv4']['address'] | default(def_storage_ip) %} +{% endif %} +{### Get the drives use swift global as default #} +{% set drives = hostvars[host]['swift_vars']['drives'] | default(swift.drives | default([])) %} +{### Loop through the drives #} +{% for drive in drives %} +{### Check if groups is defined per host or drive #} +{% set groups = drive.groups | default(hostvars[host]['swift_vars']['groups'] | default(swift.groups | default(def_groups))) %} +{### Only build the device if it is part of the group we're building the ring for #} +{% if name in groups %} +{### Build an empty device which we'll update with the appropriate details #} +{% set device = {} %} +{% set weight = drive.weight | default(hostvars[host]['swift_vars']['weight'] | default(swift.weight | default(swift_default_drive_weight))) %} +{% set region = drive.region | default(hostvars[host]['swift_vars']['region'] | default(swift.region | default(1))) %} +{% set zone = drive.zone | default(hostvars[host]['swift_vars']['zone'] | default(swift.zone | default(swift_default_host_zone))) %} +{% set repl_ip = drive.repl_ip | default( hostvars[host]['swift_vars']['repl_ip'] | default(def_repl_ip)) %} +{% set repl_port = drive.repl_port | default((hostvars[host]['swift_vars']['repl_port']) | default(port)) %} +{% set storage_ip = drive.storage_ip | default(hostvars[host]['swift_vars']['storage_ip'] | default(def_storage_ip)) %} +{% set storage_port = drive.storage_port | default((hostvars[host]['swift_vars']['storage_port']) | default(port)) %} +{### Update the device with the appropriate values #} +{% set _update = device.update({'device':drive.name}) %} +{% set _update = device.update({'weight': weight}) %} +{% set _update = device.update({'region': region}) %} +{% set _update = device.update({'zone': zone}) %} +{% set _update = device.update({'repl_ip': repl_ip}) %} +{% set _update = device.update({'repl_port': repl_port|int}) %} +{% set _update = device.update({'ip': storage_ip}) %} +{% set _update = device.update({'port': storage_port|int}) %} +{### Append the device to the drives list of the builder dict #} +{% set _update = builder.drives.append(device) %} +{% endif %} +{% endfor %} +{% endfor %} +{### Output the builder file #} +{{ builder | to_nice_json }} diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 5839958570..3a07c1c3e5 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -24,40 +24,9 @@ import threading import json import copy -USAGE = "usage: %prog -s " +USAGE = "usage: %prog -f " -DEV_KEY = "%(ip)s:%(port)d/%(device)s" - -DEFAULT_REPL = {{ swift_default_replication_number }} -DEFAULT_MIN_PART_HOURS = {{ swift_default_min_part_hours }} -DEFAULT_HOST_ZONE = {{ swift_default_host_zone }} -DEFAULT_HOST_WEIGHT = {{ swift_default_drive_weight }} -DEFAULT_ACCOUNT_PORT = {{ swift_account_port }} -DEFAULT_CONTAINER_PORT = {{ swift_container_port }} -DEFAULT_OBJECT_PORT = {{ swift_object_port }} -DEFAULT_SECTION_PORT = { - 'account': DEFAULT_ACCOUNT_PORT, - 'container': DEFAULT_CONTAINER_PORT, - 'object': DEFAULT_OBJECT_PORT, -} -DEFAULT_GROUP_MAP = { - 'account': 'account', -{% for policy in swift.storage_policies %} -{% if policy.policy.index == 0 %} - 'object': '{{ policy.policy.name }}', -{% else %} - 'object-{{ policy.policy.index}}': '{{ policy.policy.name }}', -{% endif %} -{% endfor %} - 'container': 'container' -} -DEFAULT_GROUPS= [ - 'account', -{% for policy in swift.storage_policies %} - '{{ policy.policy.name }}', -{% endfor %} - 'container' -] +DEVICE_KEY = "%(ip)s:%(port)d/%(device)s" class RingValidationError(Exception): pass @@ -98,32 +67,32 @@ def remove_host_from_ring(build_file, host): def update_host_in_ring(build_file, new_host, old_host, validate=False): if new_host.get('zone', 0) != old_host['zone']: - devstr = DEV_KEY % new_host + devstr = DEVICE_KEY % new_host raise RingValidationError('Cannot update zone on %s, this can only be ' 'done when the drive is added' % (devstr)) if new_host.get('region', 1) != old_host['region']: - devstr = DEV_KEY % new_host + devstr = DEVICE_KEY % new_host raise RingValidationError('Cannot update region on %s, this can only ' 'be done when the drive is added' % (devstr)) try: r_ip = new_host.get('repl_ip', new_host['ip']) r_port = new_host.get('repl_port', new_host['port']) - weight = new_host.get('weight', DEFAULT_HOST_WEIGHT) + weight = new_host.get('weight') if r_ip != old_host['replication_ip'] or \ r_port != old_host['replication_port']: host_d = {'r_ip': r_ip, 'r_port': r_port} host_d.update(new_host) - host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(name)s" % host_d + host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(device)s" % host_d if not validate: run_and_wait(rb_main, ["swift-ring-builder", build_file, - "set_info", DEV_KEY % new_host, + "set_info", DEVICE_KEY % new_host, host_str]) except Exception as ex: raise RingValidationError(ex) if weight != old_host['weight'] and not validate: - change_host_weight(build_file, DEV_KEY % new_host, weight) + change_host_weight(build_file, DEVICE_KEY % new_host, weight) def add_host_to_ring(build_file, host, validate=False): @@ -131,7 +100,7 @@ def add_host_to_ring(build_file, host, validate=False): try: if host.get('region') is not None: host_str += 'r%(region)d' % host - host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) + host_str += "z%d" % (host.get('zone')) host_str += "-%(ip)s:%(port)d" % host if host.get('repl_ip'): r_ip = host['repl_ip'] @@ -141,8 +110,8 @@ def add_host_to_ring(build_file, host, validate=False): r_ip = host.get('repl_ip', host['ip']) r_port = host['repl_port'] host_str += "R%s:%d" % (r_ip, r_port) - host_str += "/%(name)s" % host - weight = host.get('weight', DEFAULT_HOST_WEIGHT) + host_str += "/%(device)s" % host + weight = host.get('weight') except Exception as ex: raise RingValidationError(ex) if not validate: @@ -178,14 +147,11 @@ def get_build_file_data(build_file): return build_file_data -def build_ring(section, conf, part_power, hosts, validate=False): +def build_ring(build_name, repl, min_part_hours, part_power, hosts, validate=False): # Create the build file - build_file = "%s.builder" % (section) + build_file = "%s.builder" % (build_name) build_file_data = get_build_file_data(build_file) - repl = conf.get('repl_number', DEFAULT_REPL) - min_part_hours = conf.get('min_part_hours', - DEFAULT_MIN_PART_HOURS) update = build_file_data is not None create_buildfile(build_file, part_power, repl, min_part_hours, update, data=build_file_data, validate=validate) @@ -194,23 +160,16 @@ def build_ring(section, conf, part_power, hosts, validate=False): if update: for i, dev in enumerate(build_file_data['devs']): if dev is not None: - old_hosts[DEV_KEY % dev] = i - section_key = section.split('-')[0] - service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) + old_hosts[DEVICE_KEY % dev] = i for host in hosts: - host_vars = hosts[host] - host_vars['device'] = host_vars['name'] - host_vars['port'] = service_port - host_vars['groups'] = host_vars.get('groups', DEFAULT_GROUPS) - if DEFAULT_GROUP_MAP[section] in host_vars['groups']: - host_key = DEV_KEY % host_vars - if host_key in old_hosts: - old_host = build_file_data['devs'][old_hosts[host_key]] - update_host_in_ring(build_file, host_vars, old_host, - validate=validate) - old_hosts.pop(host_key) - else: - add_host_to_ring(build_file, host_vars, validate=validate) + host_key = DEVICE_KEY % host + if host_key in old_hosts: + old_host = build_file_data['devs'][old_hosts[host_key]] + update_host_in_ring(build_file, host, old_host, + validate=validate) + old_hosts.pop(host_key) + else: + add_host_to_ring(build_file, host, validate=validate) if old_hosts and not validate: # There are still old hosts, these hosts must've been removed @@ -225,122 +184,37 @@ def main(setup): # load the json file try: with open(setup) as json_stream: - _inventory = json.load(json_stream) + _contents_file = json.load(json_stream) except Exception as ex: print("Failed to load json string %s" % (ex)) return 1 - _hosts = {} - # Get the swift specific global vars - global_vars = _inventory['all']['vars'] - check_section(global_vars, 'swift') - swift_vars = global_vars['swift'] - if _inventory.get("swift_hosts"): - for host in _inventory['swift_hosts']['hosts']: - host_config = _inventory['_meta']['hostvars'][host] - host_vars = host_config['swift_vars'] - host_ip = host_vars.get('storage_ip', host_config['container_address']) - if not host_vars.get('drives'): - if not swift_vars.get('drives'): - continue - else: - host_vars['drives'] = copy.deepcopy(swift_vars.get('drives')) - host_drives = host_vars.get('drives') - for host_drive in host_drives: - host_drive['ip'] = host_drive.get('storage_ip', host_ip) - if host_vars.get('groups'): - host_drive['groups'] = \ - host_drive.get('groups', host_vars['groups']) - if host_vars.get('repl_ip'): - host_drive['repl_ip'] = \ - host_drive.get('repl_ip', host_vars['repl_ip']) - if host_vars.get('repl_port'): - host_drive['repl_port'] = \ - host_drive.get('repl_port', host_vars['repl_port']) - if host_vars.get('weight'): - host_drive['weight'] = \ - host_drive.get('weight', host_vars['weight']) - key = "%s/%s" % (host_drive['ip'], host_drive['name']) - if key in _hosts: - print("%s already definined - duplicate device" % key) - return 1 - _hosts[key] = host_drive + hosts = _contents_file['drives'] + kargs = {'validate': True, 'hosts': hosts} + ring_call = [ _contents_file['builder_file'], + _contents_file['repl_number'], + _contents_file['min_part_hours'], + _contents_file['part_power']] - global_vars = _inventory['all']['vars'] - check_section(global_vars, 'swift') - swift_vars = global_vars['swift'] - if not swift_vars.get('part_power'): - print('No part_power specified - please set a part_power value') - return 1 - part_power = swift_vars.get('part_power') - - # If the repl_number or min_part hours are set on a "global" level in the - # conf lets set them here - otherwise use the overall default. - default_repl_num = swift_vars.get('repl_number', DEFAULT_REPL) - default_min_part_hours = swift_vars.get('min_part_hours', - DEFAULT_MIN_PART_HOURS) - ring_calls = [] - - # Create account ring - if the section is empty create an empty dict - # so defaults are used - if not has_section(swift_vars, 'account'): - swift_vars['account'] = {'repl_number': default_repl_num, - 'min_part_hours': default_min_part_hours} - ring_calls.append(('account', swift_vars['account'], part_power)) - - # Create container ring - if the section is empty create an empty dict - # so defaults are used - if not has_section(swift_vars, 'container'): - swift_vars['container'] = {'repl_number': default_repl_num, - 'min_part_hours': default_min_part_hours} - ring_calls.append(('container', swift_vars['container'], part_power)) - - # Create object rings (storage policies) - check_section(swift_vars, 'storage_policies') - indexes = set() - for policy in swift_vars['storage_policies']: - policy = policy['policy'] - if policy['index'] in indexes: - print("Storage Policy index %d already in use" % (policy['index'])) - return 4 - if policy['index'] == 0: - buildfilename = 'object' - else: - buildfilename = 'object-%d' % (policy['index']) - indexes.add(policy['index']) - # Set default port/min_part_hours/repl_number - if 'min_part_hours' not in policy: - policy['min_part_hours'] = default_min_part_hours - if 'repl_number' not in policy: - policy['repl_number'] = default_repl_num - if 'port' not in policy: - policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT) - ring_calls.append((buildfilename, policy, part_power)) - - # Now that we have gathered all the options for building/update the rings - # lets validate them - kargs = {'validate': True, 'hosts': _hosts} - for ring_call in ring_calls: - try: - build_ring(*ring_call, **kargs) - except RingValidationError as ex: - print(ex) - return 2 + try: + build_ring(*ring_call, **kargs) + except RingValidationError as ex: + print(ex) + return 2 # If the validation passes lets go ahead and build the rings. kargs.pop('validate') - for ring_call in ring_calls: - build_ring(*ring_call, **kargs) + build_ring(*ring_call, **kargs) + if __name__ == "__main__": parser = OptionParser(USAGE) - parser.add_option("-s", "--setup", dest="setup", - help="Specify the swift setup file.", metavar="FILE", - default="/etc/rpc_deploy/rpc_inventory.json") + parser.add_option("-f", "--file", dest="setup", + help="Specify the swift ring contents file.", metavar="FILE") options, args = parser.parse_args(sys.argv[1:]) if options.setup and not exists(options.setup): - print("Swift setup file not found or doesn't exist") + print("Swift ring contents file not found or doesn't exist") parser.print_help() sys.exit(1)