From 31ce4e14a0e8539e7ecaa4fae246d43b9cc7793a Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Thu, 18 Sep 2014 07:41:19 +1000 Subject: [PATCH 01/45] Added my inital work on ansible-swift. I have tried to put all the files in where I think they will go: swift_setup.yml -> etc/rpc_deploy/ roles swift_{common,account,container,object} -> rpc_deployment/roles swift_intentory -> scripts/swift_inventory.py To generate a hosts file run: python swift_inventory.py -s etc/rpc_deploy/swift_setup.yml -d NOTE: -d is dry-run so prints to screen, it will actually place it whereever you specify it in the swift_setup.yml file. --- etc/rpc_deploy/swift_setup.yml | 100 +++ .../roles/swift_account/handlers/main.yml | 12 + .../roles/swift_account/tasks/main.yml | 20 + .../templates/account-server.conf.j2 | 194 ++++++ .../roles/swift_account/vars/main.yml | 1 + .../roles/swift_common/handlers/main.yml | 6 + .../roles/swift_common/tasks/debian.yml | 4 + .../roles/swift_common/tasks/main.yml | 34 + .../roles/swift_common/tasks/redhat.yml | 5 + .../swift_common/templates/swift.conf.j2 | 97 +++ .../roles/swift_common/vars/main.yml | 22 + .../roles/swift_container/handlers/main.yml | 12 + .../roles/swift_container/tasks/main.yml | 20 + .../templates/container-server.conf.j2 | 205 ++++++ .../roles/swift_object/handlers/main.yml | 12 + .../roles/swift_object/tasks/main.yml | 20 + .../templates/object-server.conf.j2 | 281 ++++++++ .../roles/swift_proxy/handlers/main.yml | 3 + .../roles/swift_proxy/tasks/main.yml | 11 + .../templates/proxy-server.conf.j2 | 622 ++++++++++++++++++ scripts/swift_inventory.py | 309 +++++++++ 21 files changed, 1990 insertions(+) create mode 100644 etc/rpc_deploy/swift_setup.yml create mode 100644 rpc_deployment/roles/swift_account/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_account/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_account/templates/account-server.conf.j2 create mode 100644 rpc_deployment/roles/swift_account/vars/main.yml create mode 100644 rpc_deployment/roles/swift_common/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_common/tasks/debian.yml create mode 100644 rpc_deployment/roles/swift_common/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_common/tasks/redhat.yml create mode 100644 rpc_deployment/roles/swift_common/templates/swift.conf.j2 create mode 100644 rpc_deployment/roles/swift_common/vars/main.yml create mode 100644 rpc_deployment/roles/swift_container/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_container/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_container/templates/container-server.conf.j2 create mode 100644 rpc_deployment/roles/swift_object/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_object/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_object/templates/object-server.conf.j2 create mode 100644 rpc_deployment/roles/swift_proxy/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_proxy/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 create mode 100644 scripts/swift_inventory.py diff --git a/etc/rpc_deploy/swift_setup.yml b/etc/rpc_deploy/swift_setup.yml new file mode 100644 index 0000000000..106da684fc --- /dev/null +++ b/etc/rpc_deploy/swift_setup.yml @@ -0,0 +1,100 @@ +--- +# This swift setup file, is used as a simple way of managing your swift +# ring configuration. The swift_inventory binary will generate the ansible +# iventory file for you based of this file. Giving you one place to manage your +# cluster. +# +# NOTE: The swift ansible configuration has other variables that can be tweeked, +# see group_vars/ and roles/*/vars/ for other areas to tweek. +local: + ansible_connection=local + +# Swift settings are global through out the cluster. +swift: + part_power: 8 + output_directory: /tmp/swift + output_filename: hosts + user: swift + hash_path_suffix: changeme + hash_path_prefix: changeme + syslog_host: 10.1.1.1:514 + +proxy: + memcache_servers: + - 127.0.0.1:11211 + authtoken: + active: true + delay_auth_decision: true + auth_version: v2.0 + auth_host: keystone.local.lan + auth_port: 35357 + auth_protocol: https + auth_uri: http://keystonehost:5000/ + admin_tenant_name: service + admin_user: swift + admin_password: ADMIN + hosts: + - host: 10.0.0.1 + +account: + repl_number: 3 + hosts: + - host: 10.0.0.2 + drive: /srv/disk + region: 0 + zone: 0 + weight: 100 + +container: + repl_number: 3 + hosts: + - host: 10.0.0.3 + drive: /srv/disk + region: 0 + zone: 0 + weight: 100 + +storage_policies: + default: gold + policies: + - name: gold + index: 0 + type: replication + repl_number: 3 + hosts: + - host: 10.0.0.4 + drive: /srv/disk + region: 0 + zone: 0 + weight: 100 + - host: 10.0.0.5 + drive: /srv/disk + region: 0 + zone: 1 + weight: 100 + - host: 10.0.0.6 + drive: /srv/disk + region: 1 + zone: 0 + weight: 50 + - host: 10.0.0.7 + drive: /srv/disk + region: 1 + zone: 1 + weight: 50 + - name: silver + index: 1 + type: replication + repl_number: 2 + depricated: True + hosts: + - host: 10.0.0.4 + drive: /srv/disk + region: 0 + zone: 0 + weight: 100 + - host: 10.0.0.5 + drive: /srv/disk + region: 0 + zone: 1 + weight: 100 diff --git a/rpc_deployment/roles/swift_account/handlers/main.yml b/rpc_deployment/roles/swift_account/handlers/main.yml new file mode 100644 index 0000000000..36250f33f8 --- /dev/null +++ b/rpc_deployment/roles/swift_account/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: (re)start account server + command: swift-init account-server restart + +- name: (re)start account auditor + command: swift-init account-auditor restart + +- name: (re)start account replicator + command: swift-init account-replicator restart + +- name: (re)start account reaper + command: swift-init account-reaper restart diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml new file mode 100644 index 0000000000..6517d723dc --- /dev/null +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: "swift account server configuration" + template: src=account-server.conf.j2 dest=/etc/swfit/account-server.conf.j2 owner={{ swift }} mode=0644 + notfiy: + - (re)start account server + - (re)start account auditor + - (re)start account replicator + - (re)start account reaper + +- name: "Set account server to start at boot" + cron: special_time=reboot job="swift-init account-server start" + +- name: "Set account auditor to start at boot" + cron: special_time=reboot job="swift-init account-auditor start" + +- name: "Set account replicator to start at boot" + cron: special_time=reboot job="swift-init account-replicator start" + +- name: "Set account reaper to start at boot" + cron: special_time=reboot job="swift-init account-reaper start" diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 new file mode 100644 index 0000000000..e675c56766 --- /dev/null +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -0,0 +1,194 @@ +[DEFAULT] +bind_ip = {{ inventory_hostname }} +bind_port = 6002 +# bind_timeout = 30 +# backlog = 4096 +user = {{ user }} +# swift_dir = /etc/swift +devices = {{ drive }} +# mount_check = true +# disable_fallocate = false +# +# Use an integer to override the number of pre-forked processes that will +# accept connections. +# workers = auto +# +# Maximum concurrent requests per worker +# max_clients = 1024 +# +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = +# +# If you don't mind the extra disk space usage in overhead, you can turn this +# on to preallocate disk space with SQLite databases to decrease fragmentation. +# db_preallocation = off +# +# eventlet_debug = false +# +# You can set fallocate_reserve to the number of bytes you'd like fallocate to +# reserve, whether there is space for the given file size or not. +# fallocate_reserve = 0 + +[pipeline:main] +pipeline = healthcheck recon account-server + +[app:account-server] +use = egg:swift#account +log_facility = LOG_LOCAL1 +# You can override the default log routing for this app here: +# set log_name = account-server +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_requests = true +# set log_address = /dev/log +# +# auto_create_account_prefix = . +# +# Configure parameter for creating specific server +# To handle all verbs, including replication verbs, do not specify +# "replication_server" (this is the default). To only handle replication, +# set to a True value (e.g. "True" or "1"). To handle only non-replication +# verbs, set to "False". Unless you have a separate replication network, you +# should not specify any value for "replication_server". +# replication_server = false + +[filter:healthcheck] +use = egg:swift#healthcheck +# An optional filesystem path, which if present, will cause the healthcheck +# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE" +# disable_path = + +[filter:recon] +use = egg:swift#recon +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +recon_lock_path = /var/lock/swift + +[account-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = account-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +per_diff = 10000 +# max_diffs = 100 +# concurrency = 8 +# interval = 30 +# +# How long without an error before a node's error count is reset. This will +# also be how long before a node is reenabled after suppression is triggered. +# error_suppression_interval = 60 +# +# How many errors can accumulate before a node is temporarily ignored. +# error_suppression_limit = 10 +# +# node_timeout = 10 +# conn_timeout = 0.5 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# Time in seconds to wait between replication passes +# Note: if the parameter 'interval' is defined then it will be used in place +# of run_pause. +# run_pause = 30 +# +# recon_cache_path = /var/cache/swift + +[account-auditor] +# You can override the default log routing for this app here (don't use set!): +# log_name = account-auditor +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# Will audit each account at most once per interval +# interval = 1800 +# +# log_facility = LOG_LOCAL0 +# log_level = INFO +# accounts_per_second = 200 +# recon_cache_path = /var/cache/swift + +[account-reaper] +# You can override the default log routing for this app here (don't use set!): +# log_name = account-reaper +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# concurrency = 25 +# interval = 3600 +# node_timeout = 10 +# conn_timeout = 0.5 +# +# Normally, the reaper begins deleting account information for deleted accounts +# immediately; you can set this to delay its work however. The value is in +# seconds; 2592000 = 30 days for example. +delay_reaping = 604800 +# +# If the account fails to be be reaped due to a persistent error, the +# account reaper will log a message such as: +# Account has not been reaped since +# You can search logs for this message if space is not being reclaimed +# after you delete account(s). +# Default is 2592000 seconds (30 days). This is in addition to any time +# requested by delay_reaping. +# reap_warn_after = 2592000 + +# Note: Put it at the beginning of the pipleline to profile all middleware. But +# it is safer to put this after healthcheck. +[filter:xprofile] +use = egg:swift#xprofile +# This option enable you to switch profilers which should inherit from python +# standard profiler. Currently the supported value can be 'cProfile', +# 'eventlet.green.profile' etc. +# profile_module = eventlet.green.profile +# +# This prefix will be used to combine process ID and timestamp to name the +# profile data file. Make sure the executing user has permission to write +# into this path (missing path segments will be created, if necessary). +# If you enable profiling in more than one type of daemon, you must override +# it with an unique value like: /var/log/swift/profile/account.profile +# log_filename_prefix = /tmp/log/swift/profile/default.profile +# +# the profile data will be dumped to local disk based on above naming rule +# in this interval. +# dump_interval = 5.0 +# +# Be careful, this option will enable profiler to dump data into the file with +# time stamp which means there will be lots of files piled up in the directory. +# dump_timestamp = false +# +# This is the path of the URL to access the mini web UI. +# path = /__profile__ +# +# Clear the data when the wsgi server shutdown. +# flush_at_shutdown = false +# +# unwind the iterator of applications +# unwind = false diff --git a/rpc_deployment/roles/swift_account/vars/main.yml b/rpc_deployment/roles/swift_account/vars/main.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/rpc_deployment/roles/swift_account/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/rpc_deployment/roles/swift_common/handlers/main.yml b/rpc_deployment/roles/swift_common/handlers/main.yml new file mode 100644 index 0000000000..20633b042a --- /dev/null +++ b/rpc_deployment/roles/swift_common/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: "Install swift dependencies" + pip: requirements=/opt/swift/swift/requirements.txt + +- name: "Install swift" + shell: chdir=/opt/swift/swift python setup.py install diff --git a/rpc_deployment/roles/swift_common/tasks/debian.yml b/rpc_deployment/roles/swift_common/tasks/debian.yml new file mode 100644 index 0000000000..6f82ba68c9 --- /dev/null +++ b/rpc_deployment/roles/swift_common/tasks/debian.yml @@ -0,0 +1,4 @@ +--- +- name: "Install Debian/Ubuntu common swift packages" + apt: pkg={{ item }} state=installed update_cache=yes + with_items: ${swift_common_packages.debian} diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml new file mode 100644 index 0000000000..fc4fddf415 --- /dev/null +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: "Setup hosts file" + lineinfile: dest=/etc/hosts regexp='.*{{ item }}$' line="{{ hostvars[item].ansible_default_ipv4.address }} {{item}}" state=present + when: hostvars[item].ansible_default_ipv4.address is defined + with_items: groups['all'] + +# If target is a debian/ubuntu system +- include: debian.yml + when: ansible_os_family == 'Debian' + +# If target is a centos/redhat system +- include: redhat.yml + when: ansible_os_family == 'RedHat' + +- name: "Create a swift user" + group: name={{ user }} state=present system=yes + user: name={{ user }} state=present shell=/bin/false home=/opt/swift system=yes + +- name: "Checkout git repo" + git: repo={{ swift_repo }} version={{ git_tag }} dest=/opt/swift/swift update=yes + notifiy: + - Install swift dependencies + - Install swift + +- name: "Swift directory" + file: path=/etc/swift owner={{ user }} group={{ user }} state=directory + +- name: "Swift lock directory" + file: path=/var/lock/swift owner={{ user }} group={{ user }} state=directory + +- name: "Swift log directory" + file: path=/var/log/swift owner={{ user }} group=adm state=directory + + diff --git a/rpc_deployment/roles/swift_common/tasks/redhat.yml b/rpc_deployment/roles/swift_common/tasks/redhat.yml new file mode 100644 index 0000000000..50cb7998ed --- /dev/null +++ b/rpc_deployment/roles/swift_common/tasks/redhat.yml @@ -0,0 +1,5 @@ +--- + +- name: "install Redhat/CentOS common swift packages" + yum: pkg={{ item }} state=installed + with_items: ${swift_common_packages.redhat} diff --git a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 new file mode 100644 index 0000000000..61e7a74ed6 --- /dev/null +++ b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 @@ -0,0 +1,97 @@ +[swift-hash] + +# swift_hash_path_suffix and swift_hash_path_prefix are used as part of the +# the hashing algorithm when determining data placement in the cluster. +# These values should remain secret and MUST NOT change +# once a cluster has been deployed. + +swift_hash_path_suffix = {{ hash_path_suffix }} +swift_hash_path_prefix = {{ hash_path_prefix }} + +# Storage Policies +{% for group in groups %} + {% if group.startswith("storagepolicy") %} + {% vars = hostvars[groups[group].pop] %} +[storage-policy:{{ vars["index"] }}] +name = {{ vars["policy_name"] }} + {% if "depricated" in vars %} +depricated = {{ var["depricated"] }} + {% endif %} + {% if "default" in vars %} +default = {{ var["default"] }} + {% endif %} + {% endif %} +{% endfor %} + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. It is STRONGLY recommended to leave this value at +# the default (5 * 2**30 + 2). + +#max_file_size = 5368709122 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 + +# max_header_size is the max number of bytes in the utf8 encoding of each +# header. Using 8192 as default because eventlet use 8192 as max size of +# header line. This value may need to be increased when using identity +# v3 API tokens including more than 7 catalog entries. +# See also include_service_catalog in proxy-server.conf-sample +# (documented in overview_auth.rst) + +#max_header_size = 8192 + + +# max_object_name_length is the max number of bytes in the utf8 encoding +# of an object name + +#max_object_name_length = 1024 + + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding +# of an account name + +#max_account_name_length = 256 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name + +#max_container_name_length = 256 diff --git a/rpc_deployment/roles/swift_common/vars/main.yml b/rpc_deployment/roles/swift_common/vars/main.yml new file mode 100644 index 0000000000..93f948e600 --- /dev/null +++ b/rpc_deployment/roles/swift_common/vars/main.yml @@ -0,0 +1,22 @@ +--- +swift_repo: https://github.com/openstack/swift.git +swift_repo_tag: master + +swift_common_packages: + debian: + - curl + - python-pip + - rsync + - openssh-server + - git-core + - python-setuptools + - python-dev + - gcc + + redhat: + - curl + - gcc + - rsync + - git-core + - python-setuptools + - python-pip diff --git a/rpc_deployment/roles/swift_container/handlers/main.yml b/rpc_deployment/roles/swift_container/handlers/main.yml new file mode 100644 index 0000000000..7c034f60c1 --- /dev/null +++ b/rpc_deployment/roles/swift_container/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: (re)start container server + command: swift-init container-server restart + +- name: (re)start container auditor + command: swift-init container-auditor restart + +- name: (re)start container replicator + command: swift-init container-replicator restart + +- name: (re)start container updater + command: swift-init container-updater restart diff --git a/rpc_deployment/roles/swift_container/tasks/main.yml b/rpc_deployment/roles/swift_container/tasks/main.yml new file mode 100644 index 0000000000..4efb75a794 --- /dev/null +++ b/rpc_deployment/roles/swift_container/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: "swift container server configuration" + template: src=container-server.conf.j2 dest=/etc/swfit/container-server.conf.j2 owner={{ swift }} mode=0644 + notfiy: + - (re)start container server + - (re)start container auditor + - (re)start container replicator + - (re)start container updater + +- name: "Set container server to start at boot" + cron: special_time=reboot job="swift-init container-server start" + +- name: "Set container auditor to start at boot" + cron: special_time=reboot job="swift-init container-auditor start" + +- name: "Set container replicator to start at boot" + cron: special_time=reboot job="swift-init container-replicator start" + +- name: "Set container updater to start at boot" + cron: special_time=reboot job="swift-init container-updater start" diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 new file mode 100644 index 0000000000..55aed2808f --- /dev/null +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -0,0 +1,205 @@ +[DEFAULT] +bind_ip = {{ inventory_hostname }} +bind_port = 6001 +# bind_timeout = 30 +# backlog = 4096 +user = {{ user }} +# swift_dir = /etc/swift +devices = {{ drive }} +# mount_check = true +# disable_fallocate = false +# +# Use an integer to override the number of pre-forked processes that will +# accept connections. +# workers = auto +# +# Maximum concurrent requests per worker +# max_clients = 1024 +# +# This is a comma separated list of hosts allowed in the X-Container-Sync-To +# field for containers. This is the old-style of using container sync. It is +# strongly recommended to use the new style of a separate +# container-sync-realms.conf -- see container-sync-realms.conf-sample +# allowed_sync_hosts = 127.0.0.1 +# +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = +# +# If you don't mind the extra disk space usage in overhead, you can turn this +# on to preallocate disk space with SQLite databases to decrease fragmentation. +# db_preallocation = off +# +# eventlet_debug = false +# +# You can set fallocate_reserve to the number of bytes you'd like fallocate to +# reserve, whether there is space for the given file size or not. +# fallocate_reserve = 0 + +[pipeline:main] +pipeline = healthcheck recon container-server + +[app:container-server] +use = egg:swift#container +log_facility = LOG_LOCAL1 +# You can override the default log routing for this app here: +# set log_name = container-server +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_requests = true +# set log_address = /dev/log +# +# node_timeout = 3 +# conn_timeout = 0.5 +# allow_versions = false +# auto_create_account_prefix = . +# +# Configure parameter for creating specific server +# To handle all verbs, including replication verbs, do not specify +# "replication_server" (this is the default). To only handle replication, +# set to a True value (e.g. "True" or "1"). To handle only non-replication +# verbs, set to "False". Unless you have a separate replication network, you +# should not specify any value for "replication_server". +# replication_server = false + +[filter:healthcheck] +use = egg:swift#healthcheck +# An optional filesystem path, which if present, will cause the healthcheck +# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE" +# disable_path = + +[filter:recon] +use = egg:swift#recon +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +recon_lock_path = /var/lock/swift + +[container-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = container-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +# per_diff = 1000 +# max_diffs = 100 +# concurrency = 8 +# interval = 30 +# node_timeout = 10 +# conn_timeout = 0.5 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# Time in seconds to wait between replication passes +# Note: if the parameter 'interval' is defined then it will be used in place +# of run_pause. +# run_pause = 30 +# +# recon_cache_path = /var/cache/swift + +[container-updater] +# You can override the default log routing for this app here (don't use set!): +# log_name = container-updater +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# interval = 300 +# concurrency = 4 +node_timeout = 15 +conn_timeout = 5 +# +# slowdown will sleep that amount between containers +# slowdown = 0.01 +# +# Seconds to suppress updating an account that has generated an error +# account_suppression_time = 60 +# +# recon_cache_path = /var/cache/swift + +[container-auditor] +# You can override the default log routing for this app here (don't use set!): +# log_name = container-auditor +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# Will audit each container at most once per interval +# interval = 1800 +# +# containers_per_second = 200 +# recon_cache_path = /var/cache/swift + +[container-sync] +# You can override the default log routing for this app here (don't use set!): +# log_name = container-sync +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# +# If you need to use an HTTP Proxy, set it here; defaults to no proxy. +# You can also set this to a comma separated list of HTTP Proxies and they will +# be randomly used (simple load balancing). +# sync_proxy = http://10.1.1.1:8888,http://10.1.1.2:8888 +# +# Will sync each container at most once per interval +# interval = 300 +# +# Maximum amount of time to spend syncing each container per pass +# container_time = 60 + +# Note: Put it at the beginning of the pipleline to profile all middleware. But +# it is safer to put this after healthcheck. +[filter:xprofile] +use = egg:swift#xprofile +# This option enable you to switch profilers which should inherit from python +# standard profiler. Currently the supported value can be 'cProfile', +# 'eventlet.green.profile' etc. +# profile_module = eventlet.green.profile +# +# This prefix will be used to combine process ID and timestamp to name the +# profile data file. Make sure the executing user has permission to write +# into this path (missing path segments will be created, if necessary). +# If you enable profiling in more than one type of daemon, you must override +# it with an unique value like: /var/log/swift/profile/container.profile +# log_filename_prefix = /tmp/log/swift/profile/default.profile +# +# the profile data will be dumped to local disk based on above naming rule +# in this interval. +# dump_interval = 5.0 +# +# Be careful, this option will enable profiler to dump data into the file with +# time stamp which means there will be lots of files piled up in the directory. +# dump_timestamp = false +# +# This is the path of the URL to access the mini web UI. +# path = /__profile__ +# +# Clear the data when the wsgi server shutdown. +# flush_at_shutdown = false +# +# unwind the iterator of applications +# unwind = false diff --git a/rpc_deployment/roles/swift_object/handlers/main.yml b/rpc_deployment/roles/swift_object/handlers/main.yml new file mode 100644 index 0000000000..ab7447eee2 --- /dev/null +++ b/rpc_deployment/roles/swift_object/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: (re)start object server + command: swift-init object-server restart + +- name: (re)start object auditor + command: swift-init object-auditor restart + +- name: (re)start object replicator + command: swift-init object-replicator restart + +- name: (re)start object updater + command: swift-init object-updater restart diff --git a/rpc_deployment/roles/swift_object/tasks/main.yml b/rpc_deployment/roles/swift_object/tasks/main.yml new file mode 100644 index 0000000000..18428f1f55 --- /dev/null +++ b/rpc_deployment/roles/swift_object/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: "swift object server configuration" + template: src=object-server.conf.j2 dest=/etc/swfit/object-server.conf.j2 owner={{ swift }} mode=0644 + notfiy: + - (re)start object server + - (re)start object auditor + - (re)start object replicator + - (re)start object updater + +- name: "Set object server to start at boot" + cron: special_time=reboot job="swift-init object-server start" + +- name: "Set object auditor to start at boot" + cron: special_time=reboot job="swift-init object-auditor start" + +- name: "Set object replicator to start at boot" + cron: special_time=reboot job="swift-init object-replicator start" + +- name: "Set object updater to start at boot" + cron: special_time=reboot job="swift-init object-updater start" diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 new file mode 100644 index 0000000000..96bb2bae85 --- /dev/null +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -0,0 +1,281 @@ +[DEFAULT] +bind_ip = {{ inverntory_host }} +bind_port = 6000 +# bind_timeout = 30 +# backlog = 4096 +user = {{ user }} +swift_dir = /etc/swift +devices = {{ drive }} +# mount_check = true +# disable_fallocate = false +# expiring_objects_container_divisor = 86400 +# expiring_objects_account_name = expiring_objects +# +# Use an integer to override the number of pre-forked processes that will +# accept connections. +# workers = auto +# +# Maximum concurrent requests per worker +# max_clients = 1024 +# +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = +# +# eventlet_debug = false +# +# You can set fallocate_reserve to the number of bytes you'd like fallocate to +# reserve, whether there is space for the given file size or not. +# fallocate_reserve = 0 +# +# Time to wait while attempting to connect to another backend node. +# conn_timeout = 0.5 +# Time to wait while sending each chunk of data to another backend node. +# node_timeout = 3 +# Time to wait while receiving each chunk of data from a client or another +# backend node. +# client_timeout = 60 +# +# network_chunk_size = 65536 +# disk_chunk_size = 65536 + +[pipeline:main] +pipeline = healthcheck recon object-server + +[app:object-server] +use = egg:swift#object +log_facility = LOG_LOCAL1 +# You can override the default log routing for this app here: +# set log_name = object-server +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_requests = true +# set log_address = /dev/log +# +# max_upload_time = 86400 +# slow = 0 +# +# Objects smaller than this are not evicted from the buffercache once read +# keep_cache_size = 5424880 +# +# If true, objects for authenticated GET requests may be kept in buffer cache +# if small enough +# keep_cache_private = false +# +# on PUTs, sync data every n MB +# mb_per_sync = 512 +mb_per_sync = 64 +# +# Comma separated list of headers that can be set in metadata on an object. +# This list is in addition to X-Object-Meta-* headers and cannot include +# Content-Type, etag, Content-Length, or deleted +# allowed_headers = Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object +# +# auto_create_account_prefix = . +# +# A value of 0 means "don't use thread pools". A reasonable starting point is +# 4. +# threads_per_disk = 0 +# +# Configure parameter for creating specific server +# To handle all verbs, including replication verbs, do not specify +# "replication_server" (this is the default). To only handle replication, +# set to a True value (e.g. "True" or "1"). To handle only non-replication +# verbs, set to "False". Unless you have a separate replication network, you +# should not specify any value for "replication_server". +# replication_server = false +# +# Set to restrict the number of concurrent incoming REPLICATION requests +# Set to 0 for unlimited +# Note that REPLICATION is currently an ssync only item +# replication_concurrency = 4 +# +# Restricts incoming REPLICATION requests to one per device, +# replication_currency above allowing. This can help control I/O to each +# device, but you may wish to set this to False to allow multiple REPLICATION +# requests (up to the above replication_concurrency setting) per device. +# replication_one_per_device = True +# +# Number of seconds to wait for an existing replication device lock before +# giving up. +# replication_lock_timeout = 15 +# +# These next two settings control when the REPLICATION subrequest handler will +# abort an incoming REPLICATION attempt. An abort will occur if there are at +# least threshold number of failures and the value of failures / successes +# exceeds the ratio. The defaults of 100 and 1.0 means that at least 100 +# failures have to occur and there have to be more failures than successes for +# an abort to occur. +# replication_failure_threshold = 100 +# replication_failure_ratio = 1.0 + +[filter:healthcheck] +use = egg:swift#healthcheck +# An optional filesystem path, which if present, will cause the healthcheck +# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE" +# disable_path = + +[filter:recon] +use = egg:swift#recon +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +recon_lock_path = /var/lock/swift + +[object-replicator] +# You can override the default log routing for this app here (don't use set!): +# log_name = object-replicator +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# vm_test_mode = no +# daemonize = on +# run_pause = 30 +concurrency = 6 +# stats_interval = 300 +# +# The sync method to use; default is rsync but you can use ssync to try the +# EXPERIMENTAL all-swift-code-no-rsync-callouts method. Once ssync is verified +# as having performance comparable to, or better than, rsync, we plan to +# deprecate rsync so we can move on with more features for replication. +# sync_method = rsync +# +# max duration of a partition rsync +# rsync_timeout = 900 +# +# bandwidth limit for rsync in kB/s. 0 means unlimited +# rsync_bwlimit = 0 +# +# passed to rsync for io op timeout +# rsync_io_timeout = 30 +# +# node_timeout = +# max duration of an http request; this is for REPLICATE finalization calls and +# so should be longer than node_timeout +# http_timeout = 60 +# +# attempts to kill all workers if nothing replicates for lockup_timeout seconds +# lockup_timeout = 1800 +# +# The replicator also performs reclamation +# reclaim_age = 604800 +# +# ring_check_interval = 15 +# recon_cache_path = /var/cache/swift +# +# limits how long rsync error log lines are +# 0 means to log the entire line +# rsync_error_log_line_length = 0 +# +# handoffs_first and handoff_delete are options for a special case +# such as disk full in the cluster. These two options SHOULD NOT BE +# CHANGED, except for such an extreme situations. (e.g. disks filled up +# or are about to fill up. Anyway, DO NOT let your drives fill up) +# handoffs_first is the flag to replicate handoffs prior to canonical +# partitions. It allows to force syncing and deleting handoffs quickly. +# If set to a True value(e.g. "True" or "1"), partitions +# that are not supposed to be on the node will be replicated first. +# handoffs_first = False +# +# handoff_delete is the number of replicas which are ensured in swift. +# If the number less than the number of replicas is set, object-replicator +# could delete local handoffs even if all replicas are not ensured in the +# cluster. Object-replicator would remove local handoff partition directories +# after syncing partition when the number of successful responses is greater +# than or equal to this number. By default(auto), handoff partitions will be +# removed when it has successfully replicated to all the canonical nodes. +# handoff_delete = auto + +[object-updater] +# You can override the default log routing for this app here (don't use set!): +# log_name = object-updater +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# interval = 300 +concurrency = 3 +# node_timeout = +# slowdown will sleep that amount between objects +# slowdown = 0.01 +# +# recon_cache_path = /var/cache/swift +concurrency = 3 +node_timeout = 60 +conn_timeout = 5 + +[object-auditor] +# You can override the default log routing for this app here (don't use set!): +# log_name = object-auditor +log_facility = LOG_LOCAL2 +# log_level = INFO +# log_address = /dev/log +# +# You can set the disk chunk size that the auditor uses making it larger if +# you like for more efficient local auditing of larger objects +# disk_chunk_size = 65536 +# files_per_second = 20 +# concurrency = 1 +# bytes_per_second = 10000000 +# log_time = 3600 +# zero_byte_files_per_second = 50 +# recon_cache_path = /var/cache/swift + +# Takes a comma separated list of ints. If set, the object auditor will +# increment a counter for every object whose size is <= to the given break +# points and report the result after a full scan. +# object_size_stats = + +# Note: Put it at the beginning of the pipleline to profile all middleware. But +# it is safer to put this after healthcheck. +[filter:xprofile] +use = egg:swift#xprofile +# This option enable you to switch profilers which should inherit from python +# standard profiler. Currently the supported value can be 'cProfile', +# 'eventlet.green.profile' etc. +# profile_module = eventlet.green.profile +# +# This prefix will be used to combine process ID and timestamp to name the +# profile data file. Make sure the executing user has permission to write +# into this path (missing path segments will be created, if necessary). +# If you enable profiling in more than one type of daemon, you must override +# it with an unique value like: /var/log/swift/profile/object.profile +# log_filename_prefix = /tmp/log/swift/profile/default.profile +# +# the profile data will be dumped to local disk based on above naming rule +# in this interval. +# dump_interval = 5.0 +# +# Be careful, this option will enable profiler to dump data into the file with +# time stamp which means there will be lots of files piled up in the directory. +# dump_timestamp = false +# +# This is the path of the URL to access the mini web UI. +# path = /__profile__ +# +# Clear the data when the wsgi server shutdown. +# flush_at_shutdown = false +# +# unwind the iterator of applications +# unwind = false diff --git a/rpc_deployment/roles/swift_proxy/handlers/main.yml b/rpc_deployment/roles/swift_proxy/handlers/main.yml new file mode 100644 index 0000000000..f33effd4ce --- /dev/null +++ b/rpc_deployment/roles/swift_proxy/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: (re)start proxy server + command: swift-init proxy-server restart diff --git a/rpc_deployment/roles/swift_proxy/tasks/main.yml b/rpc_deployment/roles/swift_proxy/tasks/main.yml new file mode 100644 index 0000000000..f919721f7f --- /dev/null +++ b/rpc_deployment/roles/swift_proxy/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: "install the keystone auth_token middleware" + pip: name=keystonemiddleware + +- name: "swift object server configuration" + template: src=proxy-server.conf.j2 dest=/etc/swfit/proxy-server.conf.j2 owner={{ swift }} mode=0644 + notfiy: + - (re)start proxy server + +- name: "Set proxy server to start at boot" + cron: special_time=reboot job="swift-init proxy-server start" diff --git a/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 new file mode 100644 index 0000000000..b97a9505ec --- /dev/null +++ b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 @@ -0,0 +1,622 @@ +[DEFAULT] +bind_ip = {{ inventory_host }} +bind_port = 8080 +# bind_timeout = 30 +# backlog = 4096 +# swift_dir = /etc/swift +user = {{ user }} + +# Enables exposing configuration settings via HTTP GET /info. +# expose_info = true + +# Key to use for admin calls that are HMAC signed. Default is empty, +# which will disable admin calls to /info. +# admin_key = secret_admin_key +# +# Allows the ability to withhold sections from showing up in the public calls +# to /info. You can withhold subsections by separating the dict level with a +# ".". The following would cause the sections 'container_quotas' and 'tempurl' +# to not be listed, and the key max_failed_deletes would be removed from +# bulk_delete. Default is empty, allowing all registered fetures to be listed +# via HTTP GET /info. +# disallowed_sections = container_quotas, tempurl, bulk_delete.max_failed_deletes + +# Use an integer to override the number of pre-forked processes that will +# accept connections. Should default to the number of effective cpu +# cores in the system. It's worth noting that individual workers will +# use many eventlet co-routines to service multiple concurrent requests. +# workers = auto +# +# Maximum concurrent requests per worker +# max_clients = 1024 +# +# Set the following two lines to enable SSL. This is for testing only. +# cert_file = /etc/swift/proxy.crt +# key_file = /etc/swift/proxy.key +# +# expiring_objects_container_divisor = 86400 +# expiring_objects_account_name = expiring_objects +# +# You can specify default log routing here if you want: +# log_name = swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_headers = false +# log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 +# +# This optional suffix (default is empty) that would be appended to the swift transaction +# id allows one to easily figure out from which cluster that X-Trans-Id belongs to. +# This is very useful when one is managing more than one swift cluster. +# trans_id_suffix = +# +# comma separated list of functions to call to setup custom log handlers. +# functions get passed: conf, name, log_to_console, log_route, fmt, logger, +# adapted_logger +# log_custom_handlers = +# +# If set, log_udp_host will override log_address +# log_udp_host = +# log_udp_port = 514 +# +# You can enable StatsD logging here: +# log_statsd_host = localhost +# log_statsd_port = 8125 +# log_statsd_default_sample_rate = 1.0 +# log_statsd_sample_rate_factor = 1.0 +# log_statsd_metric_prefix = +# +# Use a comma separated list of full url (http://foo.bar:1234,https://foo.bar) +# cors_allow_origin = +# strict_cors_mode = True +# +# client_timeout = 60 +# eventlet_debug = false + +[pipeline:main] +{% if authtoken_active %} +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth container-quotas account-quotas slo dlo proxy-logging proxy-server +{% else %} +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo proxy-logging proxy-server +{% endif %} +[app:proxy-server] +use = egg:swift#proxy +log_facility = LOG_LOCAL0 +# You can override the default log routing for this app here: +# set log_name = proxy-server +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_address = /dev/log +# +# log_handoffs = true +# recheck_account_existence = 60 +# recheck_container_existence = 60 +# object_chunk_size = 65536 +# client_chunk_size = 65536 +# +# How long the proxy server will wait on responses from the a/c/o servers. +node_timeout = 60 +# +# How long the proxy server will wait for an initial response and to read a +# chunk of data from the object servers while serving GET / HEAD requests. +# Timeouts from these requests can be recovered from so setting this to +# something lower than node_timeout would provide quicker error recovery +# while allowing for a longer timeout for non-recoverable requests (PUTs). +# Defaults to node_timeout, should be overriden if node_timeout is set to a +# high number to prevent client timeouts from firing before the proxy server +# has a chance to retry. +# recoverable_node_timeout = node_timeout +# +conn_timeout = 3.5 +# +# How long to wait for requests to finish after a quorum has been established. +# post_quorum_timeout = 0.5 +# +# How long without an error before a node's error count is reset. This will +# also be how long before a node is reenabled after suppression is triggered. +# error_suppression_interval = 60 +# +# How many errors can accumulate before a node is temporarily ignored. +# error_suppression_limit = 10 +# +# If set to 'true' any authorized user may create and delete accounts; if +# 'false' no one, even authorized, can. +# allow_account_management = false +# +# Set object_post_as_copy = false to turn on fast posts where only the metadata +# changes are stored anew and the original data file is kept in place. This +# makes for quicker posts; but since the container metadata isn't updated in +# this mode, features like container sync won't be able to sync posts. +# object_post_as_copy = true +# +# If set to 'true' authorized accounts that do not yet exist within the Swift +# cluster will be automatically created. +account_autocreate = true +# +# If set to a positive value, trying to create a container when the account +# already has at least this maximum containers will result in a 403 Forbidden. +# Note: This is a soft limit, meaning a user might exceed the cap for +# recheck_account_existence before the 403s kick in. +# max_containers_per_account = 0 +# +# This is a comma separated list of account hashes that ignore the +# max_containers_per_account cap. +# max_containers_whitelist = +# +# Comma separated list of Host headers to which the proxy will deny requests. +# deny_host_headers = +# +# Prefix used when automatically creating accounts. +# auto_create_account_prefix = . +# +# Depth of the proxy put queue. +# put_queue_depth = 10 +# +# Storage nodes can be chosen at random (shuffle), by using timing +# measurements (timing), or by using an explicit match (affinity). +# Using timing measurements may allow for lower overall latency, while +# using affinity allows for finer control. In both the timing and +# affinity cases, equally-sorting nodes are still randomly chosen to +# spread load. +# The valid values for sorting_method are "affinity", "shuffle", and "timing". +# sorting_method = shuffle +# +# If the "timing" sorting_method is used, the timings will only be valid for +# the number of seconds configured by timing_expiry. +# timing_expiry = 300 +# +# The maximum time (seconds) that a large object connection is allowed to last. +# max_large_object_get_time = 86400 +# +# Set to the number of nodes to contact for a normal request. You can use +# '* replicas' at the end to have it use the number given times the number of +# replicas for the ring being used for the request. +# request_node_count = 2 * replicas +# +# Which backend servers to prefer on reads. Format is r for region +# N or rz for region N, zone M. The value after the equals is +# the priority; lower numbers are higher priority. +# +# Example: first read from region 1 zone 1, then region 1 zone 2, then +# anything in region 2, then everything else: +# read_affinity = r1z1=100, r1z2=200, r2=300 +# Default is empty, meaning no preference. +# read_affinity = +# +# Which backend servers to prefer on writes. Format is r for region +# N or rz for region N, zone M. If this is set, then when +# handling an object PUT request, some number (see setting +# write_affinity_node_count) of local backend servers will be tried +# before any nonlocal ones. +# +# Example: try to write to regions 1 and 2 before writing to any other +# nodes: +# write_affinity = r1, r2 +# Default is empty, meaning no preference. +# write_affinity = +# +# The number of local (as governed by the write_affinity setting) +# nodes to attempt to contact first, before any non-local ones. You +# can use '* replicas' at the end to have it use the number given +# times the number of replicas for the ring being used for the +# request. +# write_affinity_node_count = 2 * replicas +# +# These are the headers whose values will only be shown to swift_owners. The +# exact definition of a swift_owner is up to the auth system in use, but +# usually indicates administrative responsibilities. +# swift_owner_headers = x-container-read, x-container-write, x-container-sync-key, x-container-sync-to, x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, x-account-access-control + +[filter:tempauth] +use = egg:swift#tempauth +# You can override the default log routing for this filter here: +# set log_name = tempauth +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log +# +# The reseller prefix will verify a token begins with this prefix before even +# attempting to validate it. Also, with authorization, only Swift storage +# accounts with this prefix will be authorized by this middleware. Useful if +# multiple auth systems are in use for one Swift cluster. +# reseller_prefix = AUTH +# +# The auth prefix will cause requests beginning with this prefix to be routed +# to the auth subsystem, for granting tokens, etc. +# auth_prefix = /auth/ +# token_life = 86400 +# +# This allows middleware higher in the WSGI pipeline to override auth +# processing, useful for middleware such as tempurl and formpost. If you know +# you're not going to use such middleware and you want a bit of extra security, +# you can set this to false. +# allow_overrides = true +# +# This specifies what scheme to return with storage urls: +# http, https, or default (chooses based on what the server is running as) +# This can be useful with an SSL load balancer in front of a non-SSL server. +# storage_url_scheme = default +# +# Lastly, you need to list all the accounts/users you want here. The format is: +# user__ = [group] [group] [...] [storage_url] +# or if you want underscores in or , you can base64 encode them +# (with no equal signs) and use this format: +# user64__ = [group] [group] [...] [storage_url] +# There are special groups of: +# .reseller_admin = can do anything to any account for this auth +# .admin = can do anything within the account +# If neither of these groups are specified, the user can only access containers +# that have been explicitly allowed for them by a .admin or .reseller_admin. +# The trailing optional storage_url allows you to specify an alternate url to +# hand back to the user upon authentication. If not specified, this defaults to +# $HOST/v1/_ where $HOST will do its best to resolve +# to what the requester would need to use to reach this host. +# Here are example entries, required for running the tests: +{% if not authtoken_active %} +user_admin_admin = admin .admin .reseller_admin +user_test_tester = testing .admin +user_test2_tester2 = testing2 .admin +user_test_tester3 = testing3 +{% endif %} + +# To enable Keystone authentication you need to have the auth token +# middleware first to be configured. Here is an example below, please +# refer to the keystone's documentation for details about the +# different settings. +# +# You'll need to have as well the keystoneauth middleware enabled +# and have it in your main pipeline so instead of having tempauth in +# there you can change it to: authtoken keystoneauth +# +{% if authtoken_active %} +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory +auth_host = {{ auth_host }} +auth_port = {{ auth_port }} +auth_protocol = {{ auth_protocol }} +auth_uri = {{ auth_uri }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +delay_auth_decision = {{ delay_auth_decision }} +# cache = swift.cache +# include_service_catalog = False +# +[filter:keystoneauth] +use = egg:swift#keystoneauth +# Operator roles is the role which user would be allowed to manage a +# tenant and be able to create container or give ACL to others. +operator_roles = admin, swiftoperator +# The reseller admin role has the ability to create and delete accounts +reseller_admin_role = reseller_admin + +[filter:healthcheck] +use = egg:swift#healthcheck +# An optional filesystem path, which if present, will cause the healthcheck +# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE". +# This facility may be used to temporarily remove a Swift node from a load +# balancer pool during maintenance or upgrade (remove the file to allow the +# node back into the load balancer pool). +# disable_path = + +[filter:cache] +use = egg:swift#memcache +# You can override the default log routing for this filter here: +# set log_name = cache +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log +# +# If not set here, the value for memcache_servers will be read from +# memcache.conf (see memcache.conf-sample) or lacking that file, it will +# default to the value below. You can specify multiple servers separated with +# commas, as in: 10.1.2.3:11211,10.1.2.4:11211 +memcache_servers = {{ memcache_servers }} +# +# Sets how memcache values are serialized and deserialized: +# 0 = older, insecure pickle serialization +# 1 = json serialization but pickles can still be read (still insecure) +# 2 = json serialization only (secure and the default) +# If not set here, the value for memcache_serialization_support will be read +# from /etc/swift/memcache.conf (see memcache.conf-sample). +# To avoid an instant full cache flush, existing installations should +# upgrade with 0, then set to 1 and reload, then after some time (24 hours) +# set to 2 and reload. +# In the future, the ability to use pickle serialization will be removed. +memcache_serialization_support = 2 +# +# Sets the maximum number of connections to each memcached server per worker +# memcache_max_connections = 2 + +[filter:ratelimit] +use = egg:swift#ratelimit +# You can override the default log routing for this filter here: +# set log_name = ratelimit +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log +# +# clock_accuracy should represent how accurate the proxy servers' system clocks +# are with each other. 1000 means that all the proxies' clock are accurate to +# each other within 1 millisecond. No ratelimit should be higher than the +# clock accuracy. +# clock_accuracy = 1000 +# +# max_sleep_time_seconds = 60 +# +# log_sleep_time_seconds of 0 means disabled +# log_sleep_time_seconds = 0 +# +# allows for slow rates (e.g. running up to 5 sec's behind) to catch up. +# rate_buffer_seconds = 5 +# +# account_ratelimit of 0 means disabled +# account_ratelimit = 0 + +# these are comma separated lists of account names +# account_whitelist = a,b +# account_blacklist = c,d + +# with container_limit_x = r +# for containers of size x limit write requests per second to r. The container +# rate will be linearly interpolated from the values given. With the values +# below, a container of size 5 will get a rate of 75. +# container_ratelimit_0 = 100 +# container_ratelimit_10 = 50 +# container_ratelimit_50 = 20 + +# Similarly to the above container-level write limits, the following will limit +# container GET (listing) requests. +# container_listing_ratelimit_0 = 100 +# container_listing_ratelimit_10 = 50 +# container_listing_ratelimit_50 = 20 + +[filter:domain_remap] +use = egg:swift#domain_remap +# You can override the default log routing for this filter here: +# set log_name = domain_remap +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log +# +# storage_domain = example.com +# path_root = v1 +# reseller_prefixes = AUTH + +[filter:catch_errors] +use = egg:swift#catch_errors +# You can override the default log routing for this filter here: +# set log_name = catch_errors +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log + +[filter:cname_lookup] +# Note: this middleware requires python-dnspython +use = egg:swift#cname_lookup +# You can override the default log routing for this filter here: +# set log_name = cname_lookup +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log +# +# Specify the storage_domain that match your cloud, multiple domains +# can be specified separated by a comma +# storage_domain = example.com +# +# lookup_depth = 1 + +# Note: Put staticweb just after your auth filter(s) in the pipeline +[filter:staticweb] +use = egg:swift#staticweb + +# Note: Put tempurl before dlo, slo and your auth filter(s) in the pipeline +[filter:tempurl] +use = egg:swift#tempurl +# The methods allowed with Temp URLs. +# methods = GET HEAD PUT POST DELETE +# +# The headers to remove from incoming requests. Simply a whitespace delimited +# list of header names and names can optionally end with '*' to indicate a +# prefix match. incoming_allow_headers is a list of exceptions to these +# removals. +# incoming_remove_headers = x-timestamp +# +# The headers allowed as exceptions to incoming_remove_headers. Simply a +# whitespace delimited list of header names and names can optionally end with +# '*' to indicate a prefix match. +# incoming_allow_headers = +# +# The headers to remove from outgoing responses. Simply a whitespace delimited +# list of header names and names can optionally end with '*' to indicate a +# prefix match. outgoing_allow_headers is a list of exceptions to these +# removals. +# outgoing_remove_headers = x-object-meta-* +# +# The headers allowed as exceptions to outgoing_remove_headers. Simply a +# whitespace delimited list of header names and names can optionally end with +# '*' to indicate a prefix match. +# outgoing_allow_headers = x-object-meta-public-* + +# Note: Put formpost just before your auth filter(s) in the pipeline +[filter:formpost] +use = egg:swift#formpost + +# Note: Just needs to be placed before the proxy-server in the pipeline. +[filter:name_check] +use = egg:swift#name_check +# forbidden_chars = '"`<> +# maximum_length = 255 +# forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$ + +[filter:list-endpoints] +use = egg:swift#list_endpoints +# list_endpoints_path = /endpoints/ + +[filter:proxy-logging] +use = egg:swift#proxy_logging +# If not set, logging directives from [DEFAULT] without "access_" will be used +# access_log_name = swift +# access_log_facility = LOG_LOCAL0 +# access_log_level = INFO +# access_log_address = /dev/log +# +# If set, access_log_udp_host will override access_log_address +# access_log_udp_host = +# access_log_udp_port = 514 +# +# You can use log_statsd_* from [DEFAULT] or override them here: +# access_log_statsd_host = localhost +# access_log_statsd_port = 8125 +# access_log_statsd_default_sample_rate = 1.0 +# access_log_statsd_sample_rate_factor = 1.0 +# access_log_statsd_metric_prefix = +# access_log_headers = false +# +# If access_log_headers is True and access_log_headers_only is set only +# these headers are logged. Multiple headers can be defined as comma separated +# list like this: access_log_headers_only = Host, X-Object-Meta-Mtime +# access_log_headers_only = +# +# By default, the X-Auth-Token is logged. To obscure the value, +# set reveal_sensitive_prefix to the number of characters to log. +# For example, if set to 12, only the first 12 characters of the +# token appear in the log. An unauthorized access of the log file +# won't allow unauthorized usage of the token. However, the first +# 12 or so characters is unique enough that you can trace/debug +# token usage. Set to 0 to suppress the token completely (replaced +# by '...' in the log). +# Note: reveal_sensitive_prefix will not affect the value +# logged with access_log_headers=True. +# reveal_sensitive_prefix = 16 +# +# What HTTP methods are allowed for StatsD logging (comma-sep); request methods +# not in this list will have "BAD_METHOD" for the portion of the metric. +# log_statsd_valid_http_methods = GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS +# +# Note: The double proxy-logging in the pipeline is not a mistake. The +# left-most proxy-logging is there to log requests that were handled in +# middleware and never made it through to the right-most middleware (and +# proxy server). Double logging is prevented for normal requests. See +# proxy-logging docs. + +# Note: Put before both ratelimit and auth in the pipeline. +[filter:bulk] +use = egg:swift#bulk +# max_containers_per_extraction = 10000 +# max_failed_extractions = 1000 +# max_deletes_per_request = 10000 +# max_failed_deletes = 1000 + +# In order to keep a connection active during a potentially long bulk request, +# Swift may return whitespace prepended to the actual response body. This +# whitespace will be yielded no more than every yield_frequency seconds. +# yield_frequency = 10 + +# Note: The following parameter is used during a bulk delete of objects and +# their container. This would frequently fail because it is very likely +# that all replicated objects have not been deleted by the time the middleware got a +# successful response. It can be configured the number of retries. And the +# number of seconds to wait between each retry will be 1.5**retry + +# delete_container_retry_count = 0 + +# Note: Put after auth in the pipeline. +[filter:container-quotas] +use = egg:swift#container_quotas + +# Note: Put after auth and staticweb in the pipeline. +[filter:slo] +use = egg:swift#slo +# max_manifest_segments = 1000 +# max_manifest_size = 2097152 +# min_segment_size = 1048576 +# Start rate-limiting SLO segment serving after the Nth segment of a +# segmented object. +# rate_limit_after_segment = 10 +# +# Once segment rate-limiting kicks in for an object, limit segments served +# to N per second. 0 means no rate-limiting. +# rate_limit_segments_per_sec = 0 +# +# Time limit on GET requests (seconds) +# max_get_time = 86400 + +# Note: Put after auth and staticweb in the pipeline. +# If you don't put it in the pipeline, it will be inserted for you. +[filter:dlo] +use = egg:swift#dlo +# Start rate-limiting DLO segment serving after the Nth segment of a +# segmented object. +# rate_limit_after_segment = 10 +# +# Once segment rate-limiting kicks in for an object, limit segments served +# to N per second. 0 means no rate-limiting. +# rate_limit_segments_per_sec = 1 +# +# Time limit on GET requests (seconds) +# max_get_time = 86400 + +[filter:account-quotas] +use = egg:swift#account_quotas + +[filter:gatekeeper] +use = egg:swift#gatekeeper +# You can override the default log routing for this filter here: +# set log_name = gatekeeper +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log + +[filter:container_sync] +use = egg:swift#container_sync +# Set this to false if you want to disallow any full url values to be set for +# any new X-Container-Sync-To headers. This will keep any new full urls from +# coming in, but won't change any existing values already in the cluster. +# Updating those will have to be done manually, as knowing what the true realm +# endpoint should be cannot always be guessed. +# allow_full_urls = true +# Set this to specify this clusters //realm/cluster as "current" in /info +# current = //REALM/CLUSTER + +# Note: Put it at the beginning of the pipleline to profile all middleware. But +# it is safer to put this after catch_errors, gatekeeper and healthcheck. +[filter:xprofile] +use = egg:swift#xprofile +# This option enable you to switch profilers which should inherit from python +# standard profiler. Currently the supported value can be 'cProfile', +# 'eventlet.green.profile' etc. +# profile_module = eventlet.green.profile +# +# This prefix will be used to combine process ID and timestamp to name the +# profile data file. Make sure the executing user has permission to write +# into this path (missing path segments will be created, if necessary). +# If you enable profiling in more than one type of daemon, you must override +# it with an unique value like: /var/log/swift/profile/proxy.profile +# log_filename_prefix = /tmp/log/swift/profile/default.profile +# +# the profile data will be dumped to local disk based on above naming rule +# in this interval. +# dump_interval = 5.0 +# +# Be careful, this option will enable profiler to dump data into the file with +# time stamp which means there will be lots of files piled up in the directory. +# dump_timestamp = false +# +# This is the path of the URL to access the mini web UI. +# path = /__profile__ +# +# Clear the data when the wsgi server shutdown. +# flush_at_shutdown = false +# +# unwind the iterator of applications +# unwind = false diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py new file mode 100644 index 0000000000..d6a4536df9 --- /dev/null +++ b/scripts/swift_inventory.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python +from __future__ import print_function + +import datetime +import sys +import yaml + +from optparse import OptionParser +from os.path import exists, isdir, join + +VERSION = '0.1' +USAGE = "usage: %prog [options] -s " + +DEFAULT_PART_POWER = 8 +DEFAULT_REPL_NUM = 3 +DEFAULT_REGION = 0 +DEFAULT_ZONE = 0 +DEFAULT_WEIGHT = 100 +DEFAULT_DRIVE = "/srv/disk" +DEFAULT_OUTPUT_DIR = "/etc/ansible" +DEFAULT_OUTPUT_FILENAME = "hosts" + +RETURN_NOT_DEFINED = 3 + +# FILE formatted strings +HEADER = """# This file was generated using the %s version %s at %s +[local] +localhost ansible_connection=local + +[proxy]""" + +CATCH_ALL_GROUPS = """ +[object:children] +storagepolocy + +[swift:children] +proxy +account +container +object + +[swift:vars]""" + +DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s " +DRIVE_FORMAT += "weight=%(weight)s" + +DEFAULT_AUTHTOKEN_SETTINGS = { + 'auth_version': 'v2.0', + 'auth_host': 'keystone', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'admin_tenant_name': 'service', + 'admin_user': 'swift', + 'admin_password': 'ADMIN', +} + + +def main(setup, verbose=False, dry_run=False, overwrite=True): + # Parse the setup file, which should be yaml + _swift = {} + _drives = {} + try: + with open(setup) as yaml_stream: + _swift = yaml.load(yaml_stream) + except Exception as err: + print("ERROR: Failed to yaml failure: %s", err) + return 2 + + def _section_defined(section): + if section not in _swift: + print("ERROR: no swift section defined") + return False + return True + + def _get_output_fd(filename): + if dry_run: + return None + elif not overwrite and exists(filename): + i = 1 + while exists("%s_%d" % (filename, i)): + i += 1 + return open("%s_%d" % (filename, i), 'w') + else: + return open(filename, 'w') + + def _write_to_file(fd, data): + if not fd or verbose: + print(data) + + if fd: + if not data.endswith('\n'): + data += "\n" + fd.write(data) + fd.flush() + + def _get_drive(drive): + _drive = { + 'drive': DEFAULT_DRIVE, + 'region': DEFAULT_REGION, + 'zone': DEFAULT_ZONE, + 'weight': DEFAULT_WEIGHT} + + if "drive" not in drive: + drive["drive"] = DEFAULT_DRIVE + + key = "%(host)s%(drive)s" % drive + if key in _drives: + return _drives[key] + else: + _drive.update(drive) + data = DRIVE_FORMAT % _drive + _drives[key] = data + return data + + # First attempt to get swift settings + if not _section_defined("swift"): + return RETURN_NOT_DEFINED + + swift_options = [ + "part_power=%s" % (_swift['swift'].get('part_power', + DEFAULT_PART_POWER)), + "user=%s" % (_swift['swift'].get('user', 'swift')), + "swift_hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")), + "swift_hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")), + "syslog_host=%s" % (_swift['swift'].get('syslog_host', + 'localhost:514')), + ] + output_path = _swift['swift'].get("output_directory", DEFAULT_OUTPUT_DIR) + output_file = _swift['swift'].get("output_filename", + DEFAULT_OUTPUT_FILENAME) + if not isdir(output_path): + print("Outdir path '%s' doesn't exist", output_path) + return 4 + + output_file = join(output_path, output_file) + output_fd = _get_output_fd(output_file) + + n = datetime.datetime.now() + _write_to_file(output_fd, HEADER % (__file__, VERSION, n.ctime())) + + if not _section_defined("proxy"): + return RETURN_NOT_DEFINED + + # Parse proxies + # TODO: Add read anfinity and pipeline here? + for proxy in _swift["proxy"]["hosts"]: + _write_to_file(output_fd, "%s" % (proxy["host"])) + _write_to_file(output_fd, "\n[proxy:vars]") + _mc_servers = _swift["proxy"].get('memcache_servers') + memcache_servers = ",".join(_mc_servers) if _mc_servers else \ + '127.0.0.1:11211' + _write_to_file(output_fd, "memcache_servers=%s" % (memcache_servers)) + _at = _swift["proxy"].get('authtoken') + if _at: + authtoken = DEFAULT_AUTHTOKEN_SETTINGS + authtoken.update(_at) + at_active = authtoken.get("active", False) + if at_active: + _write_to_file(output_fd, "authtoken_active=true") + _write_to_file(output_fd, "delay_auth_decision=" + "%(delay_auth_decision)s" % authtoken) + _write_to_file(output_fd, "auth_version=" + "%(auth_version)s" % authtoken) + _write_to_file(output_fd, "auth_host=" + "%(auth_host)s" % authtoken) + _write_to_file(output_fd, "auth_port=" + "%(auth_port)s" % authtoken) + _write_to_file(output_fd, "auth_protocol=" + "%(auth_protocol)s" % authtoken) + _write_to_file(output_fd, "auth_uri=" + "%(auth_uri)s" % authtoken) + _write_to_file(output_fd, "admin_tenant_name=" + "%(admin_tenant_name)s" % authtoken) + _write_to_file(output_fd, "admin_user=" + "%(admin_user)s" % authtoken) + _write_to_file(output_fd, "admin_password=" + "%(admin_password)s" % authtoken) + else: + _write_to_file(output_fd, "authtoken_active=false") + + _write_to_file(output_fd, "\n[account]") + + if not _section_defined("account"): + return RETURN_NOT_DEFINED + + for account in _swift["account"]["hosts"]: + data = _get_drive(account) + _write_to_file(output_fd, data) + + _write_to_file(output_fd, "\n[account:vars]") + repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM) + _write_to_file(output_fd, "repl_number=%d" % (repl_num)) + + # Container section + _write_to_file(output_fd, "\n[container]") + + if not _section_defined("container"): + return RETURN_NOT_DEFINED + + for container in _swift["container"]["hosts"]: + data = _get_drive(container) + _write_to_file(output_fd, data) + + _write_to_file(output_fd, "\n[container:vars]") + repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM) + _write_to_file(output_fd, "repl_number=%d" % (repl_num)) + + # Objects / Storage polices + _storage_policies = {} + _storage_policies_idx = {} + if not _section_defined("storage_policies"): + return RETURN_NOT_DEFINED + + if "policies" not in _swift["storage_policies"]: + print("ERROR: No storage policies defined") + return 4 + + for policy in _swift["storage_policies"]["policies"]: + if policy["name"] in _storage_policies: + print("ERROR: Storage policy '%s' already defined" % policy["name"]) + return 5 + + if policy["index"] in _storage_policies_idx: + print("ERROR: Storage policy index '%s' already defined" % + policy["index"]) + return 5 + + _storage_policies[policy['name']] = "storagepolicy_%(name)s" % policy + _storage_policies_idx[policy['index']] = policy["name"] + + _write_to_file(output_fd, + "\n[%s]" % (_storage_policies[policy['name']])) + + # print the storage policy hosts. + for drive in policy.get("hosts", []): + data = _get_drive(drive) + _write_to_file(output_fd, data) + + _write_to_file(output_fd, + "\n[%s:vars]" % (_storage_policies[policy['name']])) + _write_to_file(output_fd, "index=%d" % (policy['index'])) + _write_to_file(output_fd, "policy_name=%s" % (policy['name'])) + policy_type = policy.get("type", 'replication') + _write_to_file(output_fd, "type=%s" % (policy_type)) + + depricated = policy.get("depricated", False) + if depricated: + _write_to_file(output_fd, "depricated=True") + + default = policy.get("default", False) + if default: + _write_to_file(output_fd, "default=True") + + if policy_type == 'replication': + repl_num = policy.get("repl_number", DEFAULT_REPL_NUM) + _write_to_file(output_fd, "repl_num=%d" % (repl_num)) + + # Write out the storage policy catch all group + _write_to_file(output_fd, "\n[storagepolicy:children]") + for name, longname in _storage_policies.items(): + _write_to_file(output_fd, "%s" % (longname)) + + _write_to_file(output_fd, "\n[storagepolicy:vars]") + if 'default' in _swift["storage_policies"]: + default_sp = _swift["storage_policies"]["default"] + if default_sp in _storage_policies: + _write_to_file(output_fd, "default=%s" % (default_sp)) + elif default_sp in _storage_policies_idx: + _write_to_file(output_fd, + "default=%s" % (_storage_policies_idx[default_sp])) + else: + print("ERROR: Default storage policy '%s' doesn't exist", + default_sp) + + # Write out the object and swift catchall groups + _write_to_file(output_fd, CATCH_ALL_GROUPS) + + # Now write out the global swift options that is gathered in the file + for option in swift_options: + _write_to_file(output_fd, option) + + # Done + if output_fd: + output_fd.flush() + output_fd.close() + return 0 + +if __name__ == "__main__": + parser = OptionParser(USAGE) + parser.add_option("-s", "--setup", dest="setup", + help="Specify the swift setup file.", metavar="FILE") + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + default=False, help="Be more verbose") + parser.add_option("-d", "--dryrun", action="store_true", dest="dry_run", + default=False, help="Print result out to stdout.") + parser.add_option("-C", "--copy", action="store_false", dest="overwrite", + default=True, help="Make a copy if inventory file exists") + parser.add_option("-i", "--import", dest="ring_folder", metavar="FILE", + help="Attempt to build a swift setup file" + " from the Swift builder files. Pass directory here") + + options, args = parser.parse_args(sys.argv[1:]) + if not options.setup or not exists(options.setup): + print("Swift setup file not found or doesn't exist") + parser.print_help() + sys.exit(1) + + sys.exit(main(options.setup, options.verbose, options.dry_run, + options.overwrite)) From f3864a3c8f1bcc7290a8fe5466045ceb709d97be Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Thu, 18 Sep 2014 17:56:05 +1000 Subject: [PATCH 02/45] Debugging the generation of storage policy definitions. I have been debugging the hosts file being generated, doing so I found a few bugs in the swift_inventory script. --- .../swift_common/templates/swift.conf.j2 | 20 +++++++++---------- scripts/swift_inventory.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 index 61e7a74ed6..cae360565c 100644 --- a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 +++ b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 @@ -10,17 +10,17 @@ swift_hash_path_prefix = {{ hash_path_prefix }} # Storage Policies {% for group in groups %} - {% if group.startswith("storagepolicy") %} - {% vars = hostvars[groups[group].pop] %} +{% if group.startswith("storagepolicy_") %} +{% set vars = hostvars[groups[group].pop()] %} [storage-policy:{{ vars["index"] }}] -name = {{ vars["policy_name"] }} - {% if "depricated" in vars %} -depricated = {{ var["depricated"] }} - {% endif %} - {% if "default" in vars %} -default = {{ var["default"] }} - {% endif %} - {% endif %} +name = {{ vars["policy_name"] }} +{% if "depricated" in vars %} +depricated = {{ vars["depricated"] }} +{% endif %} +{% if "default" in vars and vars["default"] == vars["policy_name"] %} +default = Yes +{% endif %} +{% endif %} {% endfor %} [swift-constraints] diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py index d6a4536df9..dc948e2541 100644 --- a/scripts/swift_inventory.py +++ b/scripts/swift_inventory.py @@ -31,7 +31,7 @@ localhost ansible_connection=local CATCH_ALL_GROUPS = """ [object:children] -storagepolocy +storagepolicy [swift:children] proxy @@ -120,8 +120,8 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): "part_power=%s" % (_swift['swift'].get('part_power', DEFAULT_PART_POWER)), "user=%s" % (_swift['swift'].get('user', 'swift')), - "swift_hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")), - "swift_hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")), + "hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")), + "hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")), "syslog_host=%s" % (_swift['swift'].get('syslog_host', 'localhost:514')), ] From aeed044c80bcf4025dcdd6d845f7f123ee402cb9 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Fri, 19 Sep 2014 15:31:46 +1000 Subject: [PATCH 03/45] Swift_inventory script now concats policy data The swift_inventory script now combines the required variables to generate the storage policy definitions into a concatenated string and stored with the host. extra_data=::: E.G: extra_data=gold:0:True:Replication If the host is a member of more then one storage group then these entries are concatenated together with a ';': extra_data=::..:;:.... The swift.conf file template now takes this into account and is generated correctly. --- .../swift_common/templates/swift.conf.j2 | 27 ++++--- scripts/swift_inventory.py | 73 +++++++++++++------ 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 index cae360565c..730ba10890 100644 --- a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 +++ b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 @@ -10,19 +10,24 @@ swift_hash_path_prefix = {{ hash_path_prefix }} # Storage Policies {% for group in groups %} -{% if group.startswith("storagepolicy_") %} -{% set vars = hostvars[groups[group].pop()] %} -[storage-policy:{{ vars["index"] }}] -name = {{ vars["policy_name"] }} -{% if "depricated" in vars %} -depricated = {{ vars["depricated"] }} -{% endif %} -{% if "default" in vars and vars["default"] == vars["policy_name"] %} +{% if group.startswith("storagepolicy_") %} +{% set vars = hostvars[groups[group].pop()] %} +{% for proxy_settings in vars["extra_data"].split(';') %} +{% set items = proxy_settings.split(':') %} +{% if group.endswith(items[0]) %} +[storage-policy:{{ items[1] }}] +name = {{ items[0] }} +{% if items[2] == 'True' %} +depricated = Yes +{% endif %} +{% if items[0] == vars["default"] %} default = Yes -{% endif %} -{% endif %} -{% endfor %} +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} [swift-constraints] # max_file_size is the largest "normal" object that can be saved in diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py index dc948e2541..bb7ab31021 100644 --- a/scripts/swift_inventory.py +++ b/scripts/swift_inventory.py @@ -42,7 +42,7 @@ object [swift:vars]""" DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s " -DRIVE_FORMAT += "weight=%(weight)s" +DRIVE_FORMAT += "weight=%(weight)s extra_data=%(extra_data)s" DEFAULT_AUTHTOKEN_SETTINGS = { 'auth_version': 'v2.0', @@ -54,6 +54,18 @@ DEFAULT_AUTHTOKEN_SETTINGS = { 'admin_password': 'ADMIN', } +DATA_ORDER = [ + NAME, + INDEX, + DEPRICATED, + TYPE, +] = [ + 'name', + 'index', + 'depricated', + 'type', +] + def main(setup, verbose=False, dry_run=False, overwrite=True): # Parse the setup file, which should be yaml @@ -93,24 +105,35 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): fd.write(data) fd.flush() - def _get_drive(drive): + def _get_drive(drive, extra_data={}): _drive = { 'drive': DEFAULT_DRIVE, 'region': DEFAULT_REGION, 'zone': DEFAULT_ZONE, - 'weight': DEFAULT_WEIGHT} + 'weight': DEFAULT_WEIGHT, + 'extra_data': ''} if "drive" not in drive: drive["drive"] = DEFAULT_DRIVE + if extra_data: + data_str = ":".join([str(extra_data.get(i, '')) for i in + DATA_ORDER]) + else: + data_str = "" + key = "%(host)s%(drive)s" % drive if key in _drives: + if not _drives[key]['extra_data'] and extra_data: + _drives[key]['extra_data'] = data_str + elif _drives[key]['extra_data'] and extra_data: + _drives[key]['extra_data'] += ";%s" % (data_str) return _drives[key] else: _drive.update(drive) - data = DRIVE_FORMAT % _drive - _drives[key] = data - return data + _drive['extra_data'] = data_str + _drives[key] = _drive + return _drive # First attempt to get swift settings if not _section_defined("swift"): @@ -185,7 +208,7 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): for account in _swift["account"]["hosts"]: data = _get_drive(account) - _write_to_file(output_fd, data) + _write_to_file(output_fd, DRIVE_FORMAT % data) _write_to_file(output_fd, "\n[account:vars]") repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM) @@ -199,7 +222,7 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): for container in _swift["container"]["hosts"]: data = _get_drive(container) - _write_to_file(output_fd, data) + _write_to_file(output_fd, DRIVE_FORMAT % data) _write_to_file(output_fd, "\n[container:vars]") repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM) @@ -215,6 +238,7 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): print("ERROR: No storage policies defined") return 4 + policy_hosts = {} for policy in _swift["storage_policies"]["policies"]: if policy["name"] in _storage_policies: print("ERROR: Storage policy '%s' already defined" % policy["name"]) @@ -228,33 +252,36 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): _storage_policies[policy['name']] = "storagepolicy_%(name)s" % policy _storage_policies_idx[policy['index']] = policy["name"] - _write_to_file(output_fd, - "\n[%s]" % (_storage_policies[policy['name']])) - + policy_type = policy.get("type", 'replication') + _host_data = { + NAME: policy['name'], + INDEX: policy['index'], + TYPE: policy_type, + DEPRICATED: policy.get("depricated", False), + } # print the storage policy hosts. for drive in policy.get("hosts", []): - data = _get_drive(drive) - _write_to_file(output_fd, data) + _get_drive(drive, _host_data) + policy_hosts[policy['name']] = policy.get("hosts") _write_to_file(output_fd, "\n[%s:vars]" % (_storage_policies[policy['name']])) - _write_to_file(output_fd, "index=%d" % (policy['index'])) - _write_to_file(output_fd, "policy_name=%s" % (policy['name'])) - policy_type = policy.get("type", 'replication') - _write_to_file(output_fd, "type=%s" % (policy_type)) - - depricated = policy.get("depricated", False) - if depricated: - _write_to_file(output_fd, "depricated=True") - default = policy.get("default", False) if default: - _write_to_file(output_fd, "default=True") + _write_to_file(output_fd, "default=%s" % (policy['name'])) if policy_type == 'replication': repl_num = policy.get("repl_number", DEFAULT_REPL_NUM) _write_to_file(output_fd, "repl_num=%d" % (repl_num)) + # now write out the drives. + for policy in _swift["storage_policies"]["policies"]: + _write_to_file(output_fd, + "\n[%s]" % (_storage_policies[policy['name']])) + for drive in policy_hosts[policy['name']]: + data = _get_drive(drive) + _write_to_file(output_fd, DRIVE_FORMAT % data) + # Write out the storage policy catch all group _write_to_file(output_fd, "\n[storagepolicy:children]") for name, longname in _storage_policies.items(): From ae579b691e717ee08b8adaad87306a614cca2c19 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Tue, 23 Sep 2014 16:54:17 +1000 Subject: [PATCH 04/45] Place swift.conf The swift_common role now places the swift.conf file in /etc/swfit. The file is generated from a template. --- rpc_deployment/roles/swift_common/tasks/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index fc4fddf415..5e517806d6 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -22,6 +22,9 @@ - Install swift dependencies - Install swift +- name: "Setup swift.conf" + template: src=swift.conf.j2 dest=/etc/swift/swift.conf owner={{ swift }} mode=0644 + - name: "Swift directory" file: path=/etc/swift owner={{ user }} group={{ user }} state=directory @@ -30,5 +33,3 @@ - name: "Swift log directory" file: path=/var/log/swift owner={{ user }} group=adm state=directory - - From 474be4ca4f67d2404d3bd89f0c54657764233d0c Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 24 Sep 2014 16:22:44 +1000 Subject: [PATCH 05/45] First version of a ring builder This change actually does quite a bit. - Removes duplication in the swift yaml file, meaning you can now add hosts directly to a global hosts section, which then can be refenced by name. You can still define hosts in the other hosts areas, but this stops duplication. - Adds a bunch of extra settings to the swift yaml file to support ring building, including: * min_part_hours * port (for account, container and object (storage policies)) * drives * repl_ip (specify a replication IP to use, if it differes from main) * repl_port (a different replication port.. as above) - Passing the server ports and the correct drives value through to templates - scripts/swift_rings.py: This script reads in the swift.yaml file and generates all required rings. I have gone this approach as this why there is no annoying ansible getting in the way. The next step is to write a rings playbook that runs the swift_rings.py script locally and then copies all the rings to every memeber of the swfit group. Alternativelty we could pass the script and swift yaml to each node (via swfit_common role) and have a simple playbook that runs the script on each node to build the rings. One benefit of swfit_rings that reads in the yaml, is that we could actaully simplfy what actaully needs to be translated into an ansible inventory. (no ring specific settings). --- etc/rpc_deploy/swift_setup.yml | 59 ++++--- .../templates/account-server.conf.j2 | 4 +- .../templates/container-server.conf.j2 | 4 +- .../templates/object-server.conf.j2 | 4 +- scripts/swift_inventory.py | 48 ++++-- scripts/swift_rings.py | 158 ++++++++++++++++++ 6 files changed, 235 insertions(+), 42 deletions(-) create mode 100644 scripts/swift_rings.py diff --git a/etc/rpc_deploy/swift_setup.yml b/etc/rpc_deploy/swift_setup.yml index 106da684fc..1b90cf6ef9 100644 --- a/etc/rpc_deploy/swift_setup.yml +++ b/etc/rpc_deploy/swift_setup.yml @@ -19,6 +19,28 @@ swift: hash_path_prefix: changeme syslog_host: 10.1.1.1:514 +# Global hosts +# Hosts can be defined in the sections below or here, the advantage of +# defining them here is that they can be reused in multiple sections below. +# +# When need to reference a global host use the form: +# - name: / +hosts: + - host: 10.0.0.4 + drive: sda + drives: /srv/disk + region: 0 + zone: 0 + weight: 100 + repl_ip: 192.168.0.4 + repl_port: 55555 + - host: 10.0.0.5 + drive: sda + drives: /srv/disk + region: 0 + zone: 1 + weight: 100 + proxy: memcache_servers: - 127.0.0.1:11211 @@ -38,47 +60,44 @@ proxy: account: repl_number: 3 + min_part_hours: 1 + port: 6002 hosts: - host: 10.0.0.2 - drive: /srv/disk + drive: sdba region: 0 zone: 0 weight: 100 container: repl_number: 3 + port: 6001 hosts: - host: 10.0.0.3 - drive: /srv/disk + drive: sda region: 0 zone: 0 weight: 100 storage_policies: default: gold + port: 6000 policies: - name: gold index: 0 type: replication repl_number: 3 + min_part_hours: 1 hosts: - - host: 10.0.0.4 - drive: /srv/disk - region: 0 - zone: 0 - weight: 100 - - host: 10.0.0.5 - drive: /srv/disk - region: 0 - zone: 1 - weight: 100 + - name: 10.0.0.4/sda + - name: 10.0.0.5/sda - host: 10.0.0.6 - drive: /srv/disk + drive: sdb region: 1 zone: 0 weight: 50 - host: 10.0.0.7 - drive: /srv/disk + drive: sdb region: 1 zone: 1 weight: 50 @@ -88,13 +107,5 @@ storage_policies: repl_number: 2 depricated: True hosts: - - host: 10.0.0.4 - drive: /srv/disk - region: 0 - zone: 0 - weight: 100 - - host: 10.0.0.5 - drive: /srv/disk - region: 0 - zone: 1 - weight: 100 + - name: 10.0.0.4/sda + - name: 10.0.0.5/sda diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 index e675c56766..7c4ebd2bbc 100644 --- a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] bind_ip = {{ inventory_hostname }} -bind_port = 6002 +bind_port = {{ account_server_port }} # bind_timeout = 30 # backlog = 4096 user = {{ user }} # swift_dir = /etc/swift -devices = {{ drive }} +devices = {{ drives }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 index 55aed2808f..72242f9aad 100644 --- a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] bind_ip = {{ inventory_hostname }} -bind_port = 6001 +bind_port = {{ container_server_port }} # bind_timeout = 30 # backlog = 4096 user = {{ user }} # swift_dir = /etc/swift -devices = {{ drive }} +devices = {{ drives }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 index 96bb2bae85..9e28a7cce6 100644 --- a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] bind_ip = {{ inverntory_host }} -bind_port = 6000 +bind_port = {{ object_server_port }} # bind_timeout = 30 # backlog = 4096 user = {{ user }} swift_dir = /etc/swift -devices = {{ drive }} +devices = {{ drives }} # mount_check = true # disable_fallocate = false # expiring_objects_container_divisor = 86400 diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py index bb7ab31021..ef8ca3fcaa 100644 --- a/scripts/swift_inventory.py +++ b/scripts/swift_inventory.py @@ -16,7 +16,8 @@ DEFAULT_REPL_NUM = 3 DEFAULT_REGION = 0 DEFAULT_ZONE = 0 DEFAULT_WEIGHT = 100 -DEFAULT_DRIVE = "/srv/disk" +DEFAULT_DRIVE = "sda" +DEFAULT_DRIVES = "/srv/disk" DEFAULT_OUTPUT_DIR = "/etc/ansible" DEFAULT_OUTPUT_FILENAME = "hosts" @@ -42,7 +43,7 @@ object [swift:vars]""" DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s " -DRIVE_FORMAT += "weight=%(weight)s extra_data=%(extra_data)s" +DRIVE_FORMAT += "weight=%(weight)s drives=%(drives)s extra_data=%(extra_data)s" DEFAULT_AUTHTOKEN_SETTINGS = { 'auth_version': 'v2.0', @@ -67,6 +68,10 @@ DATA_ORDER = [ ] +class DriveException(Exception): + pass + + def main(setup, verbose=False, dry_run=False, overwrite=True): # Parse the setup file, which should be yaml _swift = {} @@ -106,23 +111,31 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): fd.flush() def _get_drive(drive, extra_data={}): - _drive = { - 'drive': DEFAULT_DRIVE, - 'region': DEFAULT_REGION, - 'zone': DEFAULT_ZONE, - 'weight': DEFAULT_WEIGHT, - 'extra_data': ''} + if 'name' in drive: + # We are dealing with a global host reference + key = drive['name'] + if key not in _drives: + raise DriveException("Drive referenced doesn't exist") + else: + # Not a referenced host. + _drive = { + 'drive': DEFAULT_DRIVE, + 'region': DEFAULT_REGION, + 'zone': DEFAULT_ZONE, + 'weight': DEFAULT_WEIGHT, + 'drives': DEFAULT_DRIVES, + 'extra_data': ''} - if "drive" not in drive: - drive["drive"] = DEFAULT_DRIVE + if "drive" not in drive: + drive["drive"] = DEFAULT_DRIVE + + key = "%(host)s/%(drive)s" % drive if extra_data: data_str = ":".join([str(extra_data.get(i, '')) for i in DATA_ORDER]) else: data_str = "" - - key = "%(host)s%(drive)s" % drive if key in _drives: if not _drives[key]['extra_data'] and extra_data: _drives[key]['extra_data'] = data_str @@ -161,6 +174,11 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): n = datetime.datetime.now() _write_to_file(output_fd, HEADER % (__file__, VERSION, n.ctime())) + # Global hosts + if _section_defined("hosts"): + for host in _swift["hosts"]: + _get_drive(host) + if not _section_defined("proxy"): return RETURN_NOT_DEFINED @@ -213,6 +231,8 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): _write_to_file(output_fd, "\n[account:vars]") repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM) _write_to_file(output_fd, "repl_number=%d" % (repl_num)) + port = _swift["account"].get("port", 6002) + _write_to_file(output_fd, "account_server_port=%d" % (port)) # Container section _write_to_file(output_fd, "\n[container]") @@ -227,6 +247,8 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): _write_to_file(output_fd, "\n[container:vars]") repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM) _write_to_file(output_fd, "repl_number=%d" % (repl_num)) + port = _swift["container"].get("port", 6001) + _write_to_file(output_fd, "container_server_port=%d" % (port)) # Objects / Storage polices _storage_policies = {} @@ -298,6 +320,8 @@ def main(setup, verbose=False, dry_run=False, overwrite=True): else: print("ERROR: Default storage policy '%s' doesn't exist", default_sp) + port = _swift["storage_policies"].get("port", 6000) + _write_to_file(output_fd, "object_server_port=%d" % (port)) # Write out the object and swift catchall groups _write_to_file(output_fd, CATCH_ALL_GROUPS) diff --git a/scripts/swift_rings.py b/scripts/swift_rings.py new file mode 100644 index 0000000000..3c32845ae1 --- /dev/null +++ b/scripts/swift_rings.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +from __future__ import print_function +from optparse import OptionParser +from os.path import exists +from swift.cli.ringbuilder import main as rb_main + +import sys +import threading +import yaml + +USAGE = "usage: %prog -s " + +DEFAULT_PART_POWER = 10 +DEFAULT_REPL = 3 +DEFAULT_MIN_PART_HOURS = 1 +DEFAULT_HOST_DRIVES = '/srv/drive/' +DEFAULT_HOST_DRIVE = '/sdb' +DEFAULT_HOST_ZONE = 0 +DEFAULT_HOST_WEIGHT = 1 +DEFAULT_ACCOUNT_PORT = 6002 +DEFAULT_CONTAINER_PORT = 6001 +DEFAULT_OBJECT_PORT = 6000 +DEFAULT_SECTION_PORT = { + 'account': DEFAULT_ACCOUNT_PORT, + 'container': DEFAULT_CONTAINER_PORT, + 'object': DEFAULT_OBJECT_PORT, +} + + +def create_buildfile(build_file, part_power, repl, min_part_hours): + run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", + part_power, repl, min_part_hours]) + + +def add_host_to_ring(build_file, host): + host_str = "" + if host.get('region') is not None: + host_str += 'r%(region)d' % host + host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) + host_str += "-%(host)s:%(port)d" % host + if host.get('repl_port'): + r_ip = host.get('repl_ip', host['host']) + host_str += "R%s:%d" % (r_ip, host['repl_port']) + host_str += "/%(drive)s" % host + + weight = host.get('weight', DEFAULT_HOST_WEIGHT) + run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', + host_str, str(weight)]) + + +def run_and_wait(func, *args): + t = threading.Thread(target=func, args=args) + t.start() + return t.join() + + +def has_section(conf, section): + return True if conf.get(section) else False + + +def check_section(conf, section): + if not has_section(conf, section): + print("Section %s doesn't exist" % (section)) + sys.exit(2) + + +def build_ring(section, conf, part_power, hosts): + # Create the build file + build_file = "%s.builder" % (section) + repl = conf.get('repl_number', DEFAULT_REPL) + min_part_hours = conf.get('min_part_hours', + DEFAULT_MIN_PART_HOURS) + create_buildfile(build_file, part_power, repl, min_part_hours) + + # Add the hosts + if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0: + print("No hosts/drives assigned to the %s ring" % section) + sys.exit(3) + + section_key = section.split('-')[0] + service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) + for host in conf['hosts']: + if 'name' in host: + if host['name'] not in hosts: + print("Host %(name) reference not found." % host) + sys.exit(3) + host = hosts[host['name']] + else: + if 'drive' not in host: + host['drive'] = DEFAULT_HOST_DRIVE + host['port'] = service_port + add_host_to_ring(build_file, host) + + # Rebalance ring + run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) + # rb_main(("swift-ring-builder", build_file, "rebalance")) + + +def main(setup): + # load the yaml file + try: + with open(setup) as yaml_stream: + _swift = yaml.load(yaml_stream) + except Exception as ex: + print("Failed to load yaml string %s" % (ex)) + return 1 + + _hosts = {} + + if _swift.get("hosts"): + for host in _swift['hosts']: + if not host.get('drive'): + host['drive'] = DEFAULT_HOST_DRIVE + key = "%(host)s/%(drive)s" % host + if key in _hosts: + print("%(host)s already definined" % host) + return 1 + _hosts[key] = host + + check_section(_swift, 'swift') + part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER) + + # Create account ring + check_section(_swift, 'account') + build_ring('account', _swift['account'], part_power, _hosts) + + # Create container ring + check_section(_swift, 'container') + build_ring('container', _swift['container'], part_power, _hosts) + + # Create object rings (storage policies) + check_section(_swift, 'storage_policies') + check_section(_swift['storage_policies'], 'policies') + indexes = set() + for sp in _swift['storage_policies']['policies']: + if sp['index'] in indexes: + print("Storage Policy index %d already in use" % (sp['index'])) + return 4 + buildfilename = 'object-%d' % (sp['index']) + indexes.add(sp['index']) + if 'port' not in sp: + sp['port'] = _swift['storage_policies'].get('port', + DEFAULT_OBJECT_PORT) + build_ring(buildfilename, sp, part_power, _hosts) + +if __name__ == "__main__": + parser = OptionParser(USAGE) + parser.add_option("-s", "--setup", dest="setup", + help="Specify the swift setup file.", metavar="FILE", + default="/etc/swift/swift_inventory.yml") + + options, args = parser.parse_args(sys.argv[1:]) + if options.setup and not exists(options.setup): + print("Swift setup file not found or doesn't exist") + parser.print_help() + sys.exit(1) + + sys.exit(main(options.setup)) From dd5b20efa698739b3b04f46057a577b2a2fe552e Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Thu, 25 Sep 2014 05:54:45 +0000 Subject: [PATCH 06/45] Added a build swift rings playbook I have written a playbook that builds the rings locally and copies them to all hosts in the swift group. In order to write it and test, I put together an ansible server and a bunch of nodes. The build script: - On local: * Runs the swift_common role (so swift source is downloaded and installed so the ringbuilder code exists). * Makes sure /etc/swift/rings exist (as a place to store the rings locally). * Runs the scripts/swift_rings.py script on the swift yaml to generate the keys. NOTE: These are hard coded locations atm (testing purposes). - Then copies the rings to /etc/swift on all the swift servers. NOTE: The swift_ringbuilder code (from swift) which we make use of automatically makes dated backups of the rings on the local ansible server (Whatever runs the code). As of this playbook these are in /etc/swift/rings/backup/ In the process of building and testing the playbook I also fixed up a bunch of errors in the swift_common role. And now seems to be working a treat. --- .../playbooks/build_swift_rings.yml | 23 +++++++++++++++++++ .../roles/swift_common/tasks/debian.yml | 2 +- .../roles/swift_common/tasks/main.yml | 14 ++++++----- .../roles/swift_common/tasks/redhat.yml | 2 +- .../roles/swift_common/vars/main.yml | 2 ++ scripts/swift_inventory.py | 1 + 6 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 rpc_deployment/playbooks/build_swift_rings.yml diff --git a/rpc_deployment/playbooks/build_swift_rings.yml b/rpc_deployment/playbooks/build_swift_rings.yml new file mode 100644 index 0000000000..0fa4f4806f --- /dev/null +++ b/rpc_deployment/playbooks/build_swift_rings.yml @@ -0,0 +1,23 @@ +--- +- hosts: local + user: root + roles: + - swift_common + +- hosts: local + user: root + tasks: + - name: "make sure rings directory exists" + file: state=directory path=/etc/swift/rings/ owner=root group=root mode=0755 + - name: "build rings" + command: /usr/bin/python /etc/ansible/scripts/swift_rings.py -s /etc/ansible/swift_setup.yml + args: + chdir: /etc/swift/rings/ + +- hosts: swift + user: root + tasks: + - name: "Make sure /etc/swift exists" + file: state=directory path=/etc/swift/ mode=0755 + - name: "Copy the rings over" + copy: src=/etc/swift/rings/ dest=/etc/swift/ mode=644 diff --git a/rpc_deployment/roles/swift_common/tasks/debian.yml b/rpc_deployment/roles/swift_common/tasks/debian.yml index 6f82ba68c9..524495564f 100644 --- a/rpc_deployment/roles/swift_common/tasks/debian.yml +++ b/rpc_deployment/roles/swift_common/tasks/debian.yml @@ -1,4 +1,4 @@ --- - name: "Install Debian/Ubuntu common swift packages" apt: pkg={{ item }} state=installed update_cache=yes - with_items: ${swift_common_packages.debian} + with_items: "{{ swift_common_packages.debian }}" diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 5e517806d6..ebd59d628c 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -12,19 +12,18 @@ - include: redhat.yml when: ansible_os_family == 'RedHat' -- name: "Create a swift user" +- name: "Create swift group" group: name={{ user }} state=present system=yes + +- name: "Create a swift user" user: name={{ user }} state=present shell=/bin/false home=/opt/swift system=yes - name: "Checkout git repo" - git: repo={{ swift_repo }} version={{ git_tag }} dest=/opt/swift/swift update=yes - notifiy: + git: repo={{ swift_repo }} version={{ swift_repo_tag }} dest=/opt/swift/swift update=yes + notify: - Install swift dependencies - Install swift -- name: "Setup swift.conf" - template: src=swift.conf.j2 dest=/etc/swift/swift.conf owner={{ swift }} mode=0644 - - name: "Swift directory" file: path=/etc/swift owner={{ user }} group={{ user }} state=directory @@ -33,3 +32,6 @@ - name: "Swift log directory" file: path=/var/log/swift owner={{ user }} group=adm state=directory + +- name: "Setup swift.conf" + template: src=swift.conf.j2 dest=/etc/swift/swift.conf owner={{ user }} mode=0644 diff --git a/rpc_deployment/roles/swift_common/tasks/redhat.yml b/rpc_deployment/roles/swift_common/tasks/redhat.yml index 50cb7998ed..bf9f8773e8 100644 --- a/rpc_deployment/roles/swift_common/tasks/redhat.yml +++ b/rpc_deployment/roles/swift_common/tasks/redhat.yml @@ -2,4 +2,4 @@ - name: "install Redhat/CentOS common swift packages" yum: pkg={{ item }} state=installed - with_items: ${swift_common_packages.redhat} + with_items: "{{ swift_common_packages.redhat }}" diff --git a/rpc_deployment/roles/swift_common/vars/main.yml b/rpc_deployment/roles/swift_common/vars/main.yml index 93f948e600..8075b6ed33 100644 --- a/rpc_deployment/roles/swift_common/vars/main.yml +++ b/rpc_deployment/roles/swift_common/vars/main.yml @@ -1,4 +1,5 @@ --- +user: swift swift_repo: https://github.com/openstack/swift.git swift_repo_tag: master @@ -12,6 +13,7 @@ swift_common_packages: - python-setuptools - python-dev - gcc + - libffi-dev redhat: - curl diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py index ef8ca3fcaa..aefbaea0d7 100644 --- a/scripts/swift_inventory.py +++ b/scripts/swift_inventory.py @@ -39,6 +39,7 @@ proxy account container object +local [swift:vars]""" From 71d5cb75ced44c2e827e3bd6d96c71b50ce5f745 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 25 Sep 2014 16:26:44 +0000 Subject: [PATCH 07/45] Adjust environment file to spin up swift-proxy containers * Add a swift_all group_vars file --- etc/rpc_deploy/rpc_environment.yml | 41 ++++++++ etc/rpc_deploy/rpc_user_config.yml | 2 +- etc/rpc_deploy/swift_setup.yml | 8 +- .../inventory/group_vars/swift_all.yml | 97 +++++++++++++++++++ 4 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 rpc_deployment/inventory/group_vars/swift_all.yml diff --git a/etc/rpc_deploy/rpc_environment.yml b/etc/rpc_deploy/rpc_environment.yml index 8c27fb6230..76bb97612c 100644 --- a/etc/rpc_deploy/rpc_environment.yml +++ b/etc/rpc_deploy/rpc_environment.yml @@ -116,6 +116,18 @@ component_skel: utility: belongs_to: - utility_all + swift_proxy: + belongs_to: + - swift_all + swift_account: + belongs_to: + - swift_all + swift_object: + belongs_to: + - swift_all + swift_container: + belongs_to: + - swift_all container_skel: cinder_api_container: belongs_to: @@ -257,6 +269,29 @@ container_skel: - infra_containers contains: - utility + swift_proxy_container: + belongs_to: + - infra_containers + contains: + - swift_proxy + swift_account_container: + is_metal: true + belongs_to: + - swift_containers + contains: + - swift_account + swift_object_container: + is_metal: true + belongs_to: + - swift_containers + contains: + - swift_object + swift_container_container: + is_metal: true + belongs_to: + - swift_containers + contains: + - swift_container physical_skel: network_containers: belongs_to: @@ -288,3 +323,9 @@ physical_skel: storage_hosts: belongs_to: - hosts + swift_containers: + belongs_to: + - all_containers + swift_hosts: + belongs_to: + - hosts diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index 068717811b..43ca5da136 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -15,7 +15,7 @@ # This is the md5 of the environment file # this will ensure consistency when deploying. -environment_version: e0955a92a761d5845520a82dcca596af +environment_version: 16faeb1250d4119c507f18f3f74e0653 # User defined CIDR used for containers # Global cidr/s used for everything. diff --git a/etc/rpc_deploy/swift_setup.yml b/etc/rpc_deploy/swift_setup.yml index 1b90cf6ef9..4ac4ead48c 100644 --- a/etc/rpc_deploy/swift_setup.yml +++ b/etc/rpc_deploy/swift_setup.yml @@ -43,20 +43,20 @@ hosts: proxy: memcache_servers: - - 127.0.0.1:11211 + - 192.168.100.7:11211 authtoken: active: true delay_auth_decision: true auth_version: v2.0 - auth_host: keystone.local.lan + auth_host: 192.168.100.7 auth_port: 35357 auth_protocol: https - auth_uri: http://keystonehost:5000/ + auth_uri: http://192.168.100.7:5000/ admin_tenant_name: service admin_user: swift admin_password: ADMIN hosts: - - host: 10.0.0.1 + - host: account: repl_number: 3 diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml new file mode 100644 index 0000000000..3021859fa9 --- /dev/null +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -0,0 +1,97 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The variables file used by the playbooks in the Cinder-api group. +# These don't have to be explicitly imported by vars_files: they are autopopulated. + +## Service Name +service_name: swift + +# Verbosity Options +debug: False +verbose: True + +# only used when the lxc vg is present on the target +container_lvm_fstype: ext4 +container_lvm_fssize: 5GB + +## General configuration +## Set this in rpc_user_config.yml UNLESS you want all hosts to use the same +## Cinder backends. See the rpc_user_config example for more on how this is done. +# cinder_backends: +# lvm: +# volume_group: cinder-volumes +# driver: cinder.volume.drivers.lvm.LVMISCSIDriver +# backend_name: LVM_iSCSI + +swift_proxy_port: "{{ swift_proxy_port|default('8080') }}" +swift_object_port: "{{ swift_object_port|default('6000') }}" +swift_container_port: "{{ swift_container_port|default('6001') }}" +swift_account_port: "{{ swift_account_port|default('6002') }}" + +## DB +container_mysql_user: swift +container_mysql_password: "{{ swift_container_mysql_password }}" +container_database: swift + +## Cinder Auth +service_admin_tenant_name: "service" +service_admin_username: "swiftr" +service_admin_password: "{{ swift_service_password }}" + +## Cinder User / Group +system_user: swift +system_group: swift + +## Service Names +service_names: + - swift-object + - swift-account + - swift-container + - swift-proxy + +## Git Source +## ALL of this has been relocated to vars/repo_packages +## TODO(someone) this should be removed once the repo bits are all figured out. +git_repo: https://git.openstack.org/openstack/swift +git_fallback_repo: https://github.com/openstack/swift +git_etc_example: etc/ +git_install_branch: b223322ed1ef44f61490f820240aa01f1047ae2e + +service_pip_dependencies: + - pywbem + - ecdsa + - MySQL-python + - python-memcached + - pycrypto + - python-cinderclient + - python-keystoneclient + - keystonemiddleware + +container_directories: + - /var/log/swift + - /tmp/swift + - /var/lock/swift + - /etc/swift + +container_packages: + - curl + - python-pip + - rsync + - openssh-server + - git-core + - python-setuptools + - python-dev + - gcc From d9c81a3287529e53ecb4e04dda3b5009ba88b3e9 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 25 Sep 2014 16:28:38 +0000 Subject: [PATCH 08/45] Fix up the swift-proxy runs * Fix some typoes * Add haproxy for swift-proxy * Add swift_vars * Remove redundant tasks that are handled by container_common Conflicts: rpc_deployment/roles/swift_common/tasks/main.yml --- etc/rpc_deploy/user_variables.yml | 3 ++ .../inventory/group_vars/swift_all.yml | 15 ++++--- .../roles/swift_account/tasks/main.yml | 2 +- .../roles/swift_common/tasks/main.yml | 40 +++++++++---------- .../roles/swift_container/tasks/main.yml | 2 +- .../roles/swift_object/tasks/main.yml | 2 +- .../roles/swift_proxy/handlers/main.yml | 16 +++++++- .../roles/swift_proxy/tasks/main.yml | 33 +++++++++++---- .../templates/proxy-server.conf.j2 | 17 ++++---- .../vars/config_vars/haproxy_config.yml | 8 ++++ 10 files changed, 91 insertions(+), 47 deletions(-) diff --git a/etc/rpc_deploy/user_variables.yml b/etc/rpc_deploy/user_variables.yml index 94642f34c9..12fe3aac5d 100644 --- a/etc/rpc_deploy/user_variables.yml +++ b/etc/rpc_deploy/user_variables.yml @@ -145,3 +145,6 @@ rpc_support_holland_password: ## Kibana Options kibana_password: +# Swift Options: +swift_service_password: + diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 3021859fa9..b39689e513 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -16,6 +16,11 @@ # The variables file used by the playbooks in the Cinder-api group. # These don't have to be explicitly imported by vars_files: they are autopopulated. +####### TODO: ADDING BOGUS VARS THAT SHOULD BE CONFIGURABLE VIA RPC_USER_CONFIG + +authtoken_active: True +delay_auth_decision: true + ## Service Name service_name: swift @@ -36,10 +41,10 @@ container_lvm_fssize: 5GB # driver: cinder.volume.drivers.lvm.LVMISCSIDriver # backend_name: LVM_iSCSI -swift_proxy_port: "{{ swift_proxy_port|default('8080') }}" -swift_object_port: "{{ swift_object_port|default('6000') }}" -swift_container_port: "{{ swift_container_port|default('6001') }}" -swift_account_port: "{{ swift_account_port|default('6002') }}" +swift_proxy_port: "8000" +swift_object_port: "6000" +swift_container_port: "6001" +swift_account_port: "6002" ## DB container_mysql_user: swift @@ -48,7 +53,7 @@ container_database: swift ## Cinder Auth service_admin_tenant_name: "service" -service_admin_username: "swiftr" +service_admin_username: "swift" service_admin_password: "{{ swift_service_password }}" ## Cinder User / Group diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml index 6517d723dc..a3da52cc23 100644 --- a/rpc_deployment/roles/swift_account/tasks/main.yml +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: "swift account server configuration" - template: src=account-server.conf.j2 dest=/etc/swfit/account-server.conf.j2 owner={{ swift }} mode=0644 + template: src=account-server.conf.j2 dest=/etc/swift/account-server.conf.j2 owner={{ swift }} mode=0644 notfiy: - (re)start account server - (re)start account auditor diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index ebd59d628c..3dab6fbeb9 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -1,16 +1,12 @@ --- -- name: "Setup hosts file" - lineinfile: dest=/etc/hosts regexp='.*{{ item }}$' line="{{ hostvars[item].ansible_default_ipv4.address }} {{item}}" state=present - when: hostvars[item].ansible_default_ipv4.address is defined - with_items: groups['all'] +#- name: "Setup hosts file" +# lineinfile: dest=/etc/hosts regexp='.*{{ item }}$' line="{{ hostvars[item].ansible_default_ipv4.address }} {{item}}" state=present +# when: hostvars[item].ansible_default_ipv4.address is defined +# with_items: groups['all'] -# If target is a debian/ubuntu system -- include: debian.yml - when: ansible_os_family == 'Debian' - -# If target is a centos/redhat system -- include: redhat.yml - when: ansible_os_family == 'RedHat' +#- name: "Install Debian/Ubuntu common swift packages" +# apt: pkg={{ item }} state=installed update_cache=yes +# with_items: ${container_packages} - name: "Create swift group" group: name={{ user }} state=present system=yes @@ -18,20 +14,20 @@ - name: "Create a swift user" user: name={{ user }} state=present shell=/bin/false home=/opt/swift system=yes -- name: "Checkout git repo" - git: repo={{ swift_repo }} version={{ swift_repo_tag }} dest=/opt/swift/swift update=yes - notify: - - Install swift dependencies - - Install swift +#- name: "Checkout git repo" +# git: repo={{ swift_repo }} version={{ git_tag }} dest=/opt/swift/swift update=yes +# notifiy: +# - Install swift dependencies +# - Install swift -- name: "Swift directory" - file: path=/etc/swift owner={{ user }} group={{ user }} state=directory +#- name: "Swift directory" +# file: path=/etc/swift owner={{ user }} group={{ user }} state=directory -- name: "Swift lock directory" - file: path=/var/lock/swift owner={{ user }} group={{ user }} state=directory +#- name: "Swift lock directory" +# file: path=/var/lock/swift owner={{ user }} group={{ user }} state=directory -- name: "Swift log directory" - file: path=/var/log/swift owner={{ user }} group=adm state=directory +#- name: "Swift log directory" +# file: path=/var/log/swift owner={{ user }} group=adm state=directory - name: "Setup swift.conf" template: src=swift.conf.j2 dest=/etc/swift/swift.conf owner={{ user }} mode=0644 diff --git a/rpc_deployment/roles/swift_container/tasks/main.yml b/rpc_deployment/roles/swift_container/tasks/main.yml index 4efb75a794..e3794f267e 100644 --- a/rpc_deployment/roles/swift_container/tasks/main.yml +++ b/rpc_deployment/roles/swift_container/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: "swift container server configuration" - template: src=container-server.conf.j2 dest=/etc/swfit/container-server.conf.j2 owner={{ swift }} mode=0644 + template: src=container-server.conf.j2 dest=/etc/swift/container-server.conf.j2 owner={{ swift }} mode=0644 notfiy: - (re)start container server - (re)start container auditor diff --git a/rpc_deployment/roles/swift_object/tasks/main.yml b/rpc_deployment/roles/swift_object/tasks/main.yml index 18428f1f55..547f550f06 100644 --- a/rpc_deployment/roles/swift_object/tasks/main.yml +++ b/rpc_deployment/roles/swift_object/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: "swift object server configuration" - template: src=object-server.conf.j2 dest=/etc/swfit/object-server.conf.j2 owner={{ swift }} mode=0644 + template: src=object-server.conf.j2 dest=/etc/swift/object-server.conf.j2 owner={{ swift }} mode=0644 notfiy: - (re)start object server - (re)start object auditor diff --git a/rpc_deployment/roles/swift_proxy/handlers/main.yml b/rpc_deployment/roles/swift_proxy/handlers/main.yml index f33effd4ce..e8e48fbd6d 100644 --- a/rpc_deployment/roles/swift_proxy/handlers/main.yml +++ b/rpc_deployment/roles/swift_proxy/handlers/main.yml @@ -1,3 +1,17 @@ --- -- name: (re)start proxy server +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Restart proxy server command: swift-init proxy-server restart diff --git a/rpc_deployment/roles/swift_proxy/tasks/main.yml b/rpc_deployment/roles/swift_proxy/tasks/main.yml index f919721f7f..54b686c567 100644 --- a/rpc_deployment/roles/swift_proxy/tasks/main.yml +++ b/rpc_deployment/roles/swift_proxy/tasks/main.yml @@ -1,11 +1,28 @@ --- -- name: "install the keystone auth_token middleware" - pip: name=keystonemiddleware +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -- name: "swift object server configuration" - template: src=proxy-server.conf.j2 dest=/etc/swfit/proxy-server.conf.j2 owner={{ swift }} mode=0644 - notfiy: - - (re)start proxy server +- name: swift proxy server configuration + template: > + src="proxy-server.conf.j2" + dest="/etc/swift/proxy-server.conf" + owner={{ service_admin_username }} + mode=0644 + notify: Restart proxy server -- name: "Set proxy server to start at boot" - cron: special_time=reboot job="swift-init proxy-server start" +- name: Set proxy server to start at boot + cron: > + name="Restart swift proxy at boot" + special_time=reboot + job="swift-init proxy-server start" diff --git a/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 index b97a9505ec..defdb00371 100644 --- a/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 +++ b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 @@ -1,10 +1,10 @@ [DEFAULT] -bind_ip = {{ inventory_host }} -bind_port = 8080 +bind_ip = 0.0.0.0 +bind_port = {{ swift_proxy_port }} # bind_timeout = 30 # backlog = 4096 # swift_dir = /etc/swift -user = {{ user }} +user = {{ system_user }} # Enables exposing configuration settings via HTTP GET /info. # expose_info = true @@ -277,13 +277,14 @@ paste.filter_factory = keystonemiddleware.auth_token:filter_factory auth_host = {{ auth_host }} auth_port = {{ auth_port }} auth_protocol = {{ auth_protocol }} -auth_uri = {{ auth_uri }} -admin_tenant_name = {{ admin_tenant_name }} -admin_user = {{ admin_user }} -admin_password = {{ admin_password }} +auth_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +admin_tenant_name = {{ service_admin_tenant_name }} +admin_user = {{ service_admin_username }} +admin_password = {{ service_admin_password }} delay_auth_decision = {{ delay_auth_decision }} # cache = swift.cache # include_service_catalog = False +{% endif %} # [filter:keystoneauth] use = egg:swift#keystoneauth @@ -315,7 +316,7 @@ use = egg:swift#memcache # memcache.conf (see memcache.conf-sample) or lacking that file, it will # default to the value below. You can specify multiple servers separated with # commas, as in: 10.1.2.3:11211,10.1.2.4:11211 -memcache_servers = {{ memcache_servers }} +memcache_servers = {{ internal_vip_address }}:{{ memcached_port }} # # Sets how memcache values are serialized and deserialized: # 0 = older, insecure pickle serialization diff --git a/rpc_deployment/vars/config_vars/haproxy_config.yml b/rpc_deployment/vars/config_vars/haproxy_config.yml index b43042cf76..b5a575307a 100644 --- a/rpc_deployment/vars/config_vars/haproxy_config.yml +++ b/rpc_deployment/vars/config_vars/haproxy_config.yml @@ -178,3 +178,11 @@ haproxy_config: hap_backend_alg: source hap_backend_options: - "ssl-hello-chk" + - service: + hap_service_name: swift_proxy + hap_backend_nodes: "{{ groups['swift_proxy'] }}" + hap_port: 8000 + hap_balance_type: http + hap_backend_alg: source + hap_backend_options: + - "ssl-hello-chk" From 28ebd132987b81892c83ee49faeb8ce0f0d1a3e4 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 25 Sep 2014 16:31:00 +0000 Subject: [PATCH 09/45] Setup the swift_common task to place the swift.conf template * Add task for swift.conf template * Remove swift_common unrequired tasks * Add further variables * Fix some typos in the conf teplate. --- etc/rpc_deploy/user_variables.yml | 2 +- .../inventory/group_vars/swift_all.yml | 2 ++ .../roles/swift_common/handlers/main.yml | 6 ----- .../roles/swift_common/tasks/debian.yml | 4 ---- .../roles/swift_common/tasks/main.yml | 13 +++++++--- .../roles/swift_common/tasks/redhat.yml | 5 ---- .../roles/swift_common/vars/main.yml | 24 ------------------- 7 files changed, 13 insertions(+), 43 deletions(-) delete mode 100644 rpc_deployment/roles/swift_common/handlers/main.yml delete mode 100644 rpc_deployment/roles/swift_common/tasks/debian.yml delete mode 100644 rpc_deployment/roles/swift_common/tasks/redhat.yml delete mode 100644 rpc_deployment/roles/swift_common/vars/main.yml diff --git a/etc/rpc_deploy/user_variables.yml b/etc/rpc_deploy/user_variables.yml index 12fe3aac5d..2b611dbb42 100644 --- a/etc/rpc_deploy/user_variables.yml +++ b/etc/rpc_deploy/user_variables.yml @@ -147,4 +147,4 @@ kibana_password: # Swift Options: swift_service_password: - +swift_container_mysql_password: diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index b39689e513..c25ac427b5 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -20,6 +20,8 @@ authtoken_active: True delay_auth_decision: true +hash_path_suffix: testsuffix +hash_path_prefix: testprefix ## Service Name service_name: swift diff --git a/rpc_deployment/roles/swift_common/handlers/main.yml b/rpc_deployment/roles/swift_common/handlers/main.yml deleted file mode 100644 index 20633b042a..0000000000 --- a/rpc_deployment/roles/swift_common/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: "Install swift dependencies" - pip: requirements=/opt/swift/swift/requirements.txt - -- name: "Install swift" - shell: chdir=/opt/swift/swift python setup.py install diff --git a/rpc_deployment/roles/swift_common/tasks/debian.yml b/rpc_deployment/roles/swift_common/tasks/debian.yml deleted file mode 100644 index 524495564f..0000000000 --- a/rpc_deployment/roles/swift_common/tasks/debian.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: "Install Debian/Ubuntu common swift packages" - apt: pkg={{ item }} state=installed update_cache=yes - with_items: "{{ swift_common_packages.debian }}" diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 3dab6fbeb9..14ad49feb0 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -8,11 +8,18 @@ # apt: pkg={{ item }} state=installed update_cache=yes # with_items: ${container_packages} -- name: "Create swift group" - group: name={{ user }} state=present system=yes +- name: "Create a swift Group" + group: name={{ system_group }} state=present system=yes - name: "Create a swift user" - user: name={{ user }} state=present shell=/bin/false home=/opt/swift system=yes + user: name={{ system_user }} state=present shell=/bin/false home=/opt/swift system=yes + +- name: "Drop swift.conf temaplte" + template: > + src="swift.conf.j2" + dest="/etc/swift/swift.conf" + owner={{ service_admin_username }} + mode=0644 #- name: "Checkout git repo" # git: repo={{ swift_repo }} version={{ git_tag }} dest=/opt/swift/swift update=yes diff --git a/rpc_deployment/roles/swift_common/tasks/redhat.yml b/rpc_deployment/roles/swift_common/tasks/redhat.yml deleted file mode 100644 index bf9f8773e8..0000000000 --- a/rpc_deployment/roles/swift_common/tasks/redhat.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: "install Redhat/CentOS common swift packages" - yum: pkg={{ item }} state=installed - with_items: "{{ swift_common_packages.redhat }}" diff --git a/rpc_deployment/roles/swift_common/vars/main.yml b/rpc_deployment/roles/swift_common/vars/main.yml deleted file mode 100644 index 8075b6ed33..0000000000 --- a/rpc_deployment/roles/swift_common/vars/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -user: swift -swift_repo: https://github.com/openstack/swift.git -swift_repo_tag: master - -swift_common_packages: - debian: - - curl - - python-pip - - rsync - - openssh-server - - git-core - - python-setuptools - - python-dev - - gcc - - libffi-dev - - redhat: - - curl - - gcc - - rsync - - git-core - - python-setuptools - - python-pip From 5e23b45790dd54ee2cf0b50f91ced32a6407cb61 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 25 Sep 2014 16:47:19 +0000 Subject: [PATCH 10/45] Fix up the container/account/object tasks/plays * Adjust the variable names * Fix some typos * Remove unrequired tasks * Adjust config files * Add playbooks for proxy/storage * Fix cron jobs --- etc/rpc_deploy/rpc_environment.yml | 19 +++++----- etc/rpc_deploy/rpc_user_config.yml | 22 +++++++++++- .../playbooks/openstack/swift-proxy.yml | 28 +++++++++++++++ .../playbooks/openstack/swift-storage.yml | 30 ++++++++++++++++ .../roles/swift_account/handlers/main.yml | 8 ++--- .../roles/swift_account/tasks/main.yml | 36 +++++++++++++------ .../templates/account-server.conf.j2 | 8 ++--- .../roles/swift_common/tasks/main.yml | 35 +----------------- .../roles/swift_container/handlers/main.yml | 8 ++--- .../roles/swift_container/tasks/main.yml | 36 +++++++++++++------ .../templates/container-server.conf.j2 | 8 ++--- .../roles/swift_object/handlers/main.yml | 8 ++--- .../roles/swift_object/tasks/main.yml | 36 +++++++++++++------ .../templates/object-server.conf.j2 | 8 ++--- 14 files changed, 192 insertions(+), 98 deletions(-) create mode 100644 rpc_deployment/playbooks/openstack/swift-proxy.yml create mode 100644 rpc_deployment/playbooks/openstack/swift-storage.yml diff --git a/etc/rpc_deploy/rpc_environment.yml b/etc/rpc_deploy/rpc_environment.yml index 76bb97612c..845b4f7c00 100644 --- a/etc/rpc_deploy/rpc_environment.yml +++ b/etc/rpc_deploy/rpc_environment.yml @@ -119,13 +119,13 @@ component_skel: swift_proxy: belongs_to: - swift_all - swift_account: + swift_acc: belongs_to: - swift_all - swift_object: + swift_obj: belongs_to: - swift_all - swift_container: + swift_cont: belongs_to: - swift_all container_skel: @@ -262,6 +262,7 @@ container_skel: - storage_containers - log_containers - network_containers + - swift_containers contains: - rsyslog utility_container: @@ -274,24 +275,24 @@ container_skel: - infra_containers contains: - swift_proxy - swift_account_container: + swift_acc_container: is_metal: true belongs_to: - swift_containers contains: - - swift_account - swift_object_container: + - swift_acc + swift_obj_container: is_metal: true belongs_to: - swift_containers contains: - - swift_object - swift_container_container: + - swift_obj + swift_cont_container: is_metal: true belongs_to: - swift_containers contains: - - swift_container + - swift_cont physical_skel: network_containers: belongs_to: diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index 43ca5da136..90b601f3ab 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -15,7 +15,7 @@ # This is the md5 of the environment file # this will ensure consistency when deploying. -environment_version: 16faeb1250d4119c507f18f3f74e0653 +environment_version: 21de6ac181f7f9d5d950709928ccded8 # User defined CIDR used for containers # Global cidr/s used for everything. @@ -157,3 +157,23 @@ log_hosts: network_hosts: network1: ip: 172.29.236.108 + +# User defined Object Storage Hosts - this is not a required group +# object_storage1: +# ip: 172.29.236.108 +# container_vars: +# swift_vars: +# policy: gold +# region: 0 +# zone: 0 +# drives: /srv/node +# +# object_storage2: +# ip: 172.29.236.109 +# container_vars: +# swift_vars: +# policy: silver +# region: 0 +# zone: 1 +# drives: /srv/node +# diff --git a/rpc_deployment/playbooks/openstack/swift-proxy.yml b/rpc_deployment/playbooks/openstack/swift-proxy.yml new file mode 100644 index 0000000000..f6c896b22a --- /dev/null +++ b/rpc_deployment/playbooks/openstack/swift-proxy.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: swift_proxy + user: root + roles: + - common + - common_sudoers + - container_common + - openstack_common + - openstack_openrc + - galera_client_cnf + - swift_common + - swift_proxy + vars_files: + - inventory/group_vars/swift_all.yml diff --git a/rpc_deployment/playbooks/openstack/swift-storage.yml b/rpc_deployment/playbooks/openstack/swift-storage.yml new file mode 100644 index 0000000000..6668b09d40 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/swift-storage.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: swift_hosts + user: root + roles: + - common + - common_sudoers + - container_common + - openstack_common + - openstack_openrc + - galera_client_cnf + - swift_common + - swift_container + - swift_object + - swift_account + vars_files: + - inventory/group_vars/swift_all.yml diff --git a/rpc_deployment/roles/swift_account/handlers/main.yml b/rpc_deployment/roles/swift_account/handlers/main.yml index 36250f33f8..5506aaf09f 100644 --- a/rpc_deployment/roles/swift_account/handlers/main.yml +++ b/rpc_deployment/roles/swift_account/handlers/main.yml @@ -1,12 +1,12 @@ --- -- name: (re)start account server +- name: Restart account server command: swift-init account-server restart -- name: (re)start account auditor +- name: Restart account auditor command: swift-init account-auditor restart -- name: (re)start account replicator +- name: Restart account replicator command: swift-init account-replicator restart -- name: (re)start account reaper +- name: Restart account reaper command: swift-init account-reaper restart diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml index a3da52cc23..914c99e090 100644 --- a/rpc_deployment/roles/swift_account/tasks/main.yml +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -1,20 +1,36 @@ --- - name: "swift account server configuration" - template: src=account-server.conf.j2 dest=/etc/swift/account-server.conf.j2 owner={{ swift }} mode=0644 - notfiy: - - (re)start account server - - (re)start account auditor - - (re)start account replicator - - (re)start account reaper + template: > + src="account-server.conf.j2" + dest="/etc/swift/account-server.conf" + owner={{ system_user }} + mode=0644 + notify: + - Restart account server + - Restart account auditor + - Restart account replicator + - Restart account reaper - name: "Set account server to start at boot" - cron: special_time=reboot job="swift-init account-server start" + cron: > + name="Restart account-sever on boot" + special_time=reboot + job="swift-init account-server start" - name: "Set account auditor to start at boot" - cron: special_time=reboot job="swift-init account-auditor start" + cron: > + name="Restart account-auditor on boot" + special_time=reboot + job="swift-init account-auditor start" - name: "Set account replicator to start at boot" - cron: special_time=reboot job="swift-init account-replicator start" + cron: > + name="Restart account-replicator on boot" + special_time=reboot + job="swift-init account-replicator start" - name: "Set account reaper to start at boot" - cron: special_time=reboot job="swift-init account-reaper start" + cron: > + name="Restart account-reaper on boot" + special_time=reboot + job="swift-init account-reaper start" diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 index 7c4ebd2bbc..545baaba9e 100644 --- a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] -bind_ip = {{ inventory_hostname }} -bind_port = {{ account_server_port }} +bind_ip = {{ container_address }} +bind_port = {{ swift_account_port }} # bind_timeout = 30 # backlog = 4096 -user = {{ user }} +user = {{ system_user }} # swift_dir = /etc/swift -devices = {{ drives }} +devices = {{ swift_vars.drives }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 14ad49feb0..05b1a451f8 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -1,40 +1,7 @@ --- -#- name: "Setup hosts file" -# lineinfile: dest=/etc/hosts regexp='.*{{ item }}$' line="{{ hostvars[item].ansible_default_ipv4.address }} {{item}}" state=present -# when: hostvars[item].ansible_default_ipv4.address is defined -# with_items: groups['all'] - -#- name: "Install Debian/Ubuntu common swift packages" -# apt: pkg={{ item }} state=installed update_cache=yes -# with_items: ${container_packages} - -- name: "Create a swift Group" - group: name={{ system_group }} state=present system=yes - -- name: "Create a swift user" - user: name={{ system_user }} state=present shell=/bin/false home=/opt/swift system=yes - - name: "Drop swift.conf temaplte" template: > src="swift.conf.j2" dest="/etc/swift/swift.conf" - owner={{ service_admin_username }} + owner={{ system_user }} mode=0644 - -#- name: "Checkout git repo" -# git: repo={{ swift_repo }} version={{ git_tag }} dest=/opt/swift/swift update=yes -# notifiy: -# - Install swift dependencies -# - Install swift - -#- name: "Swift directory" -# file: path=/etc/swift owner={{ user }} group={{ user }} state=directory - -#- name: "Swift lock directory" -# file: path=/var/lock/swift owner={{ user }} group={{ user }} state=directory - -#- name: "Swift log directory" -# file: path=/var/log/swift owner={{ user }} group=adm state=directory - -- name: "Setup swift.conf" - template: src=swift.conf.j2 dest=/etc/swift/swift.conf owner={{ user }} mode=0644 diff --git a/rpc_deployment/roles/swift_container/handlers/main.yml b/rpc_deployment/roles/swift_container/handlers/main.yml index 7c034f60c1..7d1190d703 100644 --- a/rpc_deployment/roles/swift_container/handlers/main.yml +++ b/rpc_deployment/roles/swift_container/handlers/main.yml @@ -1,12 +1,12 @@ --- -- name: (re)start container server +- name: Restart container server command: swift-init container-server restart -- name: (re)start container auditor +- name: Restart container auditor command: swift-init container-auditor restart -- name: (re)start container replicator +- name: Restart container replicator command: swift-init container-replicator restart -- name: (re)start container updater +- name: Restart container updater command: swift-init container-updater restart diff --git a/rpc_deployment/roles/swift_container/tasks/main.yml b/rpc_deployment/roles/swift_container/tasks/main.yml index e3794f267e..d0e75fa864 100644 --- a/rpc_deployment/roles/swift_container/tasks/main.yml +++ b/rpc_deployment/roles/swift_container/tasks/main.yml @@ -1,20 +1,36 @@ --- - name: "swift container server configuration" - template: src=container-server.conf.j2 dest=/etc/swift/container-server.conf.j2 owner={{ swift }} mode=0644 - notfiy: - - (re)start container server - - (re)start container auditor - - (re)start container replicator - - (re)start container updater + template: > + src="container-server.conf.j2" + dest="/etc/swift/container-server.conf" + owner={{ system_user }} + mode=0644 + notify: + - Restart container server + - Restart container auditor + - Restart container replicator + - Restart container updater - name: "Set container server to start at boot" - cron: special_time=reboot job="swift-init container-server start" + cron: > + name="Restart container-server on boot" + special_time=reboot + job="swift-init container-server start" - name: "Set container auditor to start at boot" - cron: special_time=reboot job="swift-init container-auditor start" + cron: > + name="Restart container-auditor at boot" + special_time=reboot + job="swift-init container-auditor start" - name: "Set container replicator to start at boot" - cron: special_time=reboot job="swift-init container-replicator start" + cron: > + name="Restart container-replicator at boot" + special_time=reboot + job="swift-init container-replicator start" - name: "Set container updater to start at boot" - cron: special_time=reboot job="swift-init container-updater start" + cron: > + name="Restart container-updater at boot" + special_time=reboot + job="swift-init container-updater start" diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 index 72242f9aad..5d29c5d758 100644 --- a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] -bind_ip = {{ inventory_hostname }} -bind_port = {{ container_server_port }} +bind_ip = {{ container_address }} +bind_port = {{ swift_container_port }} # bind_timeout = 30 # backlog = 4096 -user = {{ user }} +user = {{ system_user }} # swift_dir = /etc/swift -devices = {{ drives }} +devices = {{ swift_vars.drives }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_object/handlers/main.yml b/rpc_deployment/roles/swift_object/handlers/main.yml index ab7447eee2..3bce44e32e 100644 --- a/rpc_deployment/roles/swift_object/handlers/main.yml +++ b/rpc_deployment/roles/swift_object/handlers/main.yml @@ -1,12 +1,12 @@ --- -- name: (re)start object server +- name: Restart object server command: swift-init object-server restart -- name: (re)start object auditor +- name: Restart object auditor command: swift-init object-auditor restart -- name: (re)start object replicator +- name: Restart object replicator command: swift-init object-replicator restart -- name: (re)start object updater +- name: Retart object updater command: swift-init object-updater restart diff --git a/rpc_deployment/roles/swift_object/tasks/main.yml b/rpc_deployment/roles/swift_object/tasks/main.yml index 547f550f06..7008a96dc7 100644 --- a/rpc_deployment/roles/swift_object/tasks/main.yml +++ b/rpc_deployment/roles/swift_object/tasks/main.yml @@ -1,20 +1,36 @@ --- - name: "swift object server configuration" - template: src=object-server.conf.j2 dest=/etc/swift/object-server.conf.j2 owner={{ swift }} mode=0644 - notfiy: - - (re)start object server - - (re)start object auditor - - (re)start object replicator - - (re)start object updater + template: > + src="object-server.conf.j2" + dest="/etc/swift/object-server.conf" + owner={{ system_user }} + mode=0644 + notify: + - Restart object server + - Restart object auditor + - Restart object replicator + - Restart object updater - name: "Set object server to start at boot" - cron: special_time=reboot job="swift-init object-server start" + cron: > + name="Restart object-server on boot" + special_time=reboot + job="swift-init object-server start" - name: "Set object auditor to start at boot" - cron: special_time=reboot job="swift-init object-auditor start" + cron: > + name="Restart object-auditor on boot" + special_time=reboot + job="swift-init object-auditor start" - name: "Set object replicator to start at boot" - cron: special_time=reboot job="swift-init object-replicator start" + cron: > + name="Restart object-replicator on boot" + special_time=reboot + job="swift-init object-replicator start" - name: "Set object updater to start at boot" - cron: special_time=reboot job="swift-init object-updater start" + cron: > + name="Restart object-updater on boot" + special_time=reboot + job="swift-init object-updater start" diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 index 9e28a7cce6..505e9d1b9e 100644 --- a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -1,11 +1,11 @@ [DEFAULT] -bind_ip = {{ inverntory_host }} -bind_port = {{ object_server_port }} +bind_ip = {{ container_address }} +bind_port = {{ swift_object_port }} # bind_timeout = 30 # backlog = 4096 -user = {{ user }} +user = {{ system_user }} swift_dir = /etc/swift -devices = {{ drives }} +devices = {{ swift_vars.drives }} # mount_check = true # disable_fallocate = false # expiring_objects_container_divisor = 86400 From 1fa95a22697165b207bede320105261405f835e9 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 26 Sep 2014 09:55:03 +0000 Subject: [PATCH 11/45] Adjust swift.conf to work with rpc_user_config * Set storage policies appropriately * Set the storage policies up in rpc_user_config * Adjust sample rpc_user_config * Loop through the storage policies in the swift.conf --- etc/rpc_deploy/rpc_user_config.yml | 45 ++++++++++++------- etc/rpc_deploy/user_variables.yml | 2 + .../inventory/group_vars/swift_all.yml | 2 - .../swift_common/templates/swift.conf.j2 | 30 +++++-------- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index 90b601f3ab..be75b33246 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -109,6 +109,18 @@ global_overrides: type: "vlan" range: "1:1" net_name: "vlan" + # For Swift Set Storage Policies + # storage_policies: + # - policy: + # name: "gold" + # index: 0 + # repl_number: 3 + # default: "Yes" + # - policy: + # name: "silver" + # index: 1 + # repl_number: 2 + # deprecated: "Yes" # Name of load balancer lb_name: lb_name_in_core @@ -159,21 +171,22 @@ network_hosts: ip: 172.29.236.108 # User defined Object Storage Hosts - this is not a required group -# object_storage1: -# ip: 172.29.236.108 -# container_vars: -# swift_vars: -# policy: gold -# region: 0 -# zone: 0 -# drives: /srv/node +# swift_hosts: +# object_storage1: +# ip: 172.29.236.108 +# container_vars: +# swift_vars: +# policy: gold +# region: 0 +# zone: 0 +# drives: /srv/node # -# object_storage2: -# ip: 172.29.236.109 -# container_vars: -# swift_vars: -# policy: silver -# region: 0 -# zone: 1 -# drives: /srv/node +# object_storage2: +# ip: 172.29.236.109 +# container_vars: +# swift_vars: +# policy: silver +# region: 0 +# zone: 1 +# drives: /srv/node # diff --git a/etc/rpc_deploy/user_variables.yml b/etc/rpc_deploy/user_variables.yml index 2b611dbb42..19ac332310 100644 --- a/etc/rpc_deploy/user_variables.yml +++ b/etc/rpc_deploy/user_variables.yml @@ -148,3 +148,5 @@ kibana_password: # Swift Options: swift_service_password: swift_container_mysql_password: +swift_hash_path_suffix: +swift_has_path_prefix: diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index c25ac427b5..b39689e513 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -20,8 +20,6 @@ authtoken_active: True delay_auth_decision: true -hash_path_suffix: testsuffix -hash_path_prefix: testprefix ## Service Name service_name: swift diff --git a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 index 730ba10890..f7958795a6 100644 --- a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 +++ b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 @@ -5,28 +5,20 @@ # These values should remain secret and MUST NOT change # once a cluster has been deployed. -swift_hash_path_suffix = {{ hash_path_suffix }} -swift_hash_path_prefix = {{ hash_path_prefix }} +swift_hash_path_suffix = {{ swift_hash_path_suffix }} +swift_hash_path_prefix = {{ swift_hash_path_prefix }} # Storage Policies -{% for group in groups %} -{% if group.startswith("storagepolicy_") %} -{% set vars = hostvars[groups[group].pop()] %} -{% for proxy_settings in vars["extra_data"].split(';') %} -{% set items = proxy_settings.split(':') %} -{% if group.endswith(items[0]) %} -[storage-policy:{{ items[1] }}] -name = {{ items[0] }} -{% if items[2] == 'True' %} -depricated = Yes -{% endif %} -{% if items[0] == vars["default"] %} -default = Yes -{% endif %} - -{% endif %} -{% endfor %} +{% for policy in storage_policies %} +[storage-policy:{{ policy.policy.index }}] +name = {{ policy.policy.name }} +{% if policy.policy.deprecated is defined %} +deprecated = {{ policy.policy.deprecated }} {% endif %} +{% if policy.policy.default is defined %} +default = {{ policy.policy.default }} +{% endif %} + {% endfor %} [swift-constraints] From 0d2f69bc0387f079a92587e62837c9d9341c09a9 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 26 Sep 2014 14:29:48 +0000 Subject: [PATCH 12/45] Set out the ring distribution and configuration plays * Add tasks for building the ring and distributing the play * Add additional playbook to handle ring building * Add "overall" playbook for swift-all --- .../inventory/group_vars/swift_all.yml | 2 + .../playbooks/build_swift_rings.yml | 23 --- .../playbooks/openstack/swift-all.yml | 18 ++ .../playbooks/openstack/swift-build-rings.yml | 10 ++ .../swift_ring_builder/files/swift_rings.py | 158 ++++++++++++++++++ .../roles/swift_ring_builder/tasks/main.yml | 29 ++++ .../swift_ring_distribute/tasks/main.yml | 6 + 7 files changed, 223 insertions(+), 23 deletions(-) delete mode 100644 rpc_deployment/playbooks/build_swift_rings.yml create mode 100644 rpc_deployment/playbooks/openstack/swift-all.yml create mode 100644 rpc_deployment/playbooks/openstack/swift-build-rings.yml create mode 100644 rpc_deployment/roles/swift_ring_builder/files/swift_rings.py create mode 100644 rpc_deployment/roles/swift_ring_builder/tasks/main.yml create mode 100644 rpc_deployment/roles/swift_ring_distribute/tasks/main.yml diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index b39689e513..d6979fbab3 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -90,6 +90,7 @@ container_directories: - /tmp/swift - /var/lock/swift - /etc/swift + - /etc/swift/rings/ container_packages: - curl @@ -100,3 +101,4 @@ container_packages: - python-setuptools - python-dev - gcc + - libffi-dev diff --git a/rpc_deployment/playbooks/build_swift_rings.yml b/rpc_deployment/playbooks/build_swift_rings.yml deleted file mode 100644 index 0fa4f4806f..0000000000 --- a/rpc_deployment/playbooks/build_swift_rings.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- hosts: local - user: root - roles: - - swift_common - -- hosts: local - user: root - tasks: - - name: "make sure rings directory exists" - file: state=directory path=/etc/swift/rings/ owner=root group=root mode=0755 - - name: "build rings" - command: /usr/bin/python /etc/ansible/scripts/swift_rings.py -s /etc/ansible/swift_setup.yml - args: - chdir: /etc/swift/rings/ - -- hosts: swift - user: root - tasks: - - name: "Make sure /etc/swift exists" - file: state=directory path=/etc/swift/ mode=0755 - - name: "Copy the rings over" - copy: src=/etc/swift/rings/ dest=/etc/swift/ mode=644 diff --git a/rpc_deployment/playbooks/openstack/swift-all.yml b/rpc_deployment/playbooks/openstack/swift-all.yml new file mode 100644 index 0000000000..4937c69a10 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/swift-all.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: swift-proxy.yml +- include: swift-storage.yml +- include: swift-build-rings.yml diff --git a/rpc_deployment/playbooks/openstack/swift-build-rings.yml b/rpc_deployment/playbooks/openstack/swift-build-rings.yml new file mode 100644 index 0000000000..66b35597e5 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/swift-build-rings.yml @@ -0,0 +1,10 @@ +--- +- hosts: swift_all[0] + user: root + roles: + - swift_ring_builder + +- hosts: swift_all:!swift_all[0] + user: root + roles: + - swift_ring_distribute diff --git a/rpc_deployment/roles/swift_ring_builder/files/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/files/swift_rings.py new file mode 100644 index 0000000000..3c32845ae1 --- /dev/null +++ b/rpc_deployment/roles/swift_ring_builder/files/swift_rings.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +from __future__ import print_function +from optparse import OptionParser +from os.path import exists +from swift.cli.ringbuilder import main as rb_main + +import sys +import threading +import yaml + +USAGE = "usage: %prog -s " + +DEFAULT_PART_POWER = 10 +DEFAULT_REPL = 3 +DEFAULT_MIN_PART_HOURS = 1 +DEFAULT_HOST_DRIVES = '/srv/drive/' +DEFAULT_HOST_DRIVE = '/sdb' +DEFAULT_HOST_ZONE = 0 +DEFAULT_HOST_WEIGHT = 1 +DEFAULT_ACCOUNT_PORT = 6002 +DEFAULT_CONTAINER_PORT = 6001 +DEFAULT_OBJECT_PORT = 6000 +DEFAULT_SECTION_PORT = { + 'account': DEFAULT_ACCOUNT_PORT, + 'container': DEFAULT_CONTAINER_PORT, + 'object': DEFAULT_OBJECT_PORT, +} + + +def create_buildfile(build_file, part_power, repl, min_part_hours): + run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", + part_power, repl, min_part_hours]) + + +def add_host_to_ring(build_file, host): + host_str = "" + if host.get('region') is not None: + host_str += 'r%(region)d' % host + host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) + host_str += "-%(host)s:%(port)d" % host + if host.get('repl_port'): + r_ip = host.get('repl_ip', host['host']) + host_str += "R%s:%d" % (r_ip, host['repl_port']) + host_str += "/%(drive)s" % host + + weight = host.get('weight', DEFAULT_HOST_WEIGHT) + run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', + host_str, str(weight)]) + + +def run_and_wait(func, *args): + t = threading.Thread(target=func, args=args) + t.start() + return t.join() + + +def has_section(conf, section): + return True if conf.get(section) else False + + +def check_section(conf, section): + if not has_section(conf, section): + print("Section %s doesn't exist" % (section)) + sys.exit(2) + + +def build_ring(section, conf, part_power, hosts): + # Create the build file + build_file = "%s.builder" % (section) + repl = conf.get('repl_number', DEFAULT_REPL) + min_part_hours = conf.get('min_part_hours', + DEFAULT_MIN_PART_HOURS) + create_buildfile(build_file, part_power, repl, min_part_hours) + + # Add the hosts + if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0: + print("No hosts/drives assigned to the %s ring" % section) + sys.exit(3) + + section_key = section.split('-')[0] + service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) + for host in conf['hosts']: + if 'name' in host: + if host['name'] not in hosts: + print("Host %(name) reference not found." % host) + sys.exit(3) + host = hosts[host['name']] + else: + if 'drive' not in host: + host['drive'] = DEFAULT_HOST_DRIVE + host['port'] = service_port + add_host_to_ring(build_file, host) + + # Rebalance ring + run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) + # rb_main(("swift-ring-builder", build_file, "rebalance")) + + +def main(setup): + # load the yaml file + try: + with open(setup) as yaml_stream: + _swift = yaml.load(yaml_stream) + except Exception as ex: + print("Failed to load yaml string %s" % (ex)) + return 1 + + _hosts = {} + + if _swift.get("hosts"): + for host in _swift['hosts']: + if not host.get('drive'): + host['drive'] = DEFAULT_HOST_DRIVE + key = "%(host)s/%(drive)s" % host + if key in _hosts: + print("%(host)s already definined" % host) + return 1 + _hosts[key] = host + + check_section(_swift, 'swift') + part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER) + + # Create account ring + check_section(_swift, 'account') + build_ring('account', _swift['account'], part_power, _hosts) + + # Create container ring + check_section(_swift, 'container') + build_ring('container', _swift['container'], part_power, _hosts) + + # Create object rings (storage policies) + check_section(_swift, 'storage_policies') + check_section(_swift['storage_policies'], 'policies') + indexes = set() + for sp in _swift['storage_policies']['policies']: + if sp['index'] in indexes: + print("Storage Policy index %d already in use" % (sp['index'])) + return 4 + buildfilename = 'object-%d' % (sp['index']) + indexes.add(sp['index']) + if 'port' not in sp: + sp['port'] = _swift['storage_policies'].get('port', + DEFAULT_OBJECT_PORT) + build_ring(buildfilename, sp, part_power, _hosts) + +if __name__ == "__main__": + parser = OptionParser(USAGE) + parser.add_option("-s", "--setup", dest="setup", + help="Specify the swift setup file.", metavar="FILE", + default="/etc/swift/swift_inventory.yml") + + options, args = parser.parse_args(sys.argv[1:]) + if options.setup and not exists(options.setup): + print("Swift setup file not found or doesn't exist") + parser.print_help() + sys.exit(1) + + sys.exit(main(options.setup)) diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml new file mode 100644 index 0000000000..8b21a832be --- /dev/null +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: "make sure scripts directory exists" + file: > + state=directory + path=/etc/swift/scripts/ + owner=root + group=root + mode=0755 + +- name: "Retrieve the swift_rings.py file" + copy: > + src=swift_rings.py + dest="/etc/swift/scripts/swift_rings.py" + +#- name: "build rings" +# command: /usr/bin/python /etc/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_user_config.yml +# args: +# chdir: /etc/swift/rings/ + +- name: "List ring files to copy over" + command: "/bin/ls /etc/swift/rings/" + register: ringfiles + +- name: "Fetch rings to local server" + fetch: > + src="/etc/swift/rings/{{ item }}" + dest=/tmp/swift/rings/ + flat=yes + with_items: ringfiles.stdout_lines diff --git a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml new file mode 100644 index 0000000000..93297a53cc --- /dev/null +++ b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: "Copy the rings over" + copy: > + src=/tmp/swift/rings/ + dest=/etc/swift/rings/ + mode=0644 From e787a0de43919143b61701ff799fb399a0c8dc93 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 26 Sep 2014 14:42:41 +0000 Subject: [PATCH 13/45] Clean up some unrequired swift scripts * Remove the swift_inventory script * Remove the swift_rings.py (its moved to the swift_ring_builder role) * Remove the swift_setup.yml (We will use rpc_user_config.yml for these) --- etc/rpc_deploy/swift_setup.yml | 111 ---------- scripts/swift_inventory.py | 361 --------------------------------- scripts/swift_rings.py | 158 --------------- 3 files changed, 630 deletions(-) delete mode 100644 etc/rpc_deploy/swift_setup.yml delete mode 100644 scripts/swift_inventory.py delete mode 100644 scripts/swift_rings.py diff --git a/etc/rpc_deploy/swift_setup.yml b/etc/rpc_deploy/swift_setup.yml deleted file mode 100644 index 4ac4ead48c..0000000000 --- a/etc/rpc_deploy/swift_setup.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -# This swift setup file, is used as a simple way of managing your swift -# ring configuration. The swift_inventory binary will generate the ansible -# iventory file for you based of this file. Giving you one place to manage your -# cluster. -# -# NOTE: The swift ansible configuration has other variables that can be tweeked, -# see group_vars/ and roles/*/vars/ for other areas to tweek. -local: - ansible_connection=local - -# Swift settings are global through out the cluster. -swift: - part_power: 8 - output_directory: /tmp/swift - output_filename: hosts - user: swift - hash_path_suffix: changeme - hash_path_prefix: changeme - syslog_host: 10.1.1.1:514 - -# Global hosts -# Hosts can be defined in the sections below or here, the advantage of -# defining them here is that they can be reused in multiple sections below. -# -# When need to reference a global host use the form: -# - name: / -hosts: - - host: 10.0.0.4 - drive: sda - drives: /srv/disk - region: 0 - zone: 0 - weight: 100 - repl_ip: 192.168.0.4 - repl_port: 55555 - - host: 10.0.0.5 - drive: sda - drives: /srv/disk - region: 0 - zone: 1 - weight: 100 - -proxy: - memcache_servers: - - 192.168.100.7:11211 - authtoken: - active: true - delay_auth_decision: true - auth_version: v2.0 - auth_host: 192.168.100.7 - auth_port: 35357 - auth_protocol: https - auth_uri: http://192.168.100.7:5000/ - admin_tenant_name: service - admin_user: swift - admin_password: ADMIN - hosts: - - host: - -account: - repl_number: 3 - min_part_hours: 1 - port: 6002 - hosts: - - host: 10.0.0.2 - drive: sdba - region: 0 - zone: 0 - weight: 100 - -container: - repl_number: 3 - port: 6001 - hosts: - - host: 10.0.0.3 - drive: sda - region: 0 - zone: 0 - weight: 100 - -storage_policies: - default: gold - port: 6000 - policies: - - name: gold - index: 0 - type: replication - repl_number: 3 - min_part_hours: 1 - hosts: - - name: 10.0.0.4/sda - - name: 10.0.0.5/sda - - host: 10.0.0.6 - drive: sdb - region: 1 - zone: 0 - weight: 50 - - host: 10.0.0.7 - drive: sdb - region: 1 - zone: 1 - weight: 50 - - name: silver - index: 1 - type: replication - repl_number: 2 - depricated: True - hosts: - - name: 10.0.0.4/sda - - name: 10.0.0.5/sda diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py deleted file mode 100644 index aefbaea0d7..0000000000 --- a/scripts/swift_inventory.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -import datetime -import sys -import yaml - -from optparse import OptionParser -from os.path import exists, isdir, join - -VERSION = '0.1' -USAGE = "usage: %prog [options] -s " - -DEFAULT_PART_POWER = 8 -DEFAULT_REPL_NUM = 3 -DEFAULT_REGION = 0 -DEFAULT_ZONE = 0 -DEFAULT_WEIGHT = 100 -DEFAULT_DRIVE = "sda" -DEFAULT_DRIVES = "/srv/disk" -DEFAULT_OUTPUT_DIR = "/etc/ansible" -DEFAULT_OUTPUT_FILENAME = "hosts" - -RETURN_NOT_DEFINED = 3 - -# FILE formatted strings -HEADER = """# This file was generated using the %s version %s at %s -[local] -localhost ansible_connection=local - -[proxy]""" - -CATCH_ALL_GROUPS = """ -[object:children] -storagepolicy - -[swift:children] -proxy -account -container -object -local - -[swift:vars]""" - -DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s " -DRIVE_FORMAT += "weight=%(weight)s drives=%(drives)s extra_data=%(extra_data)s" - -DEFAULT_AUTHTOKEN_SETTINGS = { - 'auth_version': 'v2.0', - 'auth_host': 'keystone', - 'auth_port': '35357', - 'auth_protocol': 'https', - 'admin_tenant_name': 'service', - 'admin_user': 'swift', - 'admin_password': 'ADMIN', -} - -DATA_ORDER = [ - NAME, - INDEX, - DEPRICATED, - TYPE, -] = [ - 'name', - 'index', - 'depricated', - 'type', -] - - -class DriveException(Exception): - pass - - -def main(setup, verbose=False, dry_run=False, overwrite=True): - # Parse the setup file, which should be yaml - _swift = {} - _drives = {} - try: - with open(setup) as yaml_stream: - _swift = yaml.load(yaml_stream) - except Exception as err: - print("ERROR: Failed to yaml failure: %s", err) - return 2 - - def _section_defined(section): - if section not in _swift: - print("ERROR: no swift section defined") - return False - return True - - def _get_output_fd(filename): - if dry_run: - return None - elif not overwrite and exists(filename): - i = 1 - while exists("%s_%d" % (filename, i)): - i += 1 - return open("%s_%d" % (filename, i), 'w') - else: - return open(filename, 'w') - - def _write_to_file(fd, data): - if not fd or verbose: - print(data) - - if fd: - if not data.endswith('\n'): - data += "\n" - fd.write(data) - fd.flush() - - def _get_drive(drive, extra_data={}): - if 'name' in drive: - # We are dealing with a global host reference - key = drive['name'] - if key not in _drives: - raise DriveException("Drive referenced doesn't exist") - else: - # Not a referenced host. - _drive = { - 'drive': DEFAULT_DRIVE, - 'region': DEFAULT_REGION, - 'zone': DEFAULT_ZONE, - 'weight': DEFAULT_WEIGHT, - 'drives': DEFAULT_DRIVES, - 'extra_data': ''} - - if "drive" not in drive: - drive["drive"] = DEFAULT_DRIVE - - key = "%(host)s/%(drive)s" % drive - - if extra_data: - data_str = ":".join([str(extra_data.get(i, '')) for i in - DATA_ORDER]) - else: - data_str = "" - if key in _drives: - if not _drives[key]['extra_data'] and extra_data: - _drives[key]['extra_data'] = data_str - elif _drives[key]['extra_data'] and extra_data: - _drives[key]['extra_data'] += ";%s" % (data_str) - return _drives[key] - else: - _drive.update(drive) - _drive['extra_data'] = data_str - _drives[key] = _drive - return _drive - - # First attempt to get swift settings - if not _section_defined("swift"): - return RETURN_NOT_DEFINED - - swift_options = [ - "part_power=%s" % (_swift['swift'].get('part_power', - DEFAULT_PART_POWER)), - "user=%s" % (_swift['swift'].get('user', 'swift')), - "hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")), - "hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")), - "syslog_host=%s" % (_swift['swift'].get('syslog_host', - 'localhost:514')), - ] - output_path = _swift['swift'].get("output_directory", DEFAULT_OUTPUT_DIR) - output_file = _swift['swift'].get("output_filename", - DEFAULT_OUTPUT_FILENAME) - if not isdir(output_path): - print("Outdir path '%s' doesn't exist", output_path) - return 4 - - output_file = join(output_path, output_file) - output_fd = _get_output_fd(output_file) - - n = datetime.datetime.now() - _write_to_file(output_fd, HEADER % (__file__, VERSION, n.ctime())) - - # Global hosts - if _section_defined("hosts"): - for host in _swift["hosts"]: - _get_drive(host) - - if not _section_defined("proxy"): - return RETURN_NOT_DEFINED - - # Parse proxies - # TODO: Add read anfinity and pipeline here? - for proxy in _swift["proxy"]["hosts"]: - _write_to_file(output_fd, "%s" % (proxy["host"])) - _write_to_file(output_fd, "\n[proxy:vars]") - _mc_servers = _swift["proxy"].get('memcache_servers') - memcache_servers = ",".join(_mc_servers) if _mc_servers else \ - '127.0.0.1:11211' - _write_to_file(output_fd, "memcache_servers=%s" % (memcache_servers)) - _at = _swift["proxy"].get('authtoken') - if _at: - authtoken = DEFAULT_AUTHTOKEN_SETTINGS - authtoken.update(_at) - at_active = authtoken.get("active", False) - if at_active: - _write_to_file(output_fd, "authtoken_active=true") - _write_to_file(output_fd, "delay_auth_decision=" - "%(delay_auth_decision)s" % authtoken) - _write_to_file(output_fd, "auth_version=" - "%(auth_version)s" % authtoken) - _write_to_file(output_fd, "auth_host=" - "%(auth_host)s" % authtoken) - _write_to_file(output_fd, "auth_port=" - "%(auth_port)s" % authtoken) - _write_to_file(output_fd, "auth_protocol=" - "%(auth_protocol)s" % authtoken) - _write_to_file(output_fd, "auth_uri=" - "%(auth_uri)s" % authtoken) - _write_to_file(output_fd, "admin_tenant_name=" - "%(admin_tenant_name)s" % authtoken) - _write_to_file(output_fd, "admin_user=" - "%(admin_user)s" % authtoken) - _write_to_file(output_fd, "admin_password=" - "%(admin_password)s" % authtoken) - else: - _write_to_file(output_fd, "authtoken_active=false") - - _write_to_file(output_fd, "\n[account]") - - if not _section_defined("account"): - return RETURN_NOT_DEFINED - - for account in _swift["account"]["hosts"]: - data = _get_drive(account) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - _write_to_file(output_fd, "\n[account:vars]") - repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_number=%d" % (repl_num)) - port = _swift["account"].get("port", 6002) - _write_to_file(output_fd, "account_server_port=%d" % (port)) - - # Container section - _write_to_file(output_fd, "\n[container]") - - if not _section_defined("container"): - return RETURN_NOT_DEFINED - - for container in _swift["container"]["hosts"]: - data = _get_drive(container) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - _write_to_file(output_fd, "\n[container:vars]") - repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_number=%d" % (repl_num)) - port = _swift["container"].get("port", 6001) - _write_to_file(output_fd, "container_server_port=%d" % (port)) - - # Objects / Storage polices - _storage_policies = {} - _storage_policies_idx = {} - if not _section_defined("storage_policies"): - return RETURN_NOT_DEFINED - - if "policies" not in _swift["storage_policies"]: - print("ERROR: No storage policies defined") - return 4 - - policy_hosts = {} - for policy in _swift["storage_policies"]["policies"]: - if policy["name"] in _storage_policies: - print("ERROR: Storage policy '%s' already defined" % policy["name"]) - return 5 - - if policy["index"] in _storage_policies_idx: - print("ERROR: Storage policy index '%s' already defined" % - policy["index"]) - return 5 - - _storage_policies[policy['name']] = "storagepolicy_%(name)s" % policy - _storage_policies_idx[policy['index']] = policy["name"] - - policy_type = policy.get("type", 'replication') - _host_data = { - NAME: policy['name'], - INDEX: policy['index'], - TYPE: policy_type, - DEPRICATED: policy.get("depricated", False), - } - # print the storage policy hosts. - for drive in policy.get("hosts", []): - _get_drive(drive, _host_data) - - policy_hosts[policy['name']] = policy.get("hosts") - _write_to_file(output_fd, - "\n[%s:vars]" % (_storage_policies[policy['name']])) - default = policy.get("default", False) - if default: - _write_to_file(output_fd, "default=%s" % (policy['name'])) - - if policy_type == 'replication': - repl_num = policy.get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_num=%d" % (repl_num)) - - # now write out the drives. - for policy in _swift["storage_policies"]["policies"]: - _write_to_file(output_fd, - "\n[%s]" % (_storage_policies[policy['name']])) - for drive in policy_hosts[policy['name']]: - data = _get_drive(drive) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - # Write out the storage policy catch all group - _write_to_file(output_fd, "\n[storagepolicy:children]") - for name, longname in _storage_policies.items(): - _write_to_file(output_fd, "%s" % (longname)) - - _write_to_file(output_fd, "\n[storagepolicy:vars]") - if 'default' in _swift["storage_policies"]: - default_sp = _swift["storage_policies"]["default"] - if default_sp in _storage_policies: - _write_to_file(output_fd, "default=%s" % (default_sp)) - elif default_sp in _storage_policies_idx: - _write_to_file(output_fd, - "default=%s" % (_storage_policies_idx[default_sp])) - else: - print("ERROR: Default storage policy '%s' doesn't exist", - default_sp) - port = _swift["storage_policies"].get("port", 6000) - _write_to_file(output_fd, "object_server_port=%d" % (port)) - - # Write out the object and swift catchall groups - _write_to_file(output_fd, CATCH_ALL_GROUPS) - - # Now write out the global swift options that is gathered in the file - for option in swift_options: - _write_to_file(output_fd, option) - - # Done - if output_fd: - output_fd.flush() - output_fd.close() - return 0 - -if __name__ == "__main__": - parser = OptionParser(USAGE) - parser.add_option("-s", "--setup", dest="setup", - help="Specify the swift setup file.", metavar="FILE") - parser.add_option("-v", "--verbose", action="store_true", dest="verbose", - default=False, help="Be more verbose") - parser.add_option("-d", "--dryrun", action="store_true", dest="dry_run", - default=False, help="Print result out to stdout.") - parser.add_option("-C", "--copy", action="store_false", dest="overwrite", - default=True, help="Make a copy if inventory file exists") - parser.add_option("-i", "--import", dest="ring_folder", metavar="FILE", - help="Attempt to build a swift setup file" - " from the Swift builder files. Pass directory here") - - options, args = parser.parse_args(sys.argv[1:]) - if not options.setup or not exists(options.setup): - print("Swift setup file not found or doesn't exist") - parser.print_help() - sys.exit(1) - - sys.exit(main(options.setup, options.verbose, options.dry_run, - options.overwrite)) diff --git a/scripts/swift_rings.py b/scripts/swift_rings.py deleted file mode 100644 index 3c32845ae1..0000000000 --- a/scripts/swift_rings.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -from optparse import OptionParser -from os.path import exists -from swift.cli.ringbuilder import main as rb_main - -import sys -import threading -import yaml - -USAGE = "usage: %prog -s " - -DEFAULT_PART_POWER = 10 -DEFAULT_REPL = 3 -DEFAULT_MIN_PART_HOURS = 1 -DEFAULT_HOST_DRIVES = '/srv/drive/' -DEFAULT_HOST_DRIVE = '/sdb' -DEFAULT_HOST_ZONE = 0 -DEFAULT_HOST_WEIGHT = 1 -DEFAULT_ACCOUNT_PORT = 6002 -DEFAULT_CONTAINER_PORT = 6001 -DEFAULT_OBJECT_PORT = 6000 -DEFAULT_SECTION_PORT = { - 'account': DEFAULT_ACCOUNT_PORT, - 'container': DEFAULT_CONTAINER_PORT, - 'object': DEFAULT_OBJECT_PORT, -} - - -def create_buildfile(build_file, part_power, repl, min_part_hours): - run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", - part_power, repl, min_part_hours]) - - -def add_host_to_ring(build_file, host): - host_str = "" - if host.get('region') is not None: - host_str += 'r%(region)d' % host - host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) - host_str += "-%(host)s:%(port)d" % host - if host.get('repl_port'): - r_ip = host.get('repl_ip', host['host']) - host_str += "R%s:%d" % (r_ip, host['repl_port']) - host_str += "/%(drive)s" % host - - weight = host.get('weight', DEFAULT_HOST_WEIGHT) - run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', - host_str, str(weight)]) - - -def run_and_wait(func, *args): - t = threading.Thread(target=func, args=args) - t.start() - return t.join() - - -def has_section(conf, section): - return True if conf.get(section) else False - - -def check_section(conf, section): - if not has_section(conf, section): - print("Section %s doesn't exist" % (section)) - sys.exit(2) - - -def build_ring(section, conf, part_power, hosts): - # Create the build file - build_file = "%s.builder" % (section) - repl = conf.get('repl_number', DEFAULT_REPL) - min_part_hours = conf.get('min_part_hours', - DEFAULT_MIN_PART_HOURS) - create_buildfile(build_file, part_power, repl, min_part_hours) - - # Add the hosts - if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0: - print("No hosts/drives assigned to the %s ring" % section) - sys.exit(3) - - section_key = section.split('-')[0] - service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) - for host in conf['hosts']: - if 'name' in host: - if host['name'] not in hosts: - print("Host %(name) reference not found." % host) - sys.exit(3) - host = hosts[host['name']] - else: - if 'drive' not in host: - host['drive'] = DEFAULT_HOST_DRIVE - host['port'] = service_port - add_host_to_ring(build_file, host) - - # Rebalance ring - run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) - # rb_main(("swift-ring-builder", build_file, "rebalance")) - - -def main(setup): - # load the yaml file - try: - with open(setup) as yaml_stream: - _swift = yaml.load(yaml_stream) - except Exception as ex: - print("Failed to load yaml string %s" % (ex)) - return 1 - - _hosts = {} - - if _swift.get("hosts"): - for host in _swift['hosts']: - if not host.get('drive'): - host['drive'] = DEFAULT_HOST_DRIVE - key = "%(host)s/%(drive)s" % host - if key in _hosts: - print("%(host)s already definined" % host) - return 1 - _hosts[key] = host - - check_section(_swift, 'swift') - part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER) - - # Create account ring - check_section(_swift, 'account') - build_ring('account', _swift['account'], part_power, _hosts) - - # Create container ring - check_section(_swift, 'container') - build_ring('container', _swift['container'], part_power, _hosts) - - # Create object rings (storage policies) - check_section(_swift, 'storage_policies') - check_section(_swift['storage_policies'], 'policies') - indexes = set() - for sp in _swift['storage_policies']['policies']: - if sp['index'] in indexes: - print("Storage Policy index %d already in use" % (sp['index'])) - return 4 - buildfilename = 'object-%d' % (sp['index']) - indexes.add(sp['index']) - if 'port' not in sp: - sp['port'] = _swift['storage_policies'].get('port', - DEFAULT_OBJECT_PORT) - build_ring(buildfilename, sp, part_power, _hosts) - -if __name__ == "__main__": - parser = OptionParser(USAGE) - parser.add_option("-s", "--setup", dest="setup", - help="Specify the swift setup file.", metavar="FILE", - default="/etc/swift/swift_inventory.yml") - - options, args = parser.parse_args(sys.argv[1:]) - if options.setup and not exists(options.setup): - print("Swift setup file not found or doesn't exist") - parser.print_help() - sys.exit(1) - - sys.exit(main(options.setup)) From d3d64d75d95469d3c5c338c8ebd913995b07b92c Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 26 Sep 2014 15:20:09 +0000 Subject: [PATCH 14/45] Adjust the ring-builder to happen on the local host * This needs to be built on the host (where the inventory file is) * Copy up from local host to remote hosts * Tag directory creation from container_common * Run against only swift_hosts and swift_proxy (Avoids running against proxy containers) --- .../playbooks/openstack/swift-build-rings.yml | 8 ++++-- .../tasks/container_os_setup.yml | 2 ++ .../roles/swift_ring_builder/tasks/main.yml | 28 +++++++------------ .../swift_ring_distribute/tasks/main.yml | 2 ++ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/rpc_deployment/playbooks/openstack/swift-build-rings.yml b/rpc_deployment/playbooks/openstack/swift-build-rings.yml index 66b35597e5..da2c40f256 100644 --- a/rpc_deployment/playbooks/openstack/swift-build-rings.yml +++ b/rpc_deployment/playbooks/openstack/swift-build-rings.yml @@ -1,10 +1,14 @@ --- -- hosts: swift_all[0] +- hosts: local user: root roles: + - { role: container_common, tags: [ 'directories' ] } + - openstack_common - swift_ring_builder + vars_files: + - inventory/group_vars/swift_all.yml -- hosts: swift_all:!swift_all[0] +- hosts: swift_hosts:swift_proxy user: root roles: - swift_ring_distribute diff --git a/rpc_deployment/roles/container_common/tasks/container_os_setup.yml b/rpc_deployment/roles/container_common/tasks/container_os_setup.yml index 6ebd0357c6..4bfb6b086d 100644 --- a/rpc_deployment/roles/container_common/tasks/container_os_setup.yml +++ b/rpc_deployment/roles/container_common/tasks/container_os_setup.yml @@ -22,3 +22,5 @@ recurse=true when: container_directories is defined with_items: container_directories + tags: + - directories diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 8b21a832be..85418f0966 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -2,28 +2,20 @@ - name: "make sure scripts directory exists" file: > state=directory - path=/etc/swift/scripts/ + path={{ item }} owner=root group=root mode=0755 + with_items: + - /tmp/swift/rings + - /tmp/swift/scripts -- name: "Retrieve the swift_rings.py file" +- name: "Copy the swift_rings.py file" copy: > src=swift_rings.py - dest="/etc/swift/scripts/swift_rings.py" + dest="/tmp/swift/scripts/swift_rings.py" -#- name: "build rings" -# command: /usr/bin/python /etc/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_user_config.yml -# args: -# chdir: /etc/swift/rings/ - -- name: "List ring files to copy over" - command: "/bin/ls /etc/swift/rings/" - register: ringfiles - -- name: "Fetch rings to local server" - fetch: > - src="/etc/swift/rings/{{ item }}" - dest=/tmp/swift/rings/ - flat=yes - with_items: ringfiles.stdout_lines +- name: "build rings" + command: /usr/bin/python /tmp/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_user_config.yml + args: + chdir: /tmp/swift/rings/ diff --git a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml index 93297a53cc..09ca124e0b 100644 --- a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml @@ -4,3 +4,5 @@ src=/tmp/swift/rings/ dest=/etc/swift/rings/ mode=0644 + owner={{ system_user }} + group={{ system_group }} From 98676818d7f0c36d60d65a14f4bb5be7ebc4c512 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 30 Sep 2014 09:39:09 +0000 Subject: [PATCH 15/45] Adjust swift_rings.py to work with rpc_user_config.yml * Adjust the way hosts/storage policies are setup * Set this as a template so default port can be a variable * First pass at changing the swift_rings.py file --- .../roles/swift_ring_builder/tasks/main.yml | 3 +- .../{files => templates}/swift_rings.py | 106 +++++++++--------- 2 files changed, 54 insertions(+), 55 deletions(-) rename rpc_deployment/roles/swift_ring_builder/{files => templates}/swift_rings.py (52%) diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 85418f0966..8a6cfbe9a5 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -11,9 +11,10 @@ - /tmp/swift/scripts - name: "Copy the swift_rings.py file" - copy: > + template: > src=swift_rings.py dest="/tmp/swift/scripts/swift_rings.py" + mode=0700 - name: "build rings" command: /usr/bin/python /tmp/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_user_config.yml diff --git a/rpc_deployment/roles/swift_ring_builder/files/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py similarity index 52% rename from rpc_deployment/roles/swift_ring_builder/files/swift_rings.py rename to rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 3c32845ae1..b5af65c2d2 100644 --- a/rpc_deployment/roles/swift_ring_builder/files/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -8,18 +8,17 @@ import sys import threading import yaml -USAGE = "usage: %prog -s " +USAGE = "usage: %prog -s " DEFAULT_PART_POWER = 10 DEFAULT_REPL = 3 DEFAULT_MIN_PART_HOURS = 1 -DEFAULT_HOST_DRIVES = '/srv/drive/' -DEFAULT_HOST_DRIVE = '/sdb' +DEFAULT_HOST_DRIVE = 'sdb' DEFAULT_HOST_ZONE = 0 -DEFAULT_HOST_WEIGHT = 1 -DEFAULT_ACCOUNT_PORT = 6002 -DEFAULT_CONTAINER_PORT = 6001 -DEFAULT_OBJECT_PORT = 6000 +DEFAULT_HOST_WEIGHT = 100 +DEFAULT_ACCOUNT_PORT = {{ swift_account_port }} +DEFAULT_CONTAINER_PORT = {{ swift_container_port }} +DEFAULT_OBJECT_PORT = {{ swift_object_port }} DEFAULT_SECTION_PORT = { 'account': DEFAULT_ACCOUNT_PORT, 'container': DEFAULT_CONTAINER_PORT, @@ -37,11 +36,11 @@ def add_host_to_ring(build_file, host): if host.get('region') is not None: host_str += 'r%(region)d' % host host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) - host_str += "-%(host)s:%(port)d" % host + host_str += "-%(ip)s:%(port)d" % host if host.get('repl_port'): r_ip = host.get('repl_ip', host['host']) host_str += "R%s:%d" % (r_ip, host['repl_port']) - host_str += "/%(drive)s" % host + host_str += "/%(name)s" % host weight = host.get('weight', DEFAULT_HOST_WEIGHT) run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', @@ -72,24 +71,12 @@ def build_ring(section, conf, part_power, hosts): DEFAULT_MIN_PART_HOURS) create_buildfile(build_file, part_power, repl, min_part_hours) - # Add the hosts - if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0: - print("No hosts/drives assigned to the %s ring" % section) - sys.exit(3) - section_key = section.split('-')[0] service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) - for host in conf['hosts']: - if 'name' in host: - if host['name'] not in hosts: - print("Host %(name) reference not found." % host) - sys.exit(3) - host = hosts[host['name']] - else: - if 'drive' not in host: - host['drive'] = DEFAULT_HOST_DRIVE - host['port'] = service_port - add_host_to_ring(build_file, host) + for host in hosts: + host_vars = hosts[host] + host_vars['port'] = service_port + add_host_to_ring(build_file, host_vars) # Rebalance ring run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) @@ -106,48 +93,59 @@ def main(setup): return 1 _hosts = {} - - if _swift.get("hosts"): - for host in _swift['hosts']: - if not host.get('drive'): - host['drive'] = DEFAULT_HOST_DRIVE - key = "%(host)s/%(drive)s" % host - if key in _hosts: - print("%(host)s already definined" % host) - return 1 - _hosts[key] = host - - check_section(_swift, 'swift') - part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER) + if _swift.get("swift_hosts"): + for host in _swift['swift_hosts']: + host_vars = _swift['swift_hosts'][host]['container_vars']['swift_vars'] + if not host_vars.get('drive'): + host_vars['drive'] = DEFAULT_HOST_DRIVE + host_ip = _swift['swift_hosts'][host]['ip'] + host_drives = host_vars.get('drive') + for host_drive in host_drives: + host_drive['ip'] = host_drive.get('ip', host_ip) + if host_vars.get('repl_ip'): + host_drive['repl_ip'] = host_drives[host_drive].get('repl_ip', host_vars['repl_ip']) + if host_vars.get('repl_port'): + host_drive['repl_port'] = host_drives[host_drive].get('repl_port', host_vars['repl_port']) + if host_vars.get('weight'): + host_drive['weight'] = host_drives[host_drive].get('weight', host_vars['weight']) + key = "%s/%s" % (host_drive['ip'], host_drive['name']) + if key in _hosts: + print("%(host)s already definined" % host) + return 1 + _hosts[key] = host_drive + + global_vars = _swift['global_overrides'] + check_section(global_vars, 'swift') + swift_vars = global_vars['swift'] + part_power = swift_vars.get('part_power', DEFAULT_PART_POWER) # Create account ring - check_section(_swift, 'account') - build_ring('account', _swift['account'], part_power, _hosts) + check_section(swift_vars, 'account') + build_ring('account', swift_vars['account'], part_power, _hosts) # Create container ring - check_section(_swift, 'container') - build_ring('container', _swift['container'], part_power, _hosts) + check_section(swift_vars, 'container') + build_ring('container', swift_vars['container'], part_power, _hosts) # Create object rings (storage policies) - check_section(_swift, 'storage_policies') - check_section(_swift['storage_policies'], 'policies') + check_section(swift_vars, 'storage_policies') indexes = set() - for sp in _swift['storage_policies']['policies']: - if sp['index'] in indexes: - print("Storage Policy index %d already in use" % (sp['index'])) + for policy in swift_vars['storage_policies']: + policy = policy['policy'] + if policy['index'] in indexes: + print("Storage Policy index %d already in use" % (policy['index'])) return 4 - buildfilename = 'object-%d' % (sp['index']) - indexes.add(sp['index']) - if 'port' not in sp: - sp['port'] = _swift['storage_policies'].get('port', - DEFAULT_OBJECT_PORT) - build_ring(buildfilename, sp, part_power, _hosts) + buildfilename = 'object-%d' % (policy['index']) + indexes.add(policy['index']) + if 'port' not in policy: + policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT) + build_ring(buildfilename, policy, part_power, _hosts) if __name__ == "__main__": parser = OptionParser(USAGE) parser.add_option("-s", "--setup", dest="setup", help="Specify the swift setup file.", metavar="FILE", - default="/etc/swift/swift_inventory.yml") + default="/etc/rpc_deploy/rpc_user_config.yml") options, args = parser.parse_args(sys.argv[1:]) if options.setup and not exists(options.setup): From 4da4e7735ad90952149e6fadac0a0315a8f18dce Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 30 Sep 2014 10:33:47 +0000 Subject: [PATCH 16/45] Fix keystone endpoint creation for Swift * Setup the keystone endpoint for swift * Move swift-proxy port to 8888 (8000 is used by heat) * Typo prevented object server services from restarting * Adjust order of swift plays for swift-all.yml --- .../inventory/group_vars/swift_all.yml | 2 +- .../openstack/keystone-add-all-services.yml | 9 ++++++++ .../playbooks/openstack/swift-all.yml | 2 +- .../playbooks/openstack/swift-build-rings.yml | 1 + .../roles/swift_account/tasks/main.yml | 2 +- .../roles/swift_object/handlers/main.yml | 2 +- .../vars/config_vars/haproxy_config.yml | 2 +- .../swift_proxy_endpoint.yml | 22 +++++++++++++++++++ scripts/uklabsetup.sh | 2 ++ 9 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 rpc_deployment/vars/openstack_service_vars/swift_proxy_endpoint.yml diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index d6979fbab3..1f5494997b 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -41,7 +41,7 @@ container_lvm_fssize: 5GB # driver: cinder.volume.drivers.lvm.LVMISCSIDriver # backend_name: LVM_iSCSI -swift_proxy_port: "8000" +swift_proxy_port: "8888" swift_object_port: "6000" swift_container_port: "6001" swift_account_port: "6002" diff --git a/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml b/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml index bb146ec33a..9a262f6ee3 100644 --- a/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml +++ b/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml @@ -111,3 +111,12 @@ vars_files: - inventory/group_vars/nova_all.yml - vars/openstack_service_vars/nova_api_s3_endpoint.yml + +# Swift endpoint +- hosts: keystone[0] + user: root + roles: + - keystone_add_service + vars_files: + - inventory/group_vars/swift_all.yml + - vars/openstack_service_vars/swift_proxy_endpoint.yml diff --git a/rpc_deployment/playbooks/openstack/swift-all.yml b/rpc_deployment/playbooks/openstack/swift-all.yml index 4937c69a10..e4042d6d1b 100644 --- a/rpc_deployment/playbooks/openstack/swift-all.yml +++ b/rpc_deployment/playbooks/openstack/swift-all.yml @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: swift-build-rings.yml - include: swift-proxy.yml - include: swift-storage.yml -- include: swift-build-rings.yml diff --git a/rpc_deployment/playbooks/openstack/swift-build-rings.yml b/rpc_deployment/playbooks/openstack/swift-build-rings.yml index da2c40f256..0a1ac2e330 100644 --- a/rpc_deployment/playbooks/openstack/swift-build-rings.yml +++ b/rpc_deployment/playbooks/openstack/swift-build-rings.yml @@ -4,6 +4,7 @@ roles: - { role: container_common, tags: [ 'directories' ] } - openstack_common + - swift_common - swift_ring_builder vars_files: - inventory/group_vars/swift_all.yml diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml index 914c99e090..fbc6c4a853 100644 --- a/rpc_deployment/roles/swift_account/tasks/main.yml +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -13,7 +13,7 @@ - name: "Set account server to start at boot" cron: > - name="Restart account-sever on boot" + name="Restart account-server on boot" special_time=reboot job="swift-init account-server start" diff --git a/rpc_deployment/roles/swift_object/handlers/main.yml b/rpc_deployment/roles/swift_object/handlers/main.yml index 3bce44e32e..20f0f616ef 100644 --- a/rpc_deployment/roles/swift_object/handlers/main.yml +++ b/rpc_deployment/roles/swift_object/handlers/main.yml @@ -8,5 +8,5 @@ - name: Restart object replicator command: swift-init object-replicator restart -- name: Retart object updater +- name: Restart object updater command: swift-init object-updater restart diff --git a/rpc_deployment/vars/config_vars/haproxy_config.yml b/rpc_deployment/vars/config_vars/haproxy_config.yml index b5a575307a..e4e6a9ee3b 100644 --- a/rpc_deployment/vars/config_vars/haproxy_config.yml +++ b/rpc_deployment/vars/config_vars/haproxy_config.yml @@ -181,7 +181,7 @@ haproxy_config: - service: hap_service_name: swift_proxy hap_backend_nodes: "{{ groups['swift_proxy'] }}" - hap_port: 8000 + hap_port: 8888 hap_balance_type: http hap_backend_alg: source hap_backend_options: diff --git a/rpc_deployment/vars/openstack_service_vars/swift_proxy_endpoint.yml b/rpc_deployment/vars/openstack_service_vars/swift_proxy_endpoint.yml new file mode 100644 index 0000000000..a2ddcf9510 --- /dev/null +++ b/rpc_deployment/vars/openstack_service_vars/swift_proxy_endpoint.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +service_type: object-store +service_description: "Object Storage Service" +service_password: "{{ swift_service_password }}" + +service_publicurl: "http://{{ external_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_%(tenant_id)s" +service_adminurl: "http://{{ internal_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_%(tenant_id)s" +service_internalurl: "http://{{ internal_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_%(tenant_id)s" diff --git a/scripts/uklabsetup.sh b/scripts/uklabsetup.sh index 2cacc5eeca..31fee39707 100755 --- a/scripts/uklabsetup.sh +++ b/scripts/uklabsetup.sh @@ -32,5 +32,7 @@ pushd /root/ansible-lxc-rpc # Openstack Service Setup ansible-playbook -e @/etc/rpc_deploy/user_variables.yml playbooks/openstack/openstack-setup.yml + + ansible-playbook -e @/etc/rpc_deploy/user_variables.yml playbooks/openstack/swift-all.yml popd popd From bd8b395dd6a1f8cfbe5d80d42bfe978cadce9e37 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 30 Sep 2014 12:50:58 +0000 Subject: [PATCH 17/45] Fix haproxy endpoint for swift-proxy HAProxy settings were wrong causing swift-proxy to 503 --- rpc_deployment/vars/config_vars/haproxy_config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rpc_deployment/vars/config_vars/haproxy_config.yml b/rpc_deployment/vars/config_vars/haproxy_config.yml index e4e6a9ee3b..d421e0acdd 100644 --- a/rpc_deployment/vars/config_vars/haproxy_config.yml +++ b/rpc_deployment/vars/config_vars/haproxy_config.yml @@ -183,6 +183,7 @@ haproxy_config: hap_backend_nodes: "{{ groups['swift_proxy'] }}" hap_port: 8888 hap_balance_type: http - hap_backend_alg: source hap_backend_options: - - "ssl-hello-chk" + - "forwardfor" + - "httpchk" + - "httplog" From 366883ed193af47e2c58590f7fc1953f0bcc8bec Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 30 Sep 2014 13:32:07 +0000 Subject: [PATCH 18/45] Adjust the sample /etc/rpc_deplooy/rpc_user_config.yml * Change this to reflect the changes required for swift --- etc/rpc_deploy/rpc_user_config.yml | 45 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index be75b33246..fead9d65d5 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -109,18 +109,31 @@ global_overrides: type: "vlan" range: "1:1" net_name: "vlan" - # For Swift Set Storage Policies - # storage_policies: + # + # Setup swift group variables when using swift + # For account/container speciying min_part_hours and repl_number is all thats required + # Alternatively defaults will be used. + # For storage policies, a name and unique index is required as well as repl_number and + # min_part_hours which will be set to a default value if not specified. + # swift: + # part_power: 8 + # account: + # repl_number: 3 + # min_part_hours: 1 + # container: + # repl_number: 3 + # storage_policies: # - policy: - # name: "gold" + # name: gold # index: 0 # repl_number: 3 - # default: "Yes" + # default: yes # - policy: - # name: "silver" + # name: silver # index: 1 # repl_number: 2 - # deprecated: "Yes" + # deprecated: yes + # # Name of load balancer lb_name: lb_name_in_core @@ -176,17 +189,31 @@ network_hosts: # ip: 172.29.236.108 # container_vars: # swift_vars: -# policy: gold # region: 0 # zone: 0 + # Specify where the drives are mounted # drives: /srv/node + # Specify the drives in use +# drive: +# - name: sdb + # weight can be specified at a host level - this will override it +# weight: 100 + # ip will default to host_ip if not specified +# ip: 172.10.100.100 + # repl_ip & port are not required, but if there is a specific network/ip for replication specify it here +# repl_ip: 10.10.0.1 +# repl_port: 54321 +# - name: sdc +# - name: sdd +# - name: sde # # object_storage2: # ip: 172.29.236.109 # container_vars: # swift_vars: -# policy: silver # region: 0 # zone: 1 # drives: /srv/node -# +# drive: +# - name: sdb +# - name: sdc From ba4efd45969c346baa8859438ba6552ffd8caf90 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 30 Sep 2014 13:43:48 +0000 Subject: [PATCH 19/45] Fix haproxy vars for swift-proxy again * Checks cause haproxy service to fail shortly after starting up! --- rpc_deployment/vars/config_vars/haproxy_config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rpc_deployment/vars/config_vars/haproxy_config.yml b/rpc_deployment/vars/config_vars/haproxy_config.yml index d421e0acdd..22be6a92d5 100644 --- a/rpc_deployment/vars/config_vars/haproxy_config.yml +++ b/rpc_deployment/vars/config_vars/haproxy_config.yml @@ -183,7 +183,3 @@ haproxy_config: hap_backend_nodes: "{{ groups['swift_proxy'] }}" hap_port: 8888 hap_balance_type: http - hap_backend_options: - - "forwardfor" - - "httpchk" - - "httplog" From ce271cbda1ca7cd06215705420ecfedd56e74f80 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 10:11:43 +0000 Subject: [PATCH 20/45] Fix ring distribution * Now distributes rings to the appropriate location * Uses fileglob to ensure only appropriate files are distributed * Fixed swift_rings.py to correctly use precedence on weight/repl_ip/repl_port --- .../roles/swift_ring_builder/templates/swift_rings.py | 6 +++--- rpc_deployment/roles/swift_ring_distribute/tasks/main.yml | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index b5af65c2d2..65582885ab 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -103,11 +103,11 @@ def main(setup): for host_drive in host_drives: host_drive['ip'] = host_drive.get('ip', host_ip) if host_vars.get('repl_ip'): - host_drive['repl_ip'] = host_drives[host_drive].get('repl_ip', host_vars['repl_ip']) + host_drive['repl_ip'] = host_drive.get('repl_ip', host_vars.get['repl_ip']) if host_vars.get('repl_port'): - host_drive['repl_port'] = host_drives[host_drive].get('repl_port', host_vars['repl_port']) + host_drive['repl_port'] = host_drive.get('repl_port', host_vars['repl_port']) if host_vars.get('weight'): - host_drive['weight'] = host_drives[host_drive].get('weight', host_vars['weight']) + host_drive['weight'] = host_drive.get('weight', host_vars['weight']) key = "%s/%s" % (host_drive['ip'], host_drive['name']) if key in _hosts: print("%(host)s already definined" % host) diff --git a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml index 09ca124e0b..0af0123b59 100644 --- a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml @@ -1,8 +1,11 @@ --- - name: "Copy the rings over" copy: > - src=/tmp/swift/rings/ - dest=/etc/swift/rings/ + src={{ item }} + dest=/etc/swift/ mode=0644 owner={{ system_user }} group={{ system_group }} + with_fileglob: + - /tmp/swift/rings/*.ring.gz + - /tmp/swift/rings/*.builder From e5baba2497826be65d118db1df231c9cf44084df Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 10:43:17 +0000 Subject: [PATCH 21/45] Set object ring for index 0 to be "object.ring.gz" * This is required by swift for legacy (non-storage policy installs) * Subsequent object rings will be object-index.ring.gz --- .../roles/swift_ring_builder/templates/swift_rings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 65582885ab..9fa652da78 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -135,7 +135,10 @@ def main(setup): if policy['index'] in indexes: print("Storage Policy index %d already in use" % (policy['index'])) return 4 - buildfilename = 'object-%d' % (policy['index']) + if policy['index'] == 0: + buildfilename = 'object' + else: + buildfilename = 'object-%d' % (policy['index']) indexes.add(policy['index']) if 'port' not in policy: policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT) From 018652cd636afe0d1a97e7023f4729fb34c9f4ac Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 13:31:33 +0000 Subject: [PATCH 22/45] Allow the use of storage policies * Set the default groups to be all groups. * Allow groups to be specified on per host or per drive level * Only add specific drives to specific rings for their groups. * Fix typos on repl_port lines * Generate group list based on inventory * Allow hosts with no drives to just be skipped over * Exit if part power isn't specified --- .../templates/swift_rings.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 9fa652da78..a8c1bd2a54 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -10,10 +10,8 @@ import yaml USAGE = "usage: %prog -s " -DEFAULT_PART_POWER = 10 DEFAULT_REPL = 3 DEFAULT_MIN_PART_HOURS = 1 -DEFAULT_HOST_DRIVE = 'sdb' DEFAULT_HOST_ZONE = 0 DEFAULT_HOST_WEIGHT = 100 DEFAULT_ACCOUNT_PORT = {{ swift_account_port }} @@ -24,7 +22,24 @@ DEFAULT_SECTION_PORT = { 'container': DEFAULT_CONTAINER_PORT, 'object': DEFAULT_OBJECT_PORT, } - +DEFAULT_GROUP_MAP = { + 'account': 'account', +{% for policy in storage_policies %} +{% if policy.policy.index == 0 %} + 'object': '{{ policy.policy.name }}', +{% else %} + 'object-{{ policy.policy.index}}': '{{ policy.policy.name }}', +{% endif %} +{% endfor %} + 'container': 'container' +} +DEFAULT_GROUPS= [ + 'account', +{% for policy in storage_policies %} + '{{ policy.policy.name }}', +{% endfor %} + 'container' +] def create_buildfile(build_file, part_power, repl, min_part_hours): run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", @@ -76,7 +91,9 @@ def build_ring(section, conf, part_power, hosts): for host in hosts: host_vars = hosts[host] host_vars['port'] = service_port - add_host_to_ring(build_file, host_vars) + host_vars['groups'] = host_vars.get('groups', DEFAULT_GROUPS) + if DEFAULT_GROUP_MAP[section] in host_vars['groups']: + add_host_to_ring(build_file, host_vars) # Rebalance ring run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) @@ -96,14 +113,16 @@ def main(setup): if _swift.get("swift_hosts"): for host in _swift['swift_hosts']: host_vars = _swift['swift_hosts'][host]['container_vars']['swift_vars'] - if not host_vars.get('drive'): - host_vars['drive'] = DEFAULT_HOST_DRIVE host_ip = _swift['swift_hosts'][host]['ip'] + if not host_vars.get('drive'): + continue host_drives = host_vars.get('drive') for host_drive in host_drives: host_drive['ip'] = host_drive.get('ip', host_ip) + if host_vars.get('groups'): + host_drive['groups'] = host_drive.get('groups', host_vars['groups']) if host_vars.get('repl_ip'): - host_drive['repl_ip'] = host_drive.get('repl_ip', host_vars.get['repl_ip']) + host_drive['repl_ip'] = host_drive.get('repl_ip', host_vars['repl_ip']) if host_vars.get('repl_port'): host_drive['repl_port'] = host_drive.get('repl_port', host_vars['repl_port']) if host_vars.get('weight'): @@ -117,7 +136,10 @@ def main(setup): global_vars = _swift['global_overrides'] check_section(global_vars, 'swift') swift_vars = global_vars['swift'] - part_power = swift_vars.get('part_power', DEFAULT_PART_POWER) + if not swift_vars.get('part_power'): + print('No part_power specified - please set a part_power value') + return 1 + part_power = swift_vars.get('part_power') # Create account ring check_section(swift_vars, 'account') From 76573339091bb354e1351136dc8d5f8d923a60ef Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 14:08:36 +0000 Subject: [PATCH 23/45] Fix up the rpc_user_config sample conf to show how to use groups * Fix documentation in rpc_user_config.yml sample file * Show how to add create groups * Explain precedence --- etc/rpc_deploy/rpc_user_config.yml | 34 +++++++++++++++---- .../templates/swift_rings.py | 2 +- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index fead9d65d5..7c85338189 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -111,10 +111,15 @@ global_overrides: net_name: "vlan" # # Setup swift group variables when using swift + # part power is required under swift. # For account/container speciying min_part_hours and repl_number is all thats required - # Alternatively defaults will be used. + # Alternatively defaults will be used (repl_number of 3, and min_part_hours of 1). # For storage policies, a name and unique index is required as well as repl_number and # min_part_hours which will be set to a default value if not specified. + # There MUST be a storage policy with index 0 configured which will be the default for legacy containers (created pre-storage policies). + # You can set one policy to be "default: yes" this will be the default storage policy for non-legacy containers that are created. + # The index value must be unique. + # Storage policies can be set to "deprecated: yes" which will mean they are not used # swift: # part_power: 8 # account: @@ -184,6 +189,19 @@ network_hosts: ip: 172.29.236.108 # User defined Object Storage Hosts - this is not a required group +# Under swift_vars you can specify the host specific swift_vars. +# region - the swift region, this isn't required. +# zone - the swift zone, this isn't required either, will default to 0 +# drives - where the drives are mounted on the server +# Above 3 vars are "host specific" +# weight: a disks weight (defaults to 100 if not specified) +# repl_ip: IP specific for object replication (not required) +# repl_port: Port specific for object replication (not required) +# groups: A list of groups to add the drive to. A group is either a storage policy or the account or container servers. (If not specified defaults to all groups, so container/account/all storage policies). +# The above 4 can be specified on a per host or per drive basis +# Or both, in which case "per drive" will take precedence for the specific drive. +# ip can be specified in swift_vars to override the hosts ip +# or per drive to override all for that specific drive. # swift_hosts: # object_storage1: # ip: 172.29.236.108 @@ -191,19 +209,21 @@ network_hosts: # swift_vars: # region: 0 # zone: 0 - # Specify where the drives are mounted +# groups: +# - silver +# - account # drives: /srv/node - # Specify the drives in use # drive: # - name: sdb - # weight can be specified at a host level - this will override it -# weight: 100 - # ip will default to host_ip if not specified # ip: 172.10.100.100 - # repl_ip & port are not required, but if there is a specific network/ip for replication specify it here # repl_ip: 10.10.0.1 # repl_port: 54321 +# groups: +# - gold +# - account +# - container # - name: sdc +# weight: 150 # - name: sdd # - name: sde # diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index a8c1bd2a54..ef299e2ac5 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -113,7 +113,7 @@ def main(setup): if _swift.get("swift_hosts"): for host in _swift['swift_hosts']: host_vars = _swift['swift_hosts'][host]['container_vars']['swift_vars'] - host_ip = _swift['swift_hosts'][host]['ip'] + host_ip = host_vars.get('ip', _swift['swift_hosts'][host]['ip']) if not host_vars.get('drive'): continue host_drives = host_vars.get('drive') From d2f6a655d5b1f0ec204d84c7fe52aaa980399480 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 14:16:49 +0000 Subject: [PATCH 24/45] Change drives to "mount_point" and drive to "drives" * Avoids confusion since drives is the list of drives * mount_point is where the drives are mounted --- etc/rpc_deploy/rpc_user_config.yml | 13 +++++++------ .../swift_account/templates/account-server.conf.j2 | 2 +- .../templates/container-server.conf.j2 | 2 +- .../swift_object/templates/object-server.conf.j2 | 2 +- .../swift_ring_builder/templates/swift_rings.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index 7c85338189..e1a6f9d073 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -192,8 +192,9 @@ network_hosts: # Under swift_vars you can specify the host specific swift_vars. # region - the swift region, this isn't required. # zone - the swift zone, this isn't required either, will default to 0 -# drives - where the drives are mounted on the server -# Above 3 vars are "host specific" +# mount_point - where the drives are mounted on the server +# drives - A list of drives in the server (Must have a name as a minimum) +# Above 4 vars are "host specific" # weight: a disks weight (defaults to 100 if not specified) # repl_ip: IP specific for object replication (not required) # repl_port: Port specific for object replication (not required) @@ -212,8 +213,8 @@ network_hosts: # groups: # - silver # - account -# drives: /srv/node -# drive: +# mount_point: /srv/node +# drives: # - name: sdb # ip: 172.10.100.100 # repl_ip: 10.10.0.1 @@ -233,7 +234,7 @@ network_hosts: # swift_vars: # region: 0 # zone: 1 -# drives: /srv/node -# drive: +# mount_point: /srv/node +# drives: # - name: sdb # - name: sdc diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 index 545baaba9e..0cce735b77 100644 --- a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -5,7 +5,7 @@ bind_port = {{ swift_account_port }} # backlog = 4096 user = {{ system_user }} # swift_dir = /etc/swift -devices = {{ swift_vars.drives }} +devices = {{ swift_vars.mount_point }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 index 5d29c5d758..cb0d01b06e 100644 --- a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -5,7 +5,7 @@ bind_port = {{ swift_container_port }} # backlog = 4096 user = {{ system_user }} # swift_dir = /etc/swift -devices = {{ swift_vars.drives }} +devices = {{ swift_vars.mount_point }} # mount_check = true # disable_fallocate = false # diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 index 505e9d1b9e..716da67aea 100644 --- a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -5,7 +5,7 @@ bind_port = {{ swift_object_port }} # backlog = 4096 user = {{ system_user }} swift_dir = /etc/swift -devices = {{ swift_vars.drives }} +devices = {{ swift_vars.mount_point }} # mount_check = true # disable_fallocate = false # expiring_objects_container_divisor = 86400 diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index ef299e2ac5..90c8e9bde8 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -114,9 +114,9 @@ def main(setup): for host in _swift['swift_hosts']: host_vars = _swift['swift_hosts'][host]['container_vars']['swift_vars'] host_ip = host_vars.get('ip', _swift['swift_hosts'][host]['ip']) - if not host_vars.get('drive'): + if not host_vars.get('drives'): continue - host_drives = host_vars.get('drive') + host_drives = host_vars.get('drives') for host_drive in host_drives: host_drive['ip'] = host_drive.get('ip', host_ip) if host_vars.get('groups'): From d94ac0cc77a0f921a12661eaf7823492013ca970 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 14:26:49 +0000 Subject: [PATCH 25/45] Create variables for default values for Swift * This allows the ring builder to work based off swift_all vars --- .../inventory/group_vars/swift_all.yml | 24 ++++++++----------- .../templates/swift_rings.py | 8 +++---- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 1f5494997b..a463bbab79 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -13,11 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The variables file used by the playbooks in the Cinder-api group. +# The variables file used by the playbooks in the swift-hosts & swift-proxy groups. # These don't have to be explicitly imported by vars_files: they are autopopulated. -####### TODO: ADDING BOGUS VARS THAT SHOULD BE CONFIGURABLE VIA RPC_USER_CONFIG - authtoken_active: True delay_auth_decision: true @@ -32,31 +30,29 @@ verbose: True container_lvm_fstype: ext4 container_lvm_fssize: 5GB -## General configuration -## Set this in rpc_user_config.yml UNLESS you want all hosts to use the same -## Cinder backends. See the rpc_user_config example for more on how this is done. -# cinder_backends: -# lvm: -# volume_group: cinder-volumes -# driver: cinder.volume.drivers.lvm.LVMISCSIDriver -# backend_name: LVM_iSCSI - +# Swift default ports swift_proxy_port: "8888" swift_object_port: "6000" swift_container_port: "6001" swift_account_port: "6002" +# Swift default variables +swift_default_replication_number: 3 +swift_default_min_part_hours: 1 +swift_default_host_zone: 0 +swift_default_drive_weight: 100 + ## DB container_mysql_user: swift container_mysql_password: "{{ swift_container_mysql_password }}" container_database: swift -## Cinder Auth +## Swift Auth service_admin_tenant_name: "service" service_admin_username: "swift" service_admin_password: "{{ swift_service_password }}" -## Cinder User / Group +## Swift User / Group system_user: swift system_group: swift diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 90c8e9bde8..5bb83af579 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -10,10 +10,10 @@ import yaml USAGE = "usage: %prog -s " -DEFAULT_REPL = 3 -DEFAULT_MIN_PART_HOURS = 1 -DEFAULT_HOST_ZONE = 0 -DEFAULT_HOST_WEIGHT = 100 +DEFAULT_REPL = {{ swift_default_replication_number }} +DEFAULT_MIN_PART_HOURS = {{ swift_default_min_part_hours }} +DEFAULT_HOST_ZONE = {{ swift_default_host_zone }} +DEFAULT_HOST_WEIGHT = {{ swift_default_drive_weight }} DEFAULT_ACCOUNT_PORT = {{ swift_account_port }} DEFAULT_CONTAINER_PORT = {{ swift_container_port }} DEFAULT_OBJECT_PORT = {{ swift_object_port }} From af4710f53ace5db6656dec6b8bc096c612d9c0ec Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 14:34:03 +0000 Subject: [PATCH 26/45] Add Legal notice to swift files * Add the legal notice to all the swift files --- .../roles/swift_account/handlers/main.yml | 14 ++++++++++++++ rpc_deployment/roles/swift_account/tasks/main.yml | 14 ++++++++++++++ rpc_deployment/roles/swift_account/vars/main.yml | 1 - rpc_deployment/roles/swift_common/tasks/main.yml | 14 ++++++++++++++ .../roles/swift_container/handlers/main.yml | 14 ++++++++++++++ .../roles/swift_container/tasks/main.yml | 14 ++++++++++++++ .../roles/swift_object/handlers/main.yml | 14 ++++++++++++++ rpc_deployment/roles/swift_object/tasks/main.yml | 14 ++++++++++++++ .../roles/swift_ring_builder/tasks/main.yml | 14 ++++++++++++++ .../swift_ring_builder/templates/swift_rings.py | 14 ++++++++++++++ .../roles/swift_ring_distribute/tasks/main.yml | 14 ++++++++++++++ 11 files changed, 140 insertions(+), 1 deletion(-) delete mode 100644 rpc_deployment/roles/swift_account/vars/main.yml diff --git a/rpc_deployment/roles/swift_account/handlers/main.yml b/rpc_deployment/roles/swift_account/handlers/main.yml index 5506aaf09f..b769c70062 100644 --- a/rpc_deployment/roles/swift_account/handlers/main.yml +++ b/rpc_deployment/roles/swift_account/handlers/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Restart account server command: swift-init account-server restart diff --git a/rpc_deployment/roles/swift_account/tasks/main.yml b/rpc_deployment/roles/swift_account/tasks/main.yml index fbc6c4a853..61210c2508 100644 --- a/rpc_deployment/roles/swift_account/tasks/main.yml +++ b/rpc_deployment/roles/swift_account/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "swift account server configuration" template: > src="account-server.conf.j2" diff --git a/rpc_deployment/roles/swift_account/vars/main.yml b/rpc_deployment/roles/swift_account/vars/main.yml deleted file mode 100644 index ed97d539c0..0000000000 --- a/rpc_deployment/roles/swift_account/vars/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 05b1a451f8..3f392bb236 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "Drop swift.conf temaplte" template: > src="swift.conf.j2" diff --git a/rpc_deployment/roles/swift_container/handlers/main.yml b/rpc_deployment/roles/swift_container/handlers/main.yml index 7d1190d703..aba4fe98e6 100644 --- a/rpc_deployment/roles/swift_container/handlers/main.yml +++ b/rpc_deployment/roles/swift_container/handlers/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Restart container server command: swift-init container-server restart diff --git a/rpc_deployment/roles/swift_container/tasks/main.yml b/rpc_deployment/roles/swift_container/tasks/main.yml index d0e75fa864..8b78f5690a 100644 --- a/rpc_deployment/roles/swift_container/tasks/main.yml +++ b/rpc_deployment/roles/swift_container/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "swift container server configuration" template: > src="container-server.conf.j2" diff --git a/rpc_deployment/roles/swift_object/handlers/main.yml b/rpc_deployment/roles/swift_object/handlers/main.yml index 20f0f616ef..88ecdb9c11 100644 --- a/rpc_deployment/roles/swift_object/handlers/main.yml +++ b/rpc_deployment/roles/swift_object/handlers/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Restart object server command: swift-init object-server restart diff --git a/rpc_deployment/roles/swift_object/tasks/main.yml b/rpc_deployment/roles/swift_object/tasks/main.yml index 7008a96dc7..b092613564 100644 --- a/rpc_deployment/roles/swift_object/tasks/main.yml +++ b/rpc_deployment/roles/swift_object/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "swift object server configuration" template: > src="object-server.conf.j2" diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 8a6cfbe9a5..815de849bc 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "make sure scripts directory exists" file: > state=directory diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 5bb83af579..ccaf4bb1c7 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -1,4 +1,18 @@ #!/usr/bin/env python +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function from optparse import OptionParser from os.path import exists diff --git a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml index 0af0123b59..5ab5826564 100644 --- a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml @@ -1,4 +1,18 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: "Copy the rings over" copy: > src={{ item }} From c11a139b0524332661eb34df78bade3eabf6e144 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 15:59:47 +0000 Subject: [PATCH 27/45] Set it so that swift-proxy isn't installed by default * Move swift-proxy out the infra-hosts group, into its own group * Adjust the sample rpc_user_config to show the "swift-proxy_hosts" group --- etc/rpc_deploy/rpc_environment.yml | 6 ++++++ etc/rpc_deploy/rpc_user_config.yml | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/etc/rpc_deploy/rpc_environment.yml b/etc/rpc_deploy/rpc_environment.yml index 845b4f7c00..58e9ec3eac 100644 --- a/etc/rpc_deploy/rpc_environment.yml +++ b/etc/rpc_deploy/rpc_environment.yml @@ -330,3 +330,9 @@ physical_skel: swift_hosts: belongs_to: - hosts + swift-proxy_containers: + belongs_to: + - all_containers + swift-proxy_hosts: + belongs_to: + - hosts diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index e1a6f9d073..bd6449c2d8 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -15,7 +15,7 @@ # This is the md5 of the environment file # this will ensure consistency when deploying. -environment_version: 21de6ac181f7f9d5d950709928ccded8 +environment_version: dc4cd5fe9c07eee223e8bbb1c5bbaad5 # User defined CIDR used for containers # Global cidr/s used for everything. @@ -110,7 +110,7 @@ global_overrides: range: "1:1" net_name: "vlan" # - # Setup swift group variables when using swift + # Setup swift group variables when using swift (Not required if not using swift) # part power is required under swift. # For account/container speciying min_part_hours and repl_number is all thats required # Alternatively defaults will be used (repl_number of 3, and min_part_hours of 1). @@ -188,6 +188,17 @@ network_hosts: network1: ip: 172.29.236.108 +# User defined Swift Proxy hosts - not required when not using swift +# Will deploy a swift-proxy container on these hosts. +# Recommend mirroring the infra_hosts +# swift-proxy_hosts: +# infra1: +# ip: 172.29.236.100 +# infra2: +# ip: 172.29.236.101 +# infra3: +# ip: 172.29.236.102 + # User defined Object Storage Hosts - this is not a required group # Under swift_vars you can specify the host specific swift_vars. # region - the swift region, this isn't required. From 73eae36b9add9ab9da0d6c2652e5536cd84e6de6 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 1 Oct 2014 17:10:12 +0000 Subject: [PATCH 28/45] Fix bug when referring to storage_policies * Storage policies were moved into swift: in the user_config * Fix typo on template drop for swift_common --- rpc_deployment/roles/swift_common/tasks/main.yml | 2 +- rpc_deployment/roles/swift_common/templates/swift.conf.j2 | 2 +- .../roles/swift_ring_builder/templates/swift_rings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 3f392bb236..742d6eafe7 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: "Drop swift.conf temaplte" +- name: "Drop swift.conf template" template: > src="swift.conf.j2" dest="/etc/swift/swift.conf" diff --git a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 index f7958795a6..9f4ded08d6 100644 --- a/rpc_deployment/roles/swift_common/templates/swift.conf.j2 +++ b/rpc_deployment/roles/swift_common/templates/swift.conf.j2 @@ -9,7 +9,7 @@ swift_hash_path_suffix = {{ swift_hash_path_suffix }} swift_hash_path_prefix = {{ swift_hash_path_prefix }} # Storage Policies -{% for policy in storage_policies %} +{% for policy in swift.storage_policies %} [storage-policy:{{ policy.policy.index }}] name = {{ policy.policy.name }} {% if policy.policy.deprecated is defined %} diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index ccaf4bb1c7..ea6d65a73c 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -38,7 +38,7 @@ DEFAULT_SECTION_PORT = { } DEFAULT_GROUP_MAP = { 'account': 'account', -{% for policy in storage_policies %} +{% for policy in swift.storage_policies %} {% if policy.policy.index == 0 %} 'object': '{{ policy.policy.name }}', {% else %} @@ -49,7 +49,7 @@ DEFAULT_GROUP_MAP = { } DEFAULT_GROUPS= [ 'account', -{% for policy in storage_policies %} +{% for policy in swift.storage_policies %} '{{ policy.policy.name }}', {% endfor %} 'container' From 727563db21143473770ca5f14ffea30712118556 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 2 Oct 2014 10:18:21 +0000 Subject: [PATCH 29/45] Convert swift to use repo_packages * Fixes ordering issues for swift plays * Move to use repo packages * Remove the swift "keystone add service" for all services --- .../inventory/group_vars/swift_all.yml | 29 ------------------- .../openstack/keystone-add-all-services.yml | 9 ------ .../playbooks/openstack/swift-all.yml | 1 + .../playbooks/openstack/swift-build-rings.yml | 1 + .../playbooks/openstack/swift-common.yml | 26 +++++++++++++++++ .../playbooks/openstack/swift-proxy.yml | 14 +++++---- .../playbooks/openstack/swift-storage.yml | 6 ---- rpc_deployment/vars/repo_packages/swift.yml | 24 ++++++++++++++- 8 files changed, 59 insertions(+), 51 deletions(-) create mode 100644 rpc_deployment/playbooks/openstack/swift-common.yml diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index a463bbab79..426676f73a 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -63,38 +63,9 @@ service_names: - swift-container - swift-proxy -## Git Source -## ALL of this has been relocated to vars/repo_packages -## TODO(someone) this should be removed once the repo bits are all figured out. -git_repo: https://git.openstack.org/openstack/swift -git_fallback_repo: https://github.com/openstack/swift -git_etc_example: etc/ -git_install_branch: b223322ed1ef44f61490f820240aa01f1047ae2e - -service_pip_dependencies: - - pywbem - - ecdsa - - MySQL-python - - python-memcached - - pycrypto - - python-cinderclient - - python-keystoneclient - - keystonemiddleware - container_directories: - /var/log/swift - /tmp/swift - /var/lock/swift - /etc/swift - /etc/swift/rings/ - -container_packages: - - curl - - python-pip - - rsync - - openssh-server - - git-core - - python-setuptools - - python-dev - - gcc - - libffi-dev diff --git a/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml b/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml index 9a262f6ee3..bb146ec33a 100644 --- a/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml +++ b/rpc_deployment/playbooks/openstack/keystone-add-all-services.yml @@ -111,12 +111,3 @@ vars_files: - inventory/group_vars/nova_all.yml - vars/openstack_service_vars/nova_api_s3_endpoint.yml - -# Swift endpoint -- hosts: keystone[0] - user: root - roles: - - keystone_add_service - vars_files: - - inventory/group_vars/swift_all.yml - - vars/openstack_service_vars/swift_proxy_endpoint.yml diff --git a/rpc_deployment/playbooks/openstack/swift-all.yml b/rpc_deployment/playbooks/openstack/swift-all.yml index e4042d6d1b..77e951ee0a 100644 --- a/rpc_deployment/playbooks/openstack/swift-all.yml +++ b/rpc_deployment/playbooks/openstack/swift-all.yml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: swift-common.yml - include: swift-build-rings.yml - include: swift-proxy.yml - include: swift-storage.yml diff --git a/rpc_deployment/playbooks/openstack/swift-build-rings.yml b/rpc_deployment/playbooks/openstack/swift-build-rings.yml index 0a1ac2e330..22df415102 100644 --- a/rpc_deployment/playbooks/openstack/swift-build-rings.yml +++ b/rpc_deployment/playbooks/openstack/swift-build-rings.yml @@ -8,6 +8,7 @@ - swift_ring_builder vars_files: - inventory/group_vars/swift_all.yml + - vars/repo_packages/swift.yml - hosts: swift_hosts:swift_proxy user: root diff --git a/rpc_deployment/playbooks/openstack/swift-common.yml b/rpc_deployment/playbooks/openstack/swift-common.yml new file mode 100644 index 0000000000..6febeab4e5 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/swift-common.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: swift_proxy:swift_hosts + user: root + roles: + - common + - common_sudoers + - container_common + - openstack_common + - openstack_openrc + - galera_client_cnf + vars_files: + - vars/repo_packages/swift.yml diff --git a/rpc_deployment/playbooks/openstack/swift-proxy.yml b/rpc_deployment/playbooks/openstack/swift-proxy.yml index f6c896b22a..65b62dcb32 100644 --- a/rpc_deployment/playbooks/openstack/swift-proxy.yml +++ b/rpc_deployment/playbooks/openstack/swift-proxy.yml @@ -16,13 +16,15 @@ - hosts: swift_proxy user: root roles: - - common - - common_sudoers - - container_common - - openstack_common - - openstack_openrc - - galera_client_cnf - swift_common - swift_proxy vars_files: - inventory/group_vars/swift_all.yml + +- hosts: swift_proxy[0] + user: root + roles: + - keystone_add_service + vars_files: + - vars/openstack_service_vars/swift_proxy_endpoint.yml + diff --git a/rpc_deployment/playbooks/openstack/swift-storage.yml b/rpc_deployment/playbooks/openstack/swift-storage.yml index 6668b09d40..5bb591bd07 100644 --- a/rpc_deployment/playbooks/openstack/swift-storage.yml +++ b/rpc_deployment/playbooks/openstack/swift-storage.yml @@ -16,12 +16,6 @@ - hosts: swift_hosts user: root roles: - - common - - common_sudoers - - container_common - - openstack_common - - openstack_openrc - - galera_client_cnf - swift_common - swift_container - swift_object diff --git a/rpc_deployment/vars/repo_packages/swift.yml b/rpc_deployment/vars/repo_packages/swift.yml index 9a3142e3f5..0cefc48e94 100644 --- a/rpc_deployment/vars/repo_packages/swift.yml +++ b/rpc_deployment/vars/repo_packages/swift.yml @@ -21,6 +21,28 @@ repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }} git_repo: https://github.com/openstack/swift git_fallback_repo: https://git.openstack.org/openstack/swift git_dest: "/opt/{{ repo_path }}" -git_install_branch: master +git_etc_example: etc/ +git_install_branch: stable/juno + +service_pip_dependencies: + - pywbem + - ecdsa + - MySQL-python + - python-memcached + - pycrypto + - python-cinderclient + - python-keystoneclient + - keystonemiddleware + +container_packages: + - curl + - python-pip + - rsync + - openssh-server + - git-core + - python-setuptools + - python-dev + - gcc + - libffi-dev pip_wheel_name: swift From 706438273c236f3da094713c2d1b775eaa363d50 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 2 Oct 2014 13:18:56 +0100 Subject: [PATCH 30/45] Fix typos and adjust environment hash * Since merging in master the environment has has changed * Fix typo in user_variables.yml --- etc/rpc_deploy/rpc_user_config.yml | 2 +- etc/rpc_deploy/user_variables.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index bd6449c2d8..5d942e69b5 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -15,7 +15,7 @@ # This is the md5 of the environment file # this will ensure consistency when deploying. -environment_version: dc4cd5fe9c07eee223e8bbb1c5bbaad5 +environment_version: 701a1a44b7d77473f3b930f21f78cddf # User defined CIDR used for containers # Global cidr/s used for everything. diff --git a/etc/rpc_deploy/user_variables.yml b/etc/rpc_deploy/user_variables.yml index 19ac332310..efb9d73295 100644 --- a/etc/rpc_deploy/user_variables.yml +++ b/etc/rpc_deploy/user_variables.yml @@ -149,4 +149,4 @@ kibana_password: swift_service_password: swift_container_mysql_password: swift_hash_path_suffix: -swift_has_path_prefix: +swift_hash_path_prefix: From 2353ceeb50b3d63c0c8862273342f6b3d33b9b2c Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 2 Oct 2014 15:34:05 +0100 Subject: [PATCH 31/45] Allow account/container to be empty or not exist * Since the values specified can all be defaults this will prevent the ring builder from failing if no vars are specified for these sections. --- .../swift_ring_builder/templates/swift_rings.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index ea6d65a73c..adb7316d20 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -143,10 +143,10 @@ def main(setup): host_drive['weight'] = host_drive.get('weight', host_vars['weight']) key = "%s/%s" % (host_drive['ip'], host_drive['name']) if key in _hosts: - print("%(host)s already definined" % host) + print("%s already definined - duplicate device" % key) return 1 _hosts[key] = host_drive - + global_vars = _swift['global_overrides'] check_section(global_vars, 'swift') swift_vars = global_vars['swift'] @@ -155,12 +155,14 @@ def main(setup): return 1 part_power = swift_vars.get('part_power') - # Create account ring - check_section(swift_vars, 'account') + # Create account ring - if the section is empty create an empty dict so defaults are used + if not has_section(swift_vars, 'account'): + swift_vars['account'] = {} build_ring('account', swift_vars['account'], part_power, _hosts) - # Create container ring - check_section(swift_vars, 'container') + # Create container ring - if the section is empty create an empty dict so defaults are use + if not has_section(swift_vars, 'container'): + swift_vars['container'] = {} build_ring('container', swift_vars['container'], part_power, _hosts) # Create object rings (storage policies) From 6b951c3e425a7eae9b9b77535950982171df6542 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 2 Oct 2014 16:25:16 +0100 Subject: [PATCH 32/45] Fix service restart for swift services * swift-init returns a 1 if the service isn't started causing restart tasks to fail, but does successfully start the service. * Running this twice as an "OR" will resolve this * This is temporary until we setup init scripts properly. --- rpc_deployment/roles/swift_account/handlers/main.yml | 9 +++++---- rpc_deployment/roles/swift_container/handlers/main.yml | 9 +++++---- rpc_deployment/roles/swift_object/handlers/main.yml | 9 +++++---- rpc_deployment/roles/swift_proxy/handlers/main.yml | 3 ++- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/rpc_deployment/roles/swift_account/handlers/main.yml b/rpc_deployment/roles/swift_account/handlers/main.yml index b769c70062..fc94263300 100644 --- a/rpc_deployment/roles/swift_account/handlers/main.yml +++ b/rpc_deployment/roles/swift_account/handlers/main.yml @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: Replace these with init scripts - name: Restart account server - command: swift-init account-server restart + shell: swift-init account-server restart || swift-init account-server restart - name: Restart account auditor - command: swift-init account-auditor restart + shell: swift-init account-auditor restart || swift-init account-auditor restart - name: Restart account replicator - command: swift-init account-replicator restart + shell: swift-init account-replicator restart || swift-init account-replicator restart - name: Restart account reaper - command: swift-init account-reaper restart + shell: swift-init account-reaper restart || swift-init account-reaper restart diff --git a/rpc_deployment/roles/swift_container/handlers/main.yml b/rpc_deployment/roles/swift_container/handlers/main.yml index aba4fe98e6..a99717192b 100644 --- a/rpc_deployment/roles/swift_container/handlers/main.yml +++ b/rpc_deployment/roles/swift_container/handlers/main.yml @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: Replace these with init scripts - name: Restart container server - command: swift-init container-server restart + shell: swift-init container-server restart || swift-init container-server restart - name: Restart container auditor - command: swift-init container-auditor restart + shell: swift-init container-auditor restart || swift-init container-auditor restart - name: Restart container replicator - command: swift-init container-replicator restart + shell: swift-init container-replicator restart || swift-init container-replicator restart - name: Restart container updater - command: swift-init container-updater restart + shell: swift-init container-updater restart || swift-init container-updater restart diff --git a/rpc_deployment/roles/swift_object/handlers/main.yml b/rpc_deployment/roles/swift_object/handlers/main.yml index 88ecdb9c11..daec10411a 100644 --- a/rpc_deployment/roles/swift_object/handlers/main.yml +++ b/rpc_deployment/roles/swift_object/handlers/main.yml @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: Replace these with init scripts - name: Restart object server - command: swift-init object-server restart + shell: swift-init object-server restart || swift-init object-server restart - name: Restart object auditor - command: swift-init object-auditor restart + shell: swift-init object-auditor restart || swift-init object-auditor restart - name: Restart object replicator - command: swift-init object-replicator restart + shell: swift-init object-replicator restart || swift-init object-replicator restart - name: Restart object updater - command: swift-init object-updater restart + shell: swift-init object-updater restart || swift-init object-updater restart diff --git a/rpc_deployment/roles/swift_proxy/handlers/main.yml b/rpc_deployment/roles/swift_proxy/handlers/main.yml index e8e48fbd6d..da5ff97632 100644 --- a/rpc_deployment/roles/swift_proxy/handlers/main.yml +++ b/rpc_deployment/roles/swift_proxy/handlers/main.yml @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: Replace this with init scripts - name: Restart proxy server - command: swift-init proxy-server restart + shell: swift-init proxy-server restart || swift-init proxy-server restart From ec0e95309f8ff31615a0ec478035dec8367f474f Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 3 Oct 2014 16:20:10 +0100 Subject: [PATCH 33/45] Set the ownership to swift.swift formounted swift drives * The account/object/container servers require access to these drives * They are not accessible by default and the swift user only exists post installation of swift. --- .../playbooks/openstack/swift-storage.yml | 1 + .../roles/swift_storage_setup/tasks/main.yml | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 rpc_deployment/roles/swift_storage_setup/tasks/main.yml diff --git a/rpc_deployment/playbooks/openstack/swift-storage.yml b/rpc_deployment/playbooks/openstack/swift-storage.yml index 5bb591bd07..13649fc8b9 100644 --- a/rpc_deployment/playbooks/openstack/swift-storage.yml +++ b/rpc_deployment/playbooks/openstack/swift-storage.yml @@ -17,6 +17,7 @@ user: root roles: - swift_common + - swift_storage_setup - swift_container - swift_object - swift_account diff --git a/rpc_deployment/roles/swift_storage_setup/tasks/main.yml b/rpc_deployment/roles/swift_storage_setup/tasks/main.yml new file mode 100644 index 0000000000..ef1bdb6821 --- /dev/null +++ b/rpc_deployment/roles/swift_storage_setup/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Set ownership on mounted drives" + file: + dest: "{{ swift_vars.mount_point }}/{{ item.name }}" + owner: "{{ system_user }}" + group: "{{ system_group }}" + state: "directory" + with_items: swift_vars.drives From 1f245da7ba09d4bee9e8ca98b7b38c59914279a9 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 3 Oct 2014 17:37:23 +0100 Subject: [PATCH 34/45] Add specific log files for swift services * The log files are automatically mounted by rsyslog containers on the servers. --- .../inventory/group_vars/swift_all.yml | 1 - .../templates/account-server.conf.j2 | 2 +- .../roles/swift_common/handlers/main.yml | 17 +++++++++++ .../roles/swift_common/tasks/log_setup.yml | 28 +++++++++++++++++++ .../roles/swift_common/tasks/main.yml | 2 ++ .../templates/container-server.conf.j2 | 2 +- .../templates/object-server.conf.j2 | 2 +- .../templates/proxy-server.conf.j2 | 2 +- 8 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 rpc_deployment/roles/swift_common/handlers/main.yml create mode 100644 rpc_deployment/roles/swift_common/tasks/log_setup.yml diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 426676f73a..71f78fcd88 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -64,7 +64,6 @@ service_names: - swift-proxy container_directories: - - /var/log/swift - /tmp/swift - /var/lock/swift - /etc/swift diff --git a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 index 0cce735b77..9631ded201 100644 --- a/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 +++ b/rpc_deployment/roles/swift_account/templates/account-server.conf.j2 @@ -18,7 +18,7 @@ devices = {{ swift_vars.mount_point }} # # You can specify default log routing here if you want: # log_name = swift -# log_facility = LOG_LOCAL0 +log_facility = LOG_LOCAL2 # log_level = INFO # log_address = /dev/log # The following caps the length of log lines to the value given; no limit if diff --git a/rpc_deployment/roles/swift_common/handlers/main.yml b/rpc_deployment/roles/swift_common/handlers/main.yml new file mode 100644 index 0000000000..a302ff2790 --- /dev/null +++ b/rpc_deployment/roles/swift_common/handlers/main.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: restart rsyslog + service: name=rsyslog state=restarted diff --git a/rpc_deployment/roles/swift_common/tasks/log_setup.yml b/rpc_deployment/roles/swift_common/tasks/log_setup.yml new file mode 100644 index 0000000000..7026745d49 --- /dev/null +++ b/rpc_deployment/roles/swift_common/tasks/log_setup.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Create logging directory" + file: + path: "/openstack/log/{{ inventory_hostname }}" + state: "directory" + owner: "syslog" + group: "syslog" + +- name: "Drop swift rsyslog conf" + template: + src: "swift-rsyslog.conf.j2" + dest: "/etc/rsyslog.d/10-swift.conf" + notify: + - restart rsyslog diff --git a/rpc_deployment/roles/swift_common/tasks/main.yml b/rpc_deployment/roles/swift_common/tasks/main.yml index 742d6eafe7..b1d06f196f 100644 --- a/rpc_deployment/roles/swift_common/tasks/main.yml +++ b/rpc_deployment/roles/swift_common/tasks/main.yml @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: log_setup.yml + - name: "Drop swift.conf template" template: > src="swift.conf.j2" diff --git a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 index cb0d01b06e..aa9ea78459 100644 --- a/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 +++ b/rpc_deployment/roles/swift_container/templates/container-server.conf.j2 @@ -24,7 +24,7 @@ devices = {{ swift_vars.mount_point }} # # You can specify default log routing here if you want: # log_name = swift -# log_facility = LOG_LOCAL0 +log_facility = LOG_LOCAL3 # log_level = INFO # log_address = /dev/log # The following caps the length of log lines to the value given; no limit if diff --git a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 index 716da67aea..8fa1d6af30 100644 --- a/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 +++ b/rpc_deployment/roles/swift_object/templates/object-server.conf.j2 @@ -20,7 +20,7 @@ devices = {{ swift_vars.mount_point }} # # You can specify default log routing here if you want: # log_name = swift -# log_facility = LOG_LOCAL0 +log_facility = LOG_LOCAL4 # log_level = INFO # log_address = /dev/log # The following caps the length of log lines to the value given; no limit if diff --git a/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 index defdb00371..225a41730f 100644 --- a/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 +++ b/rpc_deployment/roles/swift_proxy/templates/proxy-server.conf.j2 @@ -39,7 +39,7 @@ user = {{ system_user }} # # You can specify default log routing here if you want: # log_name = swift -# log_facility = LOG_LOCAL0 +log_facility = LOG_LOCAL1 # log_level = INFO # log_headers = false # log_address = /dev/log From b7669495d6dd26f16fc22bac992b25ffeec7bd62 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 3 Oct 2014 17:48:47 +0100 Subject: [PATCH 35/45] Adjust logging for proxy server * Proxy server is a container so needs its logs to go in /var/log/swift * Add the rsyslog conf template which was missing from the last commit --- .../inventory/group_vars/swift_all.yml | 1 + .../roles/swift_common/tasks/log_setup.yml | 5 ++++- .../templates/swift-rsyslog.conf.j2 | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 71f78fcd88..426676f73a 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -64,6 +64,7 @@ service_names: - swift-proxy container_directories: + - /var/log/swift - /tmp/swift - /var/lock/swift - /etc/swift diff --git a/rpc_deployment/roles/swift_common/tasks/log_setup.yml b/rpc_deployment/roles/swift_common/tasks/log_setup.yml index 7026745d49..15a5cddd53 100644 --- a/rpc_deployment/roles/swift_common/tasks/log_setup.yml +++ b/rpc_deployment/roles/swift_common/tasks/log_setup.yml @@ -15,10 +15,13 @@ - name: "Create logging directory" file: - path: "/openstack/log/{{ inventory_hostname }}" + path: "{{ item }}" state: "directory" owner: "syslog" group: "syslog" + with_items: + - /var/log/swift + - /openstack/log/{{ inventory_hostname }} - name: "Drop swift rsyslog conf" template: diff --git a/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 b/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 new file mode 100644 index 0000000000..3402fc2fe7 --- /dev/null +++ b/rpc_deployment/roles/swift_common/templates/swift-rsyslog.conf.j2 @@ -0,0 +1,22 @@ +# Uncomment the following to have a log containing all logs together +#local1,local2,local3,local4.* /openstack/log/{{ inventory_hostname }}/all.log + +# Uncomment the following to have hourly proxy logs for stats processing +#$template HourlyProxyLog,"/openstack/log/{{ inventory_hostname }}/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%" +#local1.*;local1.!notice ?HourlyProxyLog + +local1.*;local1.!notice /var/log/swift/proxy.log +local1.notice /var/log/swift/proxy.error +local1.* ~ + +local2.*;local2.!notice /openstack/log/{{ inventory_hostname }}/account.log +local2.notice /openstack/log/{{ inventory_hostname }}/account.error +local2.* ~ + +local3.*;local3.!notice /openstack/log/{{ inventory_hostname }}/container.log +local3.notice /openstack/log/{{ inventory_hostname }}/container.error +local3.* ~ + +local4.*;local4.!notice /openstack/log/{{ inventory_hostname }}/object.log +local4.notice /openstack/log/{{ inventory_hostname }}/object.error +local4.* From 52b70d75d66b1a41ba4ad46323d1cc18ece0d589 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 8 Oct 2014 05:41:36 +0000 Subject: [PATCH 36/45] Update builder files when they exist This change makes the swift_rings.py script update builder files if they exist. However there are some limitations: 1. Once a ring has been defined with a partpower, this cannot be changed without rebuilding the ring. So rule is, pick a partpower based on future use of you swift cluster. 2. When modifiying hosts using swift-builder you cannot change the region or zone. These are defined only when a new host is added. As this can cause major partition placement issues on existing hosts (will cause some major movement in the swift cluster). If you want to change a hosts zone or region, the best thing to do is set the wieght to 0 and let swift move data off the drive.. then remove it and add it again as a new host. NOTE: This needs testing and debugging, as I don't have a working full rpc environment yet. --- .../templates/swift_rings.py | 114 +++++++++++++++--- 1 file changed, 98 insertions(+), 16 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index adb7316d20..f9a9817f31 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -18,12 +18,15 @@ from optparse import OptionParser from os.path import exists from swift.cli.ringbuilder import main as rb_main +import pickle import sys import threading import yaml USAGE = "usage: %prog -s " +DEV_KEY = "%(ip)s:%(port)d/%(device)s" + DEFAULT_REPL = {{ swift_default_replication_number }} DEFAULT_MIN_PART_HOURS = {{ swift_default_min_part_hours }} DEFAULT_HOST_ZONE = {{ swift_default_host_zone }} @@ -55,9 +58,51 @@ DEFAULT_GROUPS= [ 'container' ] -def create_buildfile(build_file, part_power, repl, min_part_hours): - run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", - part_power, repl, min_part_hours]) +def create_buildfile(build_file, part_power, repl, min_part_hours, + update=False, data=None): + if update: + # build file exists, so lets just update the existing build file + if not data: + data = get_build_file_data(build_file) + if data is None: + data = {} + + if repl != data.get('replicas'): + run_and_wait(rb_main, ["swift-ring-builder", build_file, + "set_replicas", repl]) + if min_part_hours != data.get('min_part_hours'): + run_and_wait(rb_main, ["swift-ring-builder", build_file, + "set_min_part_hours", min_part_hours]) + if part_power != data.get('part_power'): + print('Part power cannot be changed.. you must rebuild the ring " + "if you need to change it.') + + else: + run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", + part_power, repl, min_part_hours]) + +def change_host_weight(build_file, host_search_str, weight): + run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_weight", + host_search_str, weight]) + +def remove_host_from_ring(build_file, host): + run_and_wait(rb_main, ["swift-ring-builder", build_file, "remove", + DEV_KEY % host]) + +def update_host_in_ring(build_file, new_host, old_host): + r_ip = new_host.get('repl_ip', new_host['ip']) + r_port = new_host.get('repl_port', new_host['port']) + weight = new_host.get('weight', DEFAULT_HOST_WEIGHT) + if r_ip != old_host['replication_ip'] or \ + r_port != old_host['replication_port']: + host_d = {'r_ip': r_ip, 'r_port': r_port} + host_d.update(host) + host_str = "%(ip)s:%(port)dR%(r_ip)s:%(d_port)d/%(name)s" % host_d + run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_info", + DEV_KEY % new_host, host_str]) + + if weight != old_host['weight']: + change_host_weight(build_file, DEV_KEY % new_host, weight) def add_host_to_ring(build_file, host): @@ -67,7 +112,7 @@ def add_host_to_ring(build_file, host): host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) host_str += "-%(ip)s:%(port)d" % host if host.get('repl_port'): - r_ip = host.get('repl_ip', host['host']) + r_ip = host.get('repl_ip', host['ip']) host_str += "R%s:%d" % (r_ip, host['repl_port']) host_str += "/%(name)s" % host @@ -75,7 +120,6 @@ def add_host_to_ring(build_file, host): run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', host_str, str(weight)]) - def run_and_wait(func, *args): t = threading.Thread(target=func, args=args) t.start() @@ -92,27 +136,58 @@ def check_section(conf, section): sys.exit(2) +def get_build_file_data(build_file): + build_file_data = None + if exists(build_file): + try: + with open(build_file) as bf_stream: + build_file_data = pickle.load(bf_stream) + except Exception as ex: + print("Error: failed to load build file '%s': %s" % (build_file, + ex)) + build_file_data = None + return build_file_data + + def build_ring(section, conf, part_power, hosts): # Create the build file build_file = "%s.builder" % (section) + build_file_data = get_build_file_data(build_file) + repl = conf.get('repl_number', DEFAULT_REPL) min_part_hours = conf.get('min_part_hours', DEFAULT_MIN_PART_HOURS) - create_buildfile(build_file, part_power, repl, min_part_hours) + update = build_file_data not None + create_buildfile(build_file, part_power, repl, min_part_hours, update, + data=build_file_data) + old_hosts = {} + if update: + old_hosts = dict([(DEV_KEY % dev, i) + for i, dev in enumerate(build_file_data['devs'])]) section_key = section.split('-')[0] service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) for host in hosts: host_vars = hosts[host] + host_vars['device'] = host_vars['name'] host_vars['port'] = service_port host_vars['groups'] = host_vars.get('groups', DEFAULT_GROUPS) if DEFAULT_GROUP_MAP[section] in host_vars['groups']: - add_host_to_ring(build_file, host_vars) + host_key = DEV_KEY % host_vars + if host_key in old_hosts: + old_host = build_file_data['devs'][old_hosts[host_key]] + update_host_in_ring(build_file, host_vars, old_host) + old_hosts.pop(host_key) + else: + add_host_to_ring(build_file, host_vars) + + if old_hosts: + # There are still old hosts, these hosts must've been removed + for host in old_hosts: + remove_host_from_ring(build_file, host) # Rebalance ring run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) - # rb_main(("swift-ring-builder", build_file, "rebalance")) - def main(setup): # load the yaml file @@ -126,7 +201,8 @@ def main(setup): _hosts = {} if _swift.get("swift_hosts"): for host in _swift['swift_hosts']: - host_vars = _swift['swift_hosts'][host]['container_vars']['swift_vars'] + host_vars = \ + _swift['swift_hosts'][host]['container_vars']['swift_vars'] host_ip = host_vars.get('ip', _swift['swift_hosts'][host]['ip']) if not host_vars.get('drives'): continue @@ -134,13 +210,17 @@ def main(setup): for host_drive in host_drives: host_drive['ip'] = host_drive.get('ip', host_ip) if host_vars.get('groups'): - host_drive['groups'] = host_drive.get('groups', host_vars['groups']) + host_drive['groups'] = host_drive.get('groups', + host_vars['groups']) if host_vars.get('repl_ip'): - host_drive['repl_ip'] = host_drive.get('repl_ip', host_vars['repl_ip']) + host_drive['repl_ip'] = host_drive.get('repl_ip', + host_vars['repl_ip']) if host_vars.get('repl_port'): - host_drive['repl_port'] = host_drive.get('repl_port', host_vars['repl_port']) + host_drive['repl_port'] = \ + host_drive.get('repl_port', host_vars['repl_port']) if host_vars.get('weight'): - host_drive['weight'] = host_drive.get('weight', host_vars['weight']) + host_drive['weight'] = host_drive.get('weight', + host_vars['weight']) key = "%s/%s" % (host_drive['ip'], host_drive['name']) if key in _hosts: print("%s already definined - duplicate device" % key) @@ -155,12 +235,14 @@ def main(setup): return 1 part_power = swift_vars.get('part_power') - # Create account ring - if the section is empty create an empty dict so defaults are used + # Create account ring - if the section is empty create an empty dict + # so defaults are used if not has_section(swift_vars, 'account'): swift_vars['account'] = {} build_ring('account', swift_vars['account'], part_power, _hosts) - # Create container ring - if the section is empty create an empty dict so defaults are use + # Create container ring - if the section is empty create an empty dict + # so defaults are used if not has_section(swift_vars, 'container'): swift_vars['container'] = {} build_ring('container', swift_vars['container'], part_power, _hosts) From 0f9c9bc08c6cc22a9931387acabc427d3735bf48 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 8 Oct 2014 11:50:20 +0100 Subject: [PATCH 37/45] Fix typos on swift_rings.py * Wrong quotation marks fixed * Missing "is" added to conditional * "d_port" var should be "r_port" * "host" var doesn't exist, should be "new_host" * for removal, host is already formatted and doesn't require a map --- .../swift_ring_builder/templates/swift_rings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index f9a9817f31..724bb2809a 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -74,8 +74,8 @@ def create_buildfile(build_file, part_power, repl, min_part_hours, run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_min_part_hours", min_part_hours]) if part_power != data.get('part_power'): - print('Part power cannot be changed.. you must rebuild the ring " - "if you need to change it.') + print('Part power cannot be changed.. you must rebuild the ring ' + 'if you need to change it.') else: run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", @@ -87,7 +87,7 @@ def change_host_weight(build_file, host_search_str, weight): def remove_host_from_ring(build_file, host): run_and_wait(rb_main, ["swift-ring-builder", build_file, "remove", - DEV_KEY % host]) + host]) def update_host_in_ring(build_file, new_host, old_host): r_ip = new_host.get('repl_ip', new_host['ip']) @@ -96,8 +96,8 @@ def update_host_in_ring(build_file, new_host, old_host): if r_ip != old_host['replication_ip'] or \ r_port != old_host['replication_port']: host_d = {'r_ip': r_ip, 'r_port': r_port} - host_d.update(host) - host_str = "%(ip)s:%(port)dR%(r_ip)s:%(d_port)d/%(name)s" % host_d + host_d.update(new_host) + host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(name)s" % host_d run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_info", DEV_KEY % new_host, host_str]) @@ -157,7 +157,7 @@ def build_ring(section, conf, part_power, hosts): repl = conf.get('repl_number', DEFAULT_REPL) min_part_hours = conf.get('min_part_hours', DEFAULT_MIN_PART_HOURS) - update = build_file_data not None + update = build_file_data is not None create_buildfile(build_file, part_power, repl, min_part_hours, update, data=build_file_data) From 4e9b54bd8e02e0cfd0f7e40261935f50b8d93ecc Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 8 Oct 2014 15:16:44 +0100 Subject: [PATCH 38/45] Fix for old_hosts when devices have been removed * When a device is removed from the ring it sets the device to None instead of deleting it all together. This means the old_hosts will fail if the DEV_KEY is applied to it. * Ignore "None" hosts, since they aren't old_hosts anymore. --- .../roles/swift_ring_builder/templates/swift_rings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index 724bb2809a..c773d653f8 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -163,8 +163,9 @@ def build_ring(section, conf, part_power, hosts): old_hosts = {} if update: - old_hosts = dict([(DEV_KEY % dev, i) - for i, dev in enumerate(build_file_data['devs'])]) + for i, dev in enumerate(build_file_data['devs']): + if dev is not None: + old_hosts[DEV_KEY % dev] = i section_key = section.split('-')[0] service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) for host in hosts: From f9131dafc98ec71d05721ce1143b72e5d7003b2d Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Wed, 8 Oct 2014 15:57:29 +0100 Subject: [PATCH 39/45] Use default repl_number and min_part_hours * This allows the default to be set at the "swift:" level * Works in the same way as precedence on other vars * Update the user_config example file --- etc/rpc_deploy/rpc_user_config.d/swift.yml | 94 +++++++++++++++++++ .../templates/swift_rings.py | 17 +++- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 etc/rpc_deploy/rpc_user_config.d/swift.yml diff --git a/etc/rpc_deploy/rpc_user_config.d/swift.yml b/etc/rpc_deploy/rpc_user_config.d/swift.yml new file mode 100644 index 0000000000..76fce59b22 --- /dev/null +++ b/etc/rpc_deploy/rpc_user_config.d/swift.yml @@ -0,0 +1,94 @@ +--- +# Setup swift group variables when using swift (Not required if not using swift) +# part power is required under swift. This can't be changed once the ring is built +# For account/container speciying min_part_hours and repl_number is all that can be set. +# These 2 can be set at the "swift" level to work as a default. +# Alternatively defaults will be used (repl_number of 3, and min_part_hours of 1). +# For storage policies, a name and unique index is required as well as repl_number and +# min_part_hours which will be set to a default value if not specified. +# There MUST be a storage policy with index 0 configured which will be the default for legacy containers (created pre-storage policies). +# You can set one policy to be "default: yes" this will be the default storage policy for non-legacy containers that are created. +# The index value must be unique. +# Storage policies can be set to "deprecated: yes" which will mean they are not used + +# global_overrides: +# swift: +# part_power: 8 +# account: +# repl_number: 3 +# min_part_hours: 1 +# container: +# repl_number: 3 +# storage_policies: +# - policy: +# name: gold +# index: 0 +# repl_number: 3 +# default: yes +# - policy: +# name: silver +# index: 1 +# repl_number: 2 +# deprecated: yes + +# User defined Swift Proxy hosts - not required when not using swift +# Will deploy a swift-proxy container on these hosts. +# Recommend mirroring the infra_hosts +# swift-proxy_hosts: +# infra1: +# ip: 172.29.236.100 +# infra2: +# ip: 172.29.236.101 +# infra3: +# ip: 172.29.236.102 + +# User defined Object Storage Hosts - this is not a required group +# Under swift_vars you can specify the host specific swift_vars. +# region - the swift region, this isn't required. +# zone - the swift zone, this isn't required either, will default to 0 +# mount_point - where the drives are mounted on the server +# drives - A list of drives in the server (Must have a name as a minimum) +# Above 4 vars are "host specific" +# weight: a disks weight (defaults to 100 if not specified) +# repl_ip: IP specific for object replication (not required) +# repl_port: Port specific for object replication (not required) +# groups: A list of groups to add the drive to. A group is either a storage policy or the account or container servers. (If not specified defaults to all groups, so container/account/all storage policies). +# The above 4 can be specified on a per host or per drive basis +# Or both, in which case "per drive" will take precedence for the specific drive. +# ip can be specified in swift_vars to override the hosts ip +# or per drive to override all for that specific drive. +# swift_hosts: +# object_storage1: +# ip: 172.29.236.108 +# container_vars: +# swift_vars: +# region: 0 +# zone: 0 +# groups: +# - silver +# - account +# mount_point: /srv/node +# drives: +# - name: sdb +# ip: 172.10.100.100 +# repl_ip: 10.10.0.1 +# repl_port: 54321 +# groups: +# - gold +# - account +# - container +# - name: sdc +# weight: 150 +# - name: sdd +# - name: sde +# +# object_storage2: +# ip: 172.29.236.109 +# container_vars: +# swift_vars: +# region: 0 +# zone: 1 +# mount_point: /srv/node +# drives: +# - name: sdb +# - name: sdc diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index c773d653f8..e6e28d8ded 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -236,16 +236,24 @@ def main(setup): return 1 part_power = swift_vars.get('part_power') + # If the repl_number or min_part hours are set on a "global" level in the + # conf lets set them here - otherwise use the overall default. + default_repl_num = swift_vars.get('repl_number', DEFAULT_REPL) + default_min_part_hours = swift_vars.get('min_part_hours', + DEFAULT_MIN_PART_HOURS) + # Create account ring - if the section is empty create an empty dict # so defaults are used if not has_section(swift_vars, 'account'): - swift_vars['account'] = {} + swift_vars['account'] = {'repl_number': default_repl_num, + 'min_part_hours': default_min_part_hours} build_ring('account', swift_vars['account'], part_power, _hosts) # Create container ring - if the section is empty create an empty dict # so defaults are used if not has_section(swift_vars, 'container'): - swift_vars['container'] = {} + swift_vars['container'] = {'repl_number': default_repl_num, + 'min_part_hours': default_min_part_hours} build_ring('container', swift_vars['container'], part_power, _hosts) # Create object rings (storage policies) @@ -261,6 +269,11 @@ def main(setup): else: buildfilename = 'object-%d' % (policy['index']) indexes.add(policy['index']) + # Set default port/min_part_hours/repl_number + if 'min_part_hours' not in policy: + policy['min_part_hours'] = default_min_part_hours + if 'repl_number' not in policy: + policy['repl_number'] = default_repl_num if 'port' not in policy: policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT) build_ring(buildfilename, policy, part_power, _hosts) From 7b4ff7ade334b117c4378f8b48339b054afd83d4 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Thu, 9 Oct 2014 16:15:36 +1100 Subject: [PATCH 40/45] Add a validation step before any rings are touched This change changes the algorithm a little, the ring builder now gathers all the information about what rings to build and instead of calling build_ring straight away it adds the options for the call to a ring_calls list. Once all build calls are added to this list we loop through and call build_rings but with validate=True, which is like a dry run. When an error is detected a RingValidationError is raised. The errors its looking for is: - Modification of zone or region in existing drives. - Change of part_power. - And any exceptions raised while attempting to print integers. If this validation step passes (no error raised) the we loop and call build_ring again without the validate option. The benefit of this approach is all the rings are "validated" before any buildfiles are modified/created. So if a user gets an error we know the current state of the rings/builders are untouched. A validation error causes the script to return a 2. Seeing as this is >0 ansible _should_ see it as a failure. --- .../templates/swift_rings.py | 119 ++++++++++++------ 1 file changed, 79 insertions(+), 40 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index e6e28d8ded..d7faeec36a 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -58,8 +58,11 @@ DEFAULT_GROUPS= [ 'container' ] +class RingValidationError(Exception): + pass + def create_buildfile(build_file, part_power, repl, min_part_hours, - update=False, data=None): + update=False, data=None, validate=False): if update: # build file exists, so lets just update the existing build file if not data: @@ -67,17 +70,18 @@ def create_buildfile(build_file, part_power, repl, min_part_hours, if data is None: data = {} - if repl != data.get('replicas'): + if repl != data.get('replicas') and not validate: run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_replicas", repl]) - if min_part_hours != data.get('min_part_hours'): + if min_part_hours != data.get('min_part_hours') and not validate: run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_min_part_hours", min_part_hours]) if part_power != data.get('part_power'): - print('Part power cannot be changed.. you must rebuild the ring ' - 'if you need to change it.') + raise RingValidationError('Part power cannot be changed! ' + 'you must rebuild the ring if you need ' + 'to change it.') - else: + elif not validate: run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", part_power, repl, min_part_hours]) @@ -89,36 +93,53 @@ def remove_host_from_ring(build_file, host): run_and_wait(rb_main, ["swift-ring-builder", build_file, "remove", host]) -def update_host_in_ring(build_file, new_host, old_host): - r_ip = new_host.get('repl_ip', new_host['ip']) - r_port = new_host.get('repl_port', new_host['port']) - weight = new_host.get('weight', DEFAULT_HOST_WEIGHT) - if r_ip != old_host['replication_ip'] or \ - r_port != old_host['replication_port']: - host_d = {'r_ip': r_ip, 'r_port': r_port} - host_d.update(new_host) - host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(name)s" % host_d - run_and_wait(rb_main, ["swift-ring-builder", build_file, "set_info", - DEV_KEY % new_host, host_str]) +def update_host_in_ring(build_file, new_host, old_host, validate=False): + if new_host.get('zone', 0) != old_host['zone']: + devstr = DEV_KEY % new_host + raise RingValidationError('Cannot update zone on %s, this can only be ' + 'done when the drive is added' % (devstr)) + if new_host.get('region', 1) != old_host['region']: + devstr = DEV_KEY % new_host + raise RingValidationError('Cannot update region on %s, this can only ' + 'be done when the drive is added' % (devstr)) - if weight != old_host['weight']: + try: + r_ip = new_host.get('repl_ip', new_host['ip']) + r_port = new_host.get('repl_port', new_host['port']) + weight = new_host.get('weight', DEFAULT_HOST_WEIGHT) + if r_ip != old_host['replication_ip'] or \ + r_port != old_host['replication_port']: + host_d = {'r_ip': r_ip, 'r_port': r_port} + host_d.update(new_host) + host_str = "%(ip)s:%(port)dR%(r_ip)s:%(r_port)d/%(name)s" % host_d + if not validate: + run_and_wait(rb_main, ["swift-ring-builder", build_file, + "set_info", DEV_KEY % new_host, + host_str]) + except Exception as ex: + raise RingValidationError(ex) + + if weight != old_host['weight'] and not validate: change_host_weight(build_file, DEV_KEY % new_host, weight) -def add_host_to_ring(build_file, host): +def add_host_to_ring(build_file, host, validate=False): host_str = "" - if host.get('region') is not None: - host_str += 'r%(region)d' % host - host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) - host_str += "-%(ip)s:%(port)d" % host - if host.get('repl_port'): - r_ip = host.get('repl_ip', host['ip']) - host_str += "R%s:%d" % (r_ip, host['repl_port']) - host_str += "/%(name)s" % host - - weight = host.get('weight', DEFAULT_HOST_WEIGHT) - run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', - host_str, str(weight)]) + try: + if host.get('region') is not None: + host_str += 'r%(region)d' % host + host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) + host_str += "-%(ip)s:%(port)d" % host + if host.get('repl_port'): + r_ip = host.get('repl_ip', host['ip']) + host_str += "R%s:%d" % (r_ip, host['repl_port']) + host_str += "/%(name)s" % host + weight = host.get('weight', DEFAULT_HOST_WEIGHT) + except Exception as ex: + raise RingValidationError(ex) + if not validate: + run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', + host_str, str(weight)]) def run_and_wait(func, *args): t = threading.Thread(target=func, args=args) @@ -149,7 +170,7 @@ def get_build_file_data(build_file): return build_file_data -def build_ring(section, conf, part_power, hosts): +def build_ring(section, conf, part_power, hosts, validate=False): # Create the build file build_file = "%s.builder" % (section) build_file_data = get_build_file_data(build_file) @@ -159,7 +180,7 @@ def build_ring(section, conf, part_power, hosts): DEFAULT_MIN_PART_HOURS) update = build_file_data is not None create_buildfile(build_file, part_power, repl, min_part_hours, update, - data=build_file_data) + data=build_file_data, validate=validate) old_hosts = {} if update: @@ -177,18 +198,20 @@ def build_ring(section, conf, part_power, hosts): host_key = DEV_KEY % host_vars if host_key in old_hosts: old_host = build_file_data['devs'][old_hosts[host_key]] - update_host_in_ring(build_file, host_vars, old_host) + update_host_in_ring(build_file, host_vars, old_host, + validate=validate) old_hosts.pop(host_key) else: - add_host_to_ring(build_file, host_vars) + add_host_to_ring(build_file, host_vars, validate=validate) - if old_hosts: + if old_hosts and not validate: # There are still old hosts, these hosts must've been removed for host in old_hosts: remove_host_from_ring(build_file, host) # Rebalance ring - run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) + if not validate: + run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) def main(setup): # load the yaml file @@ -241,20 +264,21 @@ def main(setup): default_repl_num = swift_vars.get('repl_number', DEFAULT_REPL) default_min_part_hours = swift_vars.get('min_part_hours', DEFAULT_MIN_PART_HOURS) + ring_calls = [] # Create account ring - if the section is empty create an empty dict # so defaults are used if not has_section(swift_vars, 'account'): swift_vars['account'] = {'repl_number': default_repl_num, 'min_part_hours': default_min_part_hours} - build_ring('account', swift_vars['account'], part_power, _hosts) + ring_calls.append(('account', swift_vars['account'], part_power)) # Create container ring - if the section is empty create an empty dict # so defaults are used if not has_section(swift_vars, 'container'): swift_vars['container'] = {'repl_number': default_repl_num, 'min_part_hours': default_min_part_hours} - build_ring('container', swift_vars['container'], part_power, _hosts) + ring_calls.append(('container', swift_vars['container'], part_power)) # Create object rings (storage policies) check_section(swift_vars, 'storage_policies') @@ -276,7 +300,22 @@ def main(setup): policy['repl_number'] = default_repl_num if 'port' not in policy: policy['port'] = policy.get('port', DEFAULT_OBJECT_PORT) - build_ring(buildfilename, policy, part_power, _hosts) + ring_calls.append((buildfilename, policy, part_power)) + + # Now that we have gathered all the options for building/update the rings + # lets validate them + kargs = {'validate': True, 'hosts': _hosts} + for ring_call in ring_calls: + try: + build_rings(*ring_call, **kargs) + except RingValidationError as ex: + print ex + return 2 + + # If the validation passes lets go ahead and build the rings. + kargs.pop('validate') + for ring_call in ring_calls: + build_rings(*ring_call, **kargs) if __name__ == "__main__": parser = OptionParser(USAGE) From 078ff371d6e8788220ace6821c38a9a36b3350d6 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 9 Oct 2014 10:52:23 +0100 Subject: [PATCH 41/45] Fix typos on build_ring(s) def * Fix build_rings to be "build_ring". * Fix syntax around print statement * Adjust part_power message to let you know what the part_power is in the ring, and what it is in the inventory so it can be adjusted. --- .../roles/swift_ring_builder/templates/swift_rings.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index d7faeec36a..d5ea4ea5f0 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -79,7 +79,9 @@ def create_buildfile(build_file, part_power, repl, min_part_hours, if part_power != data.get('part_power'): raise RingValidationError('Part power cannot be changed! ' 'you must rebuild the ring if you need ' - 'to change it.') + 'to change it.\nRing part power: %s ' + 'Inventory part power: %s' + %(data.get('part_power'), part_power)) elif not validate: run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", @@ -307,15 +309,15 @@ def main(setup): kargs = {'validate': True, 'hosts': _hosts} for ring_call in ring_calls: try: - build_rings(*ring_call, **kargs) + build_ring(*ring_call, **kargs) except RingValidationError as ex: - print ex + print(ex) return 2 # If the validation passes lets go ahead and build the rings. kargs.pop('validate') for ring_call in ring_calls: - build_rings(*ring_call, **kargs) + build_ring(*ring_call, **kargs) if __name__ == "__main__": parser = OptionParser(USAGE) From 5d6043bfeb9b8c7eecfa8aa5266af385e602d39d Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 17 Oct 2014 11:57:22 +0100 Subject: [PATCH 42/45] Adjust the ring_builder to use the inventory file The ring builder was using the rpc_user_config.yml, but the inventory file should be the source of all truth. * Adjust to use json instead of yaml * Adjust to use the vars as they appear in the inventory file. --- .../roles/swift_ring_builder/tasks/main.yml | 2 +- .../templates/swift_rings.py | 38 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 815de849bc..0b1f605e10 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -31,6 +31,6 @@ mode=0700 - name: "build rings" - command: /usr/bin/python /tmp/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_user_config.yml + command: /usr/bin/python /tmp/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_inventory.json args: chdir: /tmp/swift/rings/ diff --git a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py index d5ea4ea5f0..a470cf8e1c 100644 --- a/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py +++ b/rpc_deployment/roles/swift_ring_builder/templates/swift_rings.py @@ -21,9 +21,9 @@ from swift.cli.ringbuilder import main as rb_main import pickle import sys import threading -import yaml +import json -USAGE = "usage: %prog -s " +USAGE = "usage: %prog -s " DEV_KEY = "%(ip)s:%(port)d/%(device)s" @@ -216,44 +216,44 @@ def build_ring(section, conf, part_power, hosts, validate=False): run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) def main(setup): - # load the yaml file + # load the json file try: - with open(setup) as yaml_stream: - _swift = yaml.load(yaml_stream) + with open(setup) as json_stream: + _inventory = json.load(json_stream) except Exception as ex: - print("Failed to load yaml string %s" % (ex)) + print("Failed to load json string %s" % (ex)) return 1 _hosts = {} - if _swift.get("swift_hosts"): - for host in _swift['swift_hosts']: - host_vars = \ - _swift['swift_hosts'][host]['container_vars']['swift_vars'] - host_ip = host_vars.get('ip', _swift['swift_hosts'][host]['ip']) + if _inventory.get("swift_hosts"): + for host in _inventory['swift_hosts']['hosts']: + host_config = _inventory['_meta']['hostvars'][host] + host_vars = host_config['swift_vars'] + host_ip = host_vars.get('ip', host_config['container_address']) if not host_vars.get('drives'): continue host_drives = host_vars.get('drives') for host_drive in host_drives: host_drive['ip'] = host_drive.get('ip', host_ip) if host_vars.get('groups'): - host_drive['groups'] = host_drive.get('groups', - host_vars['groups']) + host_drive['groups'] = \ + host_drive.get('groups', host_vars['groups']) if host_vars.get('repl_ip'): - host_drive['repl_ip'] = host_drive.get('repl_ip', - host_vars['repl_ip']) + host_drive['repl_ip'] = \ + host_drive.get('repl_ip', host_vars['repl_ip']) if host_vars.get('repl_port'): host_drive['repl_port'] = \ host_drive.get('repl_port', host_vars['repl_port']) if host_vars.get('weight'): - host_drive['weight'] = host_drive.get('weight', - host_vars['weight']) + host_drive['weight'] = \ + host_drive.get('weight', host_vars['weight']) key = "%s/%s" % (host_drive['ip'], host_drive['name']) if key in _hosts: print("%s already definined - duplicate device" % key) return 1 _hosts[key] = host_drive - global_vars = _swift['global_overrides'] + global_vars = _inventory['all']['vars'] check_section(global_vars, 'swift') swift_vars = global_vars['swift'] if not swift_vars.get('part_power'): @@ -323,7 +323,7 @@ if __name__ == "__main__": parser = OptionParser(USAGE) parser.add_option("-s", "--setup", dest="setup", help="Specify the swift setup file.", metavar="FILE", - default="/etc/rpc_deploy/rpc_user_config.yml") + default="/etc/rpc_deploy/rpc_inventory.json") options, args = parser.parse_args(sys.argv[1:]) if options.setup and not exists(options.setup): From d4d78a9d3f16a51c98c33b76cdf24b9ee8ffb952 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 17 Oct 2014 17:55:29 +0100 Subject: [PATCH 43/45] Adjust sample configs for Swift * Move rpc_user_config.d to conf.d * Move the swift variables out of rpc_user_config * Ensure the swift.yml inside conf.d is correct --- .../{rpc_user_config.d => conf.d}/swift.yml | 0 etc/rpc_deploy/rpc_user_config.yml | 92 ------------------- 2 files changed, 92 deletions(-) rename etc/rpc_deploy/{rpc_user_config.d => conf.d}/swift.yml (100%) diff --git a/etc/rpc_deploy/rpc_user_config.d/swift.yml b/etc/rpc_deploy/conf.d/swift.yml similarity index 100% rename from etc/rpc_deploy/rpc_user_config.d/swift.yml rename to etc/rpc_deploy/conf.d/swift.yml diff --git a/etc/rpc_deploy/rpc_user_config.yml b/etc/rpc_deploy/rpc_user_config.yml index 5d942e69b5..809e71bc0d 100644 --- a/etc/rpc_deploy/rpc_user_config.yml +++ b/etc/rpc_deploy/rpc_user_config.yml @@ -109,36 +109,6 @@ global_overrides: type: "vlan" range: "1:1" net_name: "vlan" - # - # Setup swift group variables when using swift (Not required if not using swift) - # part power is required under swift. - # For account/container speciying min_part_hours and repl_number is all thats required - # Alternatively defaults will be used (repl_number of 3, and min_part_hours of 1). - # For storage policies, a name and unique index is required as well as repl_number and - # min_part_hours which will be set to a default value if not specified. - # There MUST be a storage policy with index 0 configured which will be the default for legacy containers (created pre-storage policies). - # You can set one policy to be "default: yes" this will be the default storage policy for non-legacy containers that are created. - # The index value must be unique. - # Storage policies can be set to "deprecated: yes" which will mean they are not used - # swift: - # part_power: 8 - # account: - # repl_number: 3 - # min_part_hours: 1 - # container: - # repl_number: 3 - # storage_policies: - # - policy: - # name: gold - # index: 0 - # repl_number: 3 - # default: yes - # - policy: - # name: silver - # index: 1 - # repl_number: 2 - # deprecated: yes - # # Name of load balancer lb_name: lb_name_in_core @@ -187,65 +157,3 @@ log_hosts: network_hosts: network1: ip: 172.29.236.108 - -# User defined Swift Proxy hosts - not required when not using swift -# Will deploy a swift-proxy container on these hosts. -# Recommend mirroring the infra_hosts -# swift-proxy_hosts: -# infra1: -# ip: 172.29.236.100 -# infra2: -# ip: 172.29.236.101 -# infra3: -# ip: 172.29.236.102 - -# User defined Object Storage Hosts - this is not a required group -# Under swift_vars you can specify the host specific swift_vars. -# region - the swift region, this isn't required. -# zone - the swift zone, this isn't required either, will default to 0 -# mount_point - where the drives are mounted on the server -# drives - A list of drives in the server (Must have a name as a minimum) -# Above 4 vars are "host specific" -# weight: a disks weight (defaults to 100 if not specified) -# repl_ip: IP specific for object replication (not required) -# repl_port: Port specific for object replication (not required) -# groups: A list of groups to add the drive to. A group is either a storage policy or the account or container servers. (If not specified defaults to all groups, so container/account/all storage policies). -# The above 4 can be specified on a per host or per drive basis -# Or both, in which case "per drive" will take precedence for the specific drive. -# ip can be specified in swift_vars to override the hosts ip -# or per drive to override all for that specific drive. -# swift_hosts: -# object_storage1: -# ip: 172.29.236.108 -# container_vars: -# swift_vars: -# region: 0 -# zone: 0 -# groups: -# - silver -# - account -# mount_point: /srv/node -# drives: -# - name: sdb -# ip: 172.10.100.100 -# repl_ip: 10.10.0.1 -# repl_port: 54321 -# groups: -# - gold -# - account -# - container -# - name: sdc -# weight: 150 -# - name: sdd -# - name: sde -# -# object_storage2: -# ip: 172.29.236.109 -# container_vars: -# swift_vars: -# region: 0 -# zone: 1 -# mount_point: /srv/node -# drives: -# - name: sdb -# - name: sdc From ab46cc23c2010a9d2011abe2e1aaae37742c1cba Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Mon, 20 Oct 2014 16:19:39 +0100 Subject: [PATCH 44/45] Move the ring generation to /etc/swift instead of /tmp/swift This will avoid the files being removed on reboot (or other /tmp removal time for the node that runs the scripts) --- rpc_deployment/inventory/group_vars/swift_all.yml | 1 - rpc_deployment/roles/swift_ring_builder/tasks/main.yml | 10 +++++----- .../roles/swift_ring_distribute/tasks/main.yml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/rpc_deployment/inventory/group_vars/swift_all.yml b/rpc_deployment/inventory/group_vars/swift_all.yml index 426676f73a..91c6c9265d 100644 --- a/rpc_deployment/inventory/group_vars/swift_all.yml +++ b/rpc_deployment/inventory/group_vars/swift_all.yml @@ -65,7 +65,6 @@ service_names: container_directories: - /var/log/swift - - /tmp/swift - /var/lock/swift - /etc/swift - /etc/swift/rings/ diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index 0b1f605e10..daead6f06b 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -21,16 +21,16 @@ group=root mode=0755 with_items: - - /tmp/swift/rings - - /tmp/swift/scripts + - /etc/swift/rings + - /etc/swift/scripts - name: "Copy the swift_rings.py file" template: > src=swift_rings.py - dest="/tmp/swift/scripts/swift_rings.py" + dest="/etc/swift/scripts/swift_rings.py" mode=0700 - name: "build rings" - command: /usr/bin/python /tmp/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_inventory.json + command: /usr/bin/python /etc/swift/scripts/swift_rings.py -s /etc/rpc_deploy/rpc_inventory.json args: - chdir: /tmp/swift/rings/ + chdir: /etc/swift/rings/ diff --git a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml index 5ab5826564..9e5ad8fc66 100644 --- a/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_distribute/tasks/main.yml @@ -21,5 +21,5 @@ owner={{ system_user }} group={{ system_group }} with_fileglob: - - /tmp/swift/rings/*.ring.gz - - /tmp/swift/rings/*.builder + - /etc/swift/rings/*.ring.gz + - /etc/swift/rings/*.builder From f945fc25adf17ac915f9c6a62241d48a7aaa7a15 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Tue, 21 Oct 2014 14:55:33 +0100 Subject: [PATCH 45/45] Check md5sum of swift ring .builder files before adjusting ring. For safety - if the ring on the remote hosts is not empty, or the same as the local host, we should abort and ask the user to check the consistency of the rings. This will prevent a situation where the rings are inconsistent, but still allow new hosts to be added to the ring. E.g. if the rings are removed from the local host for some reason, this will cause the run to fail. Or if the rings aren't moved to the nodes in a uniform way. Allows the user to manually intervene and fix the inconsistencies. --- .../playbooks/openstack/swift-build-rings.yml | 19 ++++++++++++ .../swift_ring_builder/tasks/check_ring.yml | 30 +++++++++++++++++++ .../roles/swift_ring_builder/tasks/main.yml | 2 ++ .../roles/swift_ring_md5sum/tasks/main.yml | 22 ++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 rpc_deployment/roles/swift_ring_builder/tasks/check_ring.yml create mode 100644 rpc_deployment/roles/swift_ring_md5sum/tasks/main.yml diff --git a/rpc_deployment/playbooks/openstack/swift-build-rings.yml b/rpc_deployment/playbooks/openstack/swift-build-rings.yml index 22df415102..f1cee27595 100644 --- a/rpc_deployment/playbooks/openstack/swift-build-rings.yml +++ b/rpc_deployment/playbooks/openstack/swift-build-rings.yml @@ -1,4 +1,23 @@ --- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: swift_hosts:swift_proxy + user: root + roles: + - swift_ring_md5sum + - hosts: local user: root roles: diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/check_ring.yml b/rpc_deployment/roles/swift_ring_builder/tasks/check_ring.yml new file mode 100644 index 0000000000..a152c5a3da --- /dev/null +++ b/rpc_deployment/roles/swift_ring_builder/tasks/check_ring.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get md5sum of local builder files + shell: cat /etc/swift/rings/*.builder 2>/dev/null | md5sum | cut -d " " -f1 + register: md5sum + +- name: Get empty md5sum + shell: echo -n | md5sum | cut -d " " -f1 + register: empty_md5sum + +# Fail if the remote hosts builder files is not empty AND +# does not match the md5sum of the local host. +- name: Check md5sum of builder files + fail: + msg: "The builder files on the remote host {{ item }} do not match the local host, and are not empty on the remote host" + when: ("{{ hostvars[item]['builder_md5sum'] }}" != "{{ empty_md5sum.stdout }}") and ("{{ hostvars[item]['builder_md5sum'] }}" != "{{ md5sum.stdout }}") + with_items: groups['swift_proxy'] + groups['swift_hosts'] diff --git a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml index daead6f06b..87bf1c6046 100644 --- a/rpc_deployment/roles/swift_ring_builder/tasks/main.yml +++ b/rpc_deployment/roles/swift_ring_builder/tasks/main.yml @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: check_ring.yml + - name: "make sure scripts directory exists" file: > state=directory diff --git a/rpc_deployment/roles/swift_ring_md5sum/tasks/main.yml b/rpc_deployment/roles/swift_ring_md5sum/tasks/main.yml new file mode 100644 index 0000000000..73f371ec4f --- /dev/null +++ b/rpc_deployment/roles/swift_ring_md5sum/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get md5sum of builder files + shell: cat /etc/swift/*.builder 2>/dev/null | md5sum | cut -d " " -f1 + register: md5sum + +- name: Register a fact for the md5sum + set_fact: + builder_md5sum: "{{ md5sum.stdout }}"