From a05490885ed1805928be145afb928824b1839057 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 9 Sep 2014 02:28:36 -0500 Subject: [PATCH] added missing deps created new memcahced module resolved nova authorized_keys file distribution issues fixed remaining horizon issues and inconsistencies removed horizon from openstack common, as its no longer needed. adjusted to host name check to ensure that we only ever have a host name that is NO longer than 63 I guess I need more sleep, i missed my temp code for testing made container name shortening smarter removed container name munging We were doing container name munging when the length of the container name was > 63. This however was causing containers to not be in the correct host groups. To avoid general issues the munging code was removed and replaced with a system exit stating that the hostnames are too long. While this is not ideal it is the best way to ensure that the system is built correctly 100% of the time. updated support plays to use new key distribution fixed typos --- rpc_deployment/inventory/dynamic_inventory.py | 26 +- .../inventory/group_vars/elasticsearch.yml | 3 +- .../inventory/group_vars/galera.yml | 2 + .../inventory/group_vars/horizon.yml | 34 +- .../inventory/group_vars/keystone_all.yml | 1 + .../inventory/group_vars/kibana.yml | 4 + .../inventory/group_vars/logstash.yml | 3 + .../inventory/group_vars/memcached.yml | 4 + .../inventory/group_vars/rabbit.yml | 4 + .../inventory/group_vars/rsyslog.yml | 4 + .../inventory/group_vars/utility_all.yml | 2 + rpc_deployment/library/memcached | 599 ++++++++++++++++++ .../playbooks/openstack/horizon-all.yml | 18 + .../playbooks/openstack/horizon-common.yml | 25 + .../playbooks/openstack/horizon-ssl.yml | 53 ++ .../playbooks/openstack/horizon.yml | 25 +- .../playbooks/openstack/nova-compute-keys.yml | 48 +- .../playbooks/openstack/openstack-common.yml | 2 +- .../playbooks/openstack/openstack-setup.yml | 2 +- rpc_deployment/playbooks/rpc_support.yml | 32 + .../roles/horizon_common/tasks/main.yml | 85 ++- .../templates/horizon-manage.py | 24 + .../roles/horizon_setup/tasks/main.yml | 4 +- .../roles/horizon_ssl/tasks/main.yml | 25 + .../nova_compute_sshkey_create/tasks/main.yml | 17 +- .../nova_compute_sshkey_setup/tasks/main.yml | 5 - .../tasks/support_key_pair.yml | 28 +- rpc_deployment/vars/repo_packages/horizon.yml | 45 ++ 28 files changed, 981 insertions(+), 143 deletions(-) create mode 100644 rpc_deployment/library/memcached create mode 100644 rpc_deployment/playbooks/openstack/horizon-all.yml create mode 100644 rpc_deployment/playbooks/openstack/horizon-common.yml create mode 100644 rpc_deployment/playbooks/openstack/horizon-ssl.yml create mode 100644 rpc_deployment/roles/horizon_common/templates/horizon-manage.py create mode 100644 rpc_deployment/roles/horizon_ssl/tasks/main.yml create mode 100644 rpc_deployment/vars/repo_packages/horizon.yml diff --git a/rpc_deployment/inventory/dynamic_inventory.py b/rpc_deployment/inventory/dynamic_inventory.py index b1f7df2de1..83b4f5eae4 100755 --- a/rpc_deployment/inventory/dynamic_inventory.py +++ b/rpc_deployment/inventory/dynamic_inventory.py @@ -159,7 +159,6 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name, cuuid = cuuid.split('-')[0] container_host_name = '%s-%s' % (type_and_name, cuuid) hostvars_options = hostvars[container_host_name] = {} - if container_host_type not in inventory: inventory[container_host_type] = { "hosts": [], @@ -295,14 +294,21 @@ def _add_container_hosts(assignment, config, container_name, container_type, affinity = host_options.get('affinity', {}) container_affinity = affinity.get(container_name, 1) - # Ensures that container names are not longer than 64 - name_length = len(host_type) - if name_length > 25: - name_diff = name_length - 25 - host_name = host_type[:-name_diff] - else: - host_name = host_type - type_and_name = '%s_%s' % (host_name, container_name) + # Ensures that container names are not longer than 63 + # This section will ensure that we are not it by the following bug: + # https://bugzilla.mindrot.org/show_bug.cgi?id=2239 + type_and_name = '%s_%s' % (host_type, container_name) + max_hostname_len = 52 + if len(type_and_name) > max_hostname_len: + raise SystemExit( + 'The resulting combination of [ "%s" + "%s" ] is longer than' + ' 52 characters. This combination will result in a container' + ' name that is longer than the maximum allowable hostname of' + ' 63 characters. Before this process can continue please' + ' adjust the host entries in your "rpc_user_config.yml" to use' + ' a short hostname. The recommended hostname length is < 20' + ' characters long.' % (host_type, container_name) + ) physical_host = inventory['_meta']['hostvars'][host_type] container_host_type = '%s_containers' % host_type @@ -323,7 +329,7 @@ def _add_container_hosts(assignment, config, container_name, container_type, physical_host_type, config, is_metal, - assignment + assignment, ) # Add the physical host type to all containers from the built inventory diff --git a/rpc_deployment/inventory/group_vars/elasticsearch.yml b/rpc_deployment/inventory/group_vars/elasticsearch.yml index 88e733a146..089e66ae05 100644 --- a/rpc_deployment/inventory/group_vars/elasticsearch.yml +++ b/rpc_deployment/inventory/group_vars/elasticsearch.yml @@ -33,4 +33,5 @@ container_packages: service_pip_dependencies: - requests - + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/galera.yml b/rpc_deployment/inventory/group_vars/galera.yml index ed3bfcaae2..62314deb23 100644 --- a/rpc_deployment/inventory/group_vars/galera.yml +++ b/rpc_deployment/inventory/group_vars/galera.yml @@ -37,3 +37,5 @@ galera_gcache_size: 1G service_pip_dependencies: - MySQL-python + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/horizon.yml b/rpc_deployment/inventory/group_vars/horizon.yml index 84326b3b8a..e8c556c081 100644 --- a/rpc_deployment/inventory/group_vars/horizon.yml +++ b/rpc_deployment/inventory/group_vars/horizon.yml @@ -20,7 +20,7 @@ containerize: true ## Service Name -service_name: apache2 +service_name: horizon # Verbosity Options debug: False @@ -40,43 +40,23 @@ container_database: dash system_user: www-data system_group: www-data -## Git Source -git_repo: https://git.openstack.org/openstack/horizon -git_fallback_repo: https://github.com/openstack/horizon -git_install_branch: stable/icehouse # Installation directories -install_root_dir: /opt/horizon -install_lib_dir: /opt/horizon/lib/python2.7/site-packages - -service_pip_dependencies: - - oslo.config - - MySQL-python - - python-memcached - - django-appconf - - pycrypto - - ply - - greenlet +install_lib_dir: /usr/local/lib/python2.7/dist-packages container_directories: - - "{{ install_root_dir }}" + - "/etc/horizon" - "{{ install_lib_dir }}" -container_packages: - - apache2 - - apache2-utils - - libapache2-mod-wsgi - - libssl-dev - - libxslt1.1 - - openssl +service_pip_dependencies: + - MySQL-python + - python-memcached + - pycrypto horizon_fqdn: "{{ external_vip_address }}" horizon_server_name: "{{ container_name }}" horizon_self_signed: true -pip_install_options: "--install-option='--prefix={{ install_root_dir }}'" -service_name: horizon - ## Optional certification options # horizon_cacert_pem: /path/to/cacert.pem # horizon_ssl_cert: /etc/ssl/certs/apache.cert diff --git a/rpc_deployment/inventory/group_vars/keystone_all.yml b/rpc_deployment/inventory/group_vars/keystone_all.yml index 7c44c426ce..1f1f80d38b 100644 --- a/rpc_deployment/inventory/group_vars/keystone_all.yml +++ b/rpc_deployment/inventory/group_vars/keystone_all.yml @@ -66,6 +66,7 @@ service_pip_dependencies: - MySQL-python - pycrypto - python-memcached + - pycrypto - python-keystoneclient diff --git a/rpc_deployment/inventory/group_vars/kibana.yml b/rpc_deployment/inventory/group_vars/kibana.yml index 222f0be950..c33f6ab7c3 100644 --- a/rpc_deployment/inventory/group_vars/kibana.yml +++ b/rpc_deployment/inventory/group_vars/kibana.yml @@ -37,3 +37,7 @@ kibana_fqdn: "{{ external_vip_address }}" kibana_server_name: "{{ container_name }}" kibana_self_signed: true kibana_ssl_port: 8443 + +service_pip_dependencies: + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/logstash.yml b/rpc_deployment/inventory/group_vars/logstash.yml index 5acddbd64e..b0b045131d 100644 --- a/rpc_deployment/inventory/group_vars/logstash.yml +++ b/rpc_deployment/inventory/group_vars/logstash.yml @@ -33,3 +33,6 @@ container_packages: - logstash - logstash-contrib +service_pip_dependencies: + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/memcached.yml b/rpc_deployment/inventory/group_vars/memcached.yml index 7b165df92f..c2d0fc149a 100644 --- a/rpc_deployment/inventory/group_vars/memcached.yml +++ b/rpc_deployment/inventory/group_vars/memcached.yml @@ -18,3 +18,7 @@ service_name: memcached # only used when the lxc vg is present on the target container_lvm_fstype: ext4 container_lvm_fssize: 5GB + +service_pip_dependencies: + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/rabbit.yml b/rpc_deployment/inventory/group_vars/rabbit.yml index 10ed0f3ea2..d7242ef4e9 100644 --- a/rpc_deployment/inventory/group_vars/rabbit.yml +++ b/rpc_deployment/inventory/group_vars/rabbit.yml @@ -31,3 +31,7 @@ rabbit_cookie: "{{ rabbitmq_cookie_token }}" enable_management_plugin: true rabbit_cluster_name: rpc + +service_pip_dependencies: + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/rsyslog.yml b/rpc_deployment/inventory/group_vars/rsyslog.yml index 737156c483..e1d52065a5 100644 --- a/rpc_deployment/inventory/group_vars/rsyslog.yml +++ b/rpc_deployment/inventory/group_vars/rsyslog.yml @@ -25,3 +25,7 @@ container_lvm_fssize: 5GB apt_container_repos: - { repo: "ppa:adiscon/v8-stable", state: "present" } + +service_pip_dependencies: + - python-memcached + - pycrypto diff --git a/rpc_deployment/inventory/group_vars/utility_all.yml b/rpc_deployment/inventory/group_vars/utility_all.yml index 9319566d1f..ce6b813f5a 100644 --- a/rpc_deployment/inventory/group_vars/utility_all.yml +++ b/rpc_deployment/inventory/group_vars/utility_all.yml @@ -32,6 +32,8 @@ service_pip_dependencies: - python-neutronclient - python-novaclient - python-swiftclient + - python-memcached + - pycrypto container_packages: - ruby1.9.1 diff --git a/rpc_deployment/library/memcached b/rpc_deployment/library/memcached new file mode 100644 index 0000000000..5623c3e370 --- /dev/null +++ b/rpc_deployment/library/memcached @@ -0,0 +1,599 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2014, Kevin Carter +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import base64 +import stat +import sys + +import memcache +try: + from Crypto import Random + from Crypto.Cipher import AES + + ENCRYPT_IMPORT = True +except ImportError: + ENCRYPT_IMPORT = False + + +DOCUMENTATION = """ +--- +module: memcached +version_added: "1.6.6" +short_description: + - Add, remove, and get items from memcached +description: + - Add, remove, and get items from memcached +options: + name: + description: + - Memcached key name + required: true + content: + description: + - Add content to memcahced. Only used when state is 'present'. + required: false + file_path: + description: + - This can be used with state 'present' and 'retrieve'. When set + with state 'present' the contents of a file will be used, when + set with state 'retrieve' the contents of the memcached key will + be written to a file. + required: false + state: + description: + - ['absent', 'present', 'retrieve'] + required: true + server: + description: + - server IP address and port. This can be a comma seperated list of + servers to connect to. + required: true + encrypt_string: + description: + - Encrypt/Decrypt a memcached object using a provided value. + required: false + dir_mode: + description: + - If a directory is created when using the ``file_path`` argument + the directory will be created with a set mode. + default: '0755' + required: false + file_mode: + description: + - If a file is created when using the ``file_path`` argument + the file will be created with a set mode. + default: '0644' + required: false + expires: + description: + - Seconds until an item is expired from memcached. + default: 300 + required: false +notes: + - The "absent" state will remove an item from memcached. + - The "present" state will place an item from a string or a file into + memcached. + - The "retrieve" state will get an item from memcached and return it as a + string. If a ``file_path`` is set this module will also write the value + to a file. + - All items added into memcached are base64 encoded. + - All items retrieved will attempt base64 decode and return the string + value if not applicable. + - Items retrieve from memcached are returned within a "value" key unless + a ``file_path`` is specified which would then write the contents of the + memcached key to a file. + - The ``file_path`` and ``content`` fields are mutually exclusive. + - If you'd like to encrypt items in memcached PyCrypto is a required. +requirements: + - "python-memcached" +optional_requirements: + - "pycrypto" +author: Kevin Carter +""" + +EXAMPLES = """ +# Add an item into memcached. +- memcached: + name: "key_name" + content: "Super awesome value" + state: "present" + server: "localhost:11211" + +# Read the contents of a memcached key, returned as "memcached_phrase.value". +- memcached: + name: "key_name" + state: "retrieve" + server: "localhost:11211" + register: memcached_key + +# Add the contents of a file into memcached. +- memcached: + name: "key_name" + file_path: "/home/user_name/file.txt" + state: "present" + server: "localhost:11211" + +# Write the contents of a memcached key to a file and is returned as +# "memcached_phrase.value". +- memcached: + name: "key_name" + file_path: "/home/user_name/file.txt" + state: "retrieve" + server: "localhost:11211" + register: memcached_key + +# Delete an item from memcached. +- memcached: + name: "key_name" + state: "absent" + server: "localhost:11211" +""" + +SERVER_MAX_VALUE_LENGTH = 1024 * 256 + +MAX_MEMCACHED_CHUNKS = 256 + + +class AESCipher(object): + """Encrypt an a string in using AES. + + Solution derived from "http://stackoverflow.com/a/21928790" + """ + def __init__(self, key): + if ENCRYPT_IMPORT is False: + raise ImportError( + 'PyCrypto failed to be imported. Encryption is not supported' + ' on this system until PyCrypto is installed.' + ) + + self.bs = 32 + if len(key) >= 32: + self.key = key[:32] + else: + self.key = self._pad(key) + + def encrypt(self, raw): + """Encrypt raw message. + + :param raw: ``str`` + :returns: ``str`` Base64 encoded string. + """ + raw = self._pad(raw) + iv = Random.new().read(AES.block_size) + cipher = AES.new(self.key, AES.MODE_CBC, iv) + return base64.b64encode(iv + cipher.encrypt(raw)) + + def decrypt(self, enc): + """Decrypt an encrypted message. + + :param enc: ``str`` + :returns: ``str`` + """ + enc = base64.b64decode(enc) + iv = enc[:AES.block_size] + cipher = AES.new(self.key, AES.MODE_CBC, iv) + return self._unpad(cipher.decrypt(enc[AES.block_size:])) + + def _pad(self, string): + """Pad an AES encryption key. + + :param string: ``str`` + """ + base = (self.bs - len(string) % self.bs) + back = chr(self.bs - len(string) % self.bs) + return string + base * back + + @staticmethod + def _unpad(string): + """Un-pad an AES encryption key. + + :param string: ``str`` + """ + ordinal_range = ord(string[len(string)-1:]) + return string[:-ordinal_range] + + +class Memcached(object): + """Manage objects within memcached.""" + def __init__(self, module): + self.module = module + self.state_change = False + self.mc = None + + def router(self): + """Route all commands to their respected functions. + + If an exception happens a failure will be raised. + """ + + try: + action = getattr(self, self.module.params['state']) + self.mc = memcache.Client( + self.module.params['server'].split(','), + server_max_value_length=SERVER_MAX_VALUE_LENGTH, + debug=0 + ) + facts = action() + except Exception as exp: + self._failure(error=str(exp), rc=1, msg='general exception') + else: + self.mc.disconnect_all() + self.module.exit_json( + changed=self.state_change, **facts + ) + + def _failure(self, error, rc, msg): + """Return a Failure when running an Ansible command. + + :param error: ``str`` Error that occurred. + :param rc: ``int`` Return code while executing an Ansible command. + :param msg: ``str`` Message to report. + """ + + self.module.fail_json(msg=msg, rc=rc, err=error) + + def absent(self): + """Remove a key from memcached. + + If the value is not deleted when instructed to do so an exception will + be raised. + + :return: ``dict`` + """ + + key_name = self.module.params['name'] + get_keys = [ + '%s.%s' % (key_name, i) for i in xrange(MAX_MEMCACHED_CHUNKS) + ] + self.mc.delete_multi(get_keys) + value = self.mc.get_multi(get_keys) + if not value: + self.state_change = True + return {'absent': True, 'key': self.module.params['name']} + else: + self._failure( + error='Memcache key not deleted', + rc=1, + msg='Failed to remove an item from memcached please check your' + ' memcached server for issues. If you are load balancing' + ' memcahced, attempt to connect to a single node.' + ) + + @staticmethod + def _decode_value(value): + """Return a ``str`` from a base64 decoded value. + + If the content is not a base64 ``str`` the raw value will be returned. + + :param value: ``str`` + :return: + """ + + try: + b64_value = base64.decodestring(value) + except Exception: + return value + else: + return b64_value + + def _encode_value(self, value): + """Return a base64 encoded value. + + If the value can't be base64 encoded an excption will be raised. + + :param value: ``str`` + :return: ``str`` + """ + + try: + b64_value = base64.encodestring(value) + except Exception as exp: + self._failure( + error=str(exp), + rc=1, + msg='The value provided can not be Base64 encoded.' + ) + else: + return b64_value + + def _file_read(self, full_path, pass_on_error=False): + """Read the contents of a file. + + This will read the contents of a file. If the ``full_path`` does not + exist an exception will be raised. + + :param full_path: ``str`` + :return: ``str`` + """ + + try: + with open(full_path, 'rb') as f: + o_value = f.read() + except IOError as exp: + if pass_on_error is False: + self._failure( + error=str(exp), + rc=1, + msg="The file you've specified does not exist. Please" + " check your full path @ [ %s ]." % full_path + ) + else: + return None + else: + return o_value + + def _chown(self, path, mode_type): + """Chown a file or directory based on a given mode type. + + If the file is modified the state will be changed. + + :param path: ``str`` + :param mode_type: ``str`` + """ + mode = self.module.params.get(mode_type) + # Ensure that the mode type is a string. + mode = str(mode) + _mode = oct(stat.S_IMODE(os.stat(path).st_mode)) + if _mode != mode or _mode[1:] != mode: + os.chmod(path, int(mode, 8)) + self.state_change = True + + def _file_write(self, full_path, value): + """Write the contents of ``value`` to the ``full_path``. + + This will return True upon success and will raise an exception upon + failure. + + :param full_path: ``str`` + :param value: ``str`` + :return: ``bol`` + """ + + try: + # Ensure that the directory exists + dir_path = os.path.dirname(full_path) + try: + os.makedirs(dir_path) + except OSError as exp: + if exp.errno == errno.EEXIST and os.path.isdir(dir_path): + pass + else: + self._failure( + error=str(exp), + rc=1, + msg="The directory [ %s ] does not exist and couldn't" + " be created. Please check the path and that you" + " have permission to write the file." + ) + + # Ensure proper directory permissions + self._chown(path=dir_path, mode_type='dir_mode') + + # Write contents of a cached key to a file. + with open(full_path, 'wb') as f: + if isinstance(value, list): + f.writelines(value) + else: + f.write(value) + + # Ensure proper file permissions + self._chown(path=full_path, mode_type='file_mode') + + except IOError as exp: + self._failure( + error=str(exp), + rc=1, + msg="There was an issue while attempting to write to the" + " file [ %s ]. Please check your full path and" + " permissions." % full_path + ) + else: + return True + + def retrieve(self): + """Return a value from memcached. + + If ``file_path`` is specified the value of the memcached key will be + written to a file at the ``file_path`` location. If the value of a key + is None, an exception will be raised. + + :returns: ``dict`` + """ + + key_name = self.module.params['name'] + get_keys = [ + '%s.%s' % (key_name, i) for i in xrange(MAX_MEMCACHED_CHUNKS) + ] + multi_value = self.mc.get_multi(get_keys) + if multi_value: + value = ''.join([i for i in multi_value.values() if i is not None]) + # Get the file path if specified. + file_path = self.module.params.get('file_path') + if file_path is not None: + full_path = os.path.abspath(os.path.expanduser(file_path)) + + # Decode cached value + encrypt_string = self.module.params.get('encrypt_string') + if encrypt_string: + _d_value = AESCipher(key=encrypt_string) + d_value = _d_value.decrypt(enc=value) + if not d_value: + d_value = self._decode_value(value=value) + else: + d_value = self._decode_value(value=value) + + o_value = self._file_read( + full_path=full_path, pass_on_error=True + ) + + # compare old value to new value and write if different + if o_value != d_value: + self.state_change = True + self._file_write(full_path=full_path, value=d_value) + + return { + 'present': True, + 'key': self.module.params['name'], + 'value': value, + 'file_path': full_path + } + else: + return { + 'present': True, + 'key': self.module.params['name'], + 'value': value + } + else: + self._failure( + error='Memcache key not found', + rc=1, + msg='The key you specified was not found within memcached. ' + 'If you are load balancing memcahced, attempt to connect' + ' to a single node.' + ) + + def present(self): + """Create and or update a key within Memcached. + + The state processed here is present. This state will ensure that + content is written to a memcached server. When ``file_path`` is + specified the content will be read in from a file. + """ + + file_path = self.module.params.get('file_path') + if file_path is not None: + full_path = os.path.abspath(os.path.expanduser(file_path)) + # Read the contents of a file into memcached. + o_value = self._file_read(full_path=full_path) + else: + o_value = self.module.params['content'] + + # Encode cached value + encrypt_string = self.module.params.get('encrypt_string') + if encrypt_string: + _d_value = AESCipher(key=encrypt_string) + d_value = _d_value.encrypt(raw=o_value) + else: + d_value = self._encode_value(value=o_value) + + compare = 1024 * 128 + chunks = sys.getsizeof(d_value) / compare + if chunks == 0: + chunks = 1 + elif chunks > MAX_MEMCACHED_CHUNKS: + self._failure( + error='Memcache content too large', + rc=1, + msg='The content that you are attempting to cache is larger' + ' than [ %s ] megabytes.' + % ((compare * MAX_MEMCACHED_CHUNKS / 1024 / 1024)) + ) + + step = len(d_value) / chunks + if step == 0: + step = 1 + + key_name = self.module.params['name'] + split_d_value = {} + count = 0 + for i in range(0, len(d_value), step): + split_d_value['%s.%s' % (key_name, count)] = d_value[i:i+step] + count += 1 + + value = self.mc.set_multi( + mapping=split_d_value, + time=self.module.params['expires'], + min_compress_len=2048 + ) + + if not value: + self.state_change = True + return { + 'present': True, + 'key': self.module.params['name'] + } + else: + self._failure( + error='Memcache content not created', + rc=1, + msg='The content you attempted to place within memcached' + ' was not created. If you are load balancing' + ' memcahced, attempt to connect to a single node.' + ' Returned a value of unstored keys [ %s ].' % value + ) + + +def main(): + """Main ansible run method.""" + module = AnsibleModule( + argument_spec=dict( + name=dict( + type='str', + required=True + ), + content=dict( + type='str', + required=False + ), + file_path=dict( + type='str', + required=False + ), + state=dict( + type='str', + required=True + ), + server=dict( + type='str', + required=True + ), + expires=dict( + type='int', + default=300, + required=False + ), + file_mode=dict( + type='str', + default='0644', + required=False + ), + dir_mode=dict( + type='str', + default='0755', + required=False + ), + encrypt_string=dict( + type='str', + required=False + ) + ), + supports_check_mode=False, + mutually_exclusive=[ + ['content', 'file_path'] + ] + ) + ms = Memcached(module=module) + ms.router() + +# import module snippets +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/rpc_deployment/playbooks/openstack/horizon-all.yml b/rpc_deployment/playbooks/openstack/horizon-all.yml new file mode 100644 index 0000000000..a13aa1bd70 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/horizon-all.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: horizon-common.yml +- include: horizon-ssl.yml +- include: horizon.yml diff --git a/rpc_deployment/playbooks/openstack/horizon-common.yml b/rpc_deployment/playbooks/openstack/horizon-common.yml new file mode 100644 index 0000000000..23ef245188 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/horizon-common.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: horizon_all + user: root + roles: + - common + - container_common + - openstack_common + - openstack_openrc + - galera_client_cnf + vars_files: + - vars/repo_packages/horizon.yml diff --git a/rpc_deployment/playbooks/openstack/horizon-ssl.yml b/rpc_deployment/playbooks/openstack/horizon-ssl.yml new file mode 100644 index 0000000000..0550b2bc91 --- /dev/null +++ b/rpc_deployment/playbooks/openstack/horizon-ssl.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: horizon_all[0] + user: root + roles: + - horizon_ssl + vars_files: + - vars/repo_packages/horizon.yml + +- hosts: horizon_all[0] + user: root + gather_facts: false + tasks: + - name: Distribute apache keys for cluster consumption + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "present" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/etc/ssl/private/apache.key", name: "apache_key" } + - { src: "/etc/ssl/certs/apache.cert", name: "apache_cert" } + +- hosts: horizon_all:!horizon_all[0] + user: root + gather_facts: false + tasks: + - name: Retrieve apache keys + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "retrieve" + file_mode: "{{ item.file_mode }}" + dir_mode: "{{ item.dir_mode }}" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/etc/ssl/private/apache.key", name: "apache_key", file_mode: "0640", dir_mode: "0750" } + - { src: "/etc/ssl/certs/apache.cert", name: "apache_cert", file_mode: "0644", dir_mode: "0755" } diff --git a/rpc_deployment/playbooks/openstack/horizon.yml b/rpc_deployment/playbooks/openstack/horizon.yml index 704e8e6132..efed7ff5b2 100644 --- a/rpc_deployment/playbooks/openstack/horizon.yml +++ b/rpc_deployment/playbooks/openstack/horizon.yml @@ -13,29 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -- hosts: horizon_all - user: root - roles: - - common - - container_common - - galera_client_cnf - -- hosts: horizon_all - user: root - roles: - - openstack_common - - openstack_openrc - - horizon_common - vars_files: - - vars/openstack_service_vars/horizon.yml - - hosts: horizon_all[0] user: root roles: + - horizon_common - galera_db_setup - horizon_setup + - horizon_apache + vars_files: + - vars/openstack_service_vars/horizon.yml -- hosts: horizon_all +- hosts: horizon_all:!horizon_all[0] user: root roles: + - horizon_common - horizon_apache + vars_files: + - vars/openstack_service_vars/horizon.yml diff --git a/rpc_deployment/playbooks/openstack/nova-compute-keys.yml b/rpc_deployment/playbooks/openstack/nova-compute-keys.yml index fbdb611372..25dbdce37d 100644 --- a/rpc_deployment/playbooks/openstack/nova-compute-keys.yml +++ b/rpc_deployment/playbooks/openstack/nova-compute-keys.yml @@ -13,30 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -- hosts: localhost - user: root - gather_facts: false - tasks: - - name: Remove [ /tmp/authorized_keys ] file if found - file: - path: "/tmp/authorized_keys" - state: "absent" - - hosts: nova_compute user: root roles: - nova_compute_sshkey_create +- hosts: nova_compute[0] + user: root + gather_facts: false + tasks: + - name: Distribute authorized keys for cluster consumption + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "present" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/var/lib/nova/.ssh/authorized_keys", name: "authorized_keys" } + +- hosts: nova_compute:!nova_compute[0] + user: root + gather_facts: false + tasks: + - name: Retrieve authorized keys + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "retrieve" + file_mode: "{{ item.file_mode }}" + dir_mode: "{{ item.dir_mode }}" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/var/lib/nova/.ssh/authorized_keys", name: "authorized_keys", file_mode: "0640", dir_mode: "0750" } + - hosts: nova_compute user: root roles: - nova_compute_sshkey_setup - -- hosts: localhost - user: root - gather_facts: false - tasks: - - name: Remove [ /tmp/authorized_keys ] file if found - file: - path: "/tmp/authorized_keys" - state: "absent" diff --git a/rpc_deployment/playbooks/openstack/openstack-common.yml b/rpc_deployment/playbooks/openstack/openstack-common.yml index 6e97d5d65d..bfc1dd3493 100644 --- a/rpc_deployment/playbooks/openstack/openstack-common.yml +++ b/rpc_deployment/playbooks/openstack/openstack-common.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- hosts: keystone_all:glance_all:heat_all:neutron_all:nova_all:cinder_all:horizon +- hosts: keystone_all:glance_all:heat_all:neutron_all:nova_all:cinder_all user: root roles: - common diff --git a/rpc_deployment/playbooks/openstack/openstack-setup.yml b/rpc_deployment/playbooks/openstack/openstack-setup.yml index c2d8b53219..7fc306e539 100644 --- a/rpc_deployment/playbooks/openstack/openstack-setup.yml +++ b/rpc_deployment/playbooks/openstack/openstack-setup.yml @@ -21,6 +21,6 @@ - include: nova-all.yml - include: neutron-all.yml - include: cinder-all.yml -- include: horizon.yml +- include: horizon-all.yml - include: utility.yml - include: ../infrastructure/rsyslog-config.yml diff --git a/rpc_deployment/playbooks/rpc_support.yml b/rpc_deployment/playbooks/rpc_support.yml index 2dc73aedb4..48c9fe42b5 100644 --- a/rpc_deployment/playbooks/rpc_support.yml +++ b/rpc_deployment/playbooks/rpc_support.yml @@ -25,6 +25,38 @@ roles: - rpc_support_api +- hosts: utility[0] + user: root + gather_facts: false + tasks: + - name: Distribute authorized keys for cluster consumption + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "present" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/root/.ssh/rpc_support", name: "rpc_support" } + - { src: "/root/.ssh/rpc_support.pub", name: "rpc_support_pub" } + +- hosts: utility:utility[0] + user: root + gather_facts: false + tasks: + - name: Retrieve authorized keys + memcached: + name: "{{ item.name }}" + file_path: "{{ item.src }}" + state: "retrieve" + file_mode: "{{ item.file_mode }}" + dir_mode: "{{ item.dir_mode }}" + server: "{{ hostvars[groups['memcached'][0]]['ansible_ssh_host'] }}:11211" + encrypt_string: "{{ memcached_encryption_key }}" + with_items: + - { src: "/root/.ssh/rpc_support", name: "rpc_support", file_mode: "0600", dir_mode: "0755" } + - { src: "/root/.ssh/rpc_support.pub", name: "rpc_support_pub", file_mode: "0640", dir_mode: "0755" } + # Setup holland backup - hosts: galera user: root diff --git a/rpc_deployment/roles/horizon_common/tasks/main.yml b/rpc_deployment/roles/horizon_common/tasks/main.yml index 1a70a8968c..b645bc046b 100644 --- a/rpc_deployment/roles/horizon_common/tasks/main.yml +++ b/rpc_deployment/roles/horizon_common/tasks/main.yml @@ -13,53 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: create self-signed SSL cert - command: > - openssl req -new -nodes -x509 -subj - "/C=US/ST=Texas/L=San Antonio/O=IT/CN={{ horizon_server_name }}" - -days 3650 - -keyout /etc/ssl/private/apache.key - -out /etc/ssl/certs/apache.cert - -extensions v3_ca - creates=/etc/ssl/certs/apache.cert - when: horizon_self_signed is defined and horizon_self_signed == true +- name: Add horizon etc dir + file: + dest: "/etc/horizon" + recurse: "yes" + owner: "{{ system_group }}" + group: "{{ system_group }}" + state: "directory" -- name: Setup Horizon config - template: > - src=local_settings.py - dest={{ install_lib_dir }}/openstack_dashboard/local/local_settings.py - owner={{ system_user }} - group={{ system_group }} - -- name: Copy manage.py - command: > - creates={{ install_lib_dir }}/manage.py - cp -a "/opt/{{ service_name }}_{{ git_install_branch | replace('/', '_') }}/manage.py" "{{ install_lib_dir }}" +# The Horizon config files should be replaced for the JUNO release +# juno_revision: true +- name: Setup Horizon config(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ system_user }}" + group: "{{ system_group }}" + mode: "{{ item.mode }}" + with_items: + - { src: "local_settings.py", dest: "/etc/horizon/local_settings.py", mode: "0644" } + - { src: "horizon-manage.py", dest: "/usr/local/bin/horizon-manage.py", mode: "0755" } +# /opt/horizon/lib/python2.7/site-packages/manage.py - name: Collect static files - command: python /opt/horizon/lib/python2.7/site-packages/manage.py collectstatic --noinput + command: horizon-manage.py collectstatic --noinput - -- name: Fix missing file issues (1 of 2) - file: > - src={{ install_lib_dir }}/horizon/static/bootstrap/js - dest={{ install_lib_dir }}/openstack_dashboard/static/bootstrap/js - owner={{ system_group }} - group={{ system_group }} - state=link - -- name: Fix missing file issues (2 of 2) - file: > - src={{ install_lib_dir }}/horizon/static/horizon - dest={{ install_lib_dir }}/openstack_dashboard/static/horizon - owner={{ system_group }} - group={{ system_group }} - state=link +- name: Create hoirzon links + file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ system_group }}" + group: "{{ system_group }}" + state: "link" + with_items: + - { src: "/etc/horizon/local_settings.py", dest: "{{ install_lib_dir }}/openstack_dashboard/local/local_settings.py" } + - { src: "{{ install_lib_dir }}/horizon/static/bootstrap/js", dest: "{{ install_lib_dir }}/openstack_dashboard/static/bootstrap/js" } + - { src: "{{ install_lib_dir }}/horizon/static/horizon", dest: "{{ install_lib_dir }}/openstack_dashboard/static/horizon" } - name: Set horizon permissions - file: > - state=directory - dest={{ install_root_dir }} - recurse=yes - owner={{ system_group }} - group={{ system_group }} + file: + dest: "{{ item }}" + recurse: "yes" + owner: "{{ system_group }}" + group: "{{ system_group }}" + state: "directory" + with_items: + - "{{ install_lib_dir }}/horizon" + - "{{ install_lib_dir }}/openstack_dashboard" diff --git a/rpc_deployment/roles/horizon_common/templates/horizon-manage.py b/rpc_deployment/roles/horizon_common/templates/horizon-manage.py new file mode 100644 index 0000000000..5e255d36b8 --- /dev/null +++ b/rpc_deployment/roles/horizon_common/templates/horizon-manage.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +from django.core.management import execute_from_command_line # noqa + +if __name__ == "__main__": + os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", "openstack_dashboard.settings" + ) + execute_from_command_line(sys.argv) diff --git a/rpc_deployment/roles/horizon_setup/tasks/main.yml b/rpc_deployment/roles/horizon_setup/tasks/main.yml index 3b31675c63..c7290e13e6 100644 --- a/rpc_deployment/roles/horizon_setup/tasks/main.yml +++ b/rpc_deployment/roles/horizon_setup/tasks/main.yml @@ -14,6 +14,4 @@ # limitations under the License. - name: Run syncdb - command: > - chdir={{ install_lib_dir }} - python manage.py syncdb --noinput + command: horizon-manage.py syncdb --noinput diff --git a/rpc_deployment/roles/horizon_ssl/tasks/main.yml b/rpc_deployment/roles/horizon_ssl/tasks/main.yml new file mode 100644 index 0000000000..aa8b128ae0 --- /dev/null +++ b/rpc_deployment/roles/horizon_ssl/tasks/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: create self-signed SSL cert + command: > + openssl req -new -nodes -x509 -subj + "/C=US/ST=Texas/L=San Antonio/O=IT/CN={{ horizon_server_name }}" + -days 3650 + -keyout /etc/ssl/private/apache.key + -out /etc/ssl/certs/apache.cert + -extensions v3_ca + creates=/etc/ssl/certs/apache.cert + when: horizon_self_signed is defined and horizon_self_signed == true diff --git a/rpc_deployment/roles/nova_compute_sshkey_create/tasks/main.yml b/rpc_deployment/roles/nova_compute_sshkey_create/tasks/main.yml index 11da1bc579..0de80df900 100644 --- a/rpc_deployment/roles/nova_compute_sshkey_create/tasks/main.yml +++ b/rpc_deployment/roles/nova_compute_sshkey_create/tasks/main.yml @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Set nova users shell to /bin/bash and generate ssh_key + user: + name: "nova" + shell: "/bin/bash" + - name: Create the keys directory for the nova user file: state: "directory" @@ -21,16 +26,12 @@ owner: "nova" mode: "0700" -- name: Set nova users shell to /bin/bash and generate ssh_key - user: - name: "nova" - shell: "/bin/bash" - -- name: Remove old key if found +- name: Remove old key file(s) if found file: path: "{{ item }}" state: "absent" with_items: + - "/var/lib/nova/.ssh/authorized_keys" - "/var/lib/nova/.ssh/id_rsa" - "/var/lib/nova/.ssh/id_rsa.pub" @@ -53,5 +54,5 @@ changed_when: false - name: Build authorized keys - shell: echo "{{ nova_pub.stdout }}" | tee -a /tmp/authorized_keys - delegate_to: localhost + shell: echo "{{ nova_pub.stdout }}" | tee -a /var/lib/nova/.ssh/authorized_keys + delegate_to: "{{ groups['nova_compute'][0] }}" diff --git a/rpc_deployment/roles/nova_compute_sshkey_setup/tasks/main.yml b/rpc_deployment/roles/nova_compute_sshkey_setup/tasks/main.yml index 33a296b34f..dce386a278 100644 --- a/rpc_deployment/roles/nova_compute_sshkey_setup/tasks/main.yml +++ b/rpc_deployment/roles/nova_compute_sshkey_setup/tasks/main.yml @@ -21,11 +21,6 @@ group: "nova" mode: "0644" -- name: Sync authorized_keys file - synchronize: - src: /tmp/authorized_keys - dest: /var/lib/nova/.ssh/authorized_keys - - name: Set authorized_keys permissions file: path: "/var/lib/nova/.ssh/authorized_keys" diff --git a/rpc_deployment/roles/rpc_support_api/tasks/support_key_pair.yml b/rpc_deployment/roles/rpc_support_api/tasks/support_key_pair.yml index 6a475e20d7..593d7ce2b8 100644 --- a/rpc_deployment/roles/rpc_support_api/tasks/support_key_pair.yml +++ b/rpc_deployment/roles/rpc_support_api/tasks/support_key_pair.yml @@ -13,14 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Check for key file + shell: ls /root/.ssh/rpc_support + failed_when: false + changed_when: key_check.rc != 0 + register: key_check + - name: Create rpc_support SSH key - user: - name: root - generate_ssh_key: yes - ssh_key_bits: 2048 - ssh_key_comment: "rpc_support key added for Openstack Instances" - ssh_key_file: "/root/.ssh/rpc_support" + shell: ssh-keygen -f "/root/.ssh/rpc_support" -t rsa -q -N "" register: support_key + when: key_check|changed tags: - support_key - support_keypair @@ -29,8 +31,18 @@ shell: | . /root/openrc nova keypair-list | grep rpc_support + failed_when: false register: rpc_support_key - when: support_key|changed + tags: + - support_keypair + +- name: Delete rpc_support keypair in nova + shell: | + . /root/openrc + nova keypair-keypair-delete rpc_support + failed_when: false + register: rpc_support_key_delete + when: support_key|changed and rpc_support_key.rc == 0 tags: - support_keypair @@ -38,6 +50,6 @@ shell: | . /root/openrc nova keypair-add --pub-key /root/.ssh/rpc_support.pub rpc_support - when: support_key|changed and rpc_support_key.rc != 0 + when: rpc_support_key.rc != 0 or rpc_support_key_delete|changed tags: - support_keypair diff --git a/rpc_deployment/vars/repo_packages/horizon.yml b/rpc_deployment/vars/repo_packages/horizon.yml new file mode 100644 index 0000000000..c18a312ae7 --- /dev/null +++ b/rpc_deployment/vars/repo_packages/horizon.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repo_package_name: horizon + +repo_path: "{{ repo_package_name }}_{{ git_install_branch | replace('/', '_') }}" + +## Git Source +git_repo: https://git.openstack.org/openstack/horizon +git_fallback_repo: https://github.com/openstack/horizon +git_dest: "/opt/{{ repo_path }}" +git_install_branch: e53cc81fe554ac27c9c3797336f62f9da7d96226 + +pip_wheel_name: horizon + +container_packages: + - apache2 + - apache2-utils + - libapache2-mod-wsgi + - libssl-dev + - libxslt1.1 + - openssl + +service_pip_dependencies: + - oslo.config + - MySQL-python + - python-memcached + - django-appconf + - pycrypto + - ply + - greenlet + - python-keystoneclient + - keystonemiddleware