From bd89789730b0400654f9f797483aafb65b7db801 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sun, 8 Mar 2015 10:45:18 +0000 Subject: [PATCH 01/15] Sync charm-helpers --- hooks/charmhelpers/contrib/network/ip.py | 85 ++++++- .../charmhelpers/contrib/openstack/context.py | 35 ++- hooks/charmhelpers/contrib/openstack/utils.py | 208 +++++++----------- hooks/charmhelpers/core/services/helpers.py | 16 +- 4 files changed, 215 insertions(+), 129 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d268ea8f..2d9a95cd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from base64 import b64decode from subprocess import check_call import six +import yaml from charmhelpers.fetch import ( apt_install, @@ -104,9 +105,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -191,7 +224,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..0293c7d7 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -20,15 +20,18 @@ from collections import OrderedDict from functools import wraps +import errno import subprocess import json import os -import socket import sys +import time import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -421,77 +424,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): @@ -536,89 +472,115 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + return config('openstack-origin-git') != None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" +def git_clone_and_install(projects, core_project, + parent_dir='/mnt/openstack-git'): + """Clone/install all OpenStack repos specified in projects dictionary.""" global requirements_dir + update_reqs = True - if file_name == "None": + if not projects: return - yaml_file = os.path.join(charm_dir(), file_name) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, + installed = _git_clone_and_install_subset(projects, parent_dir, whitelist=['requirements']) if 'requirements' not in installed: - error_out('requirements git repository must be specified') + update_reqs = False # clone/install all other projects except requirements and the core project blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, + update_requirements=update_reqs) # clone/install the core project whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) + installed = _git_clone_and_install_subset(projects, parent_dir, + whitelist=whitelist, + update_requirements=update_reqs) if core_project not in installed: error_out('{} git repository must be specified'.format(core_project)) -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" +def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], + blacklist=[], update_requirements=False): + """Clone/install subset of OpenStack repos specified in projects dict.""" global requirements_dir installed = [] - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) + for proj, val in projects.items(): + # The project subset is chosen based on the following 3 rules: + # 1) If project is in blacklist, we don't clone/install it, period. + # 2) If whitelist is empty, we clone/install everything else. + # 3) If whitelist is not empty, we clone/install everything in the + # whitelist. + if proj in blacklist: + continue + if whitelist and proj not in whitelist: + continue + repo = val['repository'] + branch = val['branch'] + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements) + if proj == 'requirements': + requirements_dir = repo_dir + installed.append(proj) return installed -def _git_clone_and_install_single(repo, branch, update_requirements=False): +def _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False): """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) - - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + # Note(coreycb): The parent directory for storing git repositories can be + # shared by multiple charms via bind mount, etc, so we use exception + # handling to ensure the test for existence and mkdir are atomic. + try: + os.mkdir(parent_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + pass else: - repo_dir = dest_dir + juju_log('Host directory not mounted at {}. ' + 'Directory created.'.format(parent_dir)) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) + # Note(coreycb): Similar to above, the cloned git repositories can be shared + # by multiple charms via bind mount, etc, so we use exception handling and + # special lock directories to ensure that a repository clone is only + # attempted once. + try: + os.mkdir(lock_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Lock directory exists at {}. Skip git clone and wait ' + 'for lock removal before installing.'.format(lock_dir)) + while os.path.exists(lock_dir): + juju_log('Waiting for git clone to complete before installing.') + time.sleep(1) + pass + else: + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) + + os.rmdir(lock_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..15b21664 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + super(HttpRelation).__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + super(HttpRelation).__init__(self, *args, **kwargs) def provide_data(self): return { From 3b758c90325c5f7fac28ac9b51771c6f181ea08b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sun, 8 Mar 2015 10:49:08 +0000 Subject: [PATCH 02/15] Initial support for deploying from git --- README.md | 95 ++++++++++++++++++ charm-helpers-hooks.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 19 ++++ hooks/neutron_api_hooks.py | 21 +++- hooks/neutron_api_utils.py | 122 ++++++++++++++++++++++- templates/upstart/neutron-server.upstart | 22 ++++ tests/basic_deployment.py | 8 +- 8 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 templates/upstart/neutron-server.upstart diff --git a/README.md b/README.md index e73aafd9..973e2251 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,101 @@ This charm also supports scale out and high availability using the hacluster cha juju set neutron-api vip= juju add-relation neutron-hacluster neutron-api +# Deploying from source + +The minimal openstack-origin-git config required to deploy from source is: + + openstack-origin-git: + "{'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'stable/icehouse'}}" + +If you specify a 'requirements' repository, it will be used to update the +requirements.txt files of all other git repos that it applies to, before +they are installed: + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" + +Note that there are only two key values the charm knows about for the outermost +dictionary: 'neutron' and 'requirements'. These repositories must correspond to +these keys. If the requirements repository is specified, it will be installed +first. The neutron repository is always installed last. All other repostories +will be installed in between. + +NOTE(coreycb): The following is temporary to keep track of the full list of +current tip repos (may not be up-to-date). + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron-fwaas': + {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', + 'branch': 'master'}, + 'neutron-lbaas': + {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', + 'branch': 'master'}, + 'neutron-vpnaas': + {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', + 'branch': 'master'}, + 'keystonemiddleware: + {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', + 'branch: 'master'}, + 'oslo-concurrency': + {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', + 'branch: 'master'}, + 'oslo-config': + {'repository': 'git://git.openstack.org/openstack/oslo.config.git', + 'branch: 'master'}, + 'oslo-context': + {'repository': 'git://git.openstack.org/openstack/oslo.context.git', + 'branch: 'master'}, + 'oslo-db': + {'repository': 'git://git.openstack.org/openstack/oslo.db.git', + 'branch: 'master'}, + 'oslo-i18n': + {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', + 'branch: 'master'}, + 'oslo-messaging': + {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', + 'branch: 'master'}, + 'oslo-middleware: + {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git', + 'branch': 'master'}, + 'oslo-rootwrap': + {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git', + 'branch: 'master'}, + 'oslo-serialization': + {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', + 'branch: 'master'}, + 'oslo-utils': + {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', + 'branch: 'master'}, + 'pbr': + {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', + 'branch: 'master'}, + 'python-keystoneclient': + {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', + 'branch: 'master'}, + 'python-neutronclient': + {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git', + 'branch: 'master'}, + 'python-novaclient': + {'repository': 'git://git.openstack.org/openstack/python-novaclient.git', + 'branch: 'master'}, + 'stevedore': + {'repository': 'git://git.openstack.org/openstack/stevedore.git', + 'branch: 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" + # Restrictions This charm only support deployment with OpenStack Icehouse or better. diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 917cf211..714fd5d9 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..aaa21c31 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index aec778db..91c807cf 100644 --- a/config.yaml +++ b/config.yaml @@ -14,6 +14,25 @@ options: Note that updating this setting to a source that is known to provide a later version of OpenStack will trigger a software upgrade. + + Note that when openstack-origin-git is specified, openstack-specific + packages will be installed from source rather than from the + openstack-origin repository. + openstack-origin-git: + default: None + type: string + description: | + Specifies a YAML-formatted two-dimensional array listing the git + repositories and branches from which to install OpenStack and its + dependencies. + + Note that the installed config files will be determined based on + the OpenStack release of the openstack-origin option. + + Note also that this option is processed for the initial install + only. Setting this option after deployment is not supported. + + For more details see README.md. rabbit-user: default: neutron type: string diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py index 63431968..aaeb7c4c 100755 --- a/hooks/neutron_api_hooks.py +++ b/hooks/neutron_api_hooks.py @@ -30,6 +30,7 @@ from charmhelpers.fetch import ( from charmhelpers.contrib.openstack.utils import ( configure_installation_source, + git_install_requested, openstack_upgrade_available, sync_db_with_multi_ipv6_addresses ) @@ -40,6 +41,7 @@ from neutron_api_utils import ( determine_packages, determine_ports, do_openstack_upgrade, + git_install, register_configs, restart_map, services, @@ -103,6 +105,14 @@ def install(): apt_update() apt_install(determine_packages(config('openstack-origin')), fatal=True) + + # NOTE(coreycb): This is temporary for sstack proxy, unless we decide + # we need to code proxy support into the charms. + os.environ["http_proxy"] = "http://squid.internal:3128" + os.environ["https_proxy"] = "https://squid.internal:3128" + + git_install(config('openstack-origin-git')) + [open_port(port) for port in determine_ports()] @@ -119,8 +129,9 @@ def config_changed(): config('database-user')) global CONFIGS - if openstack_upgrade_available('neutron-server'): - do_openstack_upgrade(CONFIGS) + if not git_install_requested(): + if openstack_upgrade_available('neutron-server'): + do_openstack_upgrade(CONFIGS) configure_https() update_nrpe_config() CONFIGS.write_all() @@ -135,6 +146,12 @@ def config_changed(): [cluster_joined(rid) for rid in relation_ids('cluster')] +#TODO(coreycb): For deploy from git support, need to implement action-set +# and action-get to trigger re-install of git-installed +# services. IIUC they'd be triggered via: +# juju do + + @hooks.hook('amqp-relation-joined') def amqp_joined(relation_id=None): relation_set(relation_id=relation_id, diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 5d31a281..2ea595f6 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -1,6 +1,8 @@ from collections import OrderedDict from copy import deepcopy import os +import shutil +import yaml from base64 import b64encode from charmhelpers.contrib.openstack import context, templating from charmhelpers.contrib.openstack.neutron import ( @@ -10,6 +12,8 @@ from charmhelpers.contrib.openstack.neutron import ( from charmhelpers.contrib.openstack.utils import ( os_release, get_os_codename_install_source, + git_install_requested, + git_clone_and_install, configure_installation_source, ) @@ -26,7 +30,12 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.host import ( - lsb_release + adduser, + add_group, + add_user_to_group, + mkdir, + lsb_release, + write_file, ) import neutron_api_context @@ -52,6 +61,27 @@ KILO_PACKAGES = [ 'python-neutron-vpnaas', ] +BASE_GIT_PACKAGES = [ + 'libxml2-dev', + 'libxslt1-dev', + 'python-dev', + 'python-pip', + 'python-setuptools', + 'zlib1g-dev', +] + +# ubuntu packages that should not be installed when deploying from git +GIT_PACKAGE_BLACKLIST = [ + 'neutron-server', + 'python-keystoneclient', +] + +GIT_PACKAGE_BLACKLIST_KILO = [ + 'python-neutron-lbaas', + 'python-neutron-fwaas', + 'python-neutron-vpnaas', +] + BASE_SERVICES = [ 'neutron-server' ] @@ -110,14 +140,26 @@ def api_port(service): def determine_packages(source=None): # currently all packages match service names packages = [] + BASE_PACKAGES + for v in resource_map().values(): packages.extend(v['services']) pkgs = neutron_plugin_attribute(config('neutron-plugin'), 'server_packages', 'neutron') packages.extend(pkgs) + if get_os_codename_install_source(source) >= 'kilo': packages.extend(KILO_PACKAGES) + + if git_install_requested(): + packages.extend(BASE_GIT_PACKAGES) + # don't include packages that will be installed from git + for p in GIT_PACKAGE_BLACKLIST: + packages.remove(p) + if get_os_codename_install_source(source) >= 'kilo': + for p in GIT_PACKAGE_BLACKLIST_KILO: + packages.remove(p) + return list(set(packages)) @@ -242,3 +284,81 @@ def setup_ipv6(): ' main') apt_update() apt_install('haproxy/trusty-backports', fatal=True) + + +def git_install(projects): + """Perform setup, and install git repos specified in yaml parameter.""" + if git_install_requested(): + git_pre_install() + # NOTE(coreycb): charm-helpers needs support to take array of + # core_projects. That would allow all neutron* projects to be + # installed last. + core = ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas', 'neutron'] + git_clone_and_install(yaml.load(projects), core_projects=core) + git_post_install() + + +def git_pre_install(): + """Perform pre-install setup.""" + dirs = [ + '/etc/neutron', + '/etc/neutron/rootwrap.d', + '/etc/neutron/plugins', + '/var/lib/neutron', + '/var/lib/neutron/lock', + '/var/log/neutron', + ] + + logs = [ + '/var/log/neutron/server.log', + ] + + adduser('neutron', shell='/bin/bash', system_user=True) + add_group('neutron', system_group=True) + add_user_to_group('neutron', 'neutron') + + for d in dirs: + mkdir(d, owner='neutron', group='neutron', perms=0700, force=False) + + for l in logs: + write_file(l, '', owner='neutron', group='neutron', perms=0600) + + +def git_post_install(): + """Perform post-install setup.""" + src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc') + configs = { + 'api-paste': { + 'src': os.path.join(src_etc, 'api-paste.ini'), + 'dest': '/etc/neutron/api-paste.ini', + }, + 'debug-filters': { + 'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'), + 'dest': '/etc/neutron/rootwrap.d/debug.filters', + }, + 'policy': { + 'src': os.path.join(src_etc, 'policy.json'), + 'dest': '/etc/neutron/policy.json', + }, + 'rootwrap': { + 'src': os.path.join(src_etc, 'rootwrap.conf'), + 'dest': '/etc/neutron/rootwrap.conf', + }, + } + + for conf, files in configs.iteritems(): + shutil.copyfile(files['src'], files['dest']) + + render('neutron-server.default', '/etc/default/neutron-server', {}, perms=0o440) + render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) + + neutron_api_context = { + 'service_description': 'Neutron API server', + 'charm_name': 'neutron-api', + 'process_name': 'neutron-server', + } + + render('upstart/neutron-server.upstart', '/etc/init/neutron.conf', + neutron_api_context, perms=0o644) + + service_start('neutron-server') diff --git a/templates/upstart/neutron-server.upstart b/templates/upstart/neutron-server.upstart new file mode 100644 index 00000000..2d3600e3 --- /dev/null +++ b/templates/upstart/neutron-server.upstart @@ -0,0 +1,22 @@ +description "{{ service_description }}" +author "Juju {{ charm_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +chdir /var/run + +pre-start script + mkdir -p /var/run/neutron + chown neutron:root /var/run/neutron +end script + +script + [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }} + [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG" + exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ + --config-file /etc/neutron/neutron.conf \ + --log-file /var/log/neutron/server.log $CONF_ARG +end script diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e65a1481..9a2ca1be 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -65,11 +65,17 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): """Configure all of the services.""" + # NOTE(coreycb): Added the following temporarily to test deploy from source + neutron_api_config = {'openstack-origin-git': + "{'neutron':" + " {'repository': 'git://git.openstack.org/openstack/neutron.git'," + " 'branch': 'stable/icehouse'}}"} keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} nova_cc_config = {'network-manager': 'Quantum', 'quantum-security-groups': 'yes'} - configs = {'keystone': keystone_config, + configs = {'neutron-api': neutron_api_config, + 'keystone': keystone_config, 'nova-cloud-controller': nova_cc_config} super(NeutronAPIBasicDeployment, self)._configure_services(configs) From 6ab462e8815ef3cec377a3841065c9576c8a8fb4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 09:48:08 +0000 Subject: [PATCH 03/15] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 27 +- .../charmhelpers/contrib/openstack/context.py | 60 ++++- .../charmhelpers/contrib/openstack/neutron.py | 70 ++++++ hooks/charmhelpers/contrib/openstack/utils.py | 234 ++++++++++-------- hooks/charmhelpers/core/hookenv.py | 26 ++ hooks/charmhelpers/core/host.py | 6 +- hooks/charmhelpers/core/services/helpers.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- .../contrib/openstack/amulet/deployment.py | 27 +- 9 files changed, 344 insertions(+), 112 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 2d9a95cd..90ac6d69 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -16,6 +16,7 @@ import json import os +import re import time from base64 import b64decode from subprocess import check_call @@ -48,6 +49,8 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr, mkdir, write_file, ) @@ -65,12 +68,18 @@ from charmhelpers.contrib.hahelpers.apache import ( from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + INTERNAL, +) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_ipv4_addr, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, is_address_in_network, + is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip @@ -727,7 +736,14 @@ class ApacheSSLContext(OSContextGenerator): 'endpoints': [], 'ext_ports': []} - for cn in self.canonical_names(): + cns = self.canonical_names() + if cns: + for cn in cns: + self.configure_cert(cn) + else: + # Expect cert/key provided in config (currently assumed that ca + # uses ip for cn) + cn = resolve_address(endpoint_type=INTERNAL) self.configure_cert(cn) addresses = self.get_network_addresses() @@ -883,6 +899,48 @@ class NeutronContext(OSContextGenerator): return ctxt +class NeutronPortContext(OSContextGenerator): + NIC_PREFIXES = ['eth', 'bond'] + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(self.NIC_PREFIXES): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + resolved = [] + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in ports: + if re.match(mac_regex, entry): + # NIC is in known NICs and does NOT hace an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + resolved.append(hwaddr_to_nic[entry]) + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + resolved.append(entry) + + return resolved + + class OSConfigFlagContext(OSContextGenerator): """Provides support for user-defined config flags. diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 902757fe..f8851050 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -16,6 +16,7 @@ # Various utilies for dealing with Neutron and the renaming from Quantum. +import six from subprocess import check_output from charmhelpers.core.hookenv import ( @@ -237,3 +238,72 @@ def network_manager(): else: # ensure accurate naming for all releases post-H return 'neutron' + + +def parse_mappings(mappings): + parsed = {} + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + parsed[p[0].strip()] = p[2].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings, default_bridge='br-data'): + """Parse data port mappings. + + Mappings must be a space-delimited list of bridge:port mappings. + + Returns dict of the form {bridge:port}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {default_bridge: mappings.split(' ')[0]} + + bridges = _mappings.keys() + ports = _mappings.values() + if len(set(bridges)) != len(bridges): + raise Exception("It is not allowed to have more than one port " + "configured on the same bridge") + + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return _mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + return {} + + mappings = {} + for p, r in six.iteritems(_mappings): + mappings[p] = tuple(r.split(':')) + + return mappings diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0293c7d7..da65f6d3 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -20,18 +20,21 @@ from collections import OrderedDict from functools import wraps -import errno import subprocess import json import os import sys -import time import six import yaml from charmhelpers.contrib.network import ip +from charmhelpers.core import ( + hookenv, + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -332,6 +335,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = hookenv.execution_environment()['conf'][option] + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -471,116 +489,103 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != None + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git').lower() != "none" requirements_dir = None -def git_clone_and_install(projects, core_project, - parent_dir='/mnt/openstack-git'): - """Clone/install all OpenStack repos specified in projects dictionary.""" - global requirements_dir - update_reqs = True +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if not projects: + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + + The directory key is optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - # clone/install the requirements project first - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=['requirements']) - if 'requirements' not in installed: - update_reqs = False + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, - update_requirements=update_reqs) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=whitelist, - update_requirements=update_reqs) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) - - -def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], - blacklist=[], update_requirements=False): - """Clone/install subset of OpenStack repos specified in projects dict.""" - global requirements_dir - installed = [] - - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements) - if proj == 'requirements': + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) requirements_dir = repo_dir - installed.append(proj) - return installed - - -def _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements=False): - """Clone and install a single git repository.""" - dest_dir = os.path.join(parent_dir, os.path.basename(repo)) - lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - - # Note(coreycb): The parent directory for storing git repositories can be - # shared by multiple charms via bind mount, etc, so we use exception - # handling to ensure the test for existence and mkdir are atomic. - try: - os.mkdir(parent_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Directory already exists at {}. ' - 'No need to create directory.'.format(parent_dir)) - pass - else: - juju_log('Host directory not mounted at {}. ' - 'Directory created.'.format(parent_dir)) - - # Note(coreycb): Similar to above, the cloned git repositories can be shared - # by multiple charms via bind mount, etc, so we use exception handling and - # special lock directories to ensure that a repository clone is only - # attempted once. - try: - os.mkdir(lock_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Lock directory exists at {}. Skip git clone and wait ' - 'for lock removal before installing.'.format(lock_dir)) - while os.path.exists(lock_dir): - juju_log('Waiting for git clone to complete before installing.') - time.sleep(1) - pass - else: - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: - repo_dir = dest_dir + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) - os.rmdir(lock_dir) +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) + + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) + + +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) + + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) + + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) @@ -589,16 +594,39 @@ def _git_clone_and_install_single(repo, branch, parent_dir, def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index cf552b39..715dd4c5 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -566,3 +566,29 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" return os.environ.get('CHARM_DIR') + + +@cached +def action_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +def action_set(values): + """Sets the values to be returned after the action finishes""" + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def action_fail(message): + """Sets the action status to failed and sets the error message. + + The results set by action_set are preserved.""" + subprocess.check_call(['action-fail', message]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b771c611..830822af 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -339,12 +339,16 @@ def lsb_release(): def pwgen(length=None): """Generate a random pasword.""" if length is None: + # A random length is ok to use a weak PRNG length = random.choice(range(35, 45)) alphanumeric_chars = [ l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() random_chars = [ - random.choice(alphanumeric_chars) for _ in range(length)] + random_generator.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 15b21664..3eb5fb44 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -139,7 +139,7 @@ class MysqlRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'user', 'password', 'database'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -154,7 +154,7 @@ class HttpRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'port'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134a..406a35c5 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] From 60f3818f039e2e1b15d3012147174cd2b3468a3a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 10:20:49 +0000 Subject: [PATCH 04/15] Overall refresh to current install from source approach --- Makefile | 5 +- README.md | 166 +++++++++--------- actions.yaml | 2 + actions/git-reinstall | 1 + actions/git_reinstall.py | 40 +++++ charm-helpers-hooks.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 15 +- hooks/neutron_api_hooks.py | 9 +- hooks/neutron_api_utils.py | 16 +- tests/16-basic-trusty-icehouse-git | 9 + ...basic-trusty-juno => 17-basic-trusty-juno} | 0 tests/18-basic-trusty-juno-git | 12 ++ tests/basic_deployment.py | 24 ++- unit_tests/__init__.py | 2 + unit_tests/test_actions_git_reinstall.py | 85 +++++++++ 16 files changed, 272 insertions(+), 120 deletions(-) create mode 100644 actions.yaml create mode 120000 actions/git-reinstall create mode 100755 actions/git_reinstall.py create mode 100755 tests/16-basic-trusty-icehouse-git rename tests/{16-basic-trusty-juno => 17-basic-trusty-juno} (100%) create mode 100755 tests/18-basic-trusty-juno-git create mode 100644 unit_tests/test_actions_git_reinstall.py diff --git a/Makefile b/Makefile index adf4df7c..0574b23e 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON := /usr/bin/env python lint: - @flake8 --exclude hooks/charmhelpers hooks unit_tests tests + @flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests @charm proof unit_test: @@ -25,7 +25,8 @@ test: # https://bugs.launchpad.net/amulet/+bug/1320357 @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ - 16-basic-trusty-juno + 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ + 18-basic-trusty-juno-git publish: lint unit_test bzr push lp:charms/neutron-api diff --git a/README.md b/README.md index 973e2251..190c51eb 100644 --- a/README.md +++ b/README.md @@ -25,98 +25,90 @@ This charm also supports scale out and high availability using the hacluster cha # Deploying from source -The minimal openstack-origin-git config required to deploy from source is: +The minimum openstack-origin-git config required to deploy from source is: openstack-origin-git: - "{'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'stable/icehouse'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: stable/juno}" -If you specify a 'requirements' repository, it will be used to update the -requirements.txt files of all other git repos that it applies to, before -they are installed: +Note that there are only two 'name' values the charm knows about: 'requirements' +and 'neutron'. These repositories must correspond to these 'name' values. +Additionally, the requirements repository must be specified first and the +neutron repository must be specified last. All other repostories are installed +in the order in which they are specified. + +The following is a full list of current tip repos (may not be up-to-date): openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'master'}}" - -Note that there are only two key values the charm knows about for the outermost -dictionary: 'neutron' and 'requirements'. These repositories must correspond to -these keys. If the requirements repository is specified, it will be installed -first. The neutron repository is always installed last. All other repostories -will be installed in between. - -NOTE(coreycb): The following is temporary to keep track of the full list of -current tip repos (may not be up-to-date). - - openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'neutron-fwaas': - {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', - 'branch': 'master'}, - 'neutron-lbaas': - {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', - 'branch': 'master'}, - 'neutron-vpnaas': - {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', - 'branch': 'master'}, - 'keystonemiddleware: - {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', - 'branch: 'master'}, - 'oslo-concurrency': - {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', - 'branch: 'master'}, - 'oslo-config': - {'repository': 'git://git.openstack.org/openstack/oslo.config.git', - 'branch: 'master'}, - 'oslo-context': - {'repository': 'git://git.openstack.org/openstack/oslo.context.git', - 'branch: 'master'}, - 'oslo-db': - {'repository': 'git://git.openstack.org/openstack/oslo.db.git', - 'branch: 'master'}, - 'oslo-i18n': - {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', - 'branch: 'master'}, - 'oslo-messaging': - {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', - 'branch: 'master'}, - 'oslo-middleware: - {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git', - 'branch': 'master'}, - 'oslo-rootwrap': - {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git', - 'branch: 'master'}, - 'oslo-serialization': - {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', - 'branch: 'master'}, - 'oslo-utils': - {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', - 'branch: 'master'}, - 'pbr': - {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', - 'branch: 'master'}, - 'python-keystoneclient': - {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', - 'branch: 'master'}, - 'python-neutronclient': - {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git', - 'branch: 'master'}, - 'python-novaclient': - {'repository': 'git://git.openstack.org/openstack/python-novaclient.git', - 'branch: 'master'}, - 'stevedore': - {'repository': 'git://git.openstack.org/openstack/stevedore.git', - 'branch: 'master'}, - 'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'master'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: master} + - {name: oslo-concurrency, + repository: 'git://git.openstack.org/openstack/oslo.concurrency', + branch: master} + - {name: oslo-config, + repository: 'git://git.openstack.org/openstack/oslo.config', + branch: master} + - {name: oslo-context, + repository: 'git://git.openstack.org/openstack/oslo.context.git', + branch: master} + - {name: oslo-db, + repository: 'git://git.openstack.org/openstack/oslo.db', + branch: master} + - {name: oslo-i18n, + repository: 'git://git.openstack.org/openstack/oslo.i18n', + branch: master} + - {name: oslo-messaging, + repository: 'git://git.openstack.org/openstack/oslo.messaging.git', + branch: master} + - {name: oslo-middleware, + repository': 'git://git.openstack.org/openstack/oslo.middleware.git', + branch: master} + - {name: oslo-rootwrap', + repository: 'git://git.openstack.org/openstack/oslo.rootwrap.git', + branch: master} + - {name: oslo-serialization, + repository: 'git://git.openstack.org/openstack/oslo.serialization', + branch: master} + - {name: oslo-utils, + repository: 'git://git.openstack.org/openstack/oslo.utils', + branch: master} + - {name: pbr, + repository: 'git://git.openstack.org/openstack-dev/pbr', + branch: master} + - {name: stevedore, + repository: 'git://git.openstack.org/openstack/stevedore.git', + branch: 'master'} + - {name: python-keystoneclient, + repository: 'git://git.openstack.org/openstack/python-keystoneclient', + branch: master} + - {name: python-neutronclient, + repository: 'git://git.openstack.org/openstack/python-neutronclient.git', + branch: master} + - {name: python-novaclient, + repository': 'git://git.openstack.org/openstack/python-novaclient.git', + branch: master} + - {name: keystonemiddleware, + repository: 'git://git.openstack.org/openstack/keystonemiddleware', + branch: master} + - {name: neutron-fwaas, + repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', + branch: master} + - {name: neutron-lbaas, + repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', + branch: master} + - {name: neutron-vpnaas, + repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', + branch: master} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: master}" # Restrictions diff --git a/actions.yaml b/actions.yaml new file mode 100644 index 00000000..27ef55b8 --- /dev/null +++ b/actions.yaml @@ -0,0 +1,2 @@ +git-reinstall: + description: Reinstall neutron-api from the openstack-origin-git repositories. diff --git a/actions/git-reinstall b/actions/git-reinstall new file mode 120000 index 00000000..ff684984 --- /dev/null +++ b/actions/git-reinstall @@ -0,0 +1 @@ +git_reinstall.py \ No newline at end of file diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py new file mode 100755 index 00000000..5f05b47a --- /dev/null +++ b/actions/git_reinstall.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +import sys +import traceback + +sys.path.append('hooks/') + +from charmhelpers.contrib.openstack.utils import ( + git_install_requested, +) + +from charmhelpers.core.hookenv import ( + action_set, + action_fail, + config, +) + +from neutron_api_utils import ( + git_install, +) + + +def git_reinstall(): + """Reinstall from source and restart services. + + If the openstack-origin-git config option was used to install openstack + from source git repositories, then this action can be used to reinstall + from updated git repositories, followed by a restart of services.""" + if not git_install_requested(): + action_fail('openstack-origin-git is not configured') + return + + try: + git_install(config('openstack-origin-git')) + except: + action_set({'traceback': traceback.format_exc()}) + action_fail('git-reinstall resulted in an unexpected error') + + +if __name__ == '__main__': + git_reinstall() diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 714fd5d9..917cf211 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index aaa21c31..48b12f6f 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index 91c807cf..aae7e153 100644 --- a/config.yaml +++ b/config.yaml @@ -15,23 +15,20 @@ options: provide a later version of OpenStack will trigger a software upgrade. - Note that when openstack-origin-git is specified, openstack-specific - packages will be installed from source rather than from the - openstack-origin repository. + Note that when openstack-origin-git is specified, openstack + specific packages will be installed from source rather than + from the openstack-origin repository. openstack-origin-git: default: None type: string description: | - Specifies a YAML-formatted two-dimensional array listing the git - repositories and branches from which to install OpenStack and its - dependencies. + Specifies a YAML-formatted dictionary listing the git + repositories and branches from which to install OpenStack and + its dependencies. Note that the installed config files will be determined based on the OpenStack release of the openstack-origin option. - Note also that this option is processed for the initial install - only. Setting this option after deployment is not supported. - For more details see README.md. rabbit-user: default: neutron diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py index aaeb7c4c..1ab82453 100755 --- a/hooks/neutron_api_hooks.py +++ b/hooks/neutron_api_hooks.py @@ -29,6 +29,7 @@ from charmhelpers.fetch import ( ) from charmhelpers.contrib.openstack.utils import ( + config_value_changed, configure_installation_source, git_install_requested, openstack_upgrade_available, @@ -106,8 +107,7 @@ def install(): apt_install(determine_packages(config('openstack-origin')), fatal=True) - # NOTE(coreycb): This is temporary for sstack proxy, unless we decide - # we need to code proxy support into the charms. + # NOTE(coreycb): This is temporary until bug #1431286 is fixed. os.environ["http_proxy"] = "http://squid.internal:3128" os.environ["https_proxy"] = "https://squid.internal:3128" @@ -129,7 +129,10 @@ def config_changed(): config('database-user')) global CONFIGS - if not git_install_requested(): + if git_install_requested(): + if config_value_changed('openstack-origin-git'): + git_install(config('openstack-origin-git')) + else: if openstack_upgrade_available('neutron-server'): do_openstack_upgrade(CONFIGS) configure_https() diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 2ea595f6..5f94ab54 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -14,6 +14,7 @@ from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source, git_install_requested, git_clone_and_install, + git_src_dir, configure_installation_source, ) @@ -286,16 +287,12 @@ def setup_ipv6(): apt_install('haproxy/trusty-backports', fatal=True) -def git_install(projects): +def git_install(projects_yaml): """Perform setup, and install git repos specified in yaml parameter.""" if git_install_requested(): git_pre_install() - # NOTE(coreycb): charm-helpers needs support to take array of - # core_projects. That would allow all neutron* projects to be - # installed last. - core = ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas', 'neutron'] - git_clone_and_install(yaml.load(projects), core_projects=core) - git_post_install() + git_clone_and_install(projects_yaml, core_project='neutron') + git_post_install(projects_yaml) def git_pre_install(): @@ -324,9 +321,9 @@ def git_pre_install(): write_file(l, '', owner='neutron', group='neutron', perms=0600) -def git_post_install(): +def git_post_install(projects_yaml): """Perform post-install setup.""" - src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc') + src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') configs = { 'api-paste': { 'src': os.path.join(src_etc, 'api-paste.ini'), @@ -358,6 +355,7 @@ def git_post_install(): 'process_name': 'neutron-server', } + # NOTE(coreycb): Needs systemd support render('upstart/neutron-server.upstart', '/etc/init/neutron.conf', neutron_api_context, perms=0o644) diff --git a/tests/16-basic-trusty-icehouse-git b/tests/16-basic-trusty-icehouse-git new file mode 100755 index 00000000..51517017 --- /dev/null +++ b/tests/16-basic-trusty-icehouse-git @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-api git deployment on trusty-icehouse.""" + +from basic_deployment import NeutronAPIBasicDeployment + +if __name__ == '__main__': + deployment = NeutronAPIBasicDeployment(series='trusty', git=True) + deployment.run_tests() diff --git a/tests/16-basic-trusty-juno b/tests/17-basic-trusty-juno similarity index 100% rename from tests/16-basic-trusty-juno rename to tests/17-basic-trusty-juno diff --git a/tests/18-basic-trusty-juno-git b/tests/18-basic-trusty-juno-git new file mode 100755 index 00000000..91c9bdbd --- /dev/null +++ b/tests/18-basic-trusty-juno-git @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-api git deployment on trusty-juno.""" + +from basic_deployment import NeutronAPIBasicDeployment + +if __name__ == '__main__': + deployment = NeutronAPIBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno', + git=True) + deployment.run_tests() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 9a2ca1be..e7fca7a8 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -19,10 +19,12 @@ u = OpenStackAmuletUtils(ERROR) class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): """Amulet tests on a basic neutron-api deployment.""" - def __init__(self, series, openstack=None, source=None, stable=False): + def __init__(self, series, openstack=None, source=None, git=False, + stable=False): """Deploy the entire test environment.""" super(NeutronAPIBasicDeployment, self).__init__(series, openstack, source, stable) + self.git = git self._add_services() self._add_relations() self._configure_services() @@ -65,11 +67,21 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): """Configure all of the services.""" - # NOTE(coreycb): Added the following temporarily to test deploy from source - neutron_api_config = {'openstack-origin-git': - "{'neutron':" - " {'repository': 'git://git.openstack.org/openstack/neutron.git'," - " 'branch': 'stable/icehouse'}}"} + neutron_api_config = {} + if self.git: + branch = 'stable/' + self._get_openstack_release_string() + openstack_origin_git = { + 'repositories': [ + {'name': 'requirements', + 'repository': 'git://git.openstack.org/openstack/requirements', + 'branch': branch}, + {'name': 'neutron', + 'repository': 'git://git.openstack.org/openstack/neutron', + 'branch': branch}, + ], + 'directory': '/mnt/openstack-git', + } + neutron_api_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} nova_cc_config = {'network-manager': 'Quantum', diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 415b2110..43aa3614 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -1,2 +1,4 @@ import sys + +sys.path.append('actions/') sys.path.append('hooks/') diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py new file mode 100644 index 00000000..f23514a1 --- /dev/null +++ b/unit_tests/test_actions_git_reinstall.py @@ -0,0 +1,85 @@ +from mock import patch + +with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = 'neutron' + import neutron_api_utils as utils # noqa + +import git_reinstall + +from test_utils import ( + CharmTestCase +) + +TO_PATCH = [ + 'config', +] + + +openstack_origin_git = \ + """repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: stable/juno}""" + + +class TestNeutronAPIActions(CharmTestCase): + + def setUp(self): + super(TestNeutronAPIActions, self).setUp(git_reinstall, TO_PATCH) + self.config.side_effect = self.test_config.get + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + def test_git_reinstall(self, git_install, action_fail, action_set): + self.test_config.set('openstack-origin-git', openstack_origin_git) + + git_reinstall.git_reinstall() + + git_install.assert_called_with(openstack_origin_git) + self.assertTrue(git_install.called) + self.assertFalse(action_set.called) + self.assertFalse(action_fail.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_not_configured(self, _config, git_install, + action_fail, action_set): + _config.return_value = 'none' + + git_reinstall.git_reinstall() + + msg = 'openstack-origin-git is not configured' + action_fail.assert_called_with(msg) + self.assertFalse(git_install.called) + self.assertFalse(action_set.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_exception(self, _config, git_install, + action_fail, action_set): + _config.return_value = openstack_origin_git + e = OSError('something bad happened') + git_install.side_effect = e + traceback = ( + "Traceback (most recent call last):\n" + " File \"actions/git_reinstall.py\", line 33, in git_reinstall\n" + " git_install(config(\'openstack-origin-git\'))\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n" # noqa + " return _mock_self._mock_call(*args, **kwargs)\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n" # noqa + " raise effect\n" + "OSError: something bad happened\n") + + git_reinstall.git_reinstall() + + msg = 'git-reinstall resulted in an unexpected error' + action_fail.assert_called_with(msg) + action_set.assert_called_with({'traceback': traceback}) From bfeac4b18f5ae019c63b0d6d6a00b1fe3f66aac9 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 10:26:14 +0000 Subject: [PATCH 05/15] import yaml for amulet tests --- tests/basic_deployment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e7fca7a8..fecb74f2 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,6 +1,7 @@ #!/usr/bin/python import amulet +import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( OpenStackAmuletDeployment From 98ab277b54c689d1333069e07049d557b163c9ef Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 7 Apr 2015 12:09:36 +0000 Subject: [PATCH 06/15] Remove env from juju test in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 65430cc1..0574b23e 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ test: # coreycb note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -e trusty -v -p AMULET_HTTP_PROXY --timeout 900 \ + @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ 18-basic-trusty-juno-git From 03bdfbae23802ed687d972707d4d61b65fb9898b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 12:10:28 +0000 Subject: [PATCH 07/15] Get http_proxy from AMULET_HTTP_PROXY env var --- tests/basic_deployment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index c611623c..d2bc3c18 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,6 +1,7 @@ #!/usr/bin/python import amulet +import os import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( @@ -71,6 +72,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): neutron_api_config = {} if self.git: branch = 'stable/' + self._get_openstack_release_string() + amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY') openstack_origin_git = { 'repositories': [ {'name': 'requirements', @@ -81,8 +83,8 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): 'branch': branch}, ], 'directory': '/mnt/openstack-git', - 'http_proxy': 'http://squid.internal:3128', - 'https_proxy': 'https://squid.internal:3128', + 'http_proxy': amulet_http_proxy, + 'https_proxy': amulet_http_proxy, } neutron_api_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) keystone_config = {'admin-password': 'openstack', From 864216f5b934448afa0e411beaacd2fd0d5201f5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 14:06:14 +0000 Subject: [PATCH 08/15] Move deploy from source template files to templates/git --- hooks/neutron_api_utils.py | 5 +++-- templates/{ => git}/neutron_sudoers | 0 templates/{ => git}/upstart/neutron-server.upstart | 0 unit_tests/test_neutron_api_utils.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) rename templates/{ => git}/neutron_sudoers (100%) rename templates/{ => git}/upstart/neutron-server.upstart (100%) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 47e8d098..8663821a 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -392,7 +392,7 @@ def git_post_install(projects_yaml): for conf, files in configs.iteritems(): shutil.copyfile(files['src'], files['dest']) - render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, + render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) neutron_api_context = { @@ -402,7 +402,8 @@ def git_post_install(projects_yaml): } # NOTE(coreycb): Needs systemd support - render('upstart/neutron-server.upstart', '/etc/init/neutron-server.conf', + render('git/upstart/neutron-server.upstart', + '/etc/init/neutron-server.conf', neutron_api_context, perms=0o644) service_restart('neutron-server') diff --git a/templates/neutron_sudoers b/templates/git/neutron_sudoers similarity index 100% rename from templates/neutron_sudoers rename to templates/git/neutron_sudoers diff --git a/templates/upstart/neutron-server.upstart b/templates/git/upstart/neutron-server.upstart similarity index 100% rename from templates/upstart/neutron-server.upstart rename to templates/git/upstart/neutron-server.upstart diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index 7dcaa04b..4101b08b 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -272,9 +272,9 @@ class TestNeutronAPIUtils(CharmTestCase): 'process_name': 'neutron-server', } expected = [ - call('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, + call('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440), - call('upstart/neutron-server.upstart', + call('git/upstart/neutron-server.upstart', '/etc/init/neutron-server.conf', neutron_api_context, perms=0o644), ] From 687715ba6f22f0ea2c4be3980bd5dde9e91ff3db Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 18:46:36 +0000 Subject: [PATCH 09/15] Bulk copy files to /etc/neutron --- hooks/neutron_api_utils.py | 36 ++++++++++------------------ unit_tests/test_neutron_api_utils.py | 25 +++++++------------ 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 8663821a..4edc2785 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -343,10 +343,6 @@ def git_install(projects_yaml): def git_pre_install(): """Perform pre-install setup.""" dirs = [ - '/etc/neutron', - '/etc/neutron/rootwrap.d', - '/etc/neutron/plugins', - '/etc/neutron/plugins/ml2', '/var/lib/neutron', '/var/lib/neutron/lock', '/var/log/neutron', @@ -370,27 +366,19 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform post-install setup.""" src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') - configs = { - 'api-paste': { - 'src': os.path.join(src_etc, 'api-paste.ini'), - 'dest': '/etc/neutron/api-paste.ini', - }, - 'debug-filters': { - 'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'), - 'dest': '/etc/neutron/rootwrap.d/debug.filters', - }, - 'policy': { - 'src': os.path.join(src_etc, 'policy.json'), - 'dest': '/etc/neutron/policy.json', - }, - 'rootwrap': { - 'src': os.path.join(src_etc, 'rootwrap.conf'), - 'dest': '/etc/neutron/rootwrap.conf', - }, - } + configs = [ + {'src': src_etc, + 'dest': '/etc/neutron'}, + {'src': os.path.join(src_etc, 'neutron/plugins'), + 'dest': '/etc/neutron/plugins'}, + {'src': os.path.join(src_etc, 'neutron/rootwrap.d'), + 'dest': '/etc/neutron/rootwrap.d'}, + ] - for conf, files in configs.iteritems(): - shutil.copyfile(files['src'], files['dest']) + for c in configs: + if os.path.exists(c['dest']): + shutil.rmtree(c['dest']) + shutil.copytree(c['src'], c['dest']) render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index 4101b08b..af0083b5 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -227,14 +227,6 @@ class TestNeutronAPIUtils(CharmTestCase): add_group.assert_called_with('neutron', system_group=True) add_user_to_group.assert_called_with('neutron', 'neutron') expected = [ - call('/etc/neutron', owner='neutron', - group='neutron', perms=0700, force=False), - call('/etc/neutron/rootwrap.d', owner='neutron', - group='neutron', perms=0700, force=False), - call('/etc/neutron/plugins', owner='neutron', - group='neutron', perms=0700, force=False), - call('/etc/neutron/plugins/ml2', owner='neutron', - group='neutron', perms=0700, force=False), call('/var/lib/neutron', owner='neutron', group='neutron', perms=0700, force=False), call('/var/lib/neutron/lock', owner='neutron', @@ -253,19 +245,20 @@ class TestNeutronAPIUtils(CharmTestCase): @patch.object(nutils, 'service_restart') @patch.object(nutils, 'render') @patch('os.path.join') - @patch('shutil.copyfile') - def test_git_post_install(self, copyfile, join, render, service_restart, - git_src_dir): + @patch('os.path.exists') + @patch('shutil.copytree') + @patch('shutil.rmtree') + def test_git_post_install(self, rmtree, copytree, exists, join, render, + service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' nutils.git_post_install(projects_yaml) expected = [ - call('joined-string', '/etc/neutron/api-paste.ini'), - call('joined-string', '/etc/neutron/rootwrap.d/debug.filters'), - call('joined-string', '/etc/neutron/policy.json'), - call('joined-string', '/etc/neutron/rootwrap.conf'), + call('joined-string', '/etc/neutron'), + call('joined-string', '/etc/neutron/plugins'), + call('joined-string', '/etc/neutron/rootwrap.d'), ] - copyfile.assert_has_calls(expected, any_order=True) + copytree.assert_has_calls(expected) neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', From a30e7a14ca35ed97dd2aabe142ce592e9e470416 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 19:17:31 +0000 Subject: [PATCH 10/15] Turn DEBUG on by default for amulet tests --- tests/basic_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index d2bc3c18..ce82b18d 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -15,7 +15,7 @@ from charmhelpers.contrib.openstack.amulet.utils import ( ) # Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(ERROR) +u = OpenStackAmuletUtils(DEBUG) class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): From 371520a0054ff1b9fe884ba2da87deabcabb165e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 14 Apr 2015 00:05:30 +0000 Subject: [PATCH 11/15] Run config-changed hook after git-reinstall action installs from source --- actions/git_reinstall.py | 5 +++++ unit_tests/test_actions_git_reinstall.py | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py index 5f05b47a..dc5e4358 100755 --- a/actions/git_reinstall.py +++ b/actions/git_reinstall.py @@ -18,6 +18,10 @@ from neutron_api_utils import ( git_install, ) +from neutron_api_hooks import ( + config_changed, +) + def git_reinstall(): """Reinstall from source and restart services. @@ -38,3 +42,4 @@ def git_reinstall(): if __name__ == '__main__': git_reinstall() + config_changed() diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index c520d25e..8897f57f 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -1,11 +1,22 @@ -from mock import patch +from mock import patch, MagicMock with patch('charmhelpers.core.hookenv.config') as config: config.return_value = 'neutron' import neutron_api_utils as utils # noqa +# Need to do some early patching to get the module loaded. +_reg = utils.register_configs +_map = utils.restart_map + +utils.register_configs = MagicMock() +utils.restart_map = MagicMock() + import git_reinstall +# Unpatch it now that its loaded. +utils.register_configs = _reg +utils.restart_map = _map + from test_utils import ( CharmTestCase ) @@ -70,7 +81,7 @@ class TestNeutronAPIActions(CharmTestCase): git_install.side_effect = e traceback = ( "Traceback (most recent call last):\n" - " File \"actions/git_reinstall.py\", line 33, in git_reinstall\n" + " File \"actions/git_reinstall.py\", line 37, in git_reinstall\n" " git_install(config(\'openstack-origin-git\'))\n" " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n" # noqa " return _mock_self._mock_call(*args, **kwargs)\n" From 7c5e9fb91b1fc5af13db0cfc4dca1a44f2f432fc Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 14:57:22 +0000 Subject: [PATCH 12/15] Fixup test_git_reinstall_exception() --- unit_tests/test_actions_git_reinstall.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index 8897f57f..85550ae8 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -73,8 +73,9 @@ class TestNeutronAPIActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch('traceback.format_exc') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_exception(self, _config, git_install, + def test_git_reinstall_exception(self, _config, format_exc, git_install, action_fail, action_set): _config.return_value = openstack_origin_git e = OSError('something bad happened') @@ -88,6 +89,7 @@ class TestNeutronAPIActions(CharmTestCase): " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n" # noqa " raise effect\n" "OSError: something bad happened\n") + format_exc.return_value = traceback git_reinstall.git_reinstall() From bc259a9de7bfee85b634b395de2ffbb437caad43 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 15:25:11 +0000 Subject: [PATCH 13/15] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 15 +++++++++++++++ hooks/charmhelpers/contrib/openstack/neutron.py | 13 +++++++++++++ .../contrib/openstack/templates/git.upstart | 4 ++++ hooks/charmhelpers/core/hookenv.py | 15 ++++++++++++++- hooks/charmhelpers/core/strutils.py | 4 ++-- tests/charmhelpers/contrib/amulet/utils.py | 4 +++- .../contrib/openstack/amulet/deployment.py | 2 +- 8 files changed, 53 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0e0db566..fef96384 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index dd51bfbb..c9914d0d 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -808,6 +808,19 @@ class NeutronContext(OSContextGenerator): return ovs_ctxt + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + def nvp_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -891,6 +904,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.n1kv_ctxt()) elif self.plugin == 'Calico': ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index f8851050..02c92e9c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -180,6 +180,19 @@ def neutron_plugins(): 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart index da94ad12..4bed404b 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/git.upstart +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -9,5 +9,9 @@ respawn exec start-stop-daemon --start --chuid {{ user_name }} \ --chdir {{ start_dir }} --name {{ process_name }} \ --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} --config-file={{ config_file }} \ + {% endfor -%} + {% if log_file -%} --log-file={{ log_file }} + {% endif -%} diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 715dd4c5..86f805f1 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -20,11 +20,13 @@ # Authors: # Charm Helpers Developers +from __future__ import print_function import os import json import yaml import subprocess import sys +import errno from subprocess import CalledProcessError import six @@ -87,7 +89,18 @@ def log(message, level=None): if not isinstance(message, six.string_types): message = repr(message) command += [message] - subprocess.call(command) + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise class Serializable(UserDict): diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index efc4402e..a2a784aa 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -33,9 +33,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't']: + if value in ['y', 'yes', 'true', 't', 'on']: return True - elif value in ['n', 'no', 'false', 'f']: + elif value in ['n', 'no', 'false', 'f', 'off']: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 65219d33..5088b1d1 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -118,6 +118,9 @@ class AmuletUtils(object): longs, or can be a function that evaluate a variable and returns a bool. """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + for k, v in six.iteritems(expected): if k in actual: if (isinstance(v, six.string_types) or @@ -134,7 +137,6 @@ class AmuletUtils(object): def validate_relation_data(self, sentry_unit, relation, expected): """Validate actual relation data based on expected relation data.""" actual = sentry_unit.relation(relation[0], relation[1]) - self.log.debug('actual: {}'.format(repr(actual))) return self._validate_dict_data(expected, actual) def _validate_list_data(self, expected, actual): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0e0db566..fef96384 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: From 39189932bc32c095cdc6b32884de7f5ecca6d674 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 16:35:29 +0000 Subject: [PATCH 14/15] Move config_changed into try block --- actions/git_reinstall.py | 2 +- unit_tests/test_actions_git_reinstall.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py index dc5e4358..a0b9e5fb 100755 --- a/actions/git_reinstall.py +++ b/actions/git_reinstall.py @@ -35,6 +35,7 @@ def git_reinstall(): try: git_install(config('openstack-origin-git')) + config_changed() except: action_set({'traceback': traceback.format_exc()}) action_fail('git-reinstall resulted in an unexpected error') @@ -42,4 +43,3 @@ def git_reinstall(): if __name__ == '__main__': git_reinstall() - config_changed() diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index 85550ae8..c1483b14 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -45,22 +45,27 @@ class TestNeutronAPIActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') - def test_git_reinstall(self, git_install, action_fail, action_set): + @patch.object(git_reinstall, 'config_changed') + def test_git_reinstall(self, config_changed, git_install, action_fail, + action_set): self.test_config.set('openstack-origin-git', openstack_origin_git) git_reinstall.git_reinstall() git_install.assert_called_with(openstack_origin_git) self.assertTrue(git_install.called) + self.assertTrue(config_changed.called) self.assertFalse(action_set.called) self.assertFalse(action_fail.called) @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch.object(git_reinstall, 'config_changed') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_not_configured(self, _config, git_install, - action_fail, action_set): + def test_git_reinstall_not_configured(self, _config, config_changed, + git_install, action_fail, + action_set): _config.return_value = None git_reinstall.git_reinstall() @@ -73,10 +78,12 @@ class TestNeutronAPIActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch.object(git_reinstall, 'config_changed') @patch('traceback.format_exc') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_exception(self, _config, format_exc, git_install, - action_fail, action_set): + def test_git_reinstall_exception(self, _config, format_exc, + config_changed, git_install, action_fail, + action_set): _config.return_value = openstack_origin_git e = OSError('something bad happened') git_install.side_effect = e From 57579036fc5324e0932942bd965b4eb327ff7ff5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 16 Apr 2015 14:29:58 +0000 Subject: [PATCH 15/15] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5a12c9d6..f90a0289 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,9 +524,10 @@ def git_clone_and_install(projects_yaml, core_project): projects = yaml.load(projects_yaml) _git_validate_projects_yaml(projects, core_project) + old_environ = dict(os.environ) + if 'http_proxy' in projects.keys(): os.environ['http_proxy'] = projects['http_proxy'] - if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -544,6 +545,8 @@ def git_clone_and_install(projects_yaml, core_project): repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, update_requirements=True) + os.environ = old_environ + def _git_validate_projects_yaml(projects, core_project): """