From 60b2191e07acdb659ca5531caef7f0db994eafc2 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 25 Mar 2014 12:25:03 +0000 Subject: [PATCH 1/2] [hopem] synced charm-helpers --- .../charmhelpers/contrib/openstack/context.py | 76 ++++++++----------- .../charmhelpers/contrib/openstack/neutron.py | 32 +++++++- .../contrib/openstack/templates/ceph.conf | 13 ---- hooks/charmhelpers/contrib/openstack/utils.py | 27 ++++--- .../contrib/storage/linux/ceph.py | 8 +- hooks/charmhelpers/fetch/__init__.py | 27 +++++++ hooks/charmhelpers/fetch/archiveurl.py | 15 ++++ 7 files changed, 124 insertions(+), 74 deletions(-) delete mode 100644 hooks/charmhelpers/contrib/openstack/templates/ceph.conf diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c38eadbc..6014ee1b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -29,6 +29,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, https, + is_clustered ) from charmhelpers.contrib.hahelpers.apache import ( @@ -198,6 +199,7 @@ class AMQPContext(OSContextGenerator): ctxt = {} for rid in relation_ids('amqp'): + ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True @@ -212,16 +214,18 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + if relation_get('ha_queues', rid=rid, unit=unit) is not None: + ctxt['rabbitmq_ha_queues'] = True + + ha_vip_only = relation_get('ha-vip-only', + rid=rid, unit=unit) is not None + if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ - len(related_units(rid)) > 1: - if relation_get('ha_queues'): - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') - else: - ctxt['rabbitmq_ha_queues'] = False + if ('clustered' not in ctxt or ha_vip_only) \ + and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -240,10 +244,13 @@ class CephContext(OSContextGenerator): '''This generates context for /etc/ceph/ceph.conf templates''' if not relation_ids('ceph'): return {} + log('Generating template context for ceph') + mon_hosts = [] auth = None key = None + use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): mon_hosts.append(relation_get('private-address', rid=rid, @@ -255,6 +262,7 @@ class CephContext(OSContextGenerator): 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, + 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -391,7 +399,7 @@ class ApacheSSLContext(OSContextGenerator): return ctxt -class NeutronContext(object): +class NeutronContext(OSContextGenerator): interfaces = [] @property @@ -452,6 +460,22 @@ class NeutronContext(object): return nvp_ctxt + def neutron_ctxt(self): + if https(): + proto = 'https' + else: + proto = 'http' + if is_clustered(): + host = config('vip') + else: + host = unit_get('private-address') + url = '%s://%s:%s' % (proto, host, '9696') + ctxt = { + 'network_manager': self.network_manager, + 'neutron_url': url, + } + return ctxt + def __call__(self): self._ensure_packages() @@ -461,7 +485,7 @@ class NeutronContext(object): if not self.plugin: return {} - ctxt = {'network_manager': self.network_manager} + ctxt = self.neutron_ctxt() if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) @@ -493,41 +517,7 @@ class OSConfigFlagContext(OSContextGenerator): if not config_flags: return {} - if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", - level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = {} - for i in xrange(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - log("invalid config value(s) at index %s" % (i), - level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - + flags = config_flags_parser(config_flags) return {'user_config_flags': flags} diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 8d32bd00..5badc39c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -18,6 +18,22 @@ def headers_package(): return 'linux-headers-%s' % kver +def kernel_version(): + """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """ + kver = check_output(['uname', '-r']).strip() + kver = kver.split('.') + return (int(kver[0]), int(kver[1])) + + +def determine_dkms_package(): + """ Determine which DKMS package should be used based on kernel version """ + # NOTE: 3.13 kernels have support for GRE and VXLAN native + if kernel_version() >= (3, 13): + return [] + else: + return ['openvswitch-datapath-dkms'] + + # legacy def quantum_plugins(): from charmhelpers.contrib.openstack import context @@ -32,7 +48,7 @@ def quantum_plugins(): database=config('neutron-database'), relation_prefix='neutron')], 'services': ['quantum-plugin-openvswitch-agent'], - 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], + 'packages': [[headers_package()] + determine_dkms_package(), ['quantum-plugin-openvswitch-agent']], 'server_packages': ['quantum-server', 'quantum-plugin-openvswitch'], @@ -57,7 +73,8 @@ def quantum_plugins(): def neutron_plugins(): from charmhelpers.contrib.openstack import context - return { + release = os_release('nova-common') + plugins = { 'ovs': { 'config': '/etc/neutron/plugins/openvswitch/' 'ovs_neutron_plugin.ini', @@ -68,8 +85,8 @@ def neutron_plugins(): database=config('neutron-database'), relation_prefix='neutron')], 'services': ['neutron-plugin-openvswitch-agent'], - 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], - ['quantum-plugin-openvswitch-agent']], + 'packages': [[headers_package()] + determine_dkms_package(), + ['neutron-plugin-openvswitch-agent']], 'server_packages': ['neutron-server', 'neutron-plugin-openvswitch'], 'server_services': ['neutron-server'] @@ -89,6 +106,13 @@ def neutron_plugins(): 'server_services': ['neutron-server'] } } + # NOTE: patch in ml2 plugin for icehouse onwards + if release >= 'icehouse': + plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' + plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' + plugins['ovs']['server_packages'] = ['neutron-server', + 'neutron-plugin-ml2'] + return plugins def neutron_plugin_attribute(plugin, attr, net_manager=None): diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf deleted file mode 100644 index c860f7a8..00000000 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ /dev/null @@ -1,13 +0,0 @@ -############################################################################### -# [ WARNING ] -# cinder configuration file maintained by Juju -# local changes may be overwritten. -############################################################################### -{% if auth -%} -[global] - auth_supported = {{ auth }} - keyring = /etc/ceph/$cluster.$name.keyring - mon host = {{ mon_hosts }} -{% endif -%} - - diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 56d04245..3261adf4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -65,6 +65,9 @@ SWIFT_CODENAMES = OrderedDict([ ('1.10.0', 'havana'), ('1.9.1', 'havana'), ('1.9.0', 'havana'), + ('1.13.0', 'icehouse'), + ('1.12.0', 'icehouse'), + ('1.11.0', 'icehouse'), ]) DEFAULT_LOOPBACK_SIZE = '5G' @@ -420,19 +423,19 @@ def get_hostname(address, fqdn=True): Resolves hostname for given IP, or returns the input if it is already a hostname. """ - if not is_ip(address): - return address + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install('python-dnspython') + import dns.reversename - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address if fqdn: # strip trailing . diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 69b879ca..12417410 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,6 +49,9 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} + log to syslog = {use_syslog} + err to syslog = {use_syslog} + clog to syslog = {use_syslog} """ @@ -194,7 +197,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth): +def configure(service, key, auth, use_syslog): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -202,7 +205,8 @@ def configure(service, key, auth): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)))) + mon_hosts=",".join(map(str, hosts)), + use_syslog=use_syslog)) modprobe('rbd') diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..97a19912 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -97,6 +97,29 @@ def apt_install(packages, options=None, fatal=False): subprocess.call(cmd, env=env) +def apt_upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages""" + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + if dist: + cmd.append('dist-upgrade') + else: + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + + if fatal: + subprocess.check_call(cmd, env=env) + else: + subprocess.call(cmd, env=env) + + def apt_update(fatal=False): """Update local apt cache""" cmd = ['apt-get', 'update'] @@ -135,6 +158,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index e35b8f15..87e7071a 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,5 +1,7 @@ import os import urllib2 +import urlparse + from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource @@ -24,6 +26,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): def download(self, source, dest): # propogate all exceptions # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + if proto in ('http', 'https'): + auth, barehost = urllib2.splituser(netloc) + if auth is not None: + source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) + username, password = urllib2.splitpasswd(auth) + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) response = urllib2.urlopen(source) try: with open(dest, 'w') as dest_file: From e94a5fed8514cf06009dd864afcfa046d3cd8173 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 25 Mar 2014 18:44:22 +0000 Subject: [PATCH 2/2] [hopem] Added use-syslog cfg option to allow logging to syslog --- config.yaml | 3 +-- revision | 2 +- templates/ceph.conf | 8 ++++++-- templates/glance-registry.conf | 1 - 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/config.yaml b/config.yaml index 3fa34ff7..3b075fa6 100644 --- a/config.yaml +++ b/config.yaml @@ -40,8 +40,7 @@ options: type: boolean default: False description: | - By default, all services will log into their corresponding log files. - Setting this to True will force all services to log to the syslog. + If set to True, supporting services will log to syslog. # HA configuration settings vip: type: string diff --git a/revision b/revision index 0d667b5e..15c44e93 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -148 +149 diff --git a/templates/ceph.conf b/templates/ceph.conf index ea561f42..d30fc84d 100644 --- a/templates/ceph.conf +++ b/templates/ceph.conf @@ -3,10 +3,14 @@ # glance configuration file maintained by Juju # local changes may be overwritten. ############################################################################### -{% if auth %} [global] +{% if auth -%} auth_supported = {{ auth }} #keyring = /etc/ceph/$cluster.$name.keyring keyring = /etc/ceph/ceph.$name.keyring mon host = {{ mon_hosts }} -{% endif %} +{% endif -%} + log to syslog = {{ use_syslog }} + err to syslog = {{ use_syslog }} + clog to syslog = {{ use_syslog }} + diff --git a/templates/glance-registry.conf b/templates/glance-registry.conf index b9b90904..9fa1397f 100644 --- a/templates/glance-registry.conf +++ b/templates/glance-registry.conf @@ -12,7 +12,6 @@ sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database sql_idle_timeout = 3600 api_limit_max = 1000 limit_param_default = 25 -use_syslog = False {% if auth_host %} [paste_deploy]