From 8f01c3aec68809924c8707e3efa395e92937fd10 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 16 Jun 2016 12:55:38 +0000 Subject: [PATCH] Add defaults for openstack-origin-git config option openstack-origin-git currently only supports YAML that specifies the git repositories to deploy from. This adds support for default openstack-origin-git values. The default values supported are: icehouse, kilo, liberty, mitaka, and master. For example: openstack-origin-git=master. Change-Id: Ib2e0224f70e51d2686b6bb1f28e6ad528621eafb --- README.md | 94 ------------------- config.yaml | 35 ++++++- .../charmhelpers/contrib/hahelpers/cluster.py | 69 ++++++++++++-- .../contrib/openstack/amulet/deployment.py | 51 +++++----- hooks/charmhelpers/contrib/openstack/ip.py | 11 ++- hooks/charmhelpers/contrib/openstack/utils.py | 77 +++++++++++++++ .../contrib/storage/linux/ceph.py | 2 +- hooks/charmhelpers/core/host.py | 56 ++++++++++- hooks/neutron_utils.py | 2 + .../contrib/openstack/amulet/deployment.py | 51 +++++----- 10 files changed, 273 insertions(+), 175 deletions(-) diff --git a/README.md b/README.md index 17bfe063..e1588481 100644 --- a/README.md +++ b/README.md @@ -124,97 +124,3 @@ OpenStack upstream documentation recommends a MTU value of 1400: [OpenStack documentation](http://docs.openstack.org/admin-guide-cloud/content/openvswitch_plugin.html) Note that this option was added in Havana and will be ignored in older releases. - -Deploying from source -===================== - -The minimum openstack-origin-git config required to deploy from source is: - - openstack-origin-git: include-file://neutron-juno.yaml - - neutron-juno.yaml - ----------------- - repositories: - - {name: requirements, - repository: 'git://github.com/openstack/requirements', - branch: stable/juno} - - {name: neutron, - repository: 'git://github.com/openstack/neutron', - branch: stable/juno} - -Note that there are only two 'name' values the charm knows about: 'requirements' -and 'neutron'. These repositories must correspond to these 'name' values. -Additionally, the requirements repository must be specified first and the -neutron repository must be specified last. All other repositories are installed -in the order in which they are specified. - -The following is a full list of current tip repos (may not be up-to-date): - - openstack-origin-git: include-file://neutron-master.yaml - - neutron-master.yaml - ------------------- - repositories: - - {name: requirements, - repository: 'git://github.com/openstack/requirements', - branch: master} - - {name: oslo-concurrency, - repository: 'git://github.com/openstack/oslo.concurrency', - branch: master} - - {name: oslo-config, - repository: 'git://github.com/openstack/oslo.config', - branch: master} - - {name: oslo-context, - repository: 'git://github.com/openstack/oslo.context', - branch: master} - - {name: oslo-db, - repository: 'git://github.com/openstack/oslo.db', - branch: master} - - {name: oslo-i18n, - repository: 'git://github.com/openstack/oslo.i18n', - branch: master} - - {name: oslo-messaging, - repository: 'git://github.com/openstack/oslo.messaging', - branch: master} - - {name: oslo-middleware, - repository': 'git://github.com/openstack/oslo.middleware', - branch: master} - - {name: oslo-rootwrap', - repository: 'git://github.com/openstack/oslo.rootwrap', - branch: master} - - {name: oslo-serialization, - repository: 'git://github.com/openstack/oslo.serialization', - branch: master} - - {name: oslo-utils, - repository: 'git://github.com/openstack/oslo.utils', - branch: master} - - {name: pbr, - repository: 'git://github.com/openstack-dev/pbr', - branch: master} - - {name: stevedore, - repository: 'git://github.com/openstack/stevedore', - branch: 'master'} - - {name: python-keystoneclient, - repository: 'git://github.com/openstack/python-keystoneclient', - branch: master} - - {name: python-neutronclient, - repository: 'git://github.com/openstack/python-neutronclient', - branch: master} - - {name: python-novaclient, - repository': 'git://github.com/openstack/python-novaclient', - branch: master} - - {name: keystonemiddleware, - repository: 'git://github.com/openstack/keystonemiddleware', - branch: master} - - {name: neutron-fwaas, - repository': 'git://github.com/openstack/neutron-fwaas', - branch: master} - - {name: neutron-lbaas, - repository: 'git://github.com/openstack/neutron-lbaas', - branch: master} - - {name: neutron-vpnaas, - repository: 'git://github.com/openstack/neutron-vpnaas', - branch: master} - - {name: neutron, - repository: 'git://github.com/openstack/neutron', - branch: master} diff --git a/config.yaml b/config.yaml index f83c1545..c34d1872 100644 --- a/config.yaml +++ b/config.yaml @@ -46,14 +46,39 @@ options: default: type: string description: | - Specifies a YAML-formatted dictionary listing the git - repositories and branches from which to install OpenStack and - its dependencies. + Specifies a default OpenStack release name, or a YAML dictionary + listing the git repositories to install from. + + The default Openstack release name may be one of the following, where + the corresponding OpenStack github branch will be used: + * icehouse + * kilo + * liberty + * mitaka + * master + + The YAML must minimally include requirements, neutron-fwaas, + neutron-lbaas, neutron-vpnaas, and neutron repositories, and may + also include repositories for other dependencies: + repositories: + - {name: requirements, + repository: 'git://github.com/openstack/requirements', + branch: master} + - {name: neutron-fwaas, + repository: 'git://github.com/openstack/neutron-fwaas', + branch: master} + - {name: neutron-lbaas, + repository: 'git://github.com/openstack/neutron-lbaas', + branch: master} + - {name: neutron-vpnaas, + repository: 'git://github.com/openstack/neutron-vpnaas', + branch: master} + - {name: neutron, + repository: 'git://github.com/openstack/neutron', + branch: master} Note that the installed config files will be determined based on the OpenStack release of the openstack-origin option. - - For more details see README.md. plugin: default: ovs type: string diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index aa0b515d..92325a96 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -41,10 +41,11 @@ from charmhelpers.core.hookenv import ( relation_get, config as config_get, INFO, - ERROR, + DEBUG, WARNING, unit_get, - is_leader as juju_is_leader + is_leader as juju_is_leader, + status_set, ) from charmhelpers.core.decorators import ( retry_on_exception, @@ -60,6 +61,10 @@ class HAIncompleteConfig(Exception): pass +class HAIncorrectConfig(Exception): + pass + + class CRMResourceNotFound(Exception): pass @@ -274,27 +279,71 @@ def get_hacluster_config(exclude_keys=None): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip + ha-bindiface, ha-mcastport, vip, os-internal-hostname, + os-admin-hostname, os-public-hostname param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. - raises: HAIncompleteConfig if settings are missing. + raises: HAIncompleteConfig if settings are missing or incorrect. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname', + 'os-admin-hostname', 'os-public-hostname'] conf = {} for setting in settings: if exclude_keys and setting in exclude_keys: continue conf[setting] = config_get(setting) - missing = [] - [missing.append(s) for s, v in six.iteritems(conf) if v is None] - if missing: - log('Insufficient config data to configure hacluster.', level=ERROR) - raise HAIncompleteConfig + + if not valid_hacluster_config(): + raise HAIncorrectConfig('Insufficient or incorrect config data to ' + 'configure hacluster.') return conf +def valid_hacluster_config(): + ''' + Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname + must be set. + + Note: ha-bindiface and ha-macastport both have defaults and will always + be set. We only care that either vip or dns-ha is set. + + :returns: boolean: valid config returns true. + raises: HAIncompatibileConfig if settings conflict. + raises: HAIncompleteConfig if settings are missing. + ''' + vip = config_get('vip') + dns = config_get('dns-ha') + if not(bool(vip) ^ bool(dns)): + msg = ('HA: Either vip or dns-ha must be set but not both in order to ' + 'use high availability') + status_set('blocked', msg) + raise HAIncorrectConfig(msg) + + # If dns-ha then one of os-*-hostname must be set + if dns: + dns_settings = ['os-internal-hostname', 'os-admin-hostname', + 'os-public-hostname'] + # At this point it is unknown if one or all of the possible + # network spaces are in HA. Validate at least one is set which is + # the minimum required. + for setting in dns_settings: + if config_get(setting): + log('DNS HA: At least one hostname is set {}: {}' + ''.format(setting, config_get(setting)), + level=DEBUG) + return True + + msg = ('DNS HA: At least one os-*-hostname(s) must be set to use ' + 'DNS HA') + status_set('blocked', msg) + raise HAIncompleteConfig(msg) + + log('VIP HA: VIP is set {}'.format(vip), level=DEBUG) + return True + + def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index d21c9c78..6b917d0c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source self.stable = stable - # Note(coreycb): this needs to be changed when new next branches come - # out. - self.current_next = "trusty" def get_logger(self, name="deployment-logger", level=logging.DEBUG): """Get a logger object that will log to stdout.""" @@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.info('OpenStackAmuletDeployment: determine branch locations') - # Charms outside the lp:~openstack-charmers namespace - base_charms = ['mysql', 'mongodb', 'nrpe'] - - # Force these charms to current series even when using an older series. - # ie. Use trusty/nrpe even when series is precise, as the P charm - # does not possess the necessary external master config and hooks. - force_series_current = ['nrpe'] - - if self.series in ['precise', 'trusty']: - base_series = self.series - else: - base_series = self.current_next + # Charms outside the ~openstack-charmers + base_charms = { + 'mysql': ['precise', 'trusty'], + 'mongodb': ['precise', 'trusty'], + 'nrpe': ['precise', 'trusty'], + } for svc in other_services: - if svc['name'] in force_series_current: - base_series = self.current_next # If a location has been explicitly set, use it if svc.get('location'): continue - if self.stable: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) + if svc['name'] in base_charms: + # NOTE: not all charms have support for all series we + # want/need to test against, so fix to most recent + # that each base charm supports + target_series = self.series + if self.series not in base_charms[svc['name']]: + target_series = base_charms[svc['name']][-1] + svc['location'] = 'cs:{}/{}'.format(target_series, + svc['name']) + elif self.stable: + svc['location'] = 'cs:{}/{}'.format(self.series, + svc['name']) else: - if svc['name'] in base_charms: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) - else: - temp = 'lp:~openstack-charmers/charms/{}/{}/next' - svc['location'] = temp.format(self.current_next, - svc['name']) + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( + self.series, + svc['name'] + ) return other_services diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 532a1dc1..7875b997 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -109,7 +109,7 @@ def _get_address_override(endpoint_type=PUBLIC): return addr_override.format(service_name=service_name()) -def resolve_address(endpoint_type=PUBLIC): +def resolve_address(endpoint_type=PUBLIC, override=True): """Return unit address depending on net config. If unit is clustered with vip(s) and has net splits defined, return vip on @@ -119,10 +119,13 @@ def resolve_address(endpoint_type=PUBLIC): split if one is configured, or a Juju 2.0 extra-binding has been used. :param endpoint_type: Network endpoing type + :param override: Accept hostname overrides or not """ - resolved_address = _get_address_override(endpoint_type) - if resolved_address: - return resolved_address + resolved_address = None + if override: + resolved_address = _get_address_override(endpoint_type) + if resolved_address: + return resolved_address vips = config('vip') if vips: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index bd6efc48..8da5c5ed 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( related_units, relation_ids, relation_set, + service_name, status_set, hook_name ) @@ -207,6 +208,27 @@ PACKAGE_CODENAMES = { ]), } +GIT_DEFAULT_REPOS = { + 'requirements': 'git://github.com/openstack/requirements', + 'cinder': 'git://github.com/openstack/cinder', + 'glance': 'git://github.com/openstack/glance', + 'horizon': 'git://github.com/openstack/horizon', + 'keystone': 'git://github.com/openstack/keystone', + 'neutron': 'git://github.com/openstack/neutron', + 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas', + 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas', + 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas', + 'nova': 'git://github.com/openstack/nova', +} + +GIT_DEFAULT_BRANCHES = { + 'icehouse': 'icehouse-eol', + 'kilo': 'stable/kilo', + 'liberty': 'stable/liberty', + 'mitaka': 'stable/mitaka', + 'master': 'master', +} + DEFAULT_LOOPBACK_SIZE = '5G' @@ -703,6 +725,61 @@ def git_install_requested(): requirements_dir = None +def git_default_repos(projects_yaml): + """ + Returns default repos if a default openstack-origin-git value is specified. + """ + service = service_name() + core_project = service + + for default, branch in GIT_DEFAULT_BRANCHES.iteritems(): + if projects_yaml == default: + + # add the requirements repo first + repo = { + 'name': 'requirements', + 'repository': GIT_DEFAULT_REPOS['requirements'], + 'branch': branch, + } + repos = [repo] + + # neutron-* and nova-* charms require some additional repos + if service in ['neutron-api', 'neutron-gateway', + 'neutron-openvswitch']: + core_project = 'neutron' + for project in ['neutron-fwaas', 'neutron-lbaas', + 'neutron-vpnaas']: + repo = { + 'name': project, + 'repository': GIT_DEFAULT_REPOS[project], + 'branch': branch, + } + repos.append(repo) + + elif service in ['nova-cloud-controller', 'nova-compute']: + core_project = 'nova' + repo = { + 'name': 'neutron', + 'repository': GIT_DEFAULT_REPOS['neutron'], + 'branch': branch, + } + repos.append(repo) + elif service == 'openstack-dashboard': + core_project = 'horizon' + + # finally add the current service's core project repo + repo = { + 'name': core_project, + 'repository': GIT_DEFAULT_REPOS[core_project], + 'branch': branch, + } + repos.append(repo) + + return yaml.dump(dict(repositories=repos)) + + return projects_yaml + + def _git_yaml_load(projects_yaml): """ Load the specified yaml into a dictionary. diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 2528f5cf..b2484e78 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1231,7 +1231,7 @@ class CephConfContext(object): permitted = self.permitted_sections if permitted: - diff = set(conf.keys()).symmetric_difference(set(permitted)) + diff = set(conf.keys()).difference(set(permitted)) if diff: log("Config-flags contains invalid keys '%s' - they will be " "ignored" % (', '.join(diff)), level=WARNING) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 64b2df55..e367e450 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -176,7 +176,7 @@ def init_is_systemd(): def adduser(username, password=None, shell='/bin/bash', system_user=False, - primary_group=None, secondary_groups=None): + primary_group=None, secondary_groups=None, uid=None): """Add a user to the system. Will log but otherwise succeed if the user already exists. @@ -187,15 +187,21 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False, :param bool system_user: Whether to create a login or system user :param str primary_group: Primary group for user; defaults to username :param list secondary_groups: Optional list of additional groups + :param int uid: UID for user being created :returns: The password database entry struct, as returned by `pwd.getpwnam` """ try: user_info = pwd.getpwnam(username) log('user {0} already exists!'.format(username)) + if uid: + user_info = pwd.getpwuid(int(uid)) + log('user with uid {0} already exists!'.format(uid)) except KeyError: log('creating user {0}'.format(username)) cmd = ['useradd'] + if uid: + cmd.extend(['--uid', str(uid)]) if system_user or password is None: cmd.append('--system') else: @@ -230,14 +236,58 @@ def user_exists(username): return user_exists -def add_group(group_name, system_group=False): - """Add a group to the system""" +def uid_exists(uid): + """Check if a uid exists""" + try: + pwd.getpwuid(uid) + uid_exists = True + except KeyError: + uid_exists = False + return uid_exists + + +def group_exists(groupname): + """Check if a group exists""" + try: + grp.getgrnam(groupname) + group_exists = True + except KeyError: + group_exists = False + return group_exists + + +def gid_exists(gid): + """Check if a gid exists""" + try: + grp.getgrgid(gid) + gid_exists = True + except KeyError: + gid_exists = False + return gid_exists + + +def add_group(group_name, system_group=False, gid=None): + """Add a group to the system + + Will log but otherwise succeed if the group already exists. + + :param str group_name: group to create + :param bool system_group: Create system group + :param int gid: GID for user being created + + :returns: The password database entry struct, as returned by `grp.getgrnam` + """ try: group_info = grp.getgrnam(group_name) log('group {0} already exists!'.format(group_name)) + if gid: + group_info = grp.getgrgid(gid) + log('group with gid {0} already exists!'.format(gid)) except KeyError: log('creating group {0}'.format(group_name)) cmd = ['addgroup'] + if gid: + cmd.extend(['--gid', str(gid)]) if system_group: cmd.append('--system') else: diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index 662112d6..490ac510 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -45,6 +45,7 @@ from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source, git_install_requested, git_clone_and_install, + git_default_repos, git_src_dir, git_pip_venv_dir, get_hostname, @@ -849,6 +850,7 @@ def git_install(projects_yaml): """Perform setup, and install git repos specified in yaml parameter.""" if git_install_requested(): git_pre_install() + projects_yaml = git_default_repos(projects_yaml) git_clone_and_install(projects_yaml, core_project='neutron') git_post_install(projects_yaml) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index d21c9c78..6b917d0c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source self.stable = stable - # Note(coreycb): this needs to be changed when new next branches come - # out. - self.current_next = "trusty" def get_logger(self, name="deployment-logger", level=logging.DEBUG): """Get a logger object that will log to stdout.""" @@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.info('OpenStackAmuletDeployment: determine branch locations') - # Charms outside the lp:~openstack-charmers namespace - base_charms = ['mysql', 'mongodb', 'nrpe'] - - # Force these charms to current series even when using an older series. - # ie. Use trusty/nrpe even when series is precise, as the P charm - # does not possess the necessary external master config and hooks. - force_series_current = ['nrpe'] - - if self.series in ['precise', 'trusty']: - base_series = self.series - else: - base_series = self.current_next + # Charms outside the ~openstack-charmers + base_charms = { + 'mysql': ['precise', 'trusty'], + 'mongodb': ['precise', 'trusty'], + 'nrpe': ['precise', 'trusty'], + } for svc in other_services: - if svc['name'] in force_series_current: - base_series = self.current_next # If a location has been explicitly set, use it if svc.get('location'): continue - if self.stable: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) + if svc['name'] in base_charms: + # NOTE: not all charms have support for all series we + # want/need to test against, so fix to most recent + # that each base charm supports + target_series = self.series + if self.series not in base_charms[svc['name']]: + target_series = base_charms[svc['name']][-1] + svc['location'] = 'cs:{}/{}'.format(target_series, + svc['name']) + elif self.stable: + svc['location'] = 'cs:{}/{}'.format(self.series, + svc['name']) else: - if svc['name'] in base_charms: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) - else: - temp = 'lp:~openstack-charmers/charms/{}/{}/next' - svc['location'] = temp.format(self.current_next, - svc['name']) + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( + self.series, + svc['name'] + ) return other_services