From 511d79771dd809c4a71eb8e7d9d801f1ae593078 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 18 Jan 2016 11:50:09 +0000 Subject: [PATCH] Resync helpers for headers optimization --- .../contrib/openstack/amulet/deployment.py | 3 +- .../charmhelpers/contrib/openstack/neutron.py | 12 +-- hooks/charmhelpers/contrib/openstack/utils.py | 93 +++++++++++++------ hooks/charmhelpers/core/host.py | 64 ++++++++----- hooks/charmhelpers/fetch/giturl.py | 4 +- .../contrib/openstack/amulet/deployment.py | 3 +- 6 files changed, 114 insertions(+), 65 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 58b1a79c..cbaad10d 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -125,7 +125,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Charms which can not use openstack-origin, ie. many subordinates no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', - 'openvswitch-odl', 'neutron-api-odl', 'odl-controller'] + 'openvswitch-odl', 'neutron-api-odl', 'odl-controller', + 'cinder-backup'] if self.openstack: for svc in services: diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index d17c847e..835b602d 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -50,7 +50,7 @@ def determine_dkms_package(): if kernel_version() >= (3, 13): return [] else: - return ['openvswitch-datapath-dkms'] + return [headers_package(), 'openvswitch-datapath-dkms'] # legacy @@ -70,7 +70,7 @@ def quantum_plugins(): relation_prefix='neutron', ssl_dir=QUANTUM_CONF_DIR)], 'services': ['quantum-plugin-openvswitch-agent'], - 'packages': [[headers_package()] + determine_dkms_package(), + 'packages': [determine_dkms_package(), ['quantum-plugin-openvswitch-agent']], 'server_packages': ['quantum-server', 'quantum-plugin-openvswitch'], @@ -111,7 +111,7 @@ def neutron_plugins(): relation_prefix='neutron', ssl_dir=NEUTRON_CONF_DIR)], 'services': ['neutron-plugin-openvswitch-agent'], - 'packages': [[headers_package()] + determine_dkms_package(), + 'packages': [determine_dkms_package(), ['neutron-plugin-openvswitch-agent']], 'server_packages': ['neutron-server', 'neutron-plugin-openvswitch'], @@ -155,7 +155,7 @@ def neutron_plugins(): relation_prefix='neutron', ssl_dir=NEUTRON_CONF_DIR)], 'services': [], - 'packages': [[headers_package()] + determine_dkms_package(), + 'packages': [determine_dkms_package(), ['neutron-plugin-cisco']], 'server_packages': ['neutron-server', 'neutron-plugin-cisco'], @@ -174,7 +174,7 @@ def neutron_plugins(): 'neutron-dhcp-agent', 'nova-api-metadata', 'etcd'], - 'packages': [[headers_package()] + determine_dkms_package(), + 'packages': [determine_dkms_package(), ['calico-compute', 'bird', 'neutron-dhcp-agent', @@ -219,7 +219,7 @@ def neutron_plugins(): relation_prefix='neutron', ssl_dir=NEUTRON_CONF_DIR)], 'services': [], - 'packages': [[headers_package()] + determine_dkms_package()], + 'packages': [determine_dkms_package()], 'server_packages': ['neutron-server', 'python-neutron-plugin-midonet'], 'server_services': ['neutron-server'] diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 2af4476d..2ed7955c 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,29 +103,28 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2016.1', 'mitaka'), ]) -# The ugly duckling +# The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ - ('1.4.3', 'diablo'), - ('1.4.8', 'essex'), - ('1.7.4', 'folsom'), - ('1.8.0', 'grizzly'), - ('1.7.7', 'grizzly'), - ('1.7.6', 'grizzly'), - ('1.10.0', 'havana'), - ('1.9.1', 'havana'), - ('1.9.0', 'havana'), - ('1.13.1', 'icehouse'), - ('1.13.0', 'icehouse'), - ('1.12.0', 'icehouse'), - ('1.11.0', 'icehouse'), - ('2.0.0', 'juno'), - ('2.1.0', 'juno'), - ('2.2.0', 'juno'), - ('2.2.1', 'kilo'), - ('2.2.2', 'kilo'), - ('2.3.0', 'liberty'), - ('2.4.0', 'liberty'), - ('2.5.0', 'liberty'), + ('diablo', + ['1.4.3']), + ('essex', + ['1.4.8']), + ('folsom', + ['1.7.4']), + ('grizzly', + ['1.7.6', '1.7.7', '1.8.0']), + ('havana', + ['1.9.0', '1.9.1', '1.10.0']), + ('icehouse', + ['1.11.0', '1.12.0', '1.13.0', '1.13.1']), + ('juno', + ['2.0.0', '2.1.0', '2.2.0']), + ('kilo', + ['2.2.1', '2.2.2']), + ('liberty', + ['2.3.0', '2.4.0', '2.5.0']), + ('mitaka', + ['2.5.0']), ]) # >= Liberty version->codename mapping @@ -227,6 +226,33 @@ def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): error_out(e) +def get_os_version_codename_swift(codename): + '''Determine OpenStack version number of swift from codename.''' + for k, v in six.iteritems(SWIFT_CODENAMES): + if k == codename: + return v[-1] + e = 'Could not derive swift version for '\ + 'codename: %s' % codename + error_out(e) + + +def get_swift_codename(version): + '''Determine OpenStack codename that corresponds to swift version.''' + codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + if len(codenames) > 1: + # If more than one release codename contains this version we determine + # the actual codename based on the highest available install source. + for codename in reversed(codenames): + releases = UBUNTU_OPENSTACK_RELEASE + release = [k for k, v in six.iteritems(releases) if codename in v] + ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) + if codename in ret or release[0] in ret: + return codename + elif len(codenames) == 1: + return codenames[0] + return None + + def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' import apt_pkg as apt @@ -270,7 +296,7 @@ def get_os_codename_package(package, fatal=True): # < Liberty co-ordinated project versions try: if 'swift' in pkg.name: - return SWIFT_CODENAMES[vers] + return get_swift_codename(vers) else: return OPENSTACK_CODENAMES[vers] except KeyError: @@ -289,12 +315,14 @@ def get_os_version_package(pkg, fatal=True): if 'swift' in pkg: vers_map = SWIFT_CODENAMES + for cname, version in six.iteritems(vers_map): + if cname == codename: + return version[-1] else: vers_map = OPENSTACK_CODENAMES - - for version, cname in six.iteritems(vers_map): - if cname == codename: - return version + for version, cname in six.iteritems(vers_map): + if cname == codename: + return version # e = "Could not determine OpenStack version for package: %s" % pkg # error_out(e) @@ -460,11 +488,16 @@ def openstack_upgrade_available(package): cur_vers = get_os_version_package(package) if "swift" in package: codename = get_os_codename_install_source(src) - available_vers = get_os_version_codename(codename, SWIFT_CODENAMES) + avail_vers = get_os_version_codename_swift(codename) else: - available_vers = get_os_version_install_source(src) + avail_vers = get_os_version_install_source(src) apt.init() - return apt.version_compare(available_vers, cur_vers) == 1 + if "swift" in package: + major_cur_vers = cur_vers.split('.', 1)[0] + major_avail_vers = avail_vers.split('.', 1)[0] + major_diff = apt.version_compare(major_avail_vers, major_cur_vers) + return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0) + return apt.version_compare(avail_vers, cur_vers) == 1 def ensure_block_device(block_device): diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 710fdab9..aeada952 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -160,13 +160,13 @@ SYSTEMD_SYSTEM = '/run/systemd/system' def init_is_systemd(): + """Return True if the host system uses systemd, False otherwise.""" return os.path.isdir(SYSTEMD_SYSTEM) def adduser(username, password=None, shell='/bin/bash', system_user=False, primary_group=None, secondary_groups=None): - """ - Add a user to the system. + """Add a user to the system. Will log but otherwise succeed if the user already exists. @@ -174,7 +174,7 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False, :param str password: Password for user; if ``None``, create a system user :param str shell: The default shell for the user :param bool system_user: Whether to create a login or system user - :param str primary_group: Primary group for user; defaults to their username + :param str primary_group: Primary group for user; defaults to username :param list secondary_groups: Optional list of additional groups :returns: The password database entry struct, as returned by `pwd.getpwnam` @@ -300,14 +300,12 @@ def write_file(path, content, owner='root', group='root', perms=0o444): def fstab_remove(mp): - """Remove the given mountpoint entry from /etc/fstab - """ + """Remove the given mountpoint entry from /etc/fstab""" return Fstab.remove_by_mountpoint(mp) def fstab_add(dev, mp, fs, options=None): - """Adds the given device entry to the /etc/fstab file - """ + """Adds the given device entry to the /etc/fstab file""" return Fstab.add(dev, mp, fs, options=options) @@ -363,8 +361,7 @@ def fstab_mount(mountpoint): def file_hash(path, hash_type='md5'): - """ - Generate a hash checksum of the contents of 'path' or None if not found. + """Generate a hash checksum of the contents of 'path' or None if not found. :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, such as md5, sha1, sha256, sha512, etc. @@ -379,10 +376,9 @@ def file_hash(path, hash_type='md5'): def path_hash(path): - """ - Generate a hash checksum of all files matching 'path'. Standard wildcards - like '*' and '?' are supported, see documentation for the 'glob' module for - more information. + """Generate a hash checksum of all files matching 'path'. Standard + wildcards like '*' and '?' are supported, see documentation for the 'glob' + module for more information. :return: dict: A { filename: hash } dictionary for all matched files. Empty if none found. @@ -394,8 +390,7 @@ def path_hash(path): def check_hash(path, checksum, hash_type='md5'): - """ - Validate a file using a cryptographic checksum. + """Validate a file using a cryptographic checksum. :param str checksum: Value of the checksum used to validate the file. :param str hash_type: Hash algorithm used to generate `checksum`. @@ -410,6 +405,7 @@ def check_hash(path, checksum, hash_type='md5'): class ChecksumError(ValueError): + """A class derived from Value error to indicate the checksum failed.""" pass @@ -515,7 +511,7 @@ def get_bond_master(interface): def list_nics(nic_type=None): - '''Return a list of nics of given type(s)''' + """Return a list of nics of given type(s)""" if isinstance(nic_type, six.string_types): int_types = [nic_type] else: @@ -557,12 +553,13 @@ def list_nics(nic_type=None): def set_nic_mtu(nic, mtu): - '''Set MTU on a network interface''' + """Set the Maximum Transmission Unit (MTU) on a network interface.""" cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] subprocess.check_call(cmd) def get_nic_mtu(nic): + """Return the Maximum Transmission Unit (MTU) for a network interface.""" cmd = ['ip', 'addr', 'show', nic] ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') mtu = "" @@ -574,6 +571,7 @@ def get_nic_mtu(nic): def get_nic_hwaddr(nic): + """Return the Media Access Control (MAC) for a network interface.""" cmd = ['ip', '-o', '-0', 'addr', 'show', nic] ip_output = subprocess.check_output(cmd).decode('UTF-8') hwaddr = "" @@ -584,7 +582,7 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): - '''Compare supplied revno with the revno of the installed package + """Compare supplied revno with the revno of the installed package * 1 => Installed revno is greater than supplied arg * 0 => Installed revno is the same as supplied arg @@ -593,7 +591,7 @@ def cmp_pkgrevno(package, revno, pkgcache=None): This function imports apt_cache function from charmhelpers.fetch if the pkgcache argument is None. Be sure to add charmhelpers.fetch if you call this function, or pass an apt_pkg.Cache() instance. - ''' + """ import apt_pkg if not pkgcache: from charmhelpers.fetch import apt_cache @@ -603,19 +601,27 @@ def cmp_pkgrevno(package, revno, pkgcache=None): @contextmanager -def chdir(d): +def chdir(directory): + """Change the current working directory to a different directory for a code + block and return the previous directory after the block exits. Useful to + run commands from a specificed directory. + + :param str directory: The directory path to change to for this context. + """ cur = os.getcwd() try: - yield os.chdir(d) + yield os.chdir(directory) finally: os.chdir(cur) def chownr(path, owner, group, follow_links=True, chowntopdir=False): - """ - Recursively change user and group ownership of files and directories + """Recursively change user and group ownership of files and directories in given path. Doesn't chown path itself by default, only its children. + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. :param bool follow_links: Also Chown links if True :param bool chowntopdir: Also chown path itself if True """ @@ -639,15 +645,23 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): def lchownr(path, owner, group): + """Recursively change user and group ownership of files and directories + in a given path, not following symbolic links. See the documentation for + 'os.lchown' for more information. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + """ chownr(path, owner, group, follow_links=False) def get_total_ram(): - '''The total amount of system RAM in bytes. + """The total amount of system RAM in bytes. This is what is reported by the OS, and may be overcommitted when there are multiple containers hosted on the same machine. - ''' + """ with open('/proc/meminfo', 'r') as f: for line in f.readlines(): if line: diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index a7ce90a0..9ad8dc60 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -49,8 +49,8 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', '-C', dest, 'pull', source, branch] else: cmd = ['git', 'clone', source, dest, '--branch', branch] - if depth: - cmd.extend(['--depth', depth]) + if depth: + cmd.extend(['--depth', depth]) check_call(cmd) def install(self, source, branch="master", dest=None, depth=None): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 58b1a79c..cbaad10d 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -125,7 +125,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Charms which can not use openstack-origin, ie. many subordinates no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', - 'openvswitch-odl', 'neutron-api-odl', 'odl-controller'] + 'openvswitch-odl', 'neutron-api-odl', 'odl-controller', + 'cinder-backup'] if self.openstack: for svc in services: