diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index bc35b22..6341dbc 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -13,6 +13,6 @@ include: - payload.execd - contrib.network.ip - contrib.network.ufw - - contrib.python.packages + - contrib.python - contrib.charmsupport - contrib.hardening|inc=* diff --git a/charmhelpers/contrib/hahelpers/ceph.py b/charmhelpers/contrib/hahelpers/ceph.py deleted file mode 100644 index 8ff029a..0000000 --- a/charmhelpers/contrib/hahelpers/ceph.py +++ /dev/null @@ -1,291 +0,0 @@ -# -# Copyright 2012 Canonical Ltd. -# -# This file is sourced from lp:openstack-charm-helpers -# -# Authors: -# James Page -# Adam Gandelman -# - -import commands -import os -import shutil -import time - -from subprocess import ( - check_call, - check_output, - CalledProcessError -) - -from charmhelpers.core.hookenv import ( - relation_get, - relation_ids, - related_units, - log, - INFO, - ERROR -) - -from charmhelpers.core.host import ( - apt_install, - mount, - mounts, - service_start, - service_stop, - umount, -) - -KEYRING = '/etc/ceph/ceph.client.%s.keyring' -KEYFILE = '/etc/ceph/ceph.client.%s.key' - -CEPH_CONF = """[global] - auth supported = %(auth)s - keyring = %(keyring)s - mon host = %(mon_hosts)s -""" - - -def running(service): - # this local util can be dropped as soon the following branch lands - # in lp:charm-helpers - # https://code.launchpad.net/~gandelman-a/charm-helpers/service_running/ - try: - output = check_output(['service', service, 'status']) - except CalledProcessError: - return False - else: - if ("start/running" in output or "is running" in output): - return True - else: - return False - - -def install(): - ceph_dir = "/etc/ceph" - if not os.path.isdir(ceph_dir): - os.mkdir(ceph_dir) - apt_install('ceph-common', fatal=True) - - -def rbd_exists(service, pool, rbd_img): - (rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' % - (service, pool)) - return rbd_img in out - - -def create_rbd_image(service, pool, image, sizemb): - cmd = [ - 'rbd', - 'create', - image, - '--size', - str(sizemb), - '--id', - service, - '--pool', - pool - ] - check_call(cmd) - - -def pool_exists(service, name): - (rc, out) = commands.getstatusoutput("rados --id %s lspools" % service) - return name in out - - -def create_pool(service, name): - cmd = [ - 'rados', - '--id', - service, - 'mkpool', - name - ] - check_call(cmd) - - -def keyfile_path(service): - return KEYFILE % service - - -def keyring_path(service): - return KEYRING % service - - -def create_keyring(service, key): - keyring = keyring_path(service) - if os.path.exists(keyring): - log('ceph: Keyring exists at %s.' % keyring, level=INFO) - cmd = [ - 'ceph-authtool', - keyring, - '--create-keyring', - '--name=client.%s' % service, - '--add-key=%s' % key - ] - check_call(cmd) - log('ceph: Created new ring at %s.' % keyring, level=INFO) - - -def create_key_file(service, key): - # create a file containing the key - keyfile = keyfile_path(service) - if os.path.exists(keyfile): - log('ceph: Keyfile exists at %s.' % keyfile, level=INFO) - fd = open(keyfile, 'w') - fd.write(key) - fd.close() - log('ceph: Created new keyfile at %s.' % keyfile, level=INFO) - - -def get_ceph_nodes(): - hosts = [] - for r_id in relation_ids('ceph'): - for unit in related_units(r_id): - hosts.append(relation_get('private-address', unit=unit, rid=r_id)) - return hosts - - -def configure(service, key, auth): - create_keyring(service, key) - create_key_file(service, key) - hosts = get_ceph_nodes() - mon_hosts = ",".join(map(str, hosts)) - keyring = keyring_path(service) - with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: - ceph_conf.write(CEPH_CONF % locals()) - modprobe_kernel_module('rbd') - - -def image_mapped(image_name): - (rc, out) = commands.getstatusoutput('rbd showmapped') - return image_name in out - - -def map_block_storage(service, pool, image): - cmd = [ - 'rbd', - 'map', - '%s/%s' % (pool, image), - '--user', - service, - '--secret', - keyfile_path(service), - ] - check_call(cmd) - - -def filesystem_mounted(fs): - return fs in [f for m, f in mounts()] - - -def make_filesystem(blk_device, fstype='ext4', timeout=10): - count = 0 - e_noent = os.errno.ENOENT - while not os.path.exists(blk_device): - if count >= timeout: - log('ceph: gave up waiting on block device %s' % blk_device, - level=ERROR) - raise IOError(e_noent, os.strerror(e_noent), blk_device) - log('ceph: waiting for block device %s to appear' % blk_device, - level=INFO) - count += 1 - time.sleep(1) - else: - log('ceph: Formatting block device %s as filesystem %s.' % - (blk_device, fstype), level=INFO) - check_call(['mkfs', '-t', fstype, blk_device]) - - -def place_data_on_ceph(service, blk_device, data_src_dst, fstype='ext4'): - # mount block device into /mnt - mount(blk_device, '/mnt') - - # copy data to /mnt - try: - copy_files(data_src_dst, '/mnt') - except: - pass - - # umount block device - umount('/mnt') - - _dir = os.stat(data_src_dst) - uid = _dir.st_uid - gid = _dir.st_gid - - # re-mount where the data should originally be - mount(blk_device, data_src_dst, persist=True) - - # ensure original ownership of new mount. - cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst] - check_call(cmd) - - -# TODO: re-use -def modprobe_kernel_module(module): - log('ceph: Loading kernel module', level=INFO) - cmd = ['modprobe', module] - check_call(cmd) - cmd = 'echo %s >> /etc/modules' % module - check_call(cmd, shell=True) - - -def copy_files(src, dst, symlinks=False, ignore=None): - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - shutil.copytree(s, d, symlinks, ignore) - else: - shutil.copy2(s, d) - - -def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, - blk_device, fstype, system_services=[]): - """ - To be called from the current cluster leader. - Ensures given pool and RBD image exists, is mapped to a block device, - and the device is formatted and mounted at the given mount_point. - - If formatting a device for the first time, data existing at mount_point - will be migrated to the RBD device before being remounted. - - All services listed in system_services will be stopped prior to data - migration and restarted when complete. - """ - # Ensure pool, RBD image, RBD mappings are in place. - if not pool_exists(service, pool): - log('ceph: Creating new pool %s.' % pool, level=INFO) - create_pool(service, pool) - - if not rbd_exists(service, pool, rbd_img): - log('ceph: Creating RBD image (%s).' % rbd_img, level=INFO) - create_rbd_image(service, pool, rbd_img, sizemb) - - if not image_mapped(rbd_img): - log('ceph: Mapping RBD Image as a Block Device.', level=INFO) - map_block_storage(service, pool, rbd_img) - - # make file system - # TODO: What happens if for whatever reason this is run again and - # the data is already in the rbd device and/or is mounted?? - # When it is mounted already, it will fail to make the fs - # XXX: This is really sketchy! Need to at least add an fstab entry - # otherwise this hook will blow away existing data if its executed - # after a reboot. - if not filesystem_mounted(mount_point): - make_filesystem(blk_device, fstype) - - for svc in system_services: - if running(svc): - log('Stopping services %s prior to migrating data.' % svc, - level=INFO) - service_stop(svc) - - place_data_on_ceph(service, blk_device, mount_point, fstype) - - for svc in system_services: - service_start(svc) diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index ea1fd8f..53fa650 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -88,14 +88,14 @@ class OpenStackAmuletUtils(AmuletUtils): validation_function = self.validate_v2_endpoint_data xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens') if openstack_release and openstack_release >= xenial_queens: - validation_function = self.validate_v3_endpoint_data - expected = { - 'id': expected['id'], - 'region': expected['region'], - 'region_id': 'RegionOne', - 'url': self.valid_url, - 'interface': self.not_null, - 'service_id': expected['service_id']} + validation_function = self.validate_v3_endpoint_data + expected = { + 'id': expected['id'], + 'region': expected['region'], + 'region_id': 'RegionOne', + 'url': self.valid_url, + 'interface': self.not_null, + 'service_id': expected['service_id']} return validation_function(endpoints, admin_port, internal_port, public_port, expected) diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 8a20375..78a339f 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -1427,11 +1427,11 @@ class ZeroMQContext(OSContextGenerator): ctxt = {} if is_relation_made('zeromq-configuration', 'host'): for rid in relation_ids('zeromq-configuration'): - for unit in related_units(rid): - ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) - ctxt['zmq_host'] = relation_get('host', unit, rid) - ctxt['zmq_redis_address'] = relation_get( - 'zmq_redis_address', unit, rid) + for unit in related_units(rid): + ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) + ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/charmhelpers/contrib/openstack/templating.py b/charmhelpers/contrib/openstack/templating.py index a623315..050f8af 100644 --- a/charmhelpers/contrib/openstack/templating.py +++ b/charmhelpers/contrib/openstack/templating.py @@ -183,7 +183,7 @@ class OSConfigRenderer(object): /tmp/templates/grizzly/api-paste.ini /tmp/templates/havana/api-paste.ini - Since it was registered with the grizzly release, it first seraches + Since it was registered with the grizzly release, it first searches the grizzly directory for nova.conf, then the templates dir. When writing api-paste.ini, it will find the template in the grizzly diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 4e432a2..86b011b 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -83,7 +83,8 @@ from charmhelpers.fetch import ( add_source as fetch_add_source, SourceConfigError, GPGKeyError, - get_upstream_version + get_upstream_version, + filter_missing_packages ) from charmhelpers.fetch.snap import ( @@ -309,6 +310,15 @@ def error_out(msg): sys.exit(1) +def get_installed_semantic_versioned_packages(): + '''Get a list of installed packages which have OpenStack semantic versioning + + :returns List of installed packages + :rtype: [pkg1, pkg2, ...] + ''' + return filter_missing_packages(PACKAGE_CODENAMES.keys()) + + def get_os_codename_install_source(src): '''Derive OpenStack release codename from a given installation source.''' ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] @@ -972,7 +982,9 @@ def _ows_check_charm_func(state, message, charm_func_with_configs): """ if charm_func_with_configs: charm_state, charm_message = charm_func_with_configs() - if charm_state != 'active' and charm_state != 'unknown': + if (charm_state != 'active' and + charm_state != 'unknown' and + charm_state is not None): state = workload_state_compare(state, charm_state) if message: charm_message = charm_message.replace("Incomplete relations: ", @@ -1241,7 +1253,7 @@ def remote_restart(rel_name, remote_service=None): def check_actually_paused(services=None, ports=None): - """Check that services listed in the services object and and ports + """Check that services listed in the services object and ports are actually closed (not listened to), to verify that the unit is properly paused. diff --git a/charmhelpers/contrib/python.py b/charmhelpers/contrib/python.py new file mode 100644 index 0000000..84cba8c --- /dev/null +++ b/charmhelpers/contrib/python.py @@ -0,0 +1,21 @@ +# Copyright 2014-2019 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +# deprecated aliases for backwards compatibility +from charmhelpers.fetch.python import debug # noqa +from charmhelpers.fetch.python import packages # noqa +from charmhelpers.fetch.python import rpdb # noqa +from charmhelpers.fetch.python import version # noqa diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 7682820..63c9304 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -856,12 +856,22 @@ def _keyring_path(service): return KEYRING.format(service) -def create_keyring(service, key): - """Create a new Ceph keyring containing key.""" +def add_key(service, key): + """ + Add a key to a keyring. + + Creates the keyring if it doesn't already exist. + + Logs and returns if the key is already in the keyring. + """ keyring = _keyring_path(service) if os.path.exists(keyring): - log('Ceph keyring exists at %s.' % keyring, level=WARNING) - return + with open(keyring, 'r') as ring: + if key in ring.read(): + log('Ceph keyring exists at %s and has not changed.' % keyring, + level=DEBUG) + return + log('Updating existing keyring %s.' % keyring, level=DEBUG) cmd = ['ceph-authtool', keyring, '--create-keyring', '--name=client.{}'.format(service), '--add-key={}'.format(key)] @@ -869,6 +879,11 @@ def create_keyring(service, key): log('Created new ceph keyring at %s.' % keyring, level=DEBUG) +def create_keyring(service, key): + """Deprecated. Please use the more accurately named 'add_key'""" + return add_key(service, key) + + def delete_keyring(service): """Delete an existing Ceph keyring.""" keyring = _keyring_path(service) @@ -905,7 +920,7 @@ def get_ceph_nodes(relation='ceph'): def configure(service, key, auth, use_syslog): """Perform basic configuration of Ceph.""" - create_keyring(service, key) + add_key(service, key) create_key_file(service, key) hosts = get_ceph_nodes() with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: @@ -1068,7 +1083,7 @@ def ensure_ceph_keyring(service, user=None, group=None, if not key: return False - create_keyring(service=service, key=key) + add_key(service=service, key=key) keyring = _keyring_path(service) if user and group: check_call(['chown', '%s.%s' % (user, group), keyring]) diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 79953a4..47c1fc3 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -46,6 +46,7 @@ if __platform__ == "ubuntu": lsb_release, cmp_pkgrevno, CompareHostReleases, + get_distrib_codename, ) # flake8: noqa -- ignore F401 for this import elif __platform__ == "centos": from charmhelpers.core.host_factory.centos import ( # NOQA:F401 diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index a6d375a..d7e920e 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -72,6 +72,14 @@ def lsb_release(): return d +def get_distrib_codename(): + """Return the codename of the distribution + :returns: The codename + :rtype: str + """ + return lsb_release()['DISTRIB_CODENAME'].lower() + + def cmp_pkgrevno(package, revno, pkgcache=None): """Compare supplied revno with the revno of the installed package. diff --git a/charmhelpers/contrib/python/__init__.py b/charmhelpers/fetch/python/__init__.py similarity index 92% rename from charmhelpers/contrib/python/__init__.py rename to charmhelpers/fetch/python/__init__.py index d7567b8..bff99dc 100644 --- a/charmhelpers/contrib/python/__init__.py +++ b/charmhelpers/fetch/python/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2019 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/charmhelpers/fetch/python/debug.py b/charmhelpers/fetch/python/debug.py new file mode 100644 index 0000000..757135e --- /dev/null +++ b/charmhelpers/fetch/python/debug.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import atexit +import sys + +from charmhelpers.fetch.python.rpdb import Rpdb +from charmhelpers.core.hookenv import ( + open_port, + close_port, + ERROR, + log +) + +__author__ = "Jorge Niedbalski " + +DEFAULT_ADDR = "0.0.0.0" +DEFAULT_PORT = 4444 + + +def _error(message): + log(message, level=ERROR) + + +def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT): + """ + Set a trace point using the remote debugger + """ + atexit.register(close_port, port) + try: + log("Starting a remote python debugger session on %s:%s" % (addr, + port)) + open_port(port) + debugger = Rpdb(addr=addr, port=port) + debugger.set_trace(sys._getframe().f_back) + except Exception: + _error("Cannot start a remote debug session on %s:%s" % (addr, + port)) diff --git a/charmhelpers/contrib/python/packages.py b/charmhelpers/fetch/python/packages.py similarity index 100% rename from charmhelpers/contrib/python/packages.py rename to charmhelpers/fetch/python/packages.py diff --git a/charmhelpers/fetch/python/rpdb.py b/charmhelpers/fetch/python/rpdb.py new file mode 100644 index 0000000..9b31610 --- /dev/null +++ b/charmhelpers/fetch/python/rpdb.py @@ -0,0 +1,56 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Remote Python Debugger (pdb wrapper).""" + +import pdb +import socket +import sys + +__author__ = "Bertrand Janin " +__version__ = "0.1.3" + + +class Rpdb(pdb.Pdb): + + def __init__(self, addr="127.0.0.1", port=4444): + """Initialize the socket and initialize pdb.""" + + # Backup stdin and stdout before replacing them by the socket handle + self.old_stdout = sys.stdout + self.old_stdin = sys.stdin + + # Open a 'reusable' socket to let the webapp reload on the same port + self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.skt.bind((addr, port)) + self.skt.listen(1) + (clientsocket, address) = self.skt.accept() + handle = clientsocket.makefile('rw') + pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle) + sys.stdout = sys.stdin = handle + + def shutdown(self): + """Revert stdin and stdout, close the socket.""" + sys.stdout = self.old_stdout + sys.stdin = self.old_stdin + self.skt.close() + self.set_continue() + + def do_continue(self, arg): + """Stop all operation on ``continue``.""" + self.shutdown() + return 1 + + do_EOF = do_quit = do_exit = do_c = do_cont = do_continue diff --git a/charmhelpers/fetch/python/version.py b/charmhelpers/fetch/python/version.py new file mode 100644 index 0000000..3eb4210 --- /dev/null +++ b/charmhelpers/fetch/python/version.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +__author__ = "Jorge Niedbalski " + + +def current_version(): + """Current system python version""" + return sys.version_info + + +def current_version_string(): + """Current system python version as string major.minor.micro""" + return "{0}.{1}.{2}".format(sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro)