openstack/charm-swift-storage
Code Issues Proposed changes

Update charm-helpers-hooks.yaml and sync ch

Browse Source 
Using the new version of the sync tool which removes the charmhelpers
directory before syncing, run charm helpers sync to find any unexpected
missing dependencies.

Change-Id: I8e046f8816fd91920ba2c15f8c960b74442eb7bd
This commit is contained in:
David Ames 2019-02-12 15:58:48 -08:00
parent d94b5e9d31
commit 0b3be36674
15 changed files with 224 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charm-helpers-hooks.yaml
View File

@ -13,6 +13,6 @@ include:
- payload.execd
- contrib.network.ip
- contrib.network.ufw
- contrib.python.packages
- contrib.python
- contrib.charmsupport
- contrib.hardening|inc=*

291
charmhelpers/contrib/hahelpers/ceph.py
View File

@ -1,291 +0,0 @@
#
# Copyright 2012 Canonical Ltd.
#
# This file is sourced from lp:openstack-charm-helpers
#
# Authors:
# James Page <james.page@ubuntu.com>
# Adam Gandelman <adamg@ubuntu.com>
#
import commands
import os
import shutil
import time
from subprocess import (
check_call,
check_output,
CalledProcessError
)
from charmhelpers.core.hookenv import (
relation_get,
relation_ids,
related_units,
log,
INFO,
ERROR
)
from charmhelpers.core.host import (
apt_install,
mount,
mounts,
service_start,
service_stop,
umount,
)
KEYRING = '/etc/ceph/ceph.client.%s.keyring'
KEYFILE = '/etc/ceph/ceph.client.%s.key'
CEPH_CONF = """[global]
auth supported = %(auth)s
keyring = %(keyring)s
mon host = %(mon_hosts)s
"""
def running(service):
# this local util can be dropped as soon the following branch lands
# in lp:charm-helpers
# https://code.launchpad.net/~gandelman-a/charm-helpers/service_running/
try:
output = check_output(['service', service, 'status'])
except CalledProcessError:
return False
else:
if ("start/running" in output or "is running" in output):
return True
else:
return False
def install():
ceph_dir = "/etc/ceph"
if not os.path.isdir(ceph_dir):
os.mkdir(ceph_dir)
apt_install('ceph-common', fatal=True)
def rbd_exists(service, pool, rbd_img):
(rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' %
(service, pool))
return rbd_img in out
def create_rbd_image(service, pool, image, sizemb):
cmd = [
'rbd',
'create',
image,
'--size',
str(sizemb),
'--id',
service,
'--pool',
pool
]
check_call(cmd)
def pool_exists(service, name):
(rc, out) = commands.getstatusoutput("rados --id %s lspools" % service)
return name in out
def create_pool(service, name):
cmd = [
'rados',
'--id',
service,
'mkpool',
name
]
check_call(cmd)
def keyfile_path(service):
return KEYFILE % service
def keyring_path(service):
return KEYRING % service
def create_keyring(service, key):
keyring = keyring_path(service)
if os.path.exists(keyring):
log('ceph: Keyring exists at %s.' % keyring, level=INFO)
cmd = [
'ceph-authtool',
keyring,
'--create-keyring',
'--name=client.%s' % service,
'--add-key=%s' % key
]
check_call(cmd)
log('ceph: Created new ring at %s.' % keyring, level=INFO)
def create_key_file(service, key):
# create a file containing the key
keyfile = keyfile_path(service)
if os.path.exists(keyfile):
log('ceph: Keyfile exists at %s.' % keyfile, level=INFO)
fd = open(keyfile, 'w')
fd.write(key)
fd.close()
log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
def get_ceph_nodes():
hosts = []
for r_id in relation_ids('ceph'):
for unit in related_units(r_id):
hosts.append(relation_get('private-address', unit=unit, rid=r_id))
return hosts
def configure(service, key, auth):
create_keyring(service, key)
create_key_file(service, key)
hosts = get_ceph_nodes()
mon_hosts = ",".join(map(str, hosts))
keyring = keyring_path(service)
with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
ceph_conf.write(CEPH_CONF % locals())
modprobe_kernel_module('rbd')
def image_mapped(image_name):
(rc, out) = commands.getstatusoutput('rbd showmapped')
return image_name in out
def map_block_storage(service, pool, image):
cmd = [
'rbd',
'map',
'%s/%s' % (pool, image),
'--user',
service,
'--secret',
keyfile_path(service),
]
check_call(cmd)
def filesystem_mounted(fs):
return fs in [f for m, f in mounts()]
def make_filesystem(blk_device, fstype='ext4', timeout=10):
count = 0
e_noent = os.errno.ENOENT
while not os.path.exists(blk_device):
if count >= timeout:
log('ceph: gave up waiting on block device %s' % blk_device,
level=ERROR)
raise IOError(e_noent, os.strerror(e_noent), blk_device)
log('ceph: waiting for block device %s to appear' % blk_device,
level=INFO)
count += 1
time.sleep(1)
else:
log('ceph: Formatting block device %s as filesystem %s.' %
(blk_device, fstype), level=INFO)
check_call(['mkfs', '-t', fstype, blk_device])
def place_data_on_ceph(service, blk_device, data_src_dst, fstype='ext4'):
# mount block device into /mnt
mount(blk_device, '/mnt')
# copy data to /mnt
try:
copy_files(data_src_dst, '/mnt')
except:
pass
# umount block device
umount('/mnt')
_dir = os.stat(data_src_dst)
uid = _dir.st_uid
gid = _dir.st_gid
# re-mount where the data should originally be
mount(blk_device, data_src_dst, persist=True)
# ensure original ownership of new mount.
cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst]
check_call(cmd)
# TODO: re-use
def modprobe_kernel_module(module):
log('ceph: Loading kernel module', level=INFO)
cmd = ['modprobe', module]
check_call(cmd)
cmd = 'echo %s >> /etc/modules' % module
check_call(cmd, shell=True)
def copy_files(src, dst, symlinks=False, ignore=None):
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
blk_device, fstype, system_services=[]):
"""
To be called from the current cluster leader.
Ensures given pool and RBD image exists, is mapped to a block device,
and the device is formatted and mounted at the given mount_point.
If formatting a device for the first time, data existing at mount_point
will be migrated to the RBD device before being remounted.
All services listed in system_services will be stopped prior to data
migration and restarted when complete.
"""
# Ensure pool, RBD image, RBD mappings are in place.
if not pool_exists(service, pool):
log('ceph: Creating new pool %s.' % pool, level=INFO)
create_pool(service, pool)
if not rbd_exists(service, pool, rbd_img):
log('ceph: Creating RBD image (%s).' % rbd_img, level=INFO)
create_rbd_image(service, pool, rbd_img, sizemb)
if not image_mapped(rbd_img):
log('ceph: Mapping RBD Image as a Block Device.', level=INFO)
map_block_storage(service, pool, rbd_img)
# make file system
# TODO: What happens if for whatever reason this is run again and
# the data is already in the rbd device and/or is mounted??
# When it is mounted already, it will fail to make the fs
# XXX: This is really sketchy! Need to at least add an fstab entry
# otherwise this hook will blow away existing data if its executed
# after a reboot.
if not filesystem_mounted(mount_point):
make_filesystem(blk_device, fstype)
for svc in system_services:
if running(svc):
log('Stopping services %s prior to migrating data.' % svc,
level=INFO)
service_stop(svc)
place_data_on_ceph(service, blk_device, mount_point, fstype)
for svc in system_services:
service_start(svc)

16
charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -88,14 +88,14 @@ class OpenStackAmuletUtils(AmuletUtils):
validation_function = self.validate_v2_endpoint_data
xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
if openstack_release and openstack_release >= xenial_queens:
validation_function = self.validate_v3_endpoint_data
expected = {
'id': expected['id'],
'region': expected['region'],
'region_id': 'RegionOne',
'url': self.valid_url,
'interface': self.not_null,
'service_id': expected['service_id']}
validation_function = self.validate_v3_endpoint_data
expected = {
'id': expected['id'],
'region': expected['region'],
'region_id': 'RegionOne',
'url': self.valid_url,
'interface': self.not_null,
'service_id': expected['service_id']}
return validation_function(endpoints, admin_port, internal_port,
public_port, expected)

10
charmhelpers/contrib/openstack/context.py
View File

@ -1427,11 +1427,11 @@ class ZeroMQContext(OSContextGenerator):
ctxt = {}
if is_relation_made('zeromq-configuration', 'host'):
for rid in relation_ids('zeromq-configuration'):
for unit in related_units(rid):
ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
ctxt['zmq_host'] = relation_get('host', unit, rid)
ctxt['zmq_redis_address'] = relation_get(
'zmq_redis_address', unit, rid)
for unit in related_units(rid):
ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
ctxt['zmq_host'] = relation_get('host', unit, rid)
ctxt['zmq_redis_address'] = relation_get(
'zmq_redis_address', unit, rid)
return ctxt

2
charmhelpers/contrib/openstack/templating.py
View File

@ -183,7 +183,7 @@ class OSConfigRenderer(object):
/tmp/templates/grizzly/api-paste.ini
/tmp/templates/havana/api-paste.ini
Since it was registered with the grizzly release, it first seraches
Since it was registered with the grizzly release, it first searches
the grizzly directory for nova.conf, then the templates dir.
When writing api-paste.ini, it will find the template in the grizzly

18
charmhelpers/contrib/openstack/utils.py
View File

@ -83,7 +83,8 @@ from charmhelpers.fetch import (
add_source as fetch_add_source,
SourceConfigError,
GPGKeyError,
get_upstream_version
get_upstream_version,
filter_missing_packages
)
from charmhelpers.fetch.snap import (
@ -309,6 +310,15 @@ def error_out(msg):
sys.exit(1)
def get_installed_semantic_versioned_packages():
'''Get a list of installed packages which have OpenStack semantic versioning
:returns List of installed packages
:rtype: [pkg1, pkg2, ...]
'''
return filter_missing_packages(PACKAGE_CODENAMES.keys())
def get_os_codename_install_source(src):
'''Derive OpenStack release codename from a given installation source.'''
ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
@ -972,7 +982,9 @@ def _ows_check_charm_func(state, message, charm_func_with_configs):
"""
if charm_func_with_configs:
charm_state, charm_message = charm_func_with_configs()
if charm_state != 'active' and charm_state != 'unknown':
if (charm_state != 'active' and
charm_state != 'unknown' and
charm_state is not None):
state = workload_state_compare(state, charm_state)
if message:
charm_message = charm_message.replace("Incomplete relations: ",
@ -1241,7 +1253,7 @@ def remote_restart(rel_name, remote_service=None):
def check_actually_paused(services=None, ports=None):
"""Check that services listed in the services object and and ports
"""Check that services listed in the services object and ports
are actually closed (not listened to), to verify that the unit is
properly paused.

21
charmhelpers/contrib/python.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright 2014-2019 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
# deprecated aliases for backwards compatibility
from charmhelpers.fetch.python import debug # noqa
from charmhelpers.fetch.python import packages # noqa
from charmhelpers.fetch.python import rpdb # noqa
from charmhelpers.fetch.python import version # noqa

27
charmhelpers/contrib/storage/linux/ceph.py
View File

@ -856,12 +856,22 @@ def _keyring_path(service):
return KEYRING.format(service)
def create_keyring(service, key):
"""Create a new Ceph keyring containing key."""
def add_key(service, key):
"""
Add a key to a keyring.
Creates the keyring if it doesn't already exist.
Logs and returns if the key is already in the keyring.
"""
keyring = _keyring_path(service)
if os.path.exists(keyring):
log('Ceph keyring exists at %s.' % keyring, level=WARNING)
return
with open(keyring, 'r') as ring:
if key in ring.read():
log('Ceph keyring exists at %s and has not changed.' % keyring,
level=DEBUG)
return
log('Updating existing keyring %s.' % keyring, level=DEBUG)
cmd = ['ceph-authtool', keyring, '--create-keyring',
'--name=client.{}'.format(service), '--add-key={}'.format(key)]
@ -869,6 +879,11 @@ def create_keyring(service, key):
log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
def create_keyring(service, key):
"""Deprecated. Please use the more accurately named 'add_key'"""
return add_key(service, key)
def delete_keyring(service):
"""Delete an existing Ceph keyring."""
keyring = _keyring_path(service)
@ -905,7 +920,7 @@ def get_ceph_nodes(relation='ceph'):
def configure(service, key, auth, use_syslog):
"""Perform basic configuration of Ceph."""
create_keyring(service, key)
add_key(service, key)
create_key_file(service, key)
hosts = get_ceph_nodes()
with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
@ -1068,7 +1083,7 @@ def ensure_ceph_keyring(service, user=None, group=None,
if not key:
return False
create_keyring(service=service, key=key)
add_key(service=service, key=key)
keyring = _keyring_path(service)
if user and group:
check_call(['chown', '%s.%s' % (user, group), keyring])

1
charmhelpers/core/host.py
View File

@ -46,6 +46,7 @@ if __platform__ == "ubuntu":
lsb_release,
cmp_pkgrevno,
CompareHostReleases,
get_distrib_codename,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
from charmhelpers.core.host_factory.centos import ( # NOQA:F401

8
charmhelpers/core/host_factory/ubuntu.py
View File

@ -72,6 +72,14 @@ def lsb_release():
return d
def get_distrib_codename():
"""Return the codename of the distribution
:returns: The codename
:rtype: str
"""
return lsb_release()['DISTRIB_CODENAME'].lower()
def cmp_pkgrevno(package, revno, pkgcache=None):
"""Compare supplied revno with the revno of the installed package.

2
charmhelpers/contrib/python/__init__.py → charmhelpers/fetch/python/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright 2014-2015 Canonical Limited.
# Copyright 2014-2019 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

54
charmhelpers/fetch/python/debug.py Normal file
View File

@ -0,0 +1,54 @@
#!/usr/bin/env python
# coding: utf-8
# Copyright 2014-2015 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import atexit
import sys
from charmhelpers.fetch.python.rpdb import Rpdb
from charmhelpers.core.hookenv import (
open_port,
close_port,
ERROR,
log
)
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
DEFAULT_ADDR = "0.0.0.0"
DEFAULT_PORT = 4444
def _error(message):
log(message, level=ERROR)
def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
"""
Set a trace point using the remote debugger
"""
atexit.register(close_port, port)
try:
log("Starting a remote python debugger session on %s:%s" % (addr,
port))
open_port(port)
debugger = Rpdb(addr=addr, port=port)
debugger.set_trace(sys._getframe().f_back)
except Exception:
_error("Cannot start a remote debug session on %s:%s" % (addr,
port))

0
charmhelpers/contrib/python/packages.py → charmhelpers/fetch/python/packages.py
View File

56
charmhelpers/fetch/python/rpdb.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2014-2015 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Remote Python Debugger (pdb wrapper)."""
import pdb
import socket
import sys
__author__ = "Bertrand Janin <b@janin.com>"
__version__ = "0.1.3"
class Rpdb(pdb.Pdb):
def __init__(self, addr="127.0.0.1", port=4444):
"""Initialize the socket and initialize pdb."""
# Backup stdin and stdout before replacing them by the socket handle
self.old_stdout = sys.stdout
self.old_stdin = sys.stdin
# Open a 'reusable' socket to let the webapp reload on the same port
self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
self.skt.bind((addr, port))
self.skt.listen(1)
(clientsocket, address) = self.skt.accept()
handle = clientsocket.makefile('rw')
pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
sys.stdout = sys.stdin = handle
def shutdown(self):
"""Revert stdin and stdout, close the socket."""
sys.stdout = self.old_stdout
sys.stdin = self.old_stdin
self.skt.close()
self.set_continue()
def do_continue(self, arg):
"""Stop all operation on ``continue``."""
self.shutdown()
return 1
do_EOF = do_quit = do_exit = do_c = do_cont = do_continue

32
charmhelpers/fetch/python/version.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env python
# coding: utf-8
# Copyright 2014-2015 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
def current_version():
"""Current system python version"""
return sys.version_info
def current_version_string():
"""Current system python version as string major.minor.micro"""
return "{0}.{1}.{2}".format(sys.version_info.major,
sys.version_info.minor,
sys.version_info.micro)