openstack/charm-keystone
Code Issues Proposed changes

py3: Switch to using Python 3 for rocky or later

Browse Source 
Switch package install to Python 3 for OpenStack Rocky or later.

When upgrading, remove any python-* packages that were explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

Also drop the python2 shebang from hooks/manager.py in favor of
specifying the interpreter on the subprocess call. The python
interpreter version must match the python version of the OpenStack
payload due to the keystoneclient library imports.

Depends-On: I18996e15d2d08b1dacf0533132eae880cbb9aa32
Change-Id: If973ebc2be3b32ee3ff2122b5874dad96cda9fec
This commit is contained in:
Corey Bryant 2018-09-19 14:54:11 +00:00
parent 1c1c41c7f1
commit 9cb5636eba
16 changed files with 277 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
charmhelpers/contrib/hahelpers/apache.py
View File

@ -23,8 +23,8 @@
#
import os
import subprocess
from charmhelpers.core import host
from charmhelpers.core.hookenv import (
config as config_get,
relation_get,
@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
if ca_cert:
cert_file = ('/usr/local/share/ca-certificates/'
'keystone_juju_ca_cert.crt')
old_cert = retrieve_ca_cert(cert_file)
if old_cert and old_cert == ca_cert:
log("CA cert is the same as installed version", level=INFO)
else:
log("Installing new CA cert", level=INFO)
with open(cert_file, 'wb') as crt:
crt.write(ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')

3
charmhelpers/contrib/hardening/apache/checks/config.py
View File

@ -14,6 +14,7 @@
import os
import re
import six
import subprocess
@ -95,6 +96,8 @@ class ApacheConfContext(object):
ctxt = settings['hardening']
out = subprocess.check_output(['apache2', '-v'])
if six.PY3:
out = out.decode('utf-8')
ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
out).group(1)
ctxt['apache_icondir'] = '/usr/share/apache2/icons/'

6
charmhelpers/contrib/hardening/audits/apache.py
View File

@ -15,7 +15,7 @@
import re
import subprocess
from six import string_types
import six
from charmhelpers.core.hookenv import (
log,
@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit):
def __init__(self, modules):
if modules is None:
self.modules = []
elif isinstance(modules, string_types):
elif isinstance(modules, six.string_types):
self.modules = [modules]
else:
self.modules = modules
@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit):
def _get_loaded_modules():
"""Returns the modules which are enabled in Apache."""
output = subprocess.check_output(['apache2ctl', '-M'])
if six.PY3:
output = output.decode('utf-8')
modules = []
for line in output.splitlines():
# Each line of the enabled module output looks like:

130
charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -24,7 +24,8 @@ import urlparse
import cinderclient.v1.client as cinder_client
import cinderclient.v2.client as cinder_clientv2
import glanceclient.v1.client as glance_client
import glanceclient.v1 as glance_client
import glanceclient.v2 as glance_clientv2
import heatclient.v1.client as heat_client
from keystoneclient.v2_0 import client as keystone_client
from keystoneauth1.identity import (
@ -617,13 +618,13 @@ class OpenStackAmuletUtils(AmuletUtils):
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant)
def authenticate_glance_admin(self, keystone):
def authenticate_glance_admin(self, keystone, force_v1_client=False):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
interface='adminURL')
if keystone.session:
return glance_client.Client(ep, session=keystone.session)
if not force_v1_client and keystone.session:
return glance_clientv2.Client("2", session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
@ -679,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
def glance_create_image(self, glance, image_name, image_url,
download_dir='tests',
hypervisor_type=None,
disk_format='qcow2',
architecture='x86_64',
container_format='bare'):
"""Download an image and upload it to glance, validate its status
and return an image object pointer. KVM defaults, can override for
LXD.
:param glance: pointer to authenticated glance connection
:param glance: pointer to authenticated glance api connection
:param image_name: display name for new image
:param image_url: url to retrieve
:param download_dir: directory to store downloaded image file
:param hypervisor_type: glance image hypervisor property
:param disk_format: glance image disk format
:param architecture: glance image architecture property
:param container_format: glance image container format
:returns: glance image pointer
"""
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
self.log.debug('Creating glance image ({}) from '
'{}...'.format(image_name, image_url))
# Download cirros image
# Download image
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@ -699,22 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
local_path = os.path.join('tests', cirros_img)
if not os.path.exists(local_path):
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
opener.retrieve(cirros_url, local_path)
f.close()
abs_file_name = os.path.join(download_dir, image_name)
if not os.path.exists(abs_file_name):
opener.retrieve(image_url, abs_file_name)
# Create glance image
with open(local_path) as f:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare', data=f)
glance_properties = {
'architecture': architecture,
}
if hypervisor_type:
glance_properties['hypervisor_type'] = hypervisor_type
# Create glance image
if float(glance.version) < 2.0:
with open(abs_file_name) as f:
image = glance.images.create(
name=image_name,
is_public=True,
disk_format=disk_format,
container_format=container_format,
properties=glance_properties,
data=f)
else:
image = glance.images.create(
name=image_name,
visibility="public",
disk_format=disk_format,
container_format=container_format)
glance.images.upload(image.id, open(abs_file_name, 'rb'))
glance.images.update(image.id, **glance_properties)
# Wait for image to reach active status
img_id = image.id
@ -729,24 +754,68 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Validating image attributes...')
val_img_name = glance.images.get(img_id).name
val_img_stat = glance.images.get(img_id).status
val_img_pub = glance.images.get(img_id).is_public
val_img_cfmt = glance.images.get(img_id).container_format
val_img_dfmt = glance.images.get(img_id).disk_format
if float(glance.version) < 2.0:
val_img_pub = glance.images.get(img_id).is_public
else:
val_img_pub = glance.images.get(img_id).visibility == "public"
msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
'container fmt:{} disk fmt:{}'.format(
val_img_name, val_img_pub, img_id,
val_img_stat, val_img_cfmt, val_img_dfmt))
if val_img_name == image_name and val_img_stat == 'active' \
and val_img_pub is True and val_img_cfmt == 'bare' \
and val_img_dfmt == 'qcow2':
and val_img_pub is True and val_img_cfmt == container_format \
and val_img_dfmt == disk_format:
self.log.debug(msg_attr)
else:
msg = ('Volume validation failed, {}'.format(msg_attr))
msg = ('Image validation failed, {}'.format(msg_attr))
amulet.raise_status(amulet.FAIL, msg=msg)
return image
def create_cirros_image(self, glance, image_name, hypervisor_type=None):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
:param glance: pointer to authenticated glance connection
:param image_name: display name for new image
:param hypervisor_type: glance image hypervisor property
:returns: glance image pointer
"""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'glance_create_image instead of '
'create_cirros_image.')
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
# Get cirros image URL
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
proxies = {'http': http_proxy}
opener = urllib.FancyURLopener(proxies)
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
f.close()
return self.glance_create_image(
glance,
image_name,
cirros_url,
hypervisor_type=hypervisor_type)
def delete_image(self, glance, image):
"""Delete the specified image."""
@ -998,6 +1067,9 @@ class OpenStackAmuletUtils(AmuletUtils):
cmd, code, output))
amulet.raise_status(amulet.FAIL, msg=msg)
# For mimic ceph osd lspools output
output = output.replace("\n", ",")
# Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
for pool in str(output).split(','):
pool_id_name = pool.split(' ')

37
charmhelpers/contrib/openstack/context.py
View File

@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'enable-qos',
'default': False,
},
'enable_nsg_logging': {
'rel_key': 'enable-nsg-logging',
'default': False,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):
@ -1530,10 +1534,15 @@ class NeutronAPIContext(OSContextGenerator):
if 'l2-population' in rdata:
ctxt.update(self.get_neutron_options(rdata))
extension_drivers = []
if ctxt['enable_qos']:
ctxt['extension_drivers'] = 'qos'
else:
ctxt['extension_drivers'] = ''
extension_drivers.append('qos')
if ctxt['enable_nsg_logging']:
extension_drivers.append('log')
ctxt['extension_drivers'] = ','.join(extension_drivers)
return ctxt
@ -1893,7 +1902,7 @@ class EnsureDirContext(OSContextGenerator):
Some software requires a user to create a target directory to be
scanned for drop-in files with a specific format. This is why this
context is needed to do that before rendering a template.
'''
'''
def __init__(self, dirname, **kwargs):
'''Used merely to ensure that a given directory exists.'''
@ -1903,3 +1912,23 @@ class EnsureDirContext(OSContextGenerator):
def __call__(self):
mkdir(self.dirname, **self.kwargs)
return {}
class VersionsContext(OSContextGenerator):
"""Context to return the openstack and operating system versions.
"""
def __init__(self, pkg='python-keystone'):
"""Initialise context.
:param pkg: Package to extrapolate openstack version from.
:type pkg: str
"""
self.pkg = pkg
def __call__(self):
ostack = os_release(self.pkg, base='icehouse')
osystem = lsb_release()['DISTRIB_CODENAME'].lower()
return {
'openstack_release': ostack,
'operating_system_release': osystem}

6
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
View File

@ -1,12 +1,14 @@
{% if auth_host -%}
[keystone_authtoken]
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
auth_type = password
{% if api_version == "3" -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
project_domain_name = {{ admin_domain_name }}
user_domain_name = {{ admin_domain_name }}
{% else -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
project_domain_name = default
user_domain_name = default
{% endif -%}

3
charmhelpers/core/hookenv.py
View File

@ -48,6 +48,7 @@ INFO = "INFO"
DEBUG = "DEBUG"
TRACE = "TRACE"
MARKER = object()
SH_MAX_ARG = 131071
cache = {}
@ -98,7 +99,7 @@ def log(message, level=None):
command += ['-l', level]
if not isinstance(message, six.string_types):
message = repr(message)
command += [message]
command += [message[:SH_MAX_ARG]]
# Missing juju-log should not cause failures in unit tests
# Send log output to stderr
try:

26
charmhelpers/core/host.py
View File

@ -34,7 +34,7 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
from .hookenv import log, DEBUG, local_unit
from .hookenv import log, INFO, DEBUG, local_unit, charm_name
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
return modulo * wait
else:
return calculated_wait_time
def install_ca_cert(ca_cert, name=None):
"""
Install the given cert as a trusted CA.
The ``name`` is the stem of the filename where the cert is written, and if
not provided, it will default to ``juju-{charm_name}``.
If the cert is empty or None, or is unchanged, nothing is done.
"""
if not ca_cert:
return
if not isinstance(ca_cert, bytes):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return
log("Installing new CA cert at: {}".format(cert_file), level=INFO)
write_file(cert_file, ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])

2
charmhelpers/fetch/__init__.py
View File

@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
fetch = importlib.import_module(module)
filter_installed_packages = fetch.filter_installed_packages
filter_missing_packages = fetch.filter_missing_packages
install = fetch.apt_install
upgrade = fetch.apt_upgrade
update = _fetch_update = fetch.apt_update
@ -96,6 +97,7 @@ if __platform__ == "ubuntu":
apt_update = fetch.apt_update
apt_upgrade = fetch.apt_upgrade
apt_purge = fetch.apt_purge
apt_autoremove = fetch.apt_autoremove
apt_mark = fetch.apt_mark
apt_hold = fetch.apt_hold
apt_unhold = fetch.apt_unhold

4
charmhelpers/fetch/bzrurl.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
import os
from subprocess import check_call
from subprocess import STDOUT, check_output
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource,
@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
cmd = ['bzr', 'branch']
cmd += cmd_opts
cmd += [source, dest]
check_call(cmd)
check_output(cmd, stderr=STDOUT)
def install(self, source, dest=None, revno=None):
url_parts = self.parse_url(source)

4
charmhelpers/fetch/giturl.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
import os
from subprocess import check_call, CalledProcessError
from subprocess import check_output, CalledProcessError, STDOUT
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource,
@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
cmd = ['git', 'clone', source, dest, '--branch', branch]
if depth:
cmd.extend(['--depth', depth])
check_call(cmd)
check_output(cmd, stderr=STDOUT)
def install(self, source, branch="master", dest=None, depth=None):
url_parts = self.parse_url(source)

20
charmhelpers/fetch/ubuntu.py
View File

@ -189,6 +189,18 @@ def filter_installed_packages(packages):
return _pkgs
def filter_missing_packages(packages):
"""Return a list of packages that are installed.
:param packages: list of packages to evaluate.
:returns list: Packages that are installed.
"""
return list(
set(packages) -
set(filter_installed_packages(packages))
)
def apt_cache(in_memory=True, progress=None):
"""Build and return an apt cache."""
from apt import apt_pkg
@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False):
_run_apt_command(cmd, fatal)
def apt_autoremove(purge=True, fatal=False):
"""Purge one or more packages."""
cmd = ['apt-get', '--assume-yes', 'autoremove']
if purge:
cmd.append('--purge')
_run_apt_command(cmd, fatal)
def apt_mark(packages, mark, fatal=False):
"""Flag one or more packages using apt-mark."""
log("Marking {} as {}".format(packages, mark))

48
hooks/keystone_utils.py
View File

@ -93,7 +93,10 @@ from charmhelpers.fetch import (
apt_install,
apt_update,
apt_upgrade,
apt_purge,
apt_autoremove,
add_source,
filter_missing_packages,
)
from charmhelpers.core.host import (
@ -131,6 +134,14 @@ BASE_PACKAGES = [
'uuid',
]
PY3_PACKAGES = [
'python3-keystone',
'python3-keystoneclient',
'python3-memcache',
'python3-six',
'libapache2-mod-wsgi-py3',
]
BASE_PACKAGES_SNAP = [
'haproxy',
'openssl',
@ -594,6 +605,8 @@ def api_port(service):
def determine_packages():
release = CompareOpenStackReleases(os_release('keystone'))
# currently all packages match service names
if snap_install_requested():
pkgs = deepcopy(BASE_PACKAGES_SNAP)
@ -602,11 +615,29 @@ def determine_packages():
return sorted(pkgs)
else:
packages = set(services()).union(BASE_PACKAGES)
if run_in_apache():
if release >= 'rocky':
packages = [p for p in packages if not p.startswith('python-')]
packages.extend(PY3_PACKAGES)
elif run_in_apache():
packages.add('libapache2-mod-wsgi')
return sorted(packages)
def determine_purge_packages():
'''
Determine list of packages that where previously installed which are no
longer needed.
:returns: list of package names
'''
release = CompareOpenStackReleases(os_release('keystone'))
if release >= 'rocky':
pkgs = [p for p in BASE_PACKAGES if p.startswith('python-')]
pkgs.extend(['python-keystone', 'python-memcache'])
return pkgs
return []
def save_script_rc():
env_vars = {'OPENSTACK_SERVICE_KEYSTONE': 'keystone',
'OPENSTACK_PORT_ADMIN': determine_api_port(
@ -639,6 +670,11 @@ def do_openstack_upgrade(configs):
reset_os_release()
apt_install(packages=determine_packages(),
options=dpkg_opts, fatal=True)
installed_pkgs = filter_missing_packages(determine_purge_packages())
if installed_pkgs:
apt_purge(installed_pkgs, fatal=True)
apt_autoremove(purge=True, fatal=True)
else:
# TODO: Add support for upgrade from deb->snap
# NOTE(thedac): Setting devmode until LP#1719636 is fixed
@ -1054,6 +1090,9 @@ class ManagerServer():
def _launch_manager(self):
script = os.path.abspath(os.path.join(os.path.dirname(__file__),
'manager.py'))
release = CompareOpenStackReleases(
get_os_codename_install_source(config('openstack-origin'))
)
# need to set the environment variable PYTHONPATH to include the
# payload's directory for the manager.py to find the various keystone
# clients
@ -1072,8 +1111,13 @@ class ManagerServer():
env['PATH'] = ':'.join(
os.environ.get('PATH', '').split(':') +
[_bin_path])
# ensure python interpreter matches python version of OpenStack
if release >= 'rocky':
python = 'python3'
else:
python = 'python2'
# launch the process and return immediately
self.pvar = subprocess.Popen([script, self.socket_file],
self.pvar = subprocess.Popen([python, script, self.socket_file],
env=env, close_fds=True)
def clean_up(self):

2
hooks/manager.py
View File

@ -1,5 +1,3 @@
#!/usr/bin/env python2
#
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");

2
tests/tests.yaml
View File

@ -1,6 +1,6 @@
charm_name: keystone
smoke_bundles:
- bionic-queens
- bionic-rocky
gate_bundles:
- trusty-icehouse
- trusty-mitaka

34
unit_tests/test_keystone_utils.py
View File

@ -159,14 +159,26 @@ class TestKeystoneUtils(CharmTestCase):
self.assertEqual(set(ex), set(result))
@patch('charmhelpers.contrib.openstack.utils.config')
def test_determine_packages_mitaka(self, _config):
self.os_release.return_value = 'mitaka'
def test_determine_packages_queens(self, _config):
self.os_release.return_value = 'queens'
self.snap_install_requested.return_value = False
_config.return_value = None
result = utils.determine_packages()
ex = utils.BASE_PACKAGES + [
'keystone', 'python-keystoneclient', 'libapache2-mod-wsgi',
'memcached']
'keystone', 'python-keystoneclient', 'memcached',
'libapache2-mod-wsgi'
]
self.assertEqual(set(ex), set(result))
@patch('charmhelpers.contrib.openstack.utils.config')
def test_determine_packages_rocky(self, _config):
self.os_release.return_value = 'rocky'
self.snap_install_requested.return_value = False
_config.return_value = None
result = utils.determine_packages()
ex = list(set(
[p for p in utils.BASE_PACKAGES if not p.startswith('python-')] +
['memcached'] + utils.PY3_PACKAGES))
self.assertEqual(set(ex), set(result))
@patch('charmhelpers.contrib.openstack.utils.config')
@ -178,6 +190,19 @@ class TestKeystoneUtils(CharmTestCase):
ex = utils.BASE_PACKAGES_SNAP + ['memcached']
self.assertEqual(set(ex), set(result))
def test_determine_purge_packages(self):
'Ensure no packages are identified for purge prior to rocky'
self.os_release.return_value = 'queens'
self.assertEqual(utils.determine_purge_packages(), [])
def test_determine_purge_packages_rocky(self):
'Ensure python packages are identified for purge at rocky'
self.os_release.return_value = 'rocky'
self.assertEqual(utils.determine_purge_packages(),
[p for p in utils.BASE_PACKAGES
if p.startswith('python-')] +
['python-keystone', 'python-memcache'])
@patch.object(utils, 'is_elected_leader')
@patch.object(utils, 'disable_unused_apache_sites')
@patch('os.path.exists')
@ -190,6 +215,7 @@ class TestKeystoneUtils(CharmTestCase):
mock_is_elected_leader):
configs = MagicMock()
self.test_config.set('openstack-origin', 'cloud:xenial-newton')
self.os_release.return_value = 'ocata'
determine_packages.return_value = []
os_path_exists.return_value = True
run_in_apache.return_value = True