openstack/charm-glance
Code Issues Proposed changes
9fe6a51011
charm-glance/hooks/glance_utils.py
Alex Kavanagh e8a4b29784 Fix alphanumeric comparisons for openstack and ubuntu releases 
- sync charmhelpers with fix-alpha helpers
- fix up code where the alpha comparisons are done

Change-Id: I6851849b8107e59e9e6ff529348c7b183d36266b
Related-Bug: #1659575
2017-04-27 09:33:06 +01:00

684 lines
23 KiB
Python
Raw Blame History

#!/usr/bin/python
#
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import subprocess
from itertools import chain
import glance_contexts
from collections import OrderedDict
from charmhelpers.fetch import (
apt_upgrade,
apt_update,
apt_install,
add_source)
from charmhelpers.contrib.python.packages import (
pip_install,
)
from charmhelpers.core.hookenv import (
charm_dir,
config,
log,
relation_ids,
service_name,
)
from charmhelpers.core.host import (
adduser,
add_group,
add_user_to_group,
CompareHostReleases,
lsb_release,
mkdir,
pwgen,
service_stop,
service_start,
service_restart,
write_file,
)
from charmhelpers.contrib.openstack import (
templating,
context,)
from charmhelpers.contrib.hahelpers.cluster import (
is_elected_leader,
get_hacluster_config,
)
from charmhelpers.contrib.openstack.alternatives import install_alternative
from charmhelpers.contrib.openstack.utils import (
CompareOpenStackReleases,
configure_installation_source,
enable_memcache,
incomplete_relation_data,
is_unit_paused_set,
get_os_codename_install_source,
git_clone_and_install,
git_default_repos,
git_generate_systemd_init_files,
git_install_requested,
git_pip_venv_dir,
git_src_dir,
git_yaml_value,
make_assess_status_func,
os_application_version_set,
os_release,
pause_unit,
resume_unit,
token_cache_pkgs,
)
from charmhelpers.core.templating import render
from charmhelpers.core.decorators import (
retry_on_exception,
)
from charmhelpers.core.unitdata import kv
CLUSTER_RES = "grp_glance_vips"
PACKAGES = [
"apache2", "glance", "python-mysqldb", "python-swiftclient",
"python-psycopg2", "python-keystone", "python-six", "uuid", "haproxy", ]
VERSION_PACKAGE = 'glance-common'
BASE_GIT_PACKAGES = [
'libffi-dev',
'libmysqlclient-dev',
'libxml2-dev',
'libxslt1-dev',
'libssl-dev',
'libyaml-dev',
'openstack-pkg-tools',
'python-dev',
'python-pip',
'python-setuptools',
'zlib1g-dev',
]
SERVICES = [
"glance-api",
"glance-registry",
]
# ubuntu packages that should not be installed when deploying from git
GIT_PACKAGE_BLACKLIST = [
'glance',
'python-swiftclient',
'python-keystone',
]
CHARM = "glance"
GLANCE_CONF_DIR = "/etc/glance"
GLANCE_REGISTRY_CONF = "%s/glance-registry.conf" % GLANCE_CONF_DIR
GLANCE_API_CONF = "%s/glance-api.conf" % GLANCE_CONF_DIR
GLANCE_REGISTRY_PASTE = os.path.join(GLANCE_CONF_DIR,
'glance-registry-paste.ini')
GLANCE_API_PASTE = os.path.join(GLANCE_CONF_DIR,
'glance-api-paste.ini')
CEPH_CONF = "/etc/ceph/ceph.conf"
CHARM_CEPH_CONF = '/var/lib/charm/{}/ceph.conf'
HAPROXY_CONF = "/etc/haproxy/haproxy.cfg"
HTTPS_APACHE_CONF = "/etc/apache2/sites-available/openstack_https_frontend"
HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \
"openstack_https_frontend.conf"
MEMCACHED_CONF = '/etc/memcached.conf'
TEMPLATES = 'templates/'
# The interface is said to be satisfied if anyone of the interfaces in the
# list has a complete context.
REQUIRED_INTERFACES = {
'database': ['shared-db', 'pgsql-db'],
'identity': ['identity-service'],
}
def ceph_config_file():
return CHARM_CEPH_CONF.format(service_name())
CONFIG_FILES = OrderedDict([
(GLANCE_REGISTRY_CONF, {
'hook_contexts': [context.SharedDBContext(ssl_dir=GLANCE_CONF_DIR),
context.PostgresqlDBContext(),
context.IdentityServiceContext(
service='glance',
service_user='glance'),
context.SyslogContext(),
glance_contexts.LoggingConfigContext(),
glance_contexts.GlanceIPv6Context(),
context.WorkerConfigContext(),
context.OSConfigFlagContext(
charm_flag='registry-config-flags',
template_flag='registry_config_flags'),
context.MemcacheContext()],
'services': ['glance-registry']
}),
(GLANCE_API_CONF, {
'hook_contexts': [context.SharedDBContext(ssl_dir=GLANCE_CONF_DIR),
context.AMQPContext(ssl_dir=GLANCE_CONF_DIR),
context.PostgresqlDBContext(),
context.IdentityServiceContext(
service='glance',
service_user='glance'),
glance_contexts.CephGlanceContext(),
glance_contexts.ObjectStoreContext(),
glance_contexts.CinderStoreContext(),
glance_contexts.HAProxyContext(),
context.SyslogContext(),
glance_contexts.LoggingConfigContext(),
glance_contexts.GlanceIPv6Context(),
context.WorkerConfigContext(),
glance_contexts.MultiStoreContext(),
context.OSConfigFlagContext(
charm_flag='api-config-flags',
template_flag='api_config_flags'),
context.InternalEndpointContext(),
context.SubordinateConfigContext(
interface=['storage-backend'],
service=['glance-api'],
config_file=GLANCE_API_CONF),
context.MemcacheContext()],
'services': ['glance-api']
}),
(ceph_config_file(), {
'hook_contexts': [context.CephContext()],
'services': ['glance-api', 'glance-registry']
}),
(HAPROXY_CONF, {
'hook_contexts': [context.HAProxyContext(singlenode_mode=True),
glance_contexts.HAProxyContext()],
'services': ['haproxy'],
}),
(HTTPS_APACHE_CONF, {
'hook_contexts': [glance_contexts.ApacheSSLContext()],
'services': ['apache2'],
}),
(HTTPS_APACHE_24_CONF, {
'hook_contexts': [glance_contexts.ApacheSSLContext()],
'services': ['apache2'],
})
])
def register_configs():
# Register config files with their respective contexts.
# Regstration of some configs may not be required depending on
# existing of certain relations.
release = os_release('glance-common')
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
openstack_release=release)
confs = [GLANCE_REGISTRY_CONF,
GLANCE_API_CONF,
HAPROXY_CONF]
if relation_ids('ceph'):
mkdir(os.path.dirname(ceph_config_file()))
mkdir(os.path.dirname(CEPH_CONF))
# Install ceph config as an alternative for co-location with
# ceph and ceph-osd charms - glance ceph.conf will be
# lower priority that both of these but thats OK
if not os.path.exists(ceph_config_file()):
# touch file for pre-templated generation
open(ceph_config_file(), 'w').close()
install_alternative(os.path.basename(CEPH_CONF),
CEPH_CONF, ceph_config_file())
confs.append(ceph_config_file())
for conf in confs:
configs.register(conf, CONFIG_FILES[conf]['hook_contexts'])
if os.path.exists('/etc/apache2/conf-available'):
configs.register(HTTPS_APACHE_24_CONF,
CONFIG_FILES[HTTPS_APACHE_24_CONF]['hook_contexts'])
else:
configs.register(HTTPS_APACHE_CONF,
CONFIG_FILES[HTTPS_APACHE_CONF]['hook_contexts'])
if enable_memcache(release=release):
configs.register(MEMCACHED_CONF, [context.MemcacheContext()])
return configs
def determine_packages():
packages = set(PACKAGES)
if git_install_requested():
packages |= set(BASE_GIT_PACKAGES)
packages -= set(GIT_PACKAGE_BLACKLIST)
packages |= set(token_cache_pkgs(source=config('openstack-origin')))
return sorted(packages)
# NOTE(jamespage): Retry deals with sync issues during one-shot HA deploys.
# mysql might be restarting or suchlike.
@retry_on_exception(5, base_delay=3, exc_type=subprocess.CalledProcessError)
def migrate_database():
'''Runs glance-manage to initialize a new database
or migrate existing
'''
cmd = ['glance-manage', 'db_sync']
subprocess.check_call(cmd)
def do_openstack_upgrade(configs):
"""Perform an upgrade of glance. Takes care of upgrading
packages, rewriting configs + database migration and potentially
any other post-upgrade actions.
:param configs: The charms main OSConfigRenderer object.
"""
new_src = config('openstack-origin')
new_os_rel = get_os_codename_install_source(new_src)
log('Performing OpenStack upgrade to %s.' % (new_os_rel))
configure_installation_source(new_src)
dpkg_opts = [
'--option', 'Dpkg::Options::=--force-confnew',
'--option', 'Dpkg::Options::=--force-confdef',
]
apt_update()
apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
apt_install(determine_packages(), fatal=True)
# set CONFIGS to load templates from new release and regenerate config
configs.set_release(openstack_release=new_os_rel)
configs.write_all()
[service_stop(s) for s in services()]
if is_elected_leader(CLUSTER_RES):
migrate_database()
# Don't start services if the unit is supposed to be paused.
if not is_unit_paused_set():
[service_start(s) for s in services()]
def restart_map():
'''Determine the correct resource map to be passed to
charmhelpers.core.restart_on_change() based on the services configured.
:returns: dict: A dictionary mapping config file to lists of services
that should be restarted when file changes.
'''
_map = []
for f, ctxt in CONFIG_FILES.iteritems():
svcs = []
for svc in ctxt['services']:
svcs.append(svc)
if svcs:
_map.append((f, svcs))
if enable_memcache(source=config('openstack-origin')):
_map.append((MEMCACHED_CONF, ['memcached']))
return OrderedDict(_map)
def services():
''' Returns a list of (unique) services associate with this charm '''
return list(set(chain(*restart_map().values())))
def setup_ipv6():
ubuntu_rel = lsb_release()['DISTRIB_CODENAME'].lower()
if CompareHostReleases(ubuntu_rel) < "trusty":
raise Exception("IPv6 is not supported in the charms for Ubuntu "
"versions less than Trusty 14.04")
# Need haproxy >= 1.5.3 for ipv6 so for Trusty if we are <= Kilo we need to
# use trusty-backports otherwise we can use the UCA.
if (ubuntu_rel == 'trusty' and
CompareOpenStackReleases(os_release('glance')) < 'liberty'):
add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports '
'main')
apt_update()
apt_install('haproxy/trusty-backports', fatal=True)
def git_install(projects_yaml):
"""Perform setup, and install git repos specified in yaml parameter."""
if git_install_requested():
git_pre_install()
projects_yaml = git_default_repos(projects_yaml)
git_clone_and_install(projects_yaml, core_project='glance')
git_post_install(projects_yaml)
def git_pre_install():
"""Perform glance pre-install setup."""
dirs = [
'/var/lib/glance',
'/var/lib/glance/images',
'/var/lib/glance/image-cache',
'/var/lib/glance/image-cache/incomplete',
'/var/lib/glance/image-cache/invalid',
'/var/lib/glance/image-cache/queue',
'/var/log/glance',
]
logs = [
'/var/log/glance/glance-api.log',
'/var/log/glance/glance-registry.log',
]
adduser('glance', shell='/bin/bash', system_user=True)
add_group('glance', system_group=True)
add_user_to_group('glance', 'glance')
for d in dirs:
mkdir(d, owner='glance', group='glance', perms=0755, force=False)
for l in logs:
write_file(l, '', owner='glance', group='glance', perms=0600)
def git_post_install(projects_yaml):
"""Perform glance post-install setup."""
http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
if http_proxy:
for pkg in ['mysql-python', 'python-cephlibs']:
pip_install(pkg, proxy=http_proxy,
venv=git_pip_venv_dir(projects_yaml))
else:
for pkg in ['mysql-python', 'python-cephlibs']:
pip_install(pkg, venv=git_pip_venv_dir(projects_yaml))
src_etc = os.path.join(git_src_dir(projects_yaml, 'glance'), 'etc')
configs = {
'src': src_etc,
'dest': GLANCE_CONF_DIR,
}
if os.path.exists(configs['dest']):
shutil.rmtree(configs['dest'])
shutil.copytree(configs['src'], configs['dest'])
symlinks = [
# NOTE(coreycb): Need to find better solution than bin symlinks.
{'src': os.path.join(git_pip_venv_dir(projects_yaml),
'bin/glance-manage'),
'link': '/usr/local/bin/glance-manage'},
]
for s in symlinks:
if os.path.lexists(s['link']):
os.remove(s['link'])
os.symlink(s['src'], s['link'])
bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
# Use systemd init units/scripts from ubuntu wily onward
if lsb_release()['DISTRIB_RELEASE'] >= '15.10':
templates_dir = os.path.join(charm_dir(), TEMPLATES, 'git')
daemons = ['glance-api', 'glance-glare', 'glance-registry']
for daemon in daemons:
glance_context = {
'daemon_path': os.path.join(bin_dir, daemon),
}
template_file = 'git/{}.init.in.template'.format(daemon)
init_in_file = '{}.init.in'.format(daemon)
render(template_file, os.path.join(templates_dir, init_in_file),
glance_context, perms=0o644)
git_generate_systemd_init_files(templates_dir)
else:
glance_api_context = {
'service_description': 'Glance API server',
'service_name': 'Glance',
'user_name': 'glance',
'start_dir': '/var/lib/glance',
'process_name': 'glance-api',
'executable_name': os.path.join(bin_dir, 'glance-api'),
'config_files': [GLANCE_API_CONF],
'log_file': '/var/log/glance/api.log',
}
glance_registry_context = {
'service_description': 'Glance registry server',
'service_name': 'Glance',
'user_name': 'glance',
'start_dir': '/var/lib/glance',
'process_name': 'glance-registry',
'executable_name': os.path.join(bin_dir, 'glance-registry'),
'config_files': [GLANCE_REGISTRY_CONF],
'log_file': '/var/log/glance/registry.log',
}
templates_dir = 'hooks/charmhelpers/contrib/openstack/templates'
templates_dir = os.path.join(charm_dir(), templates_dir)
render('git.upstart', '/etc/init/glance-api.conf',
glance_api_context, perms=0o644,
templates_dir=templates_dir)
render('git.upstart', '/etc/init/glance-registry.conf',
glance_registry_context, perms=0o644,
templates_dir=templates_dir)
# Don't restart services if the unit is supposed to be paused.
if not is_unit_paused_set():
service_restart('glance-api')
service_restart('glance-registry')
def get_optional_interfaces():
"""Return the optional interfaces that should be checked if the relavent
relations have appeared.
:returns: {general_interface: [specific_int1, specific_int2, ...], ...}
"""
optional_interfaces = {}
if relation_ids('ha'):
optional_interfaces['ha'] = ['cluster']
if (relation_ids('ceph') or relation_ids('object-store') or
relation_ids('cinder-volume-service') or
relation_ids('storage-backend')):
optional_interfaces['storage-backend'] = ['ceph', 'object-store',
'cinder-volume-service',
'storage-backend']
if relation_ids('amqp'):
optional_interfaces['messaging'] = ['amqp']
return optional_interfaces
def check_optional_relations(configs):
"""Check that if we have a relation_id for high availability that we can
get the hacluster config. If we can't then we are blocked.
This function is called from assess_status/set_os_workload_status as the
charm_func and needs to return either None, None if there is no problem or
the status, message if there is a problem.
:param configs: an OSConfigRender() instance.
:return 2-tuple: (string, string) = (status, message)
"""
if relation_ids('ha'):
try:
get_hacluster_config()
except:
return ('blocked',
'hacluster missing configuration: '
'vip, vip_iface, vip_cidr')
# return 'unknown' as the lowest priority to not clobber an existing
# status.
return "unknown", ""
def swift_temp_url_key():
"""Generate a temp URL key, post it to Swift and return its value.
If it is already posted, the current value of the key will be returned.
"""
import requests
keystone_ctxt = context.IdentityServiceContext(service='glance',
service_user='glance')()
if not keystone_ctxt:
log('Missing identity-service relation. Skipping generation of '
'swift temporary url key.')
return
auth_url = '%s://%s:%s/v2.0/' % (keystone_ctxt['service_protocol'],
keystone_ctxt['service_host'],
keystone_ctxt['service_port'])
from swiftclient import client
from swiftclient import exceptions
@retry_on_exception(15, base_delay=10,
exc_type=(exceptions.ClientException,
requests.exceptions.ConnectionError))
def connect_and_post():
log('Connecting swift client...')
swift_connection = client.Connection(
authurl=auth_url, user='glance',
key=keystone_ctxt['admin_password'],
tenant_name=keystone_ctxt['admin_tenant_name'],
auth_version='2.0')
account_stats = swift_connection.head_account()
if 'x-account-meta-temp-url-key' in account_stats:
log("Temp URL key was already posted.")
return account_stats['x-account-meta-temp-url-key']
temp_url_key = pwgen(length=64)
swift_connection.post_account(headers={'x-account-meta-temp-url-key':
temp_url_key})
return temp_url_key
return connect_and_post()
def assess_status(configs):
"""Assess status of current unit
Decides what the state of the unit should be based on the current
configuration.
SIDE EFFECT: calls set_os_workload_status(...) which sets the workload
status of the unit.
Also calls status_set(...) directly if paused state isn't complete.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
assess_status_func(configs)()
os_application_version_set(VERSION_PACKAGE)
def assess_status_func(configs):
"""Helper function to create the function that will assess_status() for
the unit.
Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to
create the appropriate status function and then returns it.
Used directly by assess_status() and also for pausing and resuming
the unit.
NOTE: REQUIRED_INTERFACES is augmented with the optional interfaces
depending on the current config before being passed to the
make_assess_status_func() function.
NOTE(ajkavanagh) ports are not checked due to race hazards with services
that don't behave sychronously w.r.t their service scripts. e.g.
apache2.
@param configs: a templating.OSConfigRenderer() object
@return f() -> None : a function that assesses the unit's workload status
"""
required_interfaces = REQUIRED_INTERFACES.copy()
required_interfaces.update(get_optional_interfaces())
return make_assess_status_func(
configs, required_interfaces,
charm_func=check_optional_relations,
services=services(), ports=None)
def pause_unit_helper(configs):
"""Helper function to pause a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(pause_unit, configs)
def resume_unit_helper(configs):
"""Helper function to resume a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(resume_unit, configs)
def _pause_resume_helper(f, configs):
"""Helper function that uses the make_assess_status_func(...) from
charmhelpers.contrib.openstack.utils to create an assess_status(...)
function that can be used with the pause/resume of the unit
@param f: the function to be used with the assess_status(...) function
@returns None - this function is executed for its side-effect
"""
# TODO(ajkavanagh) - ports= has been left off because of the race hazard
# that exists due to service_start()
f(assess_status_func(configs),
services=services(),
ports=None)
PASTE_INI_MARKER = 'paste-ini-marker'
REINSTALL_OPTIONS = [
'--reinstall',
'--option=Dpkg::Options::=--force-confmiss'
]
def reinstall_paste_ini():
'''
Re-install glance-{api,registry}-paste.ini file from packages
Existing glance-{api,registry}-paste.ini file will be removed
and the original files provided by the packages will be
re-installed.
This will only ever be performed once per unit.
'''
db = kv()
if not db.get(PASTE_INI_MARKER):
for paste_file in [GLANCE_REGISTRY_PASTE,
GLANCE_API_PASTE]:
if os.path.exists(paste_file):
os.remove(paste_file)
apt_install(packages=['glance-api', 'glance-registry'],
options=REINSTALL_OPTIONS,
fatal=True)
db.set(PASTE_INI_MARKER, True)
db.flush()
def is_api_ready(configs):
return (not incomplete_relation_data(configs, REQUIRED_INTERFACES))
View Git Blame Copy Permalink