openstack/charm-cinder
Code Issues Proposed changes

synced /next

Browse Source
This commit is contained in:
Edward Hope-Morley 2014-12-15 10:17:11 +00:00
commit 4d4eb5b5f0
35 changed files with 839 additions and 433 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
charm-helpers-hooks.yaml
View File

@ -11,3 +11,4 @@ include:
- fetch
- payload.execd
- contrib.network.ip
- contrib.python.packages

22
hooks/charmhelpers/__init__.py
View File

@ -0,0 +1,22 @@
# Bootstrap charm-helpers, installing its dependencies if necessary using
# only standard libraries.
import subprocess
import sys
try:
import six # flake8: noqa
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
import six # flake8: noqa
try:
import yaml # flake8: noqa
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa

23
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@ -13,9 +13,10 @@ clustering-related helpers.
import subprocess
import os
from socket import gethostname as get_unit_hostname
import six
from charmhelpers.core.hookenv import (
log,
relation_ids,
@ -77,7 +78,7 @@ def is_crm_leader(resource):
"show", resource
]
try:
status = subprocess.check_output(cmd)
status = subprocess.check_output(cmd).decode('UTF-8')
except subprocess.CalledProcessError:
return False
else:
@ -150,34 +151,42 @@ def https():
return False
def determine_api_port(public_port):
def determine_api_port(public_port, singlenode_mode=False):
'''
Determine correct API server listening port based on
existence of HTTPS reverse proxy and/or haproxy.
public_port: int: standard public port for given service
singlenode_mode: boolean: Shuffle ports when only a single unit is present
returns: int: the correct listening port for the API service
'''
i = 0
if len(peer_units()) > 0 or is_clustered():
if singlenode_mode:
i += 1
elif len(peer_units()) > 0 or is_clustered():
i += 1
if https():
i += 1
return public_port - (i * 10)
def determine_apache_port(public_port):
def determine_apache_port(public_port, singlenode_mode=False):
'''
Description: Determine correct apache listening port based on public IP +
state of the cluster.
public_port: int: standard public port for given service
singlenode_mode: boolean: Shuffle ports when only a single unit is present
returns: int: the correct listening port for the HAProxy service
'''
i = 0
if len(peer_units()) > 0 or is_clustered():
if singlenode_mode:
i += 1
elif len(peer_units()) > 0 or is_clustered():
i += 1
return public_port - (i * 10)
@ -197,7 +206,7 @@ def get_hacluster_config():
for setting in settings:
conf[setting] = config_get(setting)
missing = []
[missing.append(s) for s, v in conf.iteritems() if v is None]
[missing.append(s) for s, v in six.iteritems(conf) if v is None]
if missing:
log('Insufficient config data to configure hacluster.', level=ERROR)
raise HAIncompleteConfig

4
hooks/charmhelpers/contrib/network/ip.py
View File

@ -228,7 +228,7 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
raise Exception("Interface '%s' doesn't have any %s addresses." %
(iface, inet_type))
return addresses
return sorted(addresses)
get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
@ -302,7 +302,7 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
if global_addrs:
# Make sure any found global addresses are not temporary
cmd = ['ip', 'addr', 'show', iface]
out = subprocess.check_output(cmd)
out = subprocess.check_output(cmd).decode('UTF-8')
if dynamic_only:
key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
else:

3
hooks/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -1,3 +1,4 @@
import six
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@ -69,7 +70,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
def _configure_services(self, configs):
"""Configure all of the services."""
for service, config in configs.iteritems():
for service, config in six.iteritems(configs):
self.d.configure(service, config)
def _get_openstack_release(self):

4
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -7,6 +7,8 @@ import glanceclient.v1.client as glance_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
import six
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -60,7 +62,7 @@ class OpenStackAmuletUtils(AmuletUtils):
expected service catalog endpoints.
"""
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in expected.iteritems():
for k, v in six.iteritems(expected):
if k in actual:
ret = self._validate_dict_data(expected[k][0], actual[k][0])
if ret:

451
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -1,18 +1,15 @@
import json
import os
import time
from base64 import b64decode
from subprocess import check_call
from subprocess import (
check_call
)
import six
from charmhelpers.fetch import (
apt_install,
filter_installed_packages,
)
from charmhelpers.core.hookenv import (
config,
is_relation_made,
@ -24,44 +21,44 @@ from charmhelpers.core.hookenv import (
relation_set,
unit_get,
unit_private_ip,
charm_name,
DEBUG,
INFO,
WARNING,
ERROR,
DEBUG
)
from charmhelpers.core.sysctl import create as sysctl_create
from charmhelpers.core.host import (
mkdir,
write_file
write_file,
)
from charmhelpers.contrib.hahelpers.cluster import (
determine_apache_port,
determine_api_port,
https,
is_clustered
is_clustered,
)
from charmhelpers.contrib.hahelpers.apache import (
get_cert,
get_ca_cert,
install_ca_cert,
)
from charmhelpers.contrib.openstack.neutron import (
neutron_plugin_attribute,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
get_ipv6_addr,
get_netmask_for_address,
format_ipv6_addr,
is_address_in_network
is_address_in_network,
)
from charmhelpers.contrib.openstack.utils import get_host_ip
from charmhelpers.contrib.openstack.utils import (
get_host_ip,
)
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
ADDRESS_TYPES = ['admin', 'internal', 'public']
class OSContextError(Exception):
@ -69,7 +66,7 @@ class OSContextError(Exception):
def ensure_packages(packages):
'''Install but do not upgrade required plugin packages'''
"""Install but do not upgrade required plugin packages."""
required = filter_installed_packages(packages)
if required:
apt_install(required, fatal=True)
@ -77,20 +74,27 @@ def ensure_packages(packages):
def context_complete(ctxt):
_missing = []
for k, v in ctxt.iteritems():
for k, v in six.iteritems(ctxt):
if v is None or v == '':
_missing.append(k)
if _missing:
log('Missing required data: %s' % ' '.join(_missing), level='INFO')
log('Missing required data: %s' % ' '.join(_missing), level=INFO)
return False
return True
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
The provided config_flags string may be a list of comma-separated values
which themselves may be comma-separated list of values.
"""
if config_flags.find('==') >= 0:
log("config_flags is not in expected format (key=value)",
level=ERROR)
log("config_flags is not in expected format (key=value)", level=ERROR)
raise OSContextError
# strip the following from each value.
post_strippers = ' ,'
# we strip any leading/trailing '=' or ' ' from the string then
@ -98,7 +102,7 @@ def config_flags_parser(config_flags):
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
for i in xrange(0, limit - 1):
for i in range(0, limit - 1):
current = split[i]
next = split[i + 1]
vindex = next.rfind(',')
@ -113,17 +117,18 @@ def config_flags_parser(config_flags):
# if this not the first entry, expect an embedded key.
index = current.rfind(',')
if index < 0:
log("invalid config value(s) at index %s" % (i),
level=ERROR)
log("Invalid config value(s) at index %s" % (i), level=ERROR)
raise OSContextError
key = current[index + 1:]
# Add to collection.
flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
return flags
class OSContextGenerator(object):
"""Base class for all context generators."""
interfaces = []
def __call__(self):
@ -135,11 +140,11 @@ class SharedDBContext(OSContextGenerator):
def __init__(self,
database=None, user=None, relation_prefix=None, ssl_dir=None):
'''
Allows inspecting relation for settings prefixed with relation_prefix.
This is useful for parsing access for multiple databases returned via
the shared-db interface (eg, nova_password, quantum_password)
'''
"""Allows inspecting relation for settings prefixed with
relation_prefix. This is useful for parsing access for multiple
databases returned via the shared-db interface (eg, nova_password,
quantum_password)
"""
self.relation_prefix = relation_prefix
self.database = database
self.user = user
@ -149,9 +154,8 @@ class SharedDBContext(OSContextGenerator):
self.database = self.database or config('database')
self.user = self.user or config('database-user')
if None in [self.database, self.user]:
log('Could not generate shared_db context. '
'Missing required charm config options. '
'(database name and user)')
log("Could not generate shared_db context. Missing required charm "
"config options. (database name and user)", level=ERROR)
raise OSContextError
ctxt = {}
@ -204,23 +208,24 @@ class PostgresqlDBContext(OSContextGenerator):
def __call__(self):
self.database = self.database or config('database')
if self.database is None:
log('Could not generate postgresql_db context. '
'Missing required charm config options. '
'(database name)')
log('Could not generate postgresql_db context. Missing required '
'charm config options. (database name)', level=ERROR)
raise OSContextError
ctxt = {}
ctxt = {}
for rid in relation_ids(self.interfaces[0]):
for unit in related_units(rid):
ctxt = {
'database_host': relation_get('host', rid=rid, unit=unit),
'database': self.database,
'database_user': relation_get('user', rid=rid, unit=unit),
'database_password': relation_get('password', rid=rid, unit=unit),
'database_type': 'postgresql',
}
rel_host = relation_get('host', rid=rid, unit=unit)
rel_user = relation_get('user', rid=rid, unit=unit)
rel_passwd = relation_get('password', rid=rid, unit=unit)
ctxt = {'database_host': rel_host,
'database': self.database,
'database_user': rel_user,
'database_password': rel_passwd,
'database_type': 'postgresql'}
if context_complete(ctxt):
return ctxt
return {}
@ -229,23 +234,29 @@ def db_ssl(rdata, ctxt, ssl_dir):
ca_path = os.path.join(ssl_dir, 'db-client.ca')
with open(ca_path, 'w') as fh:
fh.write(b64decode(rdata['ssl_ca']))
ctxt['database_ssl_ca'] = ca_path
elif 'ssl_ca' in rdata:
log("Charm not setup for ssl support but ssl ca found")
log("Charm not setup for ssl support but ssl ca found", level=INFO)
return ctxt
if 'ssl_cert' in rdata:
cert_path = os.path.join(
ssl_dir, 'db-client.cert')
if not os.path.exists(cert_path):
log("Waiting 1m for ssl client cert validity")
log("Waiting 1m for ssl client cert validity", level=INFO)
time.sleep(60)
with open(cert_path, 'w') as fh:
fh.write(b64decode(rdata['ssl_cert']))
ctxt['database_ssl_cert'] = cert_path
key_path = os.path.join(ssl_dir, 'db-client.key')
with open(key_path, 'w') as fh:
fh.write(b64decode(rdata['ssl_key']))
ctxt['database_ssl_key'] = key_path
return ctxt
@ -253,9 +264,8 @@ class IdentityServiceContext(OSContextGenerator):
interfaces = ['identity-service']
def __call__(self):
log('Generating template context for identity-service')
log('Generating template context for identity-service', level=DEBUG)
ctxt = {}
for rid in relation_ids('identity-service'):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
@ -263,26 +273,24 @@ class IdentityServiceContext(OSContextGenerator):
serv_host = format_ipv6_addr(serv_host) or serv_host
auth_host = rdata.get('auth_host')
auth_host = format_ipv6_addr(auth_host) or auth_host
ctxt = {
'service_port': rdata.get('service_port'),
'service_host': serv_host,
'auth_host': auth_host,
'auth_port': rdata.get('auth_port'),
'admin_tenant_name': rdata.get('service_tenant'),
'admin_user': rdata.get('service_username'),
'admin_password': rdata.get('service_password'),
'service_protocol':
rdata.get('service_protocol') or 'http',
'auth_protocol':
rdata.get('auth_protocol') or 'http',
}
svc_protocol = rdata.get('service_protocol') or 'http'
auth_protocol = rdata.get('auth_protocol') or 'http'
ctxt = {'service_port': rdata.get('service_port'),
'service_host': serv_host,
'auth_host': auth_host,
'auth_port': rdata.get('auth_port'),
'admin_tenant_name': rdata.get('service_tenant'),
'admin_user': rdata.get('service_username'),
'admin_password': rdata.get('service_password'),
'service_protocol': svc_protocol,
'auth_protocol': auth_protocol}
if context_complete(ctxt):
# NOTE(jamespage) this is required for >= icehouse
# so a missing value just indicates keystone needs
# upgrading
ctxt['admin_tenant_id'] = rdata.get('service_tenant_id')
return ctxt
return {}
@ -295,21 +303,23 @@ class AMQPContext(OSContextGenerator):
self.interfaces = [rel_name]
def __call__(self):
log('Generating template context for amqp')
log('Generating template context for amqp', level=DEBUG)
conf = config()
user_setting = 'rabbit-user'
vhost_setting = 'rabbit-vhost'
if self.relation_prefix:
user_setting = self.relation_prefix + '-rabbit-user'
vhost_setting = self.relation_prefix + '-rabbit-vhost'
user_setting = '%s-rabbit-user' % (self.relation_prefix)
vhost_setting = '%s-rabbit-vhost' % (self.relation_prefix)
else:
user_setting = 'rabbit-user'
vhost_setting = 'rabbit-vhost'
try:
username = conf[user_setting]
vhost = conf[vhost_setting]
except KeyError as e:
log('Could not generate shared_db context. '
'Missing required charm config options: %s.' % e)
log('Could not generate shared_db context. Missing required charm '
'config options: %s.' % e, level=ERROR)
raise OSContextError
ctxt = {}
for rid in relation_ids(self.rel_name):
ha_vip_only = False
@ -323,6 +333,7 @@ class AMQPContext(OSContextGenerator):
host = relation_get('private-address', rid=rid, unit=unit)
host = format_ipv6_addr(host) or host
ctxt['rabbitmq_host'] = host
ctxt.update({
'rabbitmq_user': username,
'rabbitmq_password': relation_get('password', rid=rid,
@ -333,6 +344,7 @@ class AMQPContext(OSContextGenerator):
ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
if ssl_port:
ctxt['rabbit_ssl_port'] = ssl_port
ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
if ssl_ca:
ctxt['rabbit_ssl_ca'] = ssl_ca
@ -346,41 +358,45 @@ class AMQPContext(OSContextGenerator):
if context_complete(ctxt):
if 'rabbit_ssl_ca' in ctxt:
if not self.ssl_dir:
log(("Charm not setup for ssl support "
"but ssl ca found"))
log("Charm not setup for ssl support but ssl ca "
"found", level=INFO)
break
ca_path = os.path.join(
self.ssl_dir, 'rabbit-client-ca.pem')
with open(ca_path, 'w') as fh:
fh.write(b64decode(ctxt['rabbit_ssl_ca']))
ctxt['rabbit_ssl_ca'] = ca_path
# Sufficient information found = break out!
break
# Used for active/active rabbitmq >= grizzly
if ('clustered' not in ctxt or ha_vip_only) \
and len(related_units(rid)) > 1:
if (('clustered' not in ctxt or ha_vip_only) and
len(related_units(rid)) > 1):
rabbitmq_hosts = []
for unit in related_units(rid):
host = relation_get('private-address', rid=rid, unit=unit)
host = format_ipv6_addr(host) or host
rabbitmq_hosts.append(host)
ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
if not context_complete(ctxt):
return {}
else:
return ctxt
return ctxt
class CephContext(OSContextGenerator):
"""Generates context for /etc/ceph/ceph.conf templates."""
interfaces = ['ceph']
def __call__(self):
'''This generates context for /etc/ceph/ceph.conf templates'''
if not relation_ids('ceph'):
return {}
log('Generating template context for ceph')
log('Generating template context for ceph', level=DEBUG)
mon_hosts = []
auth = None
key = None
@ -389,18 +405,18 @@ class CephContext(OSContextGenerator):
for unit in related_units(rid):
auth = relation_get('auth', rid=rid, unit=unit)
key = relation_get('key', rid=rid, unit=unit)
ceph_addr = \
relation_get('ceph-public-address', rid=rid, unit=unit) or \
relation_get('private-address', rid=rid, unit=unit)
ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
unit=unit)
unit_priv_addr = relation_get('private-address', rid=rid,
unit=unit)
ceph_addr = ceph_pub_addr or unit_priv_addr
ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
mon_hosts.append(ceph_addr)
ctxt = {
'mon_hosts': ' '.join(mon_hosts),
'auth': auth,
'key': key,
'use_syslog': use_syslog
}
ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
'auth': auth,
'key': key,
'use_syslog': use_syslog}
if not os.path.isdir('/etc/ceph'):
os.mkdir('/etc/ceph')
@ -409,79 +425,68 @@ class CephContext(OSContextGenerator):
return {}
ensure_packages(['ceph-common'])
return ctxt
ADDRESS_TYPES = ['admin', 'internal', 'public']
class HAProxyContext(OSContextGenerator):
"""Provides half a context for the haproxy template, which describes
all peers to be included in the cluster. Each charm needs to include
its own context generator that describes the port mapping.
"""
interfaces = ['cluster']
def __call__(self):
'''
Builds half a context for the haproxy template, which describes
all peers to be included in the cluster. Each charm needs to include
its own context generator that describes the port mapping.
'''
if not relation_ids('cluster'):
return {}
def __init__(self, singlenode_mode=False):
self.singlenode_mode = singlenode_mode
l_unit = local_unit().replace('/', '-')
def __call__(self):
if not relation_ids('cluster') and not self.singlenode_mode:
return {}
if config('prefer-ipv6'):
addr = get_ipv6_addr(exc_list=[config('vip')])[0]
else:
addr = get_host_ip(unit_get('private-address'))
l_unit = local_unit().replace('/', '-')
cluster_hosts = {}
# NOTE(jamespage): build out map of configured network endpoints
# and associated backends
for addr_type in ADDRESS_TYPES:
laddr = get_address_in_network(
config('os-{}-network'.format(addr_type)))
cfg_opt = 'os-{}-network'.format(addr_type)
laddr = get_address_in_network(config(cfg_opt))
if laddr:
cluster_hosts[laddr] = {}
cluster_hosts[laddr]['network'] = "{}/{}".format(
laddr,
get_netmask_for_address(laddr)
)
cluster_hosts[laddr]['backends'] = {}
cluster_hosts[laddr]['backends'][l_unit] = laddr
netmask = get_netmask_for_address(laddr)
cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
netmask),
'backends': {l_unit: laddr}}
for rid in relation_ids('cluster'):
for unit in related_units(rid):
_unit = unit.replace('/', '-')
_laddr = relation_get('{}-address'.format(addr_type),
rid=rid, unit=unit)
if _laddr:
_unit = unit.replace('/', '-')
cluster_hosts[laddr]['backends'][_unit] = _laddr
# NOTE(jamespage) no split configurations found, just use
# private addresses
if not cluster_hosts:
cluster_hosts[addr] = {}
cluster_hosts[addr]['network'] = "{}/{}".format(
addr,
get_netmask_for_address(addr)
)
cluster_hosts[addr]['backends'] = {}
cluster_hosts[addr]['backends'][l_unit] = addr
netmask = get_netmask_for_address(addr)
cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
'backends': {l_unit: addr}}
for rid in relation_ids('cluster'):
for unit in related_units(rid):
_unit = unit.replace('/', '-')
_laddr = relation_get('private-address',
rid=rid, unit=unit)
if _laddr:
_unit = unit.replace('/', '-')
cluster_hosts[addr]['backends'][_unit] = _laddr
ctxt = {
'frontends': cluster_hosts,
}
ctxt = {'frontends': cluster_hosts}
if config('haproxy-server-timeout'):
ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout')
if config('haproxy-client-timeout'):
ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
@ -495,13 +500,18 @@ class HAProxyContext(OSContextGenerator):
ctxt['stat_port'] = ':8888'
for frontend in cluster_hosts:
if len(cluster_hosts[frontend]['backends']) > 1:
if (len(cluster_hosts[frontend]['backends']) > 1 or
self.singlenode_mode):
# Enable haproxy when we have enough peers.
log('Ensuring haproxy enabled in /etc/default/haproxy.')
log('Ensuring haproxy enabled in /etc/default/haproxy.',
level=DEBUG)
with open('/etc/default/haproxy', 'w') as out:
out.write('ENABLED=1\n')
return ctxt
log('HAProxy context is incomplete, this unit has no peers.')
log('HAProxy context is incomplete, this unit has no peers.',
level=INFO)
return {}
@ -509,29 +519,28 @@ class ImageServiceContext(OSContextGenerator):
interfaces = ['image-service']
def __call__(self):
'''
Obtains the glance API server from the image-service relation. Useful
in nova and cinder (currently).
'''
log('Generating template context for image-service.')
"""Obtains the glance API server from the image-service relation.
Useful in nova and cinder (currently).
"""
log('Generating template context for image-service.', level=DEBUG)
rids = relation_ids('image-service')
if not rids:
return {}
for rid in rids:
for unit in related_units(rid):
api_server = relation_get('glance-api-server',
rid=rid, unit=unit)
if api_server:
return {'glance_api_servers': api_server}
log('ImageService context is incomplete. '
'Missing required relation data.')
log("ImageService context is incomplete. Missing required relation "
"data.", level=INFO)
return {}
class ApacheSSLContext(OSContextGenerator):
"""
Generates a context for an apache vhost configuration that configures
"""Generates a context for an apache vhost configuration that configures
HTTPS reverse proxying for one or many endpoints. Generated context
looks something like::
@ -565,6 +574,7 @@ class ApacheSSLContext(OSContextGenerator):
else:
cert_filename = 'cert'
key_filename = 'key'
write_file(path=os.path.join(ssl_dir, cert_filename),
content=b64decode(cert))
write_file(path=os.path.join(ssl_dir, key_filename),
@ -576,7 +586,8 @@ class ApacheSSLContext(OSContextGenerator):
install_ca_cert(b64decode(ca_cert))
def canonical_names(self):
'''Figure out which canonical names clients will access this service'''
"""Figure out which canonical names clients will access this service.
"""
cns = []
for r_id in relation_ids('identity-service'):
for unit in related_units(r_id):
@ -584,7 +595,8 @@ class ApacheSSLContext(OSContextGenerator):
for k in rdata:
if k.startswith('ssl_key_'):
cns.append(k.lstrip('ssl_key_'))
return list(set(cns))
return sorted(list(set(cns)))
def get_network_addresses(self):
"""For each network configured, return corresponding address and vip
@ -603,9 +615,10 @@ class ApacheSSLContext(OSContextGenerator):
...]
"""
addresses = []
vips = []
if config('vip'):
vips = config('vip').split()
else:
vips = []
for net_type in ['os-internal-network', 'os-admin-network',
'os-public-network']:
@ -614,7 +627,7 @@ class ApacheSSLContext(OSContextGenerator):
if len(vips) > 1 and is_clustered():
if not config(net_type):
log("Multiple networks configured but net_type "
"is None (%s)." % net_type, level='WARNING')
"is None (%s)." % net_type, level=WARNING)
continue
for vip in vips:
@ -627,35 +640,35 @@ class ApacheSSLContext(OSContextGenerator):
else:
addresses.append((addr, addr))
return addresses
return sorted(addresses)
def __call__(self):
if isinstance(self.external_ports, basestring):
if isinstance(self.external_ports, six.string_types):
self.external_ports = [self.external_ports]
if (not self.external_ports or not https()):
if not self.external_ports or not https():
return {}
self.configure_ca()
self.enable_modules()
ctxt = {
'namespace': self.service_namespace,
'endpoints': [],
'ext_ports': []
}
ctxt = {'namespace': self.service_namespace,
'endpoints': [],
'ext_ports': []}
for cn in self.canonical_names():
self.configure_cert(cn)
addresses = self.get_network_addresses()
for address, endpoint in set(addresses):
for address, endpoint in sorted(set(addresses)):
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port)
int_port = determine_api_port(api_port)
portmap = (address, endpoint, int(ext_port), int(int_port))
ctxt['endpoints'].append(portmap)
ctxt['ext_ports'].append(int(ext_port))
ctxt['ext_ports'] = list(set(ctxt['ext_ports']))
ctxt['ext_ports'] = sorted(list(set(ctxt['ext_ports'])))
return ctxt
@ -672,21 +685,23 @@ class NeutronContext(OSContextGenerator):
@property
def packages(self):
return neutron_plugin_attribute(
self.plugin, 'packages', self.network_manager)
return neutron_plugin_attribute(self.plugin, 'packages',
self.network_manager)
@property
def neutron_security_groups(self):
return None
def _ensure_packages(self):
[ensure_packages(pkgs) for pkgs in self.packages]
for pkgs in self.packages:
ensure_packages(pkgs)
def _save_flag_file(self):
if self.network_manager == 'quantum':
_file = '/etc/nova/quantum_plugin.conf'
else:
_file = '/etc/nova/neutron_plugin.conf'
with open(_file, 'wb') as out:
out.write(self.plugin + '\n')
@ -695,13 +710,11 @@ class NeutronContext(OSContextGenerator):
self.network_manager)
config = neutron_plugin_attribute(self.plugin, 'config',
self.network_manager)
ovs_ctxt = {
'core_plugin': driver,
'neutron_plugin': 'ovs',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config
}
ovs_ctxt = {'core_plugin': driver,
'neutron_plugin': 'ovs',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config}
return ovs_ctxt
@ -710,13 +723,11 @@ class NeutronContext(OSContextGenerator):
self.network_manager)
config = neutron_plugin_attribute(self.plugin, 'config',
self.network_manager)
nvp_ctxt = {
'core_plugin': driver,
'neutron_plugin': 'nvp',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config
}
nvp_ctxt = {'core_plugin': driver,
'neutron_plugin': 'nvp',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config}
return nvp_ctxt
@ -726,18 +737,17 @@ class NeutronContext(OSContextGenerator):
n1kv_config = neutron_plugin_attribute(self.plugin, 'config',
self.network_manager)
n1kv_user_config_flags = config('n1kv-config-flags')
n1kv_ctxt = {
'core_plugin': driver,
'neutron_plugin': 'n1kv',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': n1kv_config,
'vsm_ip': config('n1kv-vsm-ip'),
'vsm_username': config('n1kv-vsm-username'),
'vsm_password': config('n1kv-vsm-password'),
'restrict_policy_profiles': config(
'n1kv-restrict-policy-profiles'),
}
restrict_policy_profiles = config('n1kv-restrict-policy-profiles')
n1kv_ctxt = {'core_plugin': driver,
'neutron_plugin': 'n1kv',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': n1kv_config,
'vsm_ip': config('n1kv-vsm-ip'),
'vsm_username': config('n1kv-vsm-username'),
'vsm_password': config('n1kv-vsm-password'),
'restrict_policy_profiles': restrict_policy_profiles}
if n1kv_user_config_flags:
flags = config_flags_parser(n1kv_user_config_flags)
n1kv_ctxt['user_config_flags'] = flags
@ -749,13 +759,11 @@ class NeutronContext(OSContextGenerator):
self.network_manager)
config = neutron_plugin_attribute(self.plugin, 'config',
self.network_manager)
calico_ctxt = {
'core_plugin': driver,
'neutron_plugin': 'Calico',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config
}
calico_ctxt = {'core_plugin': driver,
'neutron_plugin': 'Calico',
'neutron_security_groups': self.neutron_security_groups,
'local_ip': unit_private_ip(),
'config': config}
return calico_ctxt
@ -764,15 +772,14 @@ class NeutronContext(OSContextGenerator):
proto = 'https'
else:
proto = 'http'
if is_clustered():
host = config('vip')
else:
host = unit_get('private-address')
url = '%s://%s:%s' % (proto, host, '9696')
ctxt = {
'network_manager': self.network_manager,
'neutron_url': url,
}
ctxt = {'network_manager': self.network_manager,
'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
return ctxt
def __call__(self):
@ -805,9 +812,7 @@ class NeutronContext(OSContextGenerator):
class OSConfigFlagContext(OSContextGenerator):
"""
Provides support for user-defined config flags.
"""Provides support for user-defined config flags.
Users can define a comma-seperated list of key=value pairs
in the charm configuration and apply them at any point in
@ -826,8 +831,9 @@ class OSConfigFlagContext(OSContextGenerator):
def __init__(self, charm_flag='config-flags',
template_flag='user_config_flags'):
"""
charm_flag: config flags in charm configuration.
template_flag: insert point for user-defined flags template file.
:param charm_flag: config flags in charm configuration.
:param template_flag: insert point for user-defined flags in template
file.
"""
super(OSConfigFlagContext, self).__init__()
self._charm_flag = charm_flag
@ -883,7 +889,6 @@ class SubordinateConfigContext(OSContextGenerator):
},
}
}
"""
def __init__(self, service, config_file, interface):
@ -913,26 +918,28 @@ class SubordinateConfigContext(OSContextGenerator):
if self.service not in sub_config:
log('Found subordinate_config on %s but it contained'
'nothing for %s service' % (rid, self.service))
'nothing for %s service' % (rid, self.service),
level=INFO)
continue
sub_config = sub_config[self.service]
if self.config_file not in sub_config:
log('Found subordinate_config on %s but it contained'
'nothing for %s' % (rid, self.config_file))
'nothing for %s' % (rid, self.config_file),
level=INFO)
continue
sub_config = sub_config[self.config_file]
for k, v in sub_config.iteritems():
for k, v in six.iteritems(sub_config):
if k == 'sections':
for section, config_dict in v.iteritems():
log("adding section '%s'" % (section))
for section, config_dict in six.iteritems(v):
log("adding section '%s'" % (section),
level=DEBUG)
ctxt[k][section] = config_dict
else:
ctxt[k] = v
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
return ctxt
@ -944,15 +951,14 @@ class LogLevelContext(OSContextGenerator):
False if config('debug') is None else config('debug')
ctxt['verbose'] = \
False if config('verbose') is None else config('verbose')
return ctxt
class SyslogContext(OSContextGenerator):
def __call__(self):
ctxt = {
'use_syslog': config('use-syslog')
}
ctxt = {'use_syslog': config('use-syslog')}
return ctxt
@ -960,13 +966,9 @@ class BindHostContext(OSContextGenerator):
def __call__(self):
if config('prefer-ipv6'):
return {
'bind_host': '::'
}
return {'bind_host': '::'}
else:
return {
'bind_host': '0.0.0.0'
}
return {'bind_host': '0.0.0.0'}
class WorkerConfigContext(OSContextGenerator):
@ -978,13 +980,12 @@ class WorkerConfigContext(OSContextGenerator):
except ImportError:
apt_install('python-psutil', fatal=True)
from psutil import NUM_CPUS
return NUM_CPUS
def __call__(self):
multiplier = config('worker-multiplier') or 1
ctxt = {
"workers": self.num_cpus * multiplier
}
multiplier = config('worker-multiplier') or 0
ctxt = {"workers": self.num_cpus * multiplier}
return ctxt
@ -998,22 +999,34 @@ class ZeroMQContext(OSContextGenerator):
for unit in related_units(rid):
ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
ctxt['zmq_host'] = relation_get('host', unit, rid)
return ctxt
class NotificationDriverContext(OSContextGenerator):
def __init__(self, zmq_relation='zeromq-configuration', amqp_relation='amqp'):
def __init__(self, zmq_relation='zeromq-configuration',
amqp_relation='amqp'):
"""
:param zmq_relation : Name of Zeromq relation to check
:param zmq_relation: Name of Zeromq relation to check
"""
self.zmq_relation = zmq_relation
self.amqp_relation = amqp_relation
def __call__(self):
ctxt = {
'notifications': 'False',
}
ctxt = {'notifications': 'False'}
if is_relation_made(self.amqp_relation):
ctxt['notifications'] = "True"
return ctxt
class SysctlContext(OSContextGenerator):
"""This context check if the 'sysctl' option exists on configuration
then creates a file with the loaded contents"""
def __call__(self):
sysctl_dict = config('sysctl')
if sysctl_dict:
sysctl_create(sysctl_dict,
'/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
return {'sysctl': sysctl_dict}

68
hooks/charmhelpers/contrib/openstack/ip.py
View File

@ -2,21 +2,19 @@ from charmhelpers.core.hookenv import (
config,
unit_get,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
is_address_in_network,
is_ipv6,
get_ipv6_addr,
)
from charmhelpers.contrib.hahelpers.cluster import is_clustered
PUBLIC = 'public'
INTERNAL = 'int'
ADMIN = 'admin'
_address_map = {
ADDRESS_MAP = {
PUBLIC: {
'config': 'os-public-network',
'fallback': 'public-address'
@ -33,16 +31,14 @@ _address_map = {
def canonical_url(configs, endpoint_type=PUBLIC):
'''
Returns the correct HTTP URL to this host given the state of HTTPS
"""Returns the correct HTTP URL to this host given the state of HTTPS
configuration, hacluster and charm configuration.
:configs OSTemplateRenderer: A config tempating object to inspect for
a complete https context.
:endpoint_type str: The endpoint type to resolve.
:returns str: Base URL for services on the current service unit.
'''
:param configs: OSTemplateRenderer config templating object to inspect
for a complete https context.
:param endpoint_type: str endpoint type to resolve.
:param returns: str base URL for services on the current service unit.
"""
scheme = 'http'
if 'https' in configs.complete_contexts():
scheme = 'https'
@ -53,27 +49,45 @@ def canonical_url(configs, endpoint_type=PUBLIC):
def resolve_address(endpoint_type=PUBLIC):
"""Return unit address depending on net config.
If unit is clustered with vip(s) and has net splits defined, return vip on
correct network. If clustered with no nets defined, return primary vip.
If not clustered, return unit address ensuring address is on configured net
split if one is configured.
:param endpoint_type: Network endpoing type
"""
resolved_address = None
if is_clustered():
if config(_address_map[endpoint_type]['config']) is None:
# Assume vip is simple and pass back directly
resolved_address = config('vip')
vips = config('vip')
if vips:
vips = vips.split()
net_type = ADDRESS_MAP[endpoint_type]['config']
net_addr = config(net_type)
net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
clustered = is_clustered()
if clustered:
if not net_addr:
# If no net-splits defined, we expect a single vip
resolved_address = vips[0]
else:
for vip in config('vip').split():
if is_address_in_network(
config(_address_map[endpoint_type]['config']),
vip):
for vip in vips:
if is_address_in_network(net_addr, vip):
resolved_address = vip
break
else:
if config('prefer-ipv6'):
fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0]
fallback_addr = get_ipv6_addr(exc_list=vips)[0]
else:
fallback_addr = unit_get(_address_map[endpoint_type]['fallback'])
resolved_address = get_address_in_network(
config(_address_map[endpoint_type]['config']), fallback_addr)
fallback_addr = unit_get(net_fallback)
resolved_address = get_address_in_network(net_addr, fallback_addr)
if resolved_address is None:
raise ValueError('Unable to resolve a suitable IP address'
' based on charm state and configuration')
else:
return resolved_address
raise ValueError("Unable to resolve a suitable IP address based on "
"charm state and configuration. (net_type=%s, "
"clustered=%s)" % (net_type, clustered))
return resolved_address

7
hooks/charmhelpers/contrib/openstack/neutron.py
View File

@ -14,7 +14,7 @@ from charmhelpers.contrib.openstack.utils import os_release
def headers_package():
"""Ensures correct linux-headers for running kernel are installed,
for building DKMS package"""
kver = check_output(['uname', '-r']).strip()
kver = check_output(['uname', '-r']).decode('UTF-8').strip()
return 'linux-headers-%s' % kver
QUANTUM_CONF_DIR = '/etc/quantum'
@ -22,7 +22,7 @@ QUANTUM_CONF_DIR = '/etc/quantum'
def kernel_version():
""" Retrieve the current major kernel version as a tuple e.g. (3, 13) """
kver = check_output(['uname', '-r']).strip()
kver = check_output(['uname', '-r']).decode('UTF-8').strip()
kver = kver.split('.')
return (int(kver[0]), int(kver[1]))
@ -177,7 +177,8 @@ def neutron_plugin_attribute(plugin, attr, net_manager=None):
elif manager == 'neutron':
plugins = neutron_plugins()
else:
log('Error: Network manager does not support plugins.')
log("Network manager '%s' does not support plugins." % (manager),
level=ERROR)
raise Exception
try:

4
hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
View File

@ -35,7 +35,7 @@ listen stats {{ stat_port }}
stats auth admin:password
{% if frontends -%}
{% for service, ports in service_ports.iteritems() -%}
{% for service, ports in service_ports.items() -%}
frontend tcp-in_{{ service }}
bind *:{{ ports[0] }}
bind :::{{ ports[0] }}
@ -46,7 +46,7 @@ frontend tcp-in_{{ service }}
{% for frontend in frontends -%}
backend {{ service }}_{{ frontend }}
balance leastconn
{% for unit, address in frontends[frontend]['backends'].iteritems() -%}
{% for unit, address in frontends[frontend]['backends'].items() -%}
server {{ unit }} {{ address }}:{{ ports[1] }} check
{% endfor %}
{% endfor -%}

10
hooks/charmhelpers/contrib/openstack/templating.py
View File

@ -1,13 +1,13 @@
import os
from charmhelpers.fetch import apt_install
import six
from charmhelpers.fetch import apt_install
from charmhelpers.core.hookenv import (
log,
ERROR,
INFO
)
from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
try:
@ -43,7 +43,7 @@ def get_loader(templates_dir, os_release):
order by OpenStack release.
"""
tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
for rel in OPENSTACK_CODENAMES.itervalues()]
for rel in six.itervalues(OPENSTACK_CODENAMES)]
if not os.path.isdir(templates_dir):
log('Templates directory not found @ %s.' % templates_dir,
@ -258,7 +258,7 @@ class OSConfigRenderer(object):
"""
Write out all registered config files.
"""
[self.write(k) for k in self.templates.iterkeys()]
[self.write(k) for k in six.iterkeys(self.templates)]
def set_release(self, openstack_release):
"""
@ -275,5 +275,5 @@ class OSConfigRenderer(object):
'''
interfaces = []
[interfaces.extend(i.complete_contexts())
for i in self.templates.itervalues()]
for i in six.itervalues(self.templates)]
return interfaces

135
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -10,11 +10,13 @@ import os
import socket
import sys
import six
import yaml
from charmhelpers.core.hookenv import (
config,
log as juju_log,
charm_dir,
ERROR,
INFO,
relation_ids,
relation_set
@ -31,7 +33,8 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.core.host import lsb_release, mounts, umount
from charmhelpers.fetch import apt_install, apt_cache
from charmhelpers.fetch import apt_install, apt_cache, install_remote
from charmhelpers.contrib.python.packages import pip_install
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@ -113,7 +116,7 @@ def get_os_codename_install_source(src):
# Best guess match based on deb string provided
if src.startswith('deb') or src.startswith('ppa'):
for k, v in OPENSTACK_CODENAMES.iteritems():
for k, v in six.iteritems(OPENSTACK_CODENAMES):
if v in src:
return v
@ -134,7 +137,7 @@ def get_os_codename_version(vers):
def get_os_version_codename(codename):
'''Determine OpenStack version number from codename.'''
for k, v in OPENSTACK_CODENAMES.iteritems():
for k, v in six.iteritems(OPENSTACK_CODENAMES):
if v == codename:
return k
e = 'Could not derive OpenStack version for '\
@ -194,7 +197,7 @@ def get_os_version_package(pkg, fatal=True):
else:
vers_map = OPENSTACK_CODENAMES
for version, cname in vers_map.iteritems():
for version, cname in six.iteritems(vers_map):
if cname == codename:
return version
# e = "Could not determine OpenStack version for package: %s" % pkg
@ -318,7 +321,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
rc_script.write(
"#!/bin/bash\n")
[rc_script.write('export %s=%s\n' % (u, p))
for u, p in env_vars.iteritems() if u != "script_path"]
for u, p in six.iteritems(env_vars) if u != "script_path"]
def openstack_upgrade_available(package):
@ -351,8 +354,8 @@ def ensure_block_device(block_device):
'''
_none = ['None', 'none', None]
if (block_device in _none):
error_out('prepare_storage(): Missing required input: '
'block_device=%s.' % block_device, level=ERROR)
error_out('prepare_storage(): Missing required input: block_device=%s.'
% block_device)
if block_device.startswith('/dev/'):
bdev = block_device
@ -368,8 +371,7 @@ def ensure_block_device(block_device):
bdev = '/dev/%s' % block_device
if not is_block_device(bdev):
error_out('Failed to locate valid block device at %s' % bdev,
level=ERROR)
error_out('Failed to locate valid block device at %s' % bdev)
return bdev
@ -418,7 +420,7 @@ def ns_query(address):
if isinstance(address, dns.name.Name):
rtype = 'PTR'
elif isinstance(address, basestring):
elif isinstance(address, six.string_types):
rtype = 'A'
else:
return None
@ -486,8 +488,7 @@ def sync_db_with_multi_ipv6_addresses(database, database_user,
'hostname': json.dumps(hosts)}
if relation_prefix:
keys = kwargs.keys()
for key in keys:
for key in list(kwargs.keys()):
kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key]
del kwargs[key]
@ -508,3 +509,111 @@ def os_requires_version(ostack_release, pkg):
f(*args)
return wrapped_f
return wrap
def git_install_requested():
"""Returns true if openstack-origin-git is specified."""
return config('openstack-origin-git') != "None"
requirements_dir = None
def git_clone_and_install(file_name, core_project):
"""Clone/install all OpenStack repos specified in yaml config file."""
global requirements_dir
if file_name == "None":
return
yaml_file = os.path.join(charm_dir(), file_name)
# clone/install the requirements project first
installed = _git_clone_and_install_subset(yaml_file,
whitelist=['requirements'])
if 'requirements' not in installed:
error_out('requirements git repository must be specified')
# clone/install all other projects except requirements and the core project
blacklist = ['requirements', core_project]
_git_clone_and_install_subset(yaml_file, blacklist=blacklist,
update_requirements=True)
# clone/install the core project
whitelist = [core_project]
installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
update_requirements=True)
if core_project not in installed:
error_out('{} git repository must be specified'.format(core_project))
def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
update_requirements=False):
"""Clone/install subset of OpenStack repos specified in yaml config file."""
global requirements_dir
installed = []
with open(yaml_file, 'r') as fd:
projects = yaml.load(fd)
for proj, val in projects.items():
# The project subset is chosen based on the following 3 rules:
# 1) If project is in blacklist, we don't clone/install it, period.
# 2) If whitelist is empty, we clone/install everything else.
# 3) If whitelist is not empty, we clone/install everything in the
# whitelist.
if proj in blacklist:
continue
if whitelist and proj not in whitelist:
continue
repo = val['repository']
branch = val['branch']
repo_dir = _git_clone_and_install_single(repo, branch,
update_requirements)
if proj == 'requirements':
requirements_dir = repo_dir
installed.append(proj)
return installed
def _git_clone_and_install_single(repo, branch, update_requirements=False):
"""Clone and install a single git repository."""
dest_parent_dir = "/mnt/openstack-git/"
dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
if not os.path.exists(dest_parent_dir):
juju_log('Host dir not mounted at {}. '
'Creating directory there instead.'.format(dest_parent_dir))
os.mkdir(dest_parent_dir)
if not os.path.exists(dest_dir):
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
else:
repo_dir = dest_dir
if update_requirements:
if not requirements_dir:
error_out('requirements repo must be cloned before '
'updating from global requirements.')
_git_update_requirements(repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
pip_install(repo_dir)
return repo_dir
def _git_update_requirements(package_dir, reqs_dir):
"""Update from global requirements.
Update an OpenStack git directory's requirements.txt and
test-requirements.txt from global-requirements.txt."""
orig_dir = os.getcwd()
os.chdir(reqs_dir)
cmd = "python update.py {}".format(package_dir)
try:
subprocess.check_call(cmd.split(' '))
except subprocess.CalledProcessError:
package = os.path.basename(package_dir)
error_out("Error updating {} from global-requirements.txt".format(package))
os.chdir(orig_dir)

0
hooks/charmhelpers/contrib/python/__init__.py Normal file
View File

77
hooks/charmhelpers/contrib/python/packages.py Normal file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env python
# coding: utf-8
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import log
try:
from pip import main as pip_execute
except ImportError:
apt_update()
apt_install('python-pip')
from pip import main as pip_execute
def parse_options(given, available):
"""Given a set of options, check if available"""
for key, value in sorted(given.items()):
if key in available:
yield "--{0}={1}".format(key, value)
def pip_install_requirements(requirements, **options):
"""Install a requirements file """
command = ["install"]
available_options = ('proxy', 'src', 'log', )
for option in parse_options(options, available_options):
command.append(option)
command.append("-r {0}".format(requirements))
log("Installing from file: {} with options: {}".format(requirements,
command))
pip_execute(command)
def pip_install(package, fatal=False, **options):
"""Install a python package"""
command = ["install"]
available_options = ('proxy', 'src', 'log', "index-url", )
for option in parse_options(options, available_options):
command.append(option)
if isinstance(package, list):
command.extend(package)
else:
command.append(package)
log("Installing {} package with options: {}".format(package,
command))
pip_execute(command)
def pip_uninstall(package, **options):
"""Uninstall a python package"""
command = ["uninstall", "-q", "-y"]
available_options = ('proxy', 'log', )
for option in parse_options(options, available_options):
command.append(option)
if isinstance(package, list):
command.extend(package)
else:
command.append(package)
log("Uninstalling {} package with options: {}".format(package,
command))
pip_execute(command)
def pip_list():
"""Returns the list of current python installed packages
"""
return pip_execute(["list"])

58
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -65,7 +65,8 @@ def install():
def rbd_exists(service, pool, rbd_img):
"""Check to see if a RADOS block device exists."""
try:
out = check_output(['rbd', 'list', '--id', service, '--pool', pool])
out = check_output(['rbd', 'list', '--id',
service, '--pool', pool]).decode('UTF-8')
except CalledProcessError:
return False
@ -82,7 +83,8 @@ def create_rbd_image(service, pool, image, sizemb):
def pool_exists(service, name):
"""Check to see if a RADOS pool already exists."""
try:
out = check_output(['rados', '--id', service, 'lspools'])
out = check_output(['rados', '--id', service,
'lspools']).decode('UTF-8')
except CalledProcessError:
return False
@ -96,7 +98,8 @@ def get_osds(service):
version = ceph_version()
if version and version >= '0.56':
return json.loads(check_output(['ceph', '--id', service,
'osd', 'ls', '--format=json']))
'osd', 'ls',
'--format=json']).decode('UTF-8'))
return None
@ -112,7 +115,7 @@ def create_pool(service, name, replicas=3):
# on upstream recommended best practices.
osds = get_osds(service)
if osds:
pgnum = (len(osds) * 100 / replicas)
pgnum = (len(osds) * 100 // replicas)
else:
# NOTE(james-page): Default to 200 for older ceph versions
# which don't support OSD query from cli
@ -193,7 +196,7 @@ def configure(service, key, auth, use_syslog):
def image_mapped(name):
"""Determine whether a RADOS block device is mapped locally."""
try:
out = check_output(['rbd', 'showmapped'])
out = check_output(['rbd', 'showmapped']).decode('UTF-8')
except CalledProcessError:
return False
@ -361,7 +364,7 @@ def ceph_version():
"""Retrieve the local version of ceph."""
if os.path.exists('/usr/bin/ceph'):
cmd = ['ceph', '-v']
output = check_output(cmd)
output = check_output(cmd).decode('US-ASCII')
output = output.split()
if len(output) > 3:
return output[2]
@ -369,46 +372,3 @@ def ceph_version():
return None
else:
return None
class CephBrokerRq(object):
"""Ceph broker request.
Multiple operations can be added to a request and sent to the Ceph broker
to be executed.
Request is json-encoded for sending over the wire.
The API is versioned and defaults to version 1.
"""
def __init__(self, api_version=1):
self.api_version = api_version
self.ops = []
def add_op_create_pool(self, name, replica_count=3):
self.ops.append({'op': 'create-pool', 'name': name,
'replicas': replica_count})
@property
def request(self):
return json.dumps({'api-version': self.api_version, 'ops': self.ops})
class CephBrokerRsp(object):
"""Ceph broker response.
Response is json-decoded and contents provided as methods/properties.
The API is versioned and defaults to version 1.
"""
def __init__(self, encoded_rsp):
self.api_version = None
self.rsp = json.loads(encoded_rsp)
@property
def exit_code(self):
return self.rsp.get('exit-code')
@property
def exit_msg(self):
return self.rsp.get('stderr')

8
hooks/charmhelpers/contrib/storage/linux/loopback.py
View File

@ -1,12 +1,12 @@
import os
import re
from subprocess import (
check_call,
check_output,
)
import six
##################################################
# loopback device helpers.
@ -37,7 +37,7 @@ def create_loopback(file_path):
'''
file_path = os.path.abspath(file_path)
check_call(['losetup', '--find', file_path])
for d, f in loopback_devices().iteritems():
for d, f in six.iteritems(loopback_devices()):
if f == file_path:
return d
@ -51,7 +51,7 @@ def ensure_loopback_device(path, size):
:returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
'''
for d, f in loopback_devices().iteritems():
for d, f in six.iteritems(loopback_devices()):
if f == path:
return d

1
hooks/charmhelpers/contrib/storage/linux/lvm.py
View File

@ -61,6 +61,7 @@ def list_lvm_volume_group(block_device):
vg = None
pvd = check_output(['pvdisplay', block_device]).splitlines()
for l in pvd:
l = l.decode('UTF-8')
if l.strip().startswith('VG Name'):
vg = ' '.join(l.strip().split()[2:])
return vg

5
hooks/charmhelpers/contrib/storage/linux/utils.py
View File

@ -30,7 +30,8 @@ def zap_disk(block_device):
# sometimes sgdisk exits non-zero; this is OK, dd will clean up
call(['sgdisk', '--zap-all', '--mbrtogpt',
'--clear', block_device])
dev_end = check_output(['blockdev', '--getsz', block_device])
dev_end = check_output(['blockdev', '--getsz',
block_device]).decode('UTF-8')
gpt_end = int(dev_end.split()[0]) - 100
check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
'bs=1M', 'count=1'])
@ -47,7 +48,7 @@ def is_device_mounted(device):
it doesn't.
'''
is_partition = bool(re.search(r".*[0-9]+\b", device))
out = check_output(['mount'])
out = check_output(['mount']).decode('UTF-8')
if is_partition:
return bool(re.search(device + r"\b", out))
return bool(re.search(device + r"[0-9]+\b", out))

18
hooks/charmhelpers/core/fstab.py
View File

@ -3,10 +3,11 @@
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
import io
import os
class Fstab(file):
class Fstab(io.FileIO):
"""This class extends file in order to implement a file reader/writer
for file `/etc/fstab`
"""
@ -24,8 +25,8 @@ class Fstab(file):
options = "defaults"
self.options = options
self.d = d
self.p = p
self.d = int(d)
self.p = int(p)
def __eq__(self, o):
return str(self) == str(o)
@ -45,7 +46,7 @@ class Fstab(file):
self._path = path
else:
self._path = self.DEFAULT_PATH
file.__init__(self, self._path, 'r+')
super(Fstab, self).__init__(self._path, 'rb+')
def _hydrate_entry(self, line):
# NOTE: use split with no arguments to split on any
@ -58,8 +59,9 @@ class Fstab(file):
def entries(self):
self.seek(0)
for line in self.readlines():
line = line.decode('us-ascii')
try:
if not line.startswith("#"):
if line.strip() and not line.startswith("#"):
yield self._hydrate_entry(line)
except ValueError:
pass
@ -75,14 +77,14 @@ class Fstab(file):
if self.get_entry_by_attr('device', entry.device):
return False
self.write(str(entry) + '\n')
self.write((str(entry) + '\n').encode('us-ascii'))
self.truncate()
return entry
def remove_entry(self, entry):
self.seek(0)
lines = self.readlines()
lines = [l.decode('us-ascii') for l in self.readlines()]
found = False
for index, line in enumerate(lines):
@ -97,7 +99,7 @@ class Fstab(file):
lines.remove(line)
self.seek(0)
self.write(''.join(lines))
self.write(''.join(lines).encode('us-ascii'))
self.truncate()
return True

52
hooks/charmhelpers/core/hookenv.py
View File

@ -9,9 +9,14 @@ import json
import yaml
import subprocess
import sys
import UserDict
from subprocess import CalledProcessError
import six
if not six.PY3:
from UserDict import UserDict
else:
from collections import UserDict
CRITICAL = "CRITICAL"
ERROR = "ERROR"
WARNING = "WARNING"
@ -63,16 +68,18 @@ def log(message, level=None):
command = ['juju-log']
if level:
command += ['-l', level]
if not isinstance(message, six.string_types):
message = repr(message)
command += [message]
subprocess.call(command)
class Serializable(UserDict.IterableUserDict):
class Serializable(UserDict):
"""Wrapper, an object that can be serialized to yaml or json"""
def __init__(self, obj):
# wrap the object
UserDict.IterableUserDict.__init__(self)
UserDict.__init__(self)
self.data = obj
def __getattr__(self, attr):
@ -218,7 +225,7 @@ class Config(dict):
prev_keys = []
if self._prev_dict is not None:
prev_keys = self._prev_dict.keys()
return list(set(prev_keys + dict.keys(self)))
return list(set(prev_keys + list(dict.keys(self))))
def load_previous(self, path=None):
"""Load previous copy of config from disk.
@ -269,7 +276,7 @@ class Config(dict):
"""
if self._prev_dict:
for k, v in self._prev_dict.iteritems():
for k, v in six.iteritems(self._prev_dict):
if k not in self:
self[k] = v
with open(self.path, 'w') as f:
@ -284,7 +291,8 @@ def config(scope=None):
config_cmd_line.append(scope)
config_cmd_line.append('--format=json')
try:
config_data = json.loads(subprocess.check_output(config_cmd_line))
config_data = json.loads(
subprocess.check_output(config_cmd_line).decode('UTF-8'))
if scope is not None:
return config_data
return Config(config_data)
@ -303,10 +311,10 @@ def relation_get(attribute=None, unit=None, rid=None):
if unit:
_args.append(unit)
try:
return json.loads(subprocess.check_output(_args))
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
except ValueError:
return None
except CalledProcessError, e:
except CalledProcessError as e:
if e.returncode == 2:
return None
raise
@ -318,7 +326,7 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
relation_cmd_line = ['relation-set']
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
for k, v in (relation_settings.items() + kwargs.items()):
for k, v in (list(relation_settings.items()) + list(kwargs.items())):
if v is None:
relation_cmd_line.append('{}='.format(k))
else:
@ -335,7 +343,8 @@ def relation_ids(reltype=None):
relid_cmd_line = ['relation-ids', '--format=json']
if reltype is not None:
relid_cmd_line.append(reltype)
return json.loads(subprocess.check_output(relid_cmd_line)) or []
return json.loads(
subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
return []
@ -346,7 +355,8 @@ def related_units(relid=None):
units_cmd_line = ['relation-list', '--format=json']
if relid is not None:
units_cmd_line.extend(('-r', relid))
return json.loads(subprocess.check_output(units_cmd_line)) or []
return json.loads(
subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
@cached
@ -385,21 +395,31 @@ def relations_of_type(reltype=None):
return relation_data
@cached
def metadata():
"""Get the current charm metadata.yaml contents as a python object"""
with open(os.path.join(charm_dir(), 'metadata.yaml')) as md:
return yaml.safe_load(md)
@cached
def relation_types():
"""Get a list of relation types supported by this charm"""
charmdir = os.environ.get('CHARM_DIR', '')
mdf = open(os.path.join(charmdir, 'metadata.yaml'))
md = yaml.safe_load(mdf)
rel_types = []
md = metadata()
for key in ('provides', 'requires', 'peers'):
section = md.get(key)
if section:
rel_types.extend(section.keys())
mdf.close()
return rel_types
@cached
def charm_name():
"""Get the name of the current charm as is specified on metadata.yaml"""
return metadata().get('name')
@cached
def relations():
"""Get a nested dictionary of relation data for all related units"""
@ -455,7 +475,7 @@ def unit_get(attribute):
"""Get the unit ID for the remote unit"""
_args = ['unit-get', '--format=json', attribute]
try:
return json.loads(subprocess.check_output(_args))
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
except ValueError:
return None

61
hooks/charmhelpers/core/host.py
View File

@ -14,11 +14,12 @@ import string
import subprocess
import hashlib
from contextlib import contextmanager
from collections import OrderedDict
from hookenv import log
from fstab import Fstab
import six
from .hookenv import log
from .fstab import Fstab
def service_start(service_name):
@ -54,7 +55,9 @@ def service(action, service_name):
def service_running(service):
"""Determine whether a system service is running"""
try:
output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
output = subprocess.check_output(
['service', service, 'status'],
stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError:
return False
else:
@ -67,7 +70,9 @@ def service_running(service):
def service_available(service_name):
"""Determine whether a system service is available"""
try:
subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
subprocess.check_output(
['service', service_name, 'status'],
stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError as e:
return 'unrecognized service' not in e.output
else:
@ -96,6 +101,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
return user_info
def add_group(group_name, system_group=False):
"""Add a group to the system"""
try:
group_info = grp.getgrnam(group_name)
log('group {0} already exists!'.format(group_name))
except KeyError:
log('creating group {0}'.format(group_name))
cmd = ['addgroup']
if system_group:
cmd.append('--system')
else:
cmd.extend([
'--group',
])
cmd.append(group_name)
subprocess.check_call(cmd)
group_info = grp.getgrnam(group_name)
return group_info
def add_user_to_group(username, group):
"""Add a user to a group"""
cmd = [
@ -115,7 +140,7 @@ def rsync(from_path, to_path, flags='-r', options=None):
cmd.append(from_path)
cmd.append(to_path)
log(" ".join(cmd))
return subprocess.check_output(cmd).strip()
return subprocess.check_output(cmd).decode('UTF-8').strip()
def symlink(source, destination):
@ -130,7 +155,7 @@ def symlink(source, destination):
subprocess.check_call(cmd)
def mkdir(path, owner='root', group='root', perms=0555, force=False):
def mkdir(path, owner='root', group='root', perms=0o555, force=False):
"""Create a directory"""
log("Making dir {} {}:{} {:o}".format(path, owner, group,
perms))
@ -146,7 +171,7 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False):
os.chown(realpath, uid, gid)
def write_file(path, content, owner='root', group='root', perms=0444):
def write_file(path, content, owner='root', group='root', perms=0o444):
"""Create or overwrite a file with the contents of a string"""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
@ -177,7 +202,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
cmd_args.extend([device, mountpoint])
try:
subprocess.check_output(cmd_args)
except subprocess.CalledProcessError, e:
except subprocess.CalledProcessError as e:
log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
return False
@ -191,7 +216,7 @@ def umount(mountpoint, persist=False):
cmd_args = ['umount', mountpoint]
try:
subprocess.check_output(cmd_args)
except subprocess.CalledProcessError, e:
except subprocess.CalledProcessError as e:
log('Error unmounting {}\n{}'.format(mountpoint, e.output))
return False
@ -218,8 +243,8 @@ def file_hash(path, hash_type='md5'):
"""
if os.path.exists(path):
h = getattr(hashlib, hash_type)()
with open(path, 'r') as source:
h.update(source.read()) # IGNORE:E1101 - it does have update
with open(path, 'rb') as source:
h.update(source.read())
return h.hexdigest()
else:
return None
@ -297,7 +322,7 @@ def pwgen(length=None):
if length is None:
length = random.choice(range(35, 45))
alphanumeric_chars = [
l for l in (string.letters + string.digits)
l for l in (string.ascii_letters + string.digits)
if l not in 'l0QD1vAEIOUaeiou']
random_chars = [
random.choice(alphanumeric_chars) for _ in range(length)]
@ -306,14 +331,14 @@ def pwgen(length=None):
def list_nics(nic_type):
'''Return a list of nics of given type(s)'''
if isinstance(nic_type, basestring):
if isinstance(nic_type, six.string_types):
int_types = [nic_type]
else:
int_types = nic_type
interfaces = []
for int_type in int_types:
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
ip_output = subprocess.check_output(cmd).split('\n')
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
@ -335,7 +360,7 @@ def set_nic_mtu(nic, mtu):
def get_nic_mtu(nic):
cmd = ['ip', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).split('\n')
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
mtu = ""
for line in ip_output:
words = line.split()
@ -346,7 +371,7 @@ def get_nic_mtu(nic):
def get_nic_hwaddr(nic):
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd)
ip_output = subprocess.check_output(cmd).decode('UTF-8')
hwaddr = ""
words = ip_output.split()
if 'link/ether' in words:
@ -363,8 +388,8 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
'''
import apt_pkg
from charmhelpers.fetch import apt_cache
if not pkgcache:
from charmhelpers.fetch import apt_cache
pkgcache = apt_cache()
pkg = pkgcache[package]
return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

12
hooks/charmhelpers/core/services/helpers.py
View File

@ -196,7 +196,7 @@ class StoredContext(dict):
if not os.path.isabs(file_name):
file_name = os.path.join(hookenv.charm_dir(), file_name)
with open(file_name, 'w') as file_stream:
os.fchmod(file_stream.fileno(), 0600)
os.fchmod(file_stream.fileno(), 0o600)
yaml.dump(config_data, file_stream)
def read_context(self, file_name):
@ -211,15 +211,19 @@ class StoredContext(dict):
class TemplateCallback(ManagerCallback):
"""
Callback class that will render a Jinja2 template, for use as a ready action.
Callback class that will render a Jinja2 template, for use as a ready
action.
:param str source: The template source file, relative to
`$CHARM_DIR/templates`
:param str source: The template source file, relative to `$CHARM_DIR/templates`
:param str target: The target to write the rendered template to
:param str owner: The owner of the rendered file
:param str group: The group of the rendered file
:param int perms: The permissions of the rendered file
"""
def __init__(self, source, target, owner='root', group='root', perms=0444):
def __init__(self, source, target,
owner='root', group='root', perms=0o444):
self.source = source
self.target = target
self.owner = owner

5
hooks/charmhelpers/core/templating.py
View File

@ -4,7 +4,8 @@ from charmhelpers.core import host
from charmhelpers.core import hookenv
def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
def render(source, target, context, owner='root', group='root',
perms=0o444, templates_dir=None):
"""
Render a template.
@ -47,5 +48,5 @@ def render(source, target, context, owner='root', group='root', perms=0444, temp
level=hookenv.ERROR)
raise e
content = template.render(context)
host.mkdir(os.path.dirname(target))
host.mkdir(os.path.dirname(target), owner, group)
host.write_file(target, content, owner, group, perms)

24
hooks/charmhelpers/fetch/__init__.py
View File

@ -5,10 +5,6 @@ from yaml import safe_load
from charmhelpers.core.host import (
lsb_release
)
from urlparse import (
urlparse,
urlunparse,
)
import subprocess
from charmhelpers.core.hookenv import (
config,
@ -16,6 +12,12 @@ from charmhelpers.core.hookenv import (
)
import os
import six
if six.PY3:
from urllib.parse import urlparse, urlunparse
else:
from urlparse import urlparse, urlunparse
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@ -149,7 +151,7 @@ def apt_install(packages, options=None, fatal=False):
cmd = ['apt-get', '--assume-yes']
cmd.extend(options)
cmd.append('install')
if isinstance(packages, basestring):
if isinstance(packages, six.string_types):
cmd.append(packages)
else:
cmd.extend(packages)
@ -182,7 +184,7 @@ def apt_update(fatal=False):
def apt_purge(packages, fatal=False):
"""Purge one or more packages"""
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, basestring):
if isinstance(packages, six.string_types):
cmd.append(packages)
else:
cmd.extend(packages)
@ -193,7 +195,7 @@ def apt_purge(packages, fatal=False):
def apt_hold(packages, fatal=False):
"""Hold one or more packages"""
cmd = ['apt-mark', 'hold']
if isinstance(packages, basestring):
if isinstance(packages, six.string_types):
cmd.append(packages)
else:
cmd.extend(packages)
@ -260,7 +262,7 @@ def add_source(source, key=None):
if key:
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
with NamedTemporaryFile() as key_file:
with NamedTemporaryFile('w+') as key_file:
key_file.write(key)
key_file.flush()
key_file.seek(0)
@ -297,14 +299,14 @@ def configure_sources(update=False,
sources = safe_load((config(sources_var) or '').strip()) or []
keys = safe_load((config(keys_var) or '').strip()) or None
if isinstance(sources, basestring):
if isinstance(sources, six.string_types):
sources = [sources]
if keys is None:
for source in sources:
add_source(source, None)
else:
if isinstance(keys, basestring):
if isinstance(keys, six.string_types):
keys = [keys]
if len(sources) != len(keys):
@ -401,7 +403,7 @@ def _run_apt_command(cmd, fatal=False):
while result is None or result == APT_NO_LOCK:
try:
result = subprocess.check_call(cmd, env=env)
except subprocess.CalledProcessError, e:
except subprocess.CalledProcessError as e:
retry_count = retry_count + 1
if retry_count > APT_NO_LOCK_RETRY_COUNT:
raise

69
hooks/charmhelpers/fetch/archiveurl.py
View File

@ -1,8 +1,23 @@
import os
import urllib2
from urllib import urlretrieve
import urlparse
import hashlib
import re
import six
if six.PY3:
from urllib.request import (
build_opener, install_opener, urlopen, urlretrieve,
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
)
from urllib.parse import urlparse, urlunparse, parse_qs
from urllib.error import URLError
else:
from urllib import urlretrieve
from urllib2 import (
build_opener, install_opener, urlopen,
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
URLError
)
from urlparse import urlparse, urlunparse, parse_qs
from charmhelpers.fetch import (
BaseFetchHandler,
@ -15,6 +30,24 @@ from charmhelpers.payload.archive import (
from charmhelpers.core.host import mkdir, check_hash
def splituser(host):
'''urllib.splituser(), but six's support of this seems broken'''
_userprog = re.compile('^(.*)@(.*)$')
match = _userprog.match(host)
if match:
return match.group(1, 2)
return None, host
def splitpasswd(user):
'''urllib.splitpasswd(), but six's support of this is missing'''
_passwdprog = re.compile('^([^:]*):(.*)$', re.S)
match = _passwdprog.match(user)
if match:
return match.group(1, 2)
return user, None
class ArchiveUrlFetchHandler(BaseFetchHandler):
"""
Handler to download archive files from arbitrary URLs.
@ -42,20 +75,20 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
"""
# propogate all exceptions
# URLError, OSError, etc
proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
proto, netloc, path, params, query, fragment = urlparse(source)
if proto in ('http', 'https'):
auth, barehost = urllib2.splituser(netloc)
auth, barehost = splituser(netloc)
if auth is not None:
source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
username, password = urllib2.splitpasswd(auth)
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
source = urlunparse((proto, barehost, path, params, query, fragment))
username, password = splitpasswd(auth)
passman = HTTPPasswordMgrWithDefaultRealm()
# Realm is set to None in add_password to force the username and password
# to be used whatever the realm
passman.add_password(None, source, username, password)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
urllib2.install_opener(opener)
response = urllib2.urlopen(source)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
response = urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
@ -91,17 +124,21 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
url_parts = self.parse_url(source)
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
mkdir(dest_dir, perms=0o755)
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
try:
self.download(source, dld_file)
except urllib2.URLError as e:
except URLError as e:
raise UnhandledSource(e.reason)
except OSError as e:
raise UnhandledSource(e.strerror)
options = urlparse.parse_qs(url_parts.fragment)
options = parse_qs(url_parts.fragment)
for key, value in options.items():
if key in hashlib.algorithms:
if not six.PY3:
algorithms = hashlib.algorithms
else:
algorithms = hashlib.algorithms_available
if key in algorithms:
check_hash(dld_file, value, key)
if checksum:
check_hash(dld_file, checksum, hash_type)

6
hooks/charmhelpers/fetch/bzrurl.py
View File

@ -5,6 +5,10 @@ from charmhelpers.fetch import (
)
from charmhelpers.core.host import mkdir
import six
if six.PY3:
raise ImportError('bzrlib does not support Python3')
try:
from bzrlib.branch import Branch
except ImportError:
@ -42,7 +46,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
branch_name)
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
mkdir(dest_dir, perms=0o755)
try:
self.branch(source, dest_dir)
except OSError as e:

17
hooks/charmhelpers/fetch/giturl.py
View File

@ -5,6 +5,10 @@ from charmhelpers.fetch import (
)
from charmhelpers.core.host import mkdir
import six
if six.PY3:
raise ImportError('GitPython does not support Python 3')
try:
from git import Repo
except ImportError:
@ -17,7 +21,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
"""Handler for git branches via generic and github URLs"""
def can_handle(self, source):
url_parts = self.parse_url(source)
#TODO (mattyw) no support for ssh git@ yet
# TODO (mattyw) no support for ssh git@ yet
if url_parts.scheme not in ('http', 'https', 'git'):
return False
else:
@ -30,13 +34,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
repo = Repo.clone_from(source, dest)
repo.git.checkout(branch)
def install(self, source, branch="master"):
def install(self, source, branch="master", dest=None):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
branch_name)
if dest:
dest_dir = os.path.join(dest, branch_name)
else:
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
branch_name)
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
mkdir(dest_dir, perms=0o755)
try:
self.clone(source, dest_dir, branch)
except OSError as e:

6
hooks/cinder_contexts.py
View File

@ -64,8 +64,10 @@ class HAProxyContext(OSContextGenerator):
Also used to extend cinder.conf context with correct api_listening_port
'''
haproxy_port = config('api-listening-port')
api_port = determine_api_port(config('api-listening-port'))
apache_port = determine_apache_port(config('api-listening-port'))
api_port = determine_api_port(config('api-listening-port'),
singlenode_mode=True)
apache_port = determine_apache_port(config('api-listening-port'),
singlenode_mode=True)
ctxt = {
'service_ports': {'cinder_api': [haproxy_port, apache_port]},

26
hooks/cinder_utils.py
View File

@ -140,7 +140,7 @@ CONFIG_FILES = OrderedDict([
'services': ['cinder-volume']
}),
(HAPROXY_CONF, {
'hook_contexts': [context.HAProxyContext(),
'hook_contexts': [context.HAProxyContext(singlenode_mode=True),
cinder_contexts.HAProxyContext()],
'services': ['haproxy'],
}),
@ -310,14 +310,17 @@ def configure_lvm_storage(block_devices, volume_group, overwrite=False,
vg_found = False
new_devices = []
for device in devices:
if (not is_lvm_physical_volume(device) or
(is_lvm_physical_volume(device) and
list_lvm_volume_group(device) != volume_group)):
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
clean_storage(device)
prepare_volume(device)
new_devices.append(device)
create_lvm_physical_volume(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) == volume_group):
# Mark vg as found
@ -338,6 +341,17 @@ def configure_lvm_storage(block_devices, volume_group, overwrite=False,
extend_lvm_volume_group(volume_group, new_device)
def prepare_volume(device):
clean_storage(device)
create_lvm_physical_volume(device)
def has_partition_table(block_device):
out = subprocess.check_output(['fdisk', '-l', block_device],
stderr=subprocess.STDOUT)
return "doesn't contain a valid partition" not in out
def clean_storage(block_device):
'''Ensures a block device is clean. That is:
- unmounted

22
tests/charmhelpers/__init__.py
View File

@ -0,0 +1,22 @@
# Bootstrap charm-helpers, installing its dependencies if necessary using
# only standard libraries.
import subprocess
import sys
try:
import six # flake8: noqa
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
import six # flake8: noqa
try:
import yaml # flake8: noqa
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa

6
tests/charmhelpers/contrib/amulet/deployment.py
View File

@ -1,6 +1,6 @@
import amulet
import os
import six
class AmuletDeployment(object):
@ -52,12 +52,12 @@ class AmuletDeployment(object):
def _add_relations(self, relations):
"""Add all of the relations for the services."""
for k, v in relations.iteritems():
for k, v in six.iteritems(relations):
self.d.relate(k, v)
def _configure_services(self, configs):
"""Configure all of the services."""
for service, config in configs.iteritems():
for service, config in six.iteritems(configs):
self.d.configure(service, config)
def _deploy(self):

10
tests/charmhelpers/contrib/amulet/utils.py
View File

@ -5,6 +5,8 @@ import re
import sys
import time
import six
class AmuletUtils(object):
"""Amulet utilities.
@ -58,7 +60,7 @@ class AmuletUtils(object):
Verify the specified services are running on the corresponding
service units.
"""
for k, v in commands.iteritems():
for k, v in six.iteritems(commands):
for cmd in v:
output, code = k.run(cmd)
if code != 0:
@ -100,11 +102,11 @@ class AmuletUtils(object):
longs, or can be a function that evaluate a variable and returns a
bool.
"""
for k, v in expected.iteritems():
for k, v in six.iteritems(expected):
if k in actual:
if (isinstance(v, basestring) or
if (isinstance(v, six.string_types) or
isinstance(v, bool) or
isinstance(v, (int, long))):
isinstance(v, six.integer_types)):
if v != actual[k]:
return "{}:{}".format(k, actual[k])
elif not v(actual[k]):

3
tests/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -1,3 +1,4 @@
import six
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@ -69,7 +70,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
def _configure_services(self, configs):
"""Configure all of the services."""
for service, config in configs.iteritems():
for service, config in six.iteritems(configs):
self.d.configure(service, config)
def _get_openstack_release(self):

4
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -7,6 +7,8 @@ import glanceclient.v1.client as glance_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
import six
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -60,7 +62,7 @@ class OpenStackAmuletUtils(AmuletUtils):
expected service catalog endpoints.
"""
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in expected.iteritems():
for k, v in six.iteritems(expected):
if k in actual:
ret = self._validate_dict_data(expected[k][0], actual[k][0])
if ret:

50
unit_tests/test_cinder_utils.py
View File

@ -56,6 +56,18 @@ DPKG_OPTIONS = [
'--option', 'Dpkg::Options::=--force-confdef',
]
FDISKDISPLAY = """
Disk /dev/vdb doesn't contain a valid partition table
Disk /dev/vdb: 21.5 GB, 21474836480 bytes
16 heads, 63 sectors/track, 41610 cylinders, total 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
"""
class TestCinderUtils(CharmTestCase):
@ -208,6 +220,13 @@ class TestCinderUtils(CharmTestCase):
self.assertTrue(cinder_utils._parse_block_device('/mnt/loop0'),
('/mnt/loop0', cinder_utils.DEFAULT_LOOPBACK_SIZE))
@patch('subprocess.check_output')
def test_has_partition_table(self, _check):
_check.return_value = FDISKDISPLAY
block_device = '/dev/vdb'
cinder_utils.has_partition_table(block_device)
_check.assert_called_with(['fdisk', '-l', '/dev/vdb'], stderr=-2)
@patch.object(cinder_utils, 'clean_storage')
@patch.object(cinder_utils, 'reduce_lvm_volume_group_missing')
@patch.object(cinder_utils, 'extend_lvm_volume_group')
@ -228,6 +247,37 @@ class TestCinderUtils(CharmTestCase):
reduce_lvm.assert_called_with('test')
extend_lvm.assert_called_with('test', '/dev/vdc')
@patch.object(cinder_utils, 'has_partition_table')
@patch.object(cinder_utils, 'clean_storage')
@patch.object(cinder_utils, 'reduce_lvm_volume_group_missing')
@patch.object(cinder_utils, 'extend_lvm_volume_group')
def test_configure_lvm_storage_unused_dev(self, extend_lvm, reduce_lvm,
clean_storage, has_part):
devices = ['/dev/vdb', '/dev/vdc']
self.is_lvm_physical_volume.return_value = False
has_part.return_value = False
cinder_utils.configure_lvm_storage(devices, 'test', False, True)
clean_storage.assert_has_calls(
[call('/dev/vdb'),
call('/dev/vdc')]
)
self.create_lvm_physical_volume.assert_has_calls(
[call('/dev/vdb'),
call('/dev/vdc')]
)
self.create_lvm_volume_group.assert_called_with('test', '/dev/vdb')
reduce_lvm.assert_called_with('test')
extend_lvm.assert_called_with('test', '/dev/vdc')
@patch.object(cinder_utils, 'has_partition_table')
@patch.object(cinder_utils, 'reduce_lvm_volume_group_missing')
def test_configure_lvm_storage_used_dev(self, reduce_lvm, has_part):
devices = ['/dev/vdb', '/dev/vdc']
self.is_lvm_physical_volume.return_value = False
has_part.return_value = True
cinder_utils.configure_lvm_storage(devices, 'test', False, True)
reduce_lvm.assert_called_with('test')
@patch.object(cinder_utils, 'clean_storage')
@patch.object(cinder_utils, 'reduce_lvm_volume_group_missing')
@patch.object(cinder_utils, 'extend_lvm_volume_group')