openstack/charm-swift-proxy
Code Issues Proposed changes

Resync le charm helpers

Browse Source
This commit is contained in:
Liam Young
2015-06-04 09:44:50 +01:00
parent 74249fc84e
commit 35f946ae07
10 changed files with 264 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
charm-helpers-hooks.yaml
View File

@@ -1,4 +1,4 @@
branch: lp:~le-charmers/charm-helpers/leadership-election
branch: lp:charm-helpers
destination: hooks/charmhelpers
include:
- core

2
charm-helpers-tests.yaml
View File

@@ -1,4 +1,4 @@
branch: lp:~le-charmers/charm-helpers/leadership-election
branch: lp:charm-helpers
destination: tests/charmhelpers
include:
- contrib.amulet

25
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@@ -53,6 +53,8 @@ from charmhelpers.core.strutils import (
bool_from_string,
)
DC_RESOURCE_NAME = 'DC'
class HAIncompleteConfig(Exception):
pass
@@ -105,6 +107,27 @@ def is_clustered():
return False
def is_crm_dc():
"""
Determine leadership by querying the pacemaker Designated Controller
"""
cmd = ['crm', 'status']
try:
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if not isinstance(status, six.text_type):
status = six.text_type(status, "utf-8")
except subprocess.CalledProcessError:
return False
current_dc = ''
for line in status.split('\n'):
if line.startswith('Current DC'):
# Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
current_dc = line.split(':')[1].split()[0]
if current_dc == get_unit_hostname():
return True
return False
@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
def is_crm_leader(resource, retry=False):
"""
@@ -114,6 +137,8 @@ def is_crm_leader(resource, retry=False):
We allow this operation to be retried to avoid the possibility of getting a
false negative. See LP #1396246 for more info.
"""
if resource == DC_RESOURCE_NAME:
return is_crm_dc()
cmd = ['crm', 'resource', 'show', resource]
try:
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)

15
hooks/charmhelpers/contrib/openstack/neutron.py
View File

@@ -256,11 +256,14 @@ def network_manager():
def parse_mappings(mappings):
parsed = {}
if mappings:
mappings = mappings.split(' ')
mappings = mappings.split()
for m in mappings:
p = m.partition(':')
if p[1] == ':':
parsed[p[0].strip()] = p[2].strip()
key = p[0].strip()
if p[1]:
parsed[key] = p[2].strip()
else:
parsed[key] = ''
return parsed
@@ -283,13 +286,13 @@ def parse_data_port_mappings(mappings, default_bridge='br-data'):
Returns dict of the form {bridge:port}.
"""
_mappings = parse_mappings(mappings)
if not _mappings:
if not _mappings or list(_mappings.values()) == ['']:
if not mappings:
return {}
# For backwards-compatibility we need to support port-only provided in
# config.
_mappings = {default_bridge: mappings.split(' ')[0]}
_mappings = {default_bridge: mappings.split()[0]}
bridges = _mappings.keys()
ports = _mappings.values()
@@ -309,6 +312,8 @@ def parse_vlan_range_mappings(mappings):
Mappings must be a space-delimited list of provider:start:end mappings.
The start:end range is optional and may be omitted.
Returns dict of the form {provider: (start, end)}.
"""
_mappings = parse_mappings(mappings)

81
hooks/charmhelpers/contrib/openstack/utils.py
View File

@@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr
)
from charmhelpers.contrib.python.packages import (
pip_create_virtualenv,
pip_install,
)
from charmhelpers.core.host import lsb_release, mounts, umount
from charmhelpers.fetch import apt_install, apt_cache, install_remote
from charmhelpers.contrib.python.packages import pip_install
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@@ -497,7 +501,17 @@ def git_install_requested():
requirements_dir = None
def git_clone_and_install(projects_yaml, core_project):
def _git_yaml_load(projects_yaml):
"""
Load the specified yaml into a dictionary.
"""
if not projects_yaml:
return None
return yaml.load(projects_yaml)
def git_clone_and_install(projects_yaml, core_project, depth=1):
"""
Clone/install all specified OpenStack repositories.
@@ -510,23 +524,22 @@ def git_clone_and_install(projects_yaml, core_project):
repository: 'git://git.openstack.org/openstack/requirements.git',
branch: 'stable/icehouse'}
directory: /mnt/openstack-git
http_proxy: http://squid.internal:3128
https_proxy: https://squid.internal:3128
http_proxy: squid-proxy-url
https_proxy: squid-proxy-url
The directory, http_proxy, and https_proxy keys are optional.
"""
global requirements_dir
parent_dir = '/mnt/openstack-git'
http_proxy = None
if not projects_yaml:
return
projects = yaml.load(projects_yaml)
projects = _git_yaml_load(projects_yaml)
_git_validate_projects_yaml(projects, core_project)
old_environ = dict(os.environ)
if 'http_proxy' in projects.keys():
http_proxy = projects['http_proxy']
os.environ['http_proxy'] = projects['http_proxy']
if 'https_proxy' in projects.keys():
os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +547,19 @@ def git_clone_and_install(projects_yaml, core_project):
if 'directory' in projects.keys():
parent_dir = projects['directory']
pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
for p in projects['repositories']:
repo = p['repository']
branch = p['branch']
if p['name'] == 'requirements':
repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
repo_dir = _git_clone_and_install_single(repo, branch, depth,
parent_dir, http_proxy,
update_requirements=False)
requirements_dir = repo_dir
else:
repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
repo_dir = _git_clone_and_install_single(repo, branch, depth,
parent_dir, http_proxy,
update_requirements=True)
os.environ = old_environ
@@ -574,7 +591,8 @@ def _git_ensure_key_exists(key, keys):
error_out('openstack-origin-git key \'{}\' is missing'.format(key))
def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
update_requirements):
"""
Clone and install a single git repository.
"""
@@ -587,7 +605,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
if not os.path.exists(dest_dir):
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
depth=depth)
else:
repo_dir = dest_dir
@@ -598,7 +617,12 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
_git_update_requirements(repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
pip_install(repo_dir)
if http_proxy:
pip_install(repo_dir, proxy=http_proxy,
venv=os.path.join(parent_dir, 'venv'))
else:
pip_install(repo_dir,
venv=os.path.join(parent_dir, 'venv'))
return repo_dir
@@ -621,16 +645,27 @@ def _git_update_requirements(package_dir, reqs_dir):
os.chdir(orig_dir)
def git_pip_venv_dir(projects_yaml):
"""
Return the pip virtualenv path.
"""
parent_dir = '/mnt/openstack-git'
projects = _git_yaml_load(projects_yaml)
if 'directory' in projects.keys():
parent_dir = projects['directory']
return os.path.join(parent_dir, 'venv')
def git_src_dir(projects_yaml, project):
"""
Return the directory where the specified project's source is located.
"""
parent_dir = '/mnt/openstack-git'
if not projects_yaml:
return
projects = yaml.load(projects_yaml)
projects = _git_yaml_load(projects_yaml)
if 'directory' in projects.keys():
parent_dir = projects['directory']
@@ -640,3 +675,15 @@ def git_src_dir(projects_yaml, project):
return os.path.join(parent_dir, os.path.basename(p['repository']))
return None
def git_yaml_value(projects_yaml, key):
"""
Return the value in projects_yaml for the specified key.
"""
projects = _git_yaml_load(projects_yaml)
if key in projects.keys():
return projects[key]
return None

50
hooks/charmhelpers/contrib/peerstorage/__init__.py
View File

@@ -68,36 +68,38 @@ def leader_get(attribute=None):
If a setting is not extant in the leader-get but is on the relation-get
peer rel, it is migrated and marked as such so that it is not re-migrated.
"""
settings = _leader_get(attribute=attribute)
migration_key = '__leader_get_migrated_settings__'
if not is_leader():
return settings
return _leader_get(attribute=attribute)
settings_migrated = False
migration_key = '__leader_get_migrated_settings__'
if attribute:
migrated = _leader_get(attribute=migration_key)
if migrated:
migrated = set(json.loads(migrated))
else:
migrated = set([])
elif migration_key in settings:
migrated = set(json.loads(settings[migration_key]))
# Remove from returned settings
del settings[migration_key]
leader_settings = _leader_get(attribute=attribute)
previously_migrated = _leader_get(attribute=migration_key)
if previously_migrated:
migrated = set(json.loads(previously_migrated))
else:
migrated = set([])
try:
if migration_key in leader_settings:
del leader_settings[migration_key]
except TypeError:
pass
if attribute:
if attribute in migrated:
return settings
return leader_settings
# Leader setting wins
if not settings:
settings = relation_get(attribute=attribute, unit=local_unit())
if settings:
leader_set(settings={attribute: settings})
# If attribute not present in leader db, check if this unit has set
# the attribute in the peer relation
if not leader_settings:
peer_setting = relation_get(attribute=attribute, unit=local_unit())
if peer_setting:
leader_set(settings={attribute: peer_setting})
leader_settings = peer_setting
if settings:
if leader_settings:
settings_migrated = True
migrated.add(attribute)
else:
@@ -105,20 +107,20 @@ def leader_get(attribute=None):
if r_settings:
for key in set(r_settings.keys()).difference(migrated):
# Leader setting wins
if not settings.get(key):
settings[key] = r_settings[key]
if not leader_settings.get(key):
leader_settings[key] = r_settings[key]
settings_migrated = True
migrated.add(key)
if settings_migrated:
leader_set(**settings)
leader_set(**leader_settings)
if migrated and settings_migrated:
migrated = json.dumps(list(migrated))
leader_set(settings={migration_key: migrated})
return settings
return leader_settings
def relation_set(relation_id=None, relation_settings=None, **kwargs):

33
hooks/charmhelpers/contrib/python/packages.py
View File

@@ -17,8 +17,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
import os
import subprocess
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import log
from charmhelpers.core.hookenv import charm_dir, log
try:
from pip import main as pip_execute
@@ -51,11 +54,15 @@ def pip_install_requirements(requirements, **options):
pip_execute(command)
def pip_install(package, fatal=False, upgrade=False, **options):
def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
"""Install a python package"""
command = ["install"]
if venv:
venv_python = os.path.join(venv, 'bin/pip')
command = [venv_python, "install"]
else:
command = ["install"]
available_options = ('proxy', 'src', 'log', "index-url", )
available_options = ('proxy', 'src', 'log', 'index-url', )
for option in parse_options(options, available_options):
command.append(option)
@@ -69,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options):
log("Installing {} package with options: {}".format(package,
command))
pip_execute(command)
if venv:
subprocess.check_call(command)
else:
pip_execute(command)
def pip_uninstall(package, **options):
@@ -94,3 +104,16 @@ def pip_list():
"""Returns the list of current python installed packages
"""
return pip_execute(["list"])
def pip_create_virtualenv(path=None):
"""Create an isolated Python environment."""
apt_install('python-virtualenv')
if path:
venv_path = path
else:
venv_path = os.path.join(charm_dir(), 'venv')
if not os.path.exists(venv_path):
subprocess.check_call(['virtualenv', venv_path])

139
hooks/charmhelpers/core/hookenv.py
View File

@@ -28,6 +28,7 @@ import yaml
import subprocess
import sys
import errno
import tempfile
from subprocess import CalledProcessError
import six
@@ -362,14 +363,34 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
"""Set relation information for the current unit"""
relation_settings = relation_settings if relation_settings else {}
relation_cmd_line = ['relation-set']
accepts_file = "--file" in subprocess.check_output(
relation_cmd_line + ["--help"], universal_newlines=True)
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
for k, v in (list(relation_settings.items()) + list(kwargs.items())):
if v is None:
relation_cmd_line.append('{}='.format(k))
else:
relation_cmd_line.append('{}={}'.format(k, v))
subprocess.check_call(relation_cmd_line)
settings = relation_settings.copy()
settings.update(kwargs)
for key, value in settings.items():
# Force value to be a string: it always should, but some call
# sites pass in things like dicts or numbers.
if value is not None:
settings[key] = "{}".format(value)
if accepts_file:
# --file was introduced in Juju 1.23.2. Use it by default if
# available, since otherwise we'll break if the relation data is
# too big. Ideally we should tell relation-set to read the data from
# stdin, but that feature is broken in 1.23.2: Bug #1454678.
with tempfile.NamedTemporaryFile(delete=False) as settings_file:
settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
subprocess.check_call(
relation_cmd_line + ["--file", settings_file.name])
os.remove(settings_file.name)
else:
for key, value in settings.items():
if value is None:
relation_cmd_line.append('{}='.format(key))
else:
relation_cmd_line.append('{}={}'.format(key, value))
subprocess.check_call(relation_cmd_line)
# Flush cache of any relation-gets for local unit
flush(local_unit())
@@ -606,67 +627,6 @@ def charm_dir():
return os.environ.get('CHARM_DIR')
def translate_exc(from_exc, to_exc):
def inner_translate_exc1(f):
def inner_translate_exc2(*args, **kwargs):
try:
return f(*args, **kwargs)
except from_exc:
raise to_exc
return inner_translate_exc2
return inner_translate_exc1
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def is_leader():
"""Does the current unit hold the juju leadership
Uses juju to determine whether the current unit is the leader of its peers
"""
try:
leader = json.loads(
subprocess.check_output(['is-leader', '--format=json'])
)
return (leader is True)
except ValueError:
raise NotImplementedError
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_get(attribute=None):
"""Juju leader get value(s)"""
cmd = ['leader-get', '--format=json'] + [attribute or '-']
try:
ret = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
log("Juju leader-get '%s' = '%s'" % (attribute, ret), level=DEBUG)
return ret
except ValueError:
return None
except CalledProcessError as e:
if e.returncode == 2:
return None
raise
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_set(settings=None, **kwargs):
"""Juju leader set value(s)"""
log("Juju leader-set '%s'" % (settings), level=DEBUG)
cmd = ['leader-set']
settings = settings or {}
settings.update(kwargs)
for k, v in settings.iteritems():
if v is None:
cmd.append('{}='.format(k))
else:
cmd.append('{}={}'.format(k, v))
subprocess.check_call(cmd)
@cached
def action_get(key=None):
"""Gets the value of an action parameter, or all key/value param pairs"""
@@ -737,3 +697,48 @@ def status_get():
return 'unknown'
else:
raise
def translate_exc(from_exc, to_exc):
def inner_translate_exc1(f):
def inner_translate_exc2(*args, **kwargs):
try:
return f(*args, **kwargs)
except from_exc:
raise to_exc
return inner_translate_exc2
return inner_translate_exc1
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def is_leader():
"""Does the current unit hold the juju leadership
Uses juju to determine whether the current unit is the leader of its peers
"""
cmd = ['is-leader', '--format=json']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_get(attribute=None):
"""Juju leader get value(s)"""
cmd = ['leader-get', '--format=json'] + [attribute or '-']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_set(settings=None, **kwargs):
"""Juju leader set value(s)"""
log("Juju leader-set '%s'" % (settings), level=DEBUG)
cmd = ['leader-set']
settings = settings or {}
settings.update(kwargs)
for k, v in settings.iteritems():
if v is None:
cmd.append('{}='.format(k))
else:
cmd.append('{}={}'.format(k, v))
subprocess.check_call(cmd)

39
hooks/charmhelpers/core/services/base.py
View File

@@ -15,8 +15,8 @@
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import json
from inspect import getargspec
from collections import Iterable, OrderedDict
from charmhelpers.core import host
@@ -132,8 +132,8 @@ class ServiceManager(object):
if hook_name == 'stop':
self.stop_services()
else:
self.provide_data()
self.reconfigure_services()
self.provide_data()
cfg = hookenv.config()
if cfg.implicit_save:
cfg.save()
@@ -145,15 +145,36 @@ class ServiceManager(object):
A provider must have a `name` attribute, which indicates which relation
to set data on, and a `provide_data()` method, which returns a dict of
data to set.
The `provide_data()` method can optionally accept two parameters:
* ``remote_service`` The name of the remote service that the data will
be provided to. The `provide_data()` method will be called once
for each connected service (not unit). This allows the method to
tailor its data to the given service.
* ``service_ready`` Whether or not the service definition had all of
its requirements met, and thus the ``data_ready`` callbacks run.
Note that the ``provided_data`` methods are now called **after** the
``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
a chance to generate any data necessary for the providing to the remote
services.
"""
hook_name = hookenv.hook_name()
for service in self.services.values():
for service_name, service in self.services.items():
service_ready = self.is_ready(service_name)
for provider in service.get('provided_data', []):
if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
data = provider.provide_data()
_ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
if _ready:
hookenv.relation_set(None, data)
for relid in hookenv.relation_ids(provider.name):
units = hookenv.related_units(relid)
if not units:
continue
remote_service = units[0].split('/')[0]
argspec = getargspec(provider.provide_data)
if len(argspec.args) > 1:
data = provider.provide_data(remote_service, service_ready)
else:
data = provider.provide_data()
if data:
hookenv.relation_set(relid, data)
def reconfigure_services(self, *service_names):
"""

12
hooks/charmhelpers/fetch/giturl.py
View File

@@ -45,14 +45,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
else:
return True
def clone(self, source, dest, branch):
def clone(self, source, dest, branch, depth=None):
if not self.can_handle(source):
raise UnhandledSource("Cannot handle {}".format(source))
repo = Repo.clone_from(source, dest)
repo.git.checkout(branch)
if depth:
Repo.clone_from(source, dest, branch=branch, depth=depth)
else:
Repo.clone_from(source, dest, branch=branch)
def install(self, source, branch="master", dest=None):
def install(self, source, branch="master", dest=None, depth=None):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
if dest:
@@ -63,7 +65,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0o755)
try:
self.clone(source, dest_dir, branch)
self.clone(source, dest_dir, branch, depth)
except GitCommandError as e:
raise UnhandledSource(e.message)
except OSError as e: