openstack/charm-swift-storage
Code Issues Proposed changes

Remove /srv/node from updatedb

Browse Source 
This stops updatedb from indexing the storage locations

Change-Id: I6ca6b8667fb06d3b52cedd151531fc0033cf2526
Closes-bug: 1520226
This commit is contained in:
Chris MacNaughton 2017-01-16 14:00:07 +01:00
parent 9efaea2956
commit 47a4ac6e48
13 changed files with 488 additions and 8 deletions

6
charmhelpers/contrib/network/ip.py

@ -424,7 +424,11 @@ def ns_query(address):
else:
return None
answers = dns.resolver.query(address, rtype)
try:
answers = dns.resolver.query(address, rtype)
except dns.resolver.NXDOMAIN as e:
return None
if answers:
return str(answers[0])
return None

67
charmhelpers/contrib/openstack/amulet/utils.py

@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
msg = 'No message retrieved.'
amulet.raise_status(amulet.FAIL, msg)
def validate_memcache(self, sentry_unit, conf, os_release,
earliest_release=5, section='keystone_authtoken',
check_kvs=None):
"""Check Memcache is running and is configured to be used
Example call from Amulet test:
def test_110_memcache(self):
u.validate_memcache(self.neutron_api_sentry,
'/etc/neutron/neutron.conf',
self._get_openstack_release())
:param sentry_unit: sentry unit
:param conf: OpenStack config file to check memcache settings
:param os_release: Current OpenStack release int code
:param earliest_release: Earliest Openstack release to check int code
:param section: OpenStack config file section to check
:param check_kvs: Dict of settings to check in config file
:returns: None
"""
if os_release < earliest_release:
self.log.debug('Skipping memcache checks for deployment. {} <'
'mitaka'.format(os_release))
return
_kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
self.log.debug('Checking memcached is running')
ret = self.validate_services_by_name({sentry_unit: ['memcached']})
if ret:
amulet.raise_status(amulet.FAIL, msg='Memcache running check'
'failed {}'.format(ret))
else:
self.log.debug('OK')
self.log.debug('Checking memcache url is configured in {}'.format(
conf))
if self.validate_config_data(sentry_unit, conf, section, _kvs):
message = "Memcache config error in: {}".format(conf)
amulet.raise_status(amulet.FAIL, msg=message)
else:
self.log.debug('OK')
self.log.debug('Checking memcache configuration in '
'/etc/memcached.conf')
contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
fatal=True)
ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
if ubuntu_release <= 'trusty':
memcache_listen_addr = 'ip6-localhost'
else:
memcache_listen_addr = '::1'
expected = {
'-p': '11211',
'-l': memcache_listen_addr}
found = []
for key, value in expected.items():
for line in contents.split('\n'):
if line.startswith(key):
self.log.debug('Checking {} is set to {}'.format(
key,
value))
assert value == line.split()[-1]
self.log.debug(line.split()[-1])
found.append(key)
if sorted(found) == sorted(expected.keys()):
self.log.debug('OK')
else:
message = "Memcache config error in: /etc/memcached.conf"
amulet.raise_status(amulet.FAIL, msg=message)

74
charmhelpers/contrib/openstack/context.py

@ -14,6 +14,7 @@
import glob
import json
import math
import os
import re
import time
@ -90,6 +91,9 @@ from charmhelpers.contrib.network.ip import (
from charmhelpers.contrib.openstack.utils import (
config_flags_parser,
get_host_ip,
git_determine_usr_bin,
git_determine_python_path,
enable_memcache,
)
from charmhelpers.core.unitdata import kv
@ -1207,6 +1211,43 @@ class WorkerConfigContext(OSContextGenerator):
return ctxt
class WSGIWorkerConfigContext(WorkerConfigContext):
def __init__(self, name=None, script=None, admin_script=None,
public_script=None, process_weight=1.00,
admin_process_weight=0.75, public_process_weight=0.25):
self.service_name = name
self.user = name
self.group = name
self.script = script
self.admin_script = admin_script
self.public_script = public_script
self.process_weight = process_weight
self.admin_process_weight = admin_process_weight
self.public_process_weight = public_process_weight
def __call__(self):
multiplier = config('worker-multiplier') or 1
total_processes = self.num_cpus * multiplier
ctxt = {
"service_name": self.service_name,
"user": self.user,
"group": self.group,
"script": self.script,
"admin_script": self.admin_script,
"public_script": self.public_script,
"processes": int(math.ceil(self.process_weight * total_processes)),
"admin_processes": int(math.ceil(self.admin_process_weight *
total_processes)),
"public_processes": int(math.ceil(self.public_process_weight *
total_processes)),
"threads": 1,
"usr_bin": git_determine_usr_bin(),
"python_path": git_determine_python_path(),
}
return ctxt
class ZeroMQContext(OSContextGenerator):
interfaces = ['zeromq-configuration']
@ -1512,3 +1553,36 @@ class AppArmorContext(OSContextGenerator):
"".format(self.ctxt['aa_profile'],
self.ctxt['aa_profile_mode']))
raise e
class MemcacheContext(OSContextGenerator):
"""Memcache context
This context provides options for configuring a local memcache client and
server
"""
def __init__(self, package=None):
"""
@param package: Package to examine to extrapolate OpenStack release.
Used when charms have no openstack-origin config
option (ie subordinates)
"""
self.package = package
def __call__(self):
ctxt = {}
ctxt['use_memcache'] = enable_memcache(package=self.package)
if ctxt['use_memcache']:
# Trusty version of memcached does not support ::1 as a listen
# address so use host file entry instead
if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty':
ctxt['memcache_server'] = '::1'
else:
ctxt['memcache_server'] = 'ip6-localhost'
ctxt['memcache_server_formatted'] = '[::1]'
ctxt['memcache_port'] = '11211'
ctxt['memcache_url'] = 'inet6:{}:{}'.format(
ctxt['memcache_server_formatted'],
ctxt['memcache_port'])
return ctxt

53
charmhelpers/contrib/openstack/templates/memcached.conf Normal file

@ -0,0 +1,53 @@
###############################################################################
# [ WARNING ]
# memcached configuration file maintained by Juju
# local changes may be overwritten.
###############################################################################
# memcached default config file
# 2003 - Jay Bonci <jaybonci@debian.org>
# This configuration file is read by the start-memcached script provided as
# part of the Debian GNU/Linux distribution.
# Run memcached as a daemon. This command is implied, and is not needed for the
# daemon to run. See the README.Debian that comes with this package for more
# information.
-d
# Log memcached's output to /var/log/memcached
logfile /var/log/memcached.log
# Be verbose
# -v
# Be even more verbose (print client commands as well)
# -vv
# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
# Note that the daemon will grow to this size, but does not start out holding this much
# memory
-m 64
# Default connection port is 11211
-p {{ memcache_port }}
# Run the daemon as root. The start-memcached will default to running as root if no
# -u command is present in this config file
-u memcache
# Specify which IP address to listen on. The default is to listen on all IP addresses
# This parameter is one of the only security measures that memcached has, so make sure
# it's listening on a firewalled interface.
-l {{ memcache_server }}
# Limit the number of simultaneous incoming connections. The daemon default is 1024
# -c 1024
# Lock down all paged memory. Consult with the README and homepage before you do this
# -k
# Return error when memory is exhausted (rather than removing items)
# -M
# Maximize core file limit
# -r

3
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@ -14,4 +14,7 @@ project_name = {{ admin_tenant_name }}
username = {{ admin_user }}
password = {{ admin_password }}
signing_dir = {{ signing_dir }}
{% if use_memcache == true %}
memcached_servers = {{ memcache_url }}
{% endif -%}
{% endif -%}

100
charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf Normal file

@ -0,0 +1,100 @@
# Configuration file maintained by Juju. Local changes may be overwritten.
{% if port -%}
Listen {{ port }}
{% endif -%}
{% if admin_port -%}
Listen {{ admin_port }}
{% endif -%}
{% if public_port -%}
Listen {{ public_port }}
{% endif -%}
{% if port -%}
<VirtualHost *:{{ port }}>
WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
{% if python_path -%}
python-path={{ python_path }} \
{% endif -%}
display-name=%{GROUP}
WSGIProcessGroup {{ service_name }}
WSGIScriptAlias / {{ script }}
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/{{ service_name }}_error.log
CustomLog /var/log/apache2/{{ service_name }}_access.log combined
<Directory {{ usr_bin }}>
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
</Directory>
</VirtualHost>
{% endif -%}
{% if admin_port -%}
<VirtualHost *:{{ admin_port }}>
WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
{% if python_path -%}
python-path={{ python_path }} \
{% endif -%}
display-name=%{GROUP}
WSGIProcessGroup {{ service_name }}-admin
WSGIScriptAlias / {{ admin_script }}
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/{{ service_name }}_error.log
CustomLog /var/log/apache2/{{ service_name }}_access.log combined
<Directory {{ usr_bin }}>
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
</Directory>
</VirtualHost>
{% endif -%}
{% if public_port -%}
<VirtualHost *:{{ public_port }}>
WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
{% if python_path -%}
python-path={{ python_path }} \
{% endif -%}
display-name=%{GROUP}
WSGIProcessGroup {{ service_name }}-public
WSGIScriptAlias / {{ public_script }}
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/{{ service_name }}_error.log
CustomLog /var/log/apache2/{{ service_name }}_access.log combined
<Directory {{ usr_bin }}>
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
</Directory>
</VirtualHost>
{% endif -%}

64
charmhelpers/contrib/openstack/utils.py

@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([
('newton',
['2.8.0', '2.9.0', '2.10.0']),
('ocata',
['2.11.0']),
['2.11.0', '2.12.0']),
])
# >= Liberty version->codename mapping
@ -1119,6 +1119,35 @@ def git_generate_systemd_init_files(templates_dir):
shutil.copyfile(service_source, service_dest)
def git_determine_usr_bin():
"""Return the /usr/bin path for Apache2 config.
The /usr/bin path will be located in the virtualenv if the charm
is configured to deploy from source.
"""
if git_install_requested():
projects_yaml = config('openstack-origin-git')
projects_yaml = git_default_repos(projects_yaml)
return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
else:
return '/usr/bin'
def git_determine_python_path():
"""Return the python-path for Apache2 config.
Returns 'None' unless the charm is configured to deploy from source,
in which case the path of the virtualenv's site-packages is returned.
"""
if git_install_requested():
projects_yaml = config('openstack-origin-git')
projects_yaml = git_default_repos(projects_yaml)
return os.path.join(git_pip_venv_dir(projects_yaml),
'lib/python2.7/site-packages')
else:
return None
def os_workload_status(configs, required_interfaces, charm_func=None):
"""
Decorator to set workload status based on complete contexts
@ -1925,3 +1954,36 @@ def os_application_version_set(package):
application_version_set(os_release(package))
else:
application_version_set(application_version)
def enable_memcache(source=None, release=None, package=None):
"""Determine if memcache should be enabled on the local unit
@param release: release of OpenStack currently deployed
@param package: package to derive OpenStack version deployed
@returns boolean Whether memcache should be enabled
"""
_release = None
if release:
_release = release
else:
_release = os_release(package, base='icehouse')
if not _release:
_release = get_os_codename_install_source(source)
# TODO: this should be changed to a numeric comparison using a known list
# of releases and comparing by index.
return _release >= 'mitaka'
def token_cache_pkgs(source=None, release=None):
"""Determine additional packages needed for token caching
@param source: source string for charm
@param release: release of OpenStack currently deployed
@returns List of package to enable token caching
"""
packages = []
if enable_memcache(source=source, release=release):
packages.extend(['memcached', 'python-memcache'])
return packages

14
charmhelpers/core/hookenv.py

@ -616,6 +616,20 @@ def close_port(port, protocol="TCP"):
subprocess.check_call(_args)
def open_ports(start, end, protocol="TCP"):
"""Opens a range of service network ports"""
_args = ['open-port']
_args.append('{}-{}/{}'.format(start, end, protocol))
subprocess.check_call(_args)
def close_ports(start, end, protocol="TCP"):
"""Close a range of service network ports"""
_args = ['close-port']
_args.append('{}-{}/{}'.format(start, end, protocol))
subprocess.check_call(_args)
@cached
def unit_get(attribute):
"""Get the unit ID for the remote unit"""

33
charmhelpers/core/host.py

@ -54,6 +54,7 @@ elif __platform__ == "centos":
cmp_pkgrevno,
) # flake8: noqa -- ignore F401 for this import
UPDATEDB_PATH = '/etc/updatedb.conf'
def service_start(service_name):
"""Start a system service"""
@ -306,15 +307,17 @@ def add_user_to_group(username, group):
subprocess.check_call(cmd)
def rsync(from_path, to_path, flags='-r', options=None):
def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
cmd = ['/usr/bin/rsync', flags]
if timeout:
cmd = ['timeout', str(timeout)] + cmd
cmd.extend(options)
cmd.append(from_path)
cmd.append(to_path)
log(" ".join(cmd))
return subprocess.check_output(cmd).decode('UTF-8').strip()
return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
def symlink(source, destination):
@ -684,7 +687,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
:param str path: The string path to start changing ownership.
:param str owner: The owner string to use when looking up the uid.
:param str group: The group string to use when looking up the gid.
:param bool follow_links: Also Chown links if True
:param bool follow_links: Also follow and chown links if True
:param bool chowntopdir: Also chown path itself if True
"""
uid = pwd.getpwnam(owner).pw_uid
@ -698,7 +701,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
broken_symlink = os.path.lexists(path) and not os.path.exists(path)
if not broken_symlink:
chown(path, uid, gid)
for root, dirs, files in os.walk(path):
for root, dirs, files in os.walk(path, followlinks=follow_links):
for name in dirs + files:
full = os.path.join(root, name)
broken_symlink = os.path.lexists(full) and not os.path.exists(full)
@ -749,3 +752,25 @@ def is_container():
else:
# Detect using upstart container file marker
return os.path.exists(UPSTART_CONTAINER_TYPE)
def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
with open(updatedb_path, 'r+') as f_id:
updatedb_text = f_id.read()
output = updatedb(updatedb_text, path)
f_id.seek(0)
f_id.write(output)
f_id.truncate()
def updatedb(updatedb_text, new_path):
lines = [line for line in updatedb_text.split("\n")]
for i, line in enumerate(lines):
if line.startswith("PRUNEPATHS="):
paths_line = line.split("=")[1].replace('"', '')
paths = paths_line.split(" ")
if new_path not in paths:
paths.append(new_path)
lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
output = "\n".join(lines)
return output

7
hooks/swift_storage_hooks.py

@ -55,7 +55,10 @@ from charmhelpers.fetch import (
apt_update,
filter_installed_packages
)
from charmhelpers.core.host import rsync
from charmhelpers.core.host import (
add_to_updatedb_prunepath,
rsync,
)
from charmhelpers.payload.execd import execd_preinstall
from charmhelpers.contrib.openstack.utils import (
@ -76,6 +79,7 @@ hooks = Hooks()
CONFIGS = register_configs()
NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
SUDOERS_D = '/etc/sudoers.d'
STORAGE_MOUNT_PATH = '/srv/node'
@hooks.hook('install.real')
@ -118,6 +122,7 @@ def config_changed():
save_script_rc()
if relations_of_type('nrpe-external-master'):
update_nrpe_config()
add_to_updatedb_prunepath(STORAGE_MOUNT_PATH)
@hooks.hook('upgrade-charm')

3
tests/charmhelpers/contrib/amulet/utils.py

@ -148,7 +148,8 @@ class AmuletUtils(object):
for service_name in services_list:
if (self.ubuntu_releases.index(release) >= systemd_switch or
service_name in ['rabbitmq-server', 'apache2']):
service_name in ['rabbitmq-server', 'apache2',
'memcached']):
# init is systemd (or regular sysv)
cmd = 'sudo service {} status'.format(service_name)
output, code = sentry_unit.run(cmd)

67
tests/charmhelpers/contrib/openstack/amulet/utils.py

@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
msg = 'No message retrieved.'
amulet.raise_status(amulet.FAIL, msg)
def validate_memcache(self, sentry_unit, conf, os_release,
earliest_release=5, section='keystone_authtoken',
check_kvs=None):
"""Check Memcache is running and is configured to be used
Example call from Amulet test:
def test_110_memcache(self):
u.validate_memcache(self.neutron_api_sentry,
'/etc/neutron/neutron.conf',
self._get_openstack_release())
:param sentry_unit: sentry unit
:param conf: OpenStack config file to check memcache settings
:param os_release: Current OpenStack release int code
:param earliest_release: Earliest Openstack release to check int code
:param section: OpenStack config file section to check
:param check_kvs: Dict of settings to check in config file
:returns: None
"""
if os_release < earliest_release:
self.log.debug('Skipping memcache checks for deployment. {} <'
'mitaka'.format(os_release))
return
_kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
self.log.debug('Checking memcached is running')
ret = self.validate_services_by_name({sentry_unit: ['memcached']})
if ret:
amulet.raise_status(amulet.FAIL, msg='Memcache running check'
'failed {}'.format(ret))
else:
self.log.debug('OK')
self.log.debug('Checking memcache url is configured in {}'.format(
conf))
if self.validate_config_data(sentry_unit, conf, section, _kvs):
message = "Memcache config error in: {}".format(conf)
amulet.raise_status(amulet.FAIL, msg=message)
else:
self.log.debug('OK')
self.log.debug('Checking memcache configuration in '
'/etc/memcached.conf')
contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
fatal=True)
ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
if ubuntu_release <= 'trusty':
memcache_listen_addr = 'ip6-localhost'
else:
memcache_listen_addr = '::1'
expected = {
'-p': '11211',
'-l': memcache_listen_addr}
found = []
for key, value in expected.items():
for line in contents.split('\n'):
if line.startswith(key):
self.log.debug('Checking {} is set to {}'.format(
key,
value))
assert value == line.split()[-1]
self.log.debug(line.split()[-1])
found.append(key)
if sorted(found) == sorted(expected.keys()):
self.log.debug('OK')
else:
message = "Memcache config error in: /etc/memcached.conf"
amulet.raise_status(amulet.FAIL, msg=message)

5
unit_tests/test_swift_storage_relations.py

@ -60,6 +60,7 @@ TO_PATCH = [
'status_set',
'set_os_workload_status',
'os_application_version_set',
'add_to_updatedb_prunepath',
]
@ -71,6 +72,10 @@ class SwiftStorageRelationsTests(CharmTestCase):
self.config.side_effect = self.test_config.get
self.relation_get.side_effect = self.test_relation.get
def test_prunepath(self):
hooks.config_changed()
self.add_to_updatedb_prunepath.assert_called_with("/srv/node")
def test_install_hook(self):
self.test_config.set('openstack-origin', 'cloud:precise-havana')
hooks.install()