diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index b900d125..8c8c790b 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:~openstack-charmers/charm-helpers/0mq +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/config.yaml b/config.yaml index b91fd990..fdcab2bd 100644 --- a/config.yaml +++ b/config.yaml @@ -130,4 +130,28 @@ options: default: description: | YAML-formatted associative array of sysctl key/value pairs to be set - persistently e.g. '{ kernel.pid_max : 4194303 }'. \ No newline at end of file + persistently e.g. '{ kernel.pid_max : 4194303 }'. + # Legacy (Icehouse) HA + ha-legacy-mode: + type: boolean + default: False + description: | + If True will enable Pacemaker to monitor the neutron-ha-monitor daemon + on every neutron-gateway unit, which detects neutron agents status and + reschedule resources hosting on failed agents, detects local errors and + release resources when network is unreachable or do neccessary recover + tasks. This feature targets to < Juno which doesn't natively support HA + in Neutron itself. + ha-bindiface: + type: string + default: eth0 + description: | + Default network interface on which HA cluster will bind to communicate + with the other members of the HA Cluster. + ha-mcastport: + type: int + default: 5409 + description: | + Default multicast port number that will be used to communicate between + HA Cluster nodes. + diff --git a/files/NeutronAgentMon b/files/NeutronAgentMon new file mode 100755 index 00000000..048955f1 --- /dev/null +++ b/files/NeutronAgentMon @@ -0,0 +1,155 @@ +#!/bin/sh +# +# +# NeutronAgentMon OCF RA. +# Starts crm_mon in background which logs cluster status as +# html to the specified file. +# +# Copyright 2014 Canonical Ltd. +# +# Authors: Hui Xiang +# Edward Hope-Morley +# +# OCF instance parameters: +# OCF_RESKEY_file + +####################################################################### +# Initialization: +: ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs} +. ${OCF_FUNCTIONS} +: ${__OCF_ACTION=$1} + +####################################################################### + +meta_data() { + cat < + + +1.0 + + +This is a NeutronAgentMon Resource Agent. +It monitors the 'neutron-ha-monitor daemon' status. + +Monitor '/usr/local/bin/neutron-ha-monitor.py' in the background. + + + + + +The file we want to run as a daemon. + +The file we want to run as a daemon. + + + + + + + + + + + + + +END +} + +####################################################################### + +NeutronAgentMon_usage() { + cat <> /dev/null 2>&1 & echo $! + sleep 5 + else + ocf_log warn "[NeutronAgentMon_start] Monitor daemon already running." + fi + NeutronAgentMon_exit $? +} + +NeutronAgentMon_stop() { + pid=`sudo ps -aux | grep neutron-ha-m\[o\]nitor.py | awk -F' ' '{print $2}'` + if [ ! -z $pid ]; then + sudo kill -s 9 $pid + ocf_log info "[NeutronAgentMon_stop] Pid $pid is killed." + else + ocf_log warn "[NeutronAgentMon_stop] Monitor daemon already stopped." + fi + NeutronAgentMon_exit 0 +} + +NeutronAgentMon_monitor() { + pid=`sudo ps -aux | grep neutron-ha-m\[o\]nitor.py | awk -F' ' '{print $2}'` + if [ ! -z $pid ]; then + ocf_log info "[NeutronAgentMon_monitor] success." + exit $OCF_SUCCESS + fi + exit $OCF_NOT_RUNNING +} + +NeutronAgentMon_validate() { +# Existence of the user + if [ -f $OCF_RESKEY_file ]; then + echo "Validate OK" + return $OCF_SUCCESS + else + ocf_log err "The file $OCF_RESKEY_file does not exist!" + exit $OCF_ERR_ARGS + fi +} + +if [ $# -ne 1 ]; then + NeutronAgentMon_usage + exit $OCF_ERR_ARGS +fi + +: ${OCF_RESKEY_update:="15000"} +: ${OCF_RESKEY_pidfile:="/tmp/NeutronAgentMon_${OCF_RESOURCE_INSTANCE}.pid"} +: ${OCF_RESKEY_htmlfile:="/tmp/NeutronAgentMon_${OCF_RESOURCE_INSTANCE}.html"} + +OCF_RESKEY_update=`expr $OCF_RESKEY_update / 1000` + +case $__OCF_ACTION in +meta-data) meta_data + exit $OCF_SUCCESS + ;; +start) NeutronAgentMon_start + ;; +stop) NeutronAgentMon_stop + ;; +monitor) NeutronAgentMon_monitor + ;; +validate-all) NeutronAgentMon_validate + ;; +usage|help) NeutronAgentMon_usage + exit $OCF_SUCCESS + ;; +*) NeutronAgentMon_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac + +exit $? diff --git a/files/neutron-ha-monitor.conf b/files/neutron-ha-monitor.conf new file mode 100644 index 00000000..d62ad10a --- /dev/null +++ b/files/neutron-ha-monitor.conf @@ -0,0 +1,4 @@ +[DEFAULT] +verbose=True +#debug=True +check_interval=8 diff --git a/files/neutron-ha-monitor.py b/files/neutron-ha-monitor.py new file mode 100644 index 00000000..63f518d0 --- /dev/null +++ b/files/neutron-ha-monitor.py @@ -0,0 +1,436 @@ +# Copyright 2014 Canonical Ltd. +# +# Authors: Hui Xiang +# Joshua Zhang +# Edward Hope-Morley +# + +""" +Helpers for monitoring Neutron agents, reschedule failed agents, +cleaned resources on failed nodes. +""" + +import os +import re +import sys +import signal +import socket +import subprocess +import time + +from oslo.config import cfg +from neutron.agent.linux import ovs_lib +from neutron.agent.linux import ip_lib +from neutron.common import exceptions +from neutron.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +class Daemon(object): + """A generic daemon class. + + Usage: subclass the Daemon class and override the run() method + """ + def __init__(self, stdin='/dev/null', stdout='/dev/null', + stderr='/dev/null', procname='python'): + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self.procname = procname + + def _fork(self): + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError: + LOG.exception('Fork failed') + sys.exit(1) + + def daemonize(self): + """Daemonize process by doing Stevens double fork.""" + # fork first time + self._fork() + + # decouple from parent environment + os.chdir("/") + os.setsid() + os.umask(0) + # fork second time + self._fork() + + # redirect standard file descriptors + sys.stdout.flush() + sys.stderr.flush() + stdin = open(self.stdin, 'r') + stdout = open(self.stdout, 'a+') + stderr = open(self.stderr, 'a+', 0) + os.dup2(stdin.fileno(), sys.stdin.fileno()) + os.dup2(stdout.fileno(), sys.stdout.fileno()) + os.dup2(stderr.fileno(), sys.stderr.fileno()) + + signal.signal(signal.SIGTERM, self.handle_sigterm) + + def handle_sigterm(self, signum, frame): + sys.exit(0) + + def start(self): + """Start the daemon.""" + self.daemonize() + self.run() + + def run(self): + """Override this method when subclassing Daemon. + + start() will call this method after the process has daemonized. + """ + pass + + +class MonitorNeutronAgentsDaemon(Daemon): + def __init__(self): + super(MonitorNeutronAgentsDaemon, self).__init__() + logging.setup('Neuron-HA-Monitor') + LOG.info('Monitor Neutron Agent Loop Init') + self.hostname = None + self.env = {} + + def get_env(self): + envrc_f = '/etc/legacy_ha_envrc' + envrc_f_m = False + if os.path.isfile(envrc_f): + ctime = time.ctime(os.stat(envrc_f).st_ctime) + mtime = time.ctime(os.stat(envrc_f).st_mtime) + if ctime != mtime: + envrc_f_m = True + + if not self.env or envrc_f_m: + with open(envrc_f, 'r') as f: + for line in f: + data = line.strip().split('=') + if data and data[0] and data[1]: + self.env[data[0]] = data[1] + else: + raise Exception("OpenStack env data uncomplete.") + return self.env + + def get_hostname(self): + if not self.hostname: + self.hostname = socket.gethostname() + return self.hostname + + def get_root_helper(self): + return 'sudo' + + def list_monitor_res(self): + # List crm resource 'cl_monitor' running node + nodes = [] + cmd = ['crm', 'resource', 'show', 'cl_monitor'] + output = subprocess.check_output(cmd) + pattern = re.compile('resource cl_monitor is running on: (.*) ') + nodes = pattern.findall(output) + return nodes + + def get_crm_res_lead_node(self): + nodes = self.list_monitor_res() + if nodes: + return nodes[0].strip() + else: + LOG.error('Failed to get crm resource.') + return None + + def unplug_device(self, device): + try: + device.link.delete() + except RuntimeError: + root_helper = self.get_root_helper() + # Maybe the device is OVS port, so try to delete + bridge_name = ovs_lib.get_bridge_for_iface(root_helper, + device.name) + if bridge_name: + bridge = ovs_lib.OVSBridge(bridge_name, root_helper) + bridge.delete_port(device.name) + else: + LOG.debug('Unable to find bridge for device: %s', device.name) + + def get_pattern(self, key, text): + if not key or not text: + LOG.debug('Invalid key(%s) or text(%s)' % (key, text)) + return None + + pattern = re.compile('%s' % key) + result = pattern.findall(text) + return result + + def _cleanup(self, key1, key2): + namespaces = [] + if key1: + for k in key1.iterkeys(): + namespaces.append(key2 + '-' + k) + else: + try: + cmd = ['sudo', 'ip', 'netns'] + ns = subprocess.check_output(cmd) + namespaces = self.get_pattern('(%s.*)' % key2, ns) + except RuntimeError as e: + LOG.error('Failed to list namespace, (%s)' % e) + + if namespaces: + LOG.info('Namespaces: %s is going to be deleted.' % namespaces) + self.destroy_namespaces(namespaces) + + def cleanup_dhcp(self, networks): + self._cleanup(networks, 'qdhcp') + + def cleanup_router(self, routers): + self._cleanup(routers, 'qrouter') + + def destroy_namespaces(self, namespaces): + try: + root_helper = self.get_root_helper() + for namespace in namespaces: + ip = ip_lib.IPWrapper(root_helper, namespace) + if ip.netns.exists(namespace): + for device in ip.get_devices(exclude_loopback=True): + self.unplug_device(device) + + ip.garbage_collect_namespace() + except Exception: + LOG.exception('Error unable to destroy namespace: %s', namespace) + + def is_same_host(self, host): + return str(host).strip() == self.get_hostname() + + def validate_reschedule(self): + crm_no_1_node = self.get_crm_res_lead_node() + if not crm_no_1_node: + LOG.error('No crm first node could be found.') + return False + + if not self.is_same_host(crm_no_1_node): + LOG.warn('Only the first crm node %s could reschedule. ' + % crm_no_1_node) + return False + return True + + def l3_agents_reschedule(self, l3_agents, routers, quantum): + if not self.validate_reschedule(): + return + + index = 0 + for router_id in routers: + agent = index % len(l3_agents) + LOG.info('Moving router %s from %s to %s' % + (router_id, routers[router_id], l3_agents[agent])) + try: + quantum.remove_router_from_l3_agent(l3_agent=routers[router_id], + router_id=router_id) + except exceptions.NeutronException as e: + LOG.error('Remove router raised exception: %s' % e) + try: + quantum.add_router_to_l3_agent(l3_agent=l3_agents[agent], + body={'router_id': router_id}) + except exceptions.NeutronException as e: + LOG.error('Add router raised exception: %s' % e) + index += 1 + + def dhcp_agents_reschedule(self, dhcp_agents, networks, quantum): + if not self.validate_reschedule(): + return + + index = 0 + for network_id in networks: + agent = index % len(dhcp_agents) + LOG.info('Moving network %s from %s to %s' % (network_id, + networks[network_id], dhcp_agents[agent])) + try: + quantum.remove_network_from_dhcp_agent( + dhcp_agent=networks[network_id], network_id=network_id) + except exceptions.NeutronException as e: + LOG.error('Remove network raised exception: %s' % e) + try: + quantum.add_network_to_dhcp_agent( + dhcp_agent=dhcp_agents[agent], + body={'network_id': network_id}) + except exceptions.NeutronException as e: + LOG.error('Add network raised exception: %s' % e) + index += 1 + + def get_quantum_client(self): + env = self.get_env() + if not env: + LOG.info('Unable to re-assign resources at this time') + return None + + try: + from quantumclient.v2_0 import client + except ImportError: + # Try to import neutronclient instead for havana+ + from neutronclient.v2_0 import client + + auth_url = '%(auth_protocol)s://%(keystone_host)s:%(auth_port)s/v2.0' \ + % env + quantum = client.Client(username=env['service_username'], + password=env['service_password'], + tenant_name=env['service_tenant'], + auth_url=auth_url, + region_name=env['region']) + return quantum + + def reassign_agent_resources(self, quantum=None): + """Use agent scheduler API to detect down agents and re-schedule""" + if not quantum: + LOG.error('Failed to get quantum client.') + return + + try: + DHCP_AGENT = "DHCP Agent" + L3_AGENT = "L3 Agent" + agents = quantum.list_agents(agent_type=DHCP_AGENT) + except exceptions.NeutronException as e: + LOG.error('Failed to get quantum agents, %s' % e) + return + + dhcp_agents = [] + l3_agents = [] + networks = {} + for agent in agents['agents']: + hosted_networks = quantum.list_networks_on_dhcp_agent( + agent['id'])['networks'] + if not agent['alive']: + LOG.info('DHCP Agent %s down' % agent['id']) + for network in hosted_networks: + networks[network['id']] = agent['id'] + if self.is_same_host(agent['host']): + self.cleanup_dhcp(networks) + else: + dhcp_agents.append(agent['id']) + LOG.info('Active dhcp agents: %s' % agent['id']) + if not hosted_networks and self.is_same_host(agent['host']): + self.cleanup_dhcp(None) + + agents = quantum.list_agents(agent_type=L3_AGENT) + routers = {} + for agent in agents['agents']: + hosted_routers = quantum.list_routers_on_l3_agent( + agent['id'])['routers'] + if not agent['alive']: + LOG.info('L3 Agent %s down' % agent['id']) + for router in hosted_routers: + routers[router['id']] = agent['id'] + if self.is_same_host(agent['host']): + self.cleanup_router(routers) + else: + l3_agents.append(agent['id']) + LOG.info('Active l3 agents: %s' % agent['id']) + if not hosted_routers and self.is_same_host(agent['host']): + self.cleanup_router(None) + + if not networks and not routers: + LOG.info('No networks and routers hosted on failed agents.') + return + + if len(dhcp_agents) == 0 and len(l3_agents) == 0: + LOG.error('Unable to relocate resources, there are %s dhcp_agents ' + 'and %s l3_agents in this cluster' % (len(dhcp_agents), + len(l3_agents))) + return + + if len(l3_agents) > 0: + self.l3_agents_reschedule(l3_agents, routers, quantum) + # new l3 node will not create a tunnel if don't restart ovs process + + if len(dhcp_agents) > 0: + self.dhcp_agents_reschedule(dhcp_agents, networks, quantum) + + + def check_ovs_tunnel(self, quantum=None): + ''' + Work around for Bug #1411163 + No fdb entries added when failover dhcp and l3 agent together. + ''' + if not quantum: + LOG.error('Failed to get quantum client.') + return + + try: + OVS_AGENT = 'Open vSwitch agent' + agents = quantum.list_agents(agent_type=OVS_AGENT) + except exceptions.NeutronException as e: + LOG.error('No ovs agent found on localhost, error:%s.' % e) + return + + for agent in agents['agents']: + if self.is_same_host(agent['host']) and agent['alive']: + conf = agent['configurations'] + if 'gre' in conf['tunnel_types'] and conf['l2_population'] \ + and conf['devices']: + LOG.debug('local ovs agent:%s' % agent) + ovs_output = subprocess.check_output(['ovs-vsctl', + 'list-ports', 'br-tun']) + ports = ovs_output.strip().split('\n') + look_up_gre_port = False + for port in ports: + if port.startswith('gre-'): + look_up_gre_port = True + break + if not look_up_gre_port: + try: + LOG.error('Local agent has devices, but no ovs tunnel is created,' + 'restart ovs agent.') + cmd = ['sudo', 'service', 'neutron-plugin-openvswitch-agent', + 'restart'] + subprocess.call(cmd) + except subprocess.CalledProcessError: + LOG.error('Failed to restart neutron-plugin-openvswitch-agent.') + + def check_local_agents(self): + services = ['openvswitch-switch', 'neutron-dhcp-agent', + 'neutron-metadata-agent', 'neutron-vpn-agent'] + for s in services: + status = ['sudo', 'service', s, 'status'] + restart = ['sudo', 'service', s, 'restart'] + start = ['sudo', 'service', s, 'start'] + stop = '%s stop/waiting' % s + try: + output = subprocess.check_output(status) + if output.strip() == stop: + subprocess.check_output(start) + LOG.error('Restart service: %s' % s) + if s == 'neutron-metadata-agent': + subprocess.check_output(['sudo', 'service', + 'neutron-vpn-agent', + 'restart']) + LOG.error('Restart neutron-vpn-agent') + except subprocess.CalledProcessError: + LOG.error('Restart service: %s' % s) + subprocess.check_output(restart) + if s == 'neutron-metadata-agent': + subprocess.check_output(['sudo', 'service', + 'neutron-vpn-agent', + 'restart']) + + def run(self): + while True: + LOG.info('Monitor Neutron HA Agent Loop Start') + quantum = self.get_quantum_client() + self.reassign_agent_resources(quantum=quantum) + self.check_ovs_tunnel(quantum=quantum) + self.check_local_agents() + LOG.info('sleep %s' % cfg.CONF.check_interval) + time.sleep(float(cfg.CONF.check_interval)) + + +if __name__ == '__main__': + opts = [ + cfg.StrOpt('check_interval', + default=8, + help='Check Neutron Agents interval.'), + ] + + cfg.CONF.register_cli_opts(opts) + cfg.CONF(project='monitor_neutron_agents', default_config_files=[]) + logging.setup('Neuron-HA-Monitor') + monitor_daemon = MonitorNeutronAgentsDaemon() + monitor_daemon.start() diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/hooks/charmhelpers/__init__.py +++ b/hooks/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/__init__.py +++ b/hooks/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/charmsupport/__init__.py +++ b/hooks/charmhelpers/contrib/charmsupport/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index f3a936d0..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Compatibility with the nrpe-external-master charm""" # Copyright 2012 Canonical Ltd. # @@ -8,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -145,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -161,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -195,10 +210,10 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -306,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py index d61aa47f..320961b9 100644 --- a/hooks/charmhelpers/contrib/charmsupport/volumes.py +++ b/hooks/charmhelpers/contrib/charmsupport/volumes.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Functions for managing volumes in juju units. One volume is supported per unit. Subordinates may have their own storage, provided it is on its own partition. diff --git a/hooks/charmhelpers/contrib/hahelpers/__init__.py b/hooks/charmhelpers/contrib/hahelpers/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/hahelpers/__init__.py +++ b/hooks/charmhelpers/contrib/hahelpers/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 6616ffff..00917195 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 912b2fe3..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # @@ -32,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -148,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True @@ -205,19 +225,23 @@ def determine_apache_port(public_port, singlenode_mode=False): return public_port - (i * 10) -def get_hacluster_config(): +def get_hacluster_config(exclude_keys=None): ''' Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: ha-bindiface, ha-mcastport, vip + param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: + if exclude_keys and setting in exclude_keys: + continue + conf[setting] = config_get(setting) missing = [] [missing.append(s) for s, v in six.iteritems(conf) if v is None] diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/network/__init__.py +++ b/hooks/charmhelpers/contrib/network/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 8dc83165..98b17544 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import glob import re import subprocess diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index 8f8a5230..77e2db7f 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Helpers for interacting with OpenvSwitch ''' import subprocess import os diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py index b51689f2..560e6a03 100644 --- a/hooks/charmhelpers/contrib/network/ufw.py +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """ This module contains helpers to add and remove ufw rules. @@ -21,14 +37,21 @@ Examples: >>> ufw.enable() >>> ufw.service('4949', 'close') # munin """ - -__author__ = "Felipe Reyes " - import re import os import subprocess from charmhelpers.core import hookenv +__author__ = "Felipe Reyes " + + +class UFWError(Exception): + pass + + +class UFWIPv6Error(UFWError): + pass + def is_enabled(): """ @@ -37,6 +60,7 @@ def is_enabled(): :returns: True if ufw is enabled """ output = subprocess.check_output(['ufw', 'status'], + universal_newlines=True, env={'LANG': 'en_US', 'PATH': os.environ['PATH']}) @@ -45,27 +69,76 @@ def is_enabled(): return len(m) >= 1 -def enable(): +def is_ipv6_ok(soft_fail=False): + """ + Check if IPv6 support is present and ip6tables functional + + :param soft_fail: If set to True and IPv6 support is broken, then reports + that the host doesn't have IPv6 support, otherwise a + UFWIPv6Error exception is raised. + :returns: True if IPv6 is working, False otherwise + """ + + # do we have IPv6 in the machine? + if os.path.isdir('/proc/sys/net/ipv6'): + # is ip6tables kernel module loaded? + lsmod = subprocess.check_output(['lsmod'], universal_newlines=True) + matches = re.findall('^ip6_tables[ ]+', lsmod, re.M) + if len(matches) == 0: + # ip6tables support isn't complete, let's try to load it + try: + subprocess.check_output(['modprobe', 'ip6_tables'], + universal_newlines=True) + # great, we could load the module + return True + except subprocess.CalledProcessError as ex: + hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, + level="WARN") + # we are in a world where ip6tables isn't working + if soft_fail: + # so we inform that the machine doesn't have IPv6 + return False + else: + raise UFWIPv6Error("IPv6 firewall support broken") + else: + # the module is present :) + return True + + else: + # the system doesn't have IPv6 + return False + + +def disable_ipv6(): + """ + Disable ufw IPv6 support in /etc/default/ufw + """ + exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + if exit_code == 0: + hookenv.log('IPv6 support in ufw disabled', level='INFO') + else: + hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") + raise UFWError("Couldn't disable IPv6 support in ufw") + + +def enable(soft_fail=False): """ Enable ufw + :param soft_fail: If set to True silently disables IPv6 support in ufw, + otherwise a UFWIPv6Error exception is raised when IP6 + support is broken. :returns: True if ufw is successfully enabled """ if is_enabled(): return True - if not os.path.isdir('/proc/sys/net/ipv6'): - # disable IPv6 support in ufw - hookenv.log("This machine doesn't have IPv6 enabled", level="INFO") - exit_code = subprocess.call(['sed', '-i', 's/IPV6=yes/IPV6=no/g', - '/etc/default/ufw']) - if exit_code == 0: - hookenv.log('IPv6 support in ufw disabled', level='INFO') - else: - hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") - raise Exception("Couldn't disable IPv6 support in ufw") + if not is_ipv6_ok(soft_fail): + disable_ipv6() output = subprocess.check_output(['ufw', 'enable'], + universal_newlines=True, env={'LANG': 'en_US', 'PATH': os.environ['PATH']}) @@ -91,6 +164,7 @@ def disable(): return True output = subprocess.check_output(['ufw', 'disable'], + universal_newlines=True, env={'LANG': 'en_US', 'PATH': os.environ['PATH']}) @@ -135,7 +209,7 @@ def modify_access(src, dst='any', port=None, proto=None, action='allow'): cmd += ['to', dst] if port is not None: - cmd += ['port', port] + cmd += ['port', str(port)] if proto is not None: cmd += ['proto', proto] @@ -192,9 +266,11 @@ def service(name, action): :param action: `open` or `close` """ if action == 'open': - subprocess.check_output(['ufw', 'allow', name]) + subprocess.check_output(['ufw', 'allow', str(name)], + universal_newlines=True) elif action == 'close': - subprocess.check_output(['ufw', 'delete', 'allow', name]) + subprocess.check_output(['ufw', 'delete', 'allow', str(name)], + universal_newlines=True) else: - raise Exception(("'{}' not supported, use 'allow' " - "or 'delete'").format(action)) + raise UFWError(("'{}' not supported, use 'allow' " + "or 'delete'").format(action)) diff --git a/hooks/charmhelpers/contrib/openstack/__init__.py b/hooks/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py index b413259c..ef77caf3 100644 --- a/hooks/charmhelpers/contrib/openstack/alternatives.py +++ b/hooks/charmhelpers/contrib/openstack/alternatives.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Helper for managing alternatives for file conflict resolution ''' import subprocess diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment @@ -55,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index f63f0078..753f5829 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import json import os import time diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index f062c807..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from charmhelpers.core.hookenv import ( config, unit_get, @@ -10,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -91,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 095cc24b..902757fe 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Various utilies for dealing with Neutron and the renaming from Quantum. from subprocess import check_output diff --git a/hooks/charmhelpers/contrib/openstack/templates/__init__.py b/hooks/charmhelpers/contrib/openstack/templates/__init__.py index 0b49ad28..75876796 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/templates/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # dummy __init__.py to fool syncer into thinking this is a syncable python # module diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 33df0675..24cb272b 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import six diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ddd40ce5..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1,5 +1,21 @@ #!/usr/bin/python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Common python helper functions used for OpenStack charms. from collections import OrderedDict from functools import wraps @@ -87,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/python/__init__.py +++ b/hooks/charmhelpers/contrib/python/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 78162b1b..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -1,7 +1,21 @@ #!/usr/bin/env python # coding: utf-8 -__author__ = "Jorge Niedbalski " +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -13,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" @@ -35,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -43,6 +59,9 @@ def pip_install(package, fatal=False, **options): for option in parse_options(options, available_options): command.append(option) + if upgrade: + command.append('--upgrade') + if isinstance(package, list): command.extend(package) else: diff --git a/hooks/charmhelpers/contrib/storage/__init__.py b/hooks/charmhelpers/contrib/storage/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/__init__.py +++ b/hooks/charmhelpers/contrib/storage/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/__init__.py b/hooks/charmhelpers/contrib/storage/linux/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/linux/__init__.py +++ b/hooks/charmhelpers/contrib/storage/linux/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 6ebeab5c..31ea7f9e 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py index a22c3d7b..c296f098 100644 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re from subprocess import ( diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py index 0aa65f4f..34b5f71a 100644 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from subprocess import ( CalledProcessError, check_call, diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c6a15e14..c8373b72 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re from stat import S_ISBLK diff --git a/hooks/charmhelpers/core/__init__.py b/hooks/charmhelpers/core/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/core/__init__.py +++ b/hooks/charmhelpers/core/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/core/decorators.py b/hooks/charmhelpers/core/decorators.py index 029a4ef4..bb05620b 100644 --- a/hooks/charmhelpers/core/decorators.py +++ b/hooks/charmhelpers/core/decorators.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2014 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index 0adf0db3..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -1,11 +1,27 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__author__ = 'Jorge Niedbalski R. ' +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer @@ -61,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -88,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 69ae4564..cf552b39 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Interactions with the Juju environment" # Copyright 2013 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 5221120c..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Tools for working with the host system""" # Copyright 2012 Canonical Ltd. # @@ -168,18 +184,18 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): log("Removing non-directory file {} prior to mkdir()".format(path)) os.unlink(realpath) os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) elif not path_exists: os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) + os.chown(realpath, uid, gid) + os.chmod(realpath, perms) def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -289,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): @@ -345,7 +361,7 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) if matched: interface = matched.groups()[0] else: @@ -389,6 +405,9 @@ def cmp_pkgrevno(package, revno, pkgcache=None): * 0 => Installed revno is the same as supplied arg * -1 => Installed revno is less than supplied arg + This function imports apt_cache function from charmhelpers.fetch if + the pkgcache argument is None. Be sure to add charmhelpers.fetch if + you call this function, or pass an apt_pkg.Cache() instance. ''' import apt_pkg if not pkgcache: @@ -407,13 +426,21 @@ def chdir(d): os.chdir(cur) -def chownr(path, owner, group): +def chownr(path, owner, group, follow_links=True): uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid + if follow_links: + chown = os.chown + else: + chown = os.lchown for root, dirs, files in os.walk(path): for name in dirs + files: full = os.path.join(root, name) broken_symlink = os.path.lexists(full) and not os.path.exists(full) if not broken_symlink: - os.chown(full, uid, gid) + chown(full, uid, gid) + + +def lchownr(path, owner, group): + chownr(path, owner, group, follow_links=False) diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py index 69dde79a..0928158b 100644 --- a/hooks/charmhelpers/core/services/__init__.py +++ b/hooks/charmhelpers/core/services/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from .base import * # NOQA from .helpers import * # NOQA diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py index 87ecb130..c5534e4c 100644 --- a/hooks/charmhelpers/core/services/base.py +++ b/hooks/charmhelpers/core/services/base.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re import json diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 163a7932..5e3af9da 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import yaml from charmhelpers.core import hookenv diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py new file mode 100644 index 00000000..efc4402e --- /dev/null +++ b/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in ['y', 'yes', 'true', 't']: + return True + elif value in ['n', 'no', 'false', 'f']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 0f299630..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -1,7 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__author__ = 'Jorge Niedbalski R. ' +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . import yaml @@ -10,25 +24,33 @@ from subprocess import check_call from charmhelpers.core.hookenv import ( log, DEBUG, + ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } - :type sysctl_dict: dict + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - sysctl_dict = yaml.load(sysctl_dict) + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return with open(sysctl_file, "w") as fd: - for key, value in sysctl_dict.items(): + for key, value in sysctl_dict_parsed.items(): fd.write("{}={}\n".format(key, value)) - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), level=DEBUG) check_call(["sysctl", "-p", sysctl_file]) diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 569eaed6..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.core import host @@ -5,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -48,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py new file mode 100644 index 00000000..3000134a --- /dev/null +++ b/hooks/charmhelpers/core/unitdata.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . +# +# +# Authors: +# Kapil Thangavelu +# +""" +Intro +----- + +A simple way to store state in units. This provides a key value +storage with support for versioned, transactional operation, +and can calculate deltas from previous values to simplify unit logic +when processing changes. + + +Hook Integration +---------------- + +There are several extant frameworks for hook execution, including + + - charmhelpers.core.hookenv.Hooks + - charmhelpers.core.services.ServiceManager + +The storage classes are framework agnostic, one simple integration is +via the HookData contextmanager. It will record the current hook +execution environment (including relation data, config data, etc.), +setup a transaction and allow easy access to the changes from +previously seen values. One consequence of the integration is the +reservation of particular keys ('rels', 'unit', 'env', 'config', +'charm_revisions') for their respective values. + +Here's a fully worked integration example using hookenv.Hooks:: + + from charmhelper.core import hookenv, unitdata + + hook_data = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # Print all changes to configuration from previously seen + # values. + for changed, (prev, cur) in hook_data.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + # Directly access all charm config as a mapping. + conf = db.getrange('config', True) + + # Directly access all relation data as a mapping + rels = db.getrange('rels', True) + + if __name__ == '__main__': + with hook_data(): + hook.execute() + + +A more basic integration is via the hook_scope context manager which simply +manages transaction scope (and records hook name, and timestamp):: + + >>> from unitdata import kv + >>> db = kv() + >>> with db.hook_scope('install'): + ... # do work, in transactional scope. + ... db.set('x', 1) + >>> db.get('x') + 1 + + +Usage +----- + +Values are automatically json de/serialized to preserve basic typing +and complex data struct capabilities (dicts, lists, ints, booleans, etc). + +Individual values can be manipulated via get/set:: + + >>> kv.set('y', True) + >>> kv.get('y') + True + + # We can set complex values (dicts, lists) as a single key. + >>> kv.set('config', {'a': 1, 'b': True'}) + + # Also supports returning dictionaries as a record which + # provides attribute access. + >>> config = kv.get('config', record=True) + >>> config.b + True + + +Groups of keys can be manipulated with update/getrange:: + + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") + >>> kv.getrange('gui.', strip=True) + {'z': 1, 'y': 2} + +When updating values, its very helpful to understand which values +have actually changed and how have they changed. The storage +provides a delta method to provide for this:: + + >>> data = {'debug': True, 'option': 2} + >>> delta = kv.delta(data, 'config.') + >>> delta.debug.previous + None + >>> delta.debug.current + True + >>> delta + {'debug': (None, True), 'option': (None, 2)} + +Note the delta method does not persist the actual change, it needs to +be explicitly saved via 'update' method:: + + >>> kv.update(data, 'config.') + +Values modified in the context of a hook scope retain historical values +associated to the hookname. + + >>> with db.hook_scope('config-changed'): + ... db.set('x', 42) + >>> db.gethistory('x') + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] + +""" + +import collections +import contextlib +import datetime +import json +import os +import pprint +import sqlite3 +import sys + +__author__ = 'Kapil Thangavelu ' + + +class Storage(object): + """Simple key value database for local unit state within charms. + + Modifications are automatically committed at hook exit. That's + currently regardless of exit code. + + To support dicts, lists, integer, floats, and booleans values + are automatically json encoded/decoded. + """ + def __init__(self, path=None): + self.db_path = path + if path is None: + self.db_path = os.path.join( + os.environ.get('CHARM_DIR', ''), '.unit-state.db') + self.conn = sqlite3.connect('%s' % self.db_path) + self.cursor = self.conn.cursor() + self.revision = None + self._closed = False + self._init() + + def close(self): + if self._closed: + return + self.flush(False) + self.cursor.close() + self.conn.close() + self._closed = True + + def _scoped_query(self, stmt, params=None): + if params is None: + params = [] + return stmt, params + + def get(self, key, default=None, record=False): + self.cursor.execute( + *self._scoped_query( + 'select data from kv where key=?', [key])) + result = self.cursor.fetchone() + if not result: + return default + if record: + return Record(json.loads(result[0])) + return json.loads(result[0]) + + def getrange(self, key_prefix, strip=False): + stmt = "select key, data from kv where key like '%s%%'" % key_prefix + self.cursor.execute(*self._scoped_query(stmt)) + result = self.cursor.fetchall() + + if not result: + return None + if not strip: + key_prefix = '' + return dict([ + (k[len(key_prefix):], json.loads(v)) for k, v in result]) + + def update(self, mapping, prefix=""): + for k, v in mapping.items(): + self.set("%s%s" % (prefix, k), v) + + def unset(self, key): + self.cursor.execute('delete from kv where key=?', [key]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + [key, self.revision, json.dumps('DELETED')]) + + def set(self, key, value): + serialized = json.dumps(value) + + self.cursor.execute( + 'select data from kv where key=?', [key]) + exists = self.cursor.fetchone() + + # Skip mutations to the same value + if exists: + if exists[0] == serialized: + return value + + if not exists: + self.cursor.execute( + 'insert into kv (key, data) values (?, ?)', + (key, serialized)) + else: + self.cursor.execute(''' + update kv + set data = ? + where key = ?''', [serialized, key]) + + # Save + if not self.revision: + return value + + self.cursor.execute( + 'select 1 from kv_revisions where key=? and revision=?', + [key, self.revision]) + exists = self.cursor.fetchone() + + if not exists: + self.cursor.execute( + '''insert into kv_revisions ( + revision, key, data) values (?, ?, ?)''', + (self.revision, key, serialized)) + else: + self.cursor.execute( + ''' + update kv_revisions + set data = ? + where key = ? + and revision = ?''', + [serialized, key, self.revision]) + + return value + + def delta(self, mapping, prefix): + """ + return a delta containing values that have changed. + """ + previous = self.getrange(prefix, strip=True) + if not previous: + pk = set() + else: + pk = set(previous.keys()) + ck = set(mapping.keys()) + delta = DeltaSet() + + # added + for k in ck.difference(pk): + delta[k] = Delta(None, mapping[k]) + + # removed + for k in pk.difference(ck): + delta[k] = Delta(previous[k], None) + + # changed + for k in pk.intersection(ck): + c = mapping[k] + p = previous[k] + if c != p: + delta[k] = Delta(p, c) + + return delta + + @contextlib.contextmanager + def hook_scope(self, name=""): + """Scope all future interactions to the current hook execution + revision.""" + assert not self.revision + self.cursor.execute( + 'insert into hooks (hook, date) values (?, ?)', + (name or sys.argv[0], + datetime.datetime.utcnow().isoformat())) + self.revision = self.cursor.lastrowid + try: + yield self.revision + self.revision = None + except: + self.flush(False) + self.revision = None + raise + else: + self.flush() + + def flush(self, save=True): + if save: + self.conn.commit() + elif self._closed: + return + else: + self.conn.rollback() + + def _init(self): + self.cursor.execute(''' + create table if not exists kv ( + key text, + data text, + primary key (key) + )''') + self.cursor.execute(''' + create table if not exists kv_revisions ( + key text, + revision integer, + data text, + primary key (key, revision) + )''') + self.cursor.execute(''' + create table if not exists hooks ( + version integer primary key autoincrement, + hook text, + date text + )''') + self.conn.commit() + + def gethistory(self, key, deserialize=False): + self.cursor.execute( + ''' + select kv.revision, kv.key, kv.data, h.hook, h.date + from kv_revisions kv, + hooks h + where kv.key=? + and kv.revision = h.version + ''', [key]) + if deserialize is False: + return self.cursor.fetchall() + return map(_parse_history, self.cursor.fetchall()) + + def debug(self, fh=sys.stderr): + self.cursor.execute('select * from kv') + pprint.pprint(self.cursor.fetchall(), stream=fh) + self.cursor.execute('select * from kv_revisions') + pprint.pprint(self.cursor.fetchall(), stream=fh) + + +def _parse_history(d): + return (d[0], d[1], json.loads(d[2]), d[3], + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) + + +class HookData(object): + """Simple integration for existing hook exec frameworks. + + Records all unit information, and stores deltas for processing + by the hook. + + Sample:: + + from charmhelper.core import hookenv, unitdata + + changes = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # View all changes to configuration + for changed, (prev, cur) in changes.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + if __name__ == '__main__': + with changes(): + hook.execute() + + """ + def __init__(self): + self.kv = kv() + self.conf = None + self.rels = None + + @contextlib.contextmanager + def __call__(self): + from charmhelpers.core import hookenv + hook_name = hookenv.hook_name() + + with self.kv.hook_scope(hook_name): + self._record_charm_version(hookenv.charm_dir()) + delta_config, delta_relation = self._record_hook(hookenv) + yield self.kv, delta_config, delta_relation + + def _record_charm_version(self, charm_dir): + # Record revisions.. charm revisions are meaningless + # to charm authors as they don't control the revision. + # so logic dependnent on revision is not particularly + # useful, however it is useful for debugging analysis. + charm_rev = open( + os.path.join(charm_dir, 'revision')).read().strip() + charm_rev = charm_rev or '0' + revs = self.kv.get('charm_revisions', []) + if charm_rev not in revs: + revs.append(charm_rev.strip() or '0') + self.kv.set('charm_revisions', revs) + + def _record_hook(self, hookenv): + data = hookenv.execution_environment() + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') + self.kv.set('env', data['env']) + self.kv.set('unit', data['unit']) + self.kv.set('relid', data.get('relid')) + return conf_delta, rels_delta + + +class Record(dict): + + __slots__ = () + + def __getattr__(self, k): + if k in self: + return self[k] + raise AttributeError(k) + + +class DeltaSet(Record): + + __slots__ = () + + +Delta = collections.namedtuple('Delta', ['previous', 'current']) + + +_KV = None + + +def kv(): + global _KV + if _KV is None: + _KV = Storage() + return _KV diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index aceadea4..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import importlib from tempfile import NamedTemporaryFile import time diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 8a4624b2..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,7 +1,33 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -19,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 8ef48f30..3531315a 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -11,10 +27,12 @@ if six.PY3: try: from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors except ImportError: from charmhelpers.fetch import apt_install apt_install("python-bzrlib") from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors class BzrUrlFetchHandler(BaseFetchHandler): @@ -34,9 +52,15 @@ class BzrUrlFetchHandler(BaseFetchHandler): if url_parts.scheme == "lp": from bzrlib.plugin import load_plugins load_plugins() + try: + local_branch = bzrdir.BzrDir.create_branch_convenience(dest) + except errors.AlreadyControlDirError: + local_branch = Branch.open(dest) try: remote_branch = Branch.open(source) - remote_branch.bzrdir.sprout(dest).open_branch() + remote_branch.push(local_branch) + tree = workingtree.WorkingTree.open(dest) + tree.update() except Exception as e: raise e diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index f3aa2821..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -16,6 +32,8 @@ except ImportError: apt_install("python-git") from git import Repo +from git.exc import GitCommandError # noqa E402 + class GitUrlFetchHandler(BaseFetchHandler): """Handler for git branches via generic and github URLs""" @@ -46,6 +64,8 @@ class GitUrlFetchHandler(BaseFetchHandler): mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch) + except GitCommandError as e: + raise UnhandledSource(e.message) except OSError as e: raise UnhandledSource(e.strerror) return dest_dir diff --git a/hooks/charmhelpers/payload/__init__.py b/hooks/charmhelpers/payload/__init__.py index fc9fbc08..e6f42497 100644 --- a/hooks/charmhelpers/payload/__init__.py +++ b/hooks/charmhelpers/payload/__init__.py @@ -1 +1,17 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Tools for working with files injected into a charm just before deployment." diff --git a/hooks/charmhelpers/payload/execd.py b/hooks/charmhelpers/payload/execd.py index 6476a75f..4d4d81a6 100644 --- a/hooks/charmhelpers/payload/execd.py +++ b/hooks/charmhelpers/payload/execd.py @@ -1,5 +1,21 @@ #!/usr/bin/env python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import sys import subprocess diff --git a/hooks/ha-relation-departed b/hooks/ha-relation-departed new file mode 120000 index 00000000..9a2da58e --- /dev/null +++ b/hooks/ha-relation-departed @@ -0,0 +1 @@ +quantum_hooks.py \ No newline at end of file diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index 21761319..86ea0bab 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -23,6 +23,7 @@ from charmhelpers.core.host import ( lsb_release, ) from charmhelpers.contrib.hahelpers.cluster import( + get_hacluster_config, eligible_leader ) from charmhelpers.contrib.hahelpers.apache import( @@ -50,8 +51,14 @@ from quantum_utils import ( get_topics, valid_plugin, configure_ovs, + stop_services, + cache_env_data, + update_legacy_ha_files, + remove_legacy_ha_files, + install_legacy_ha_files, + cleanup_ovs_netns, reassign_agent_resources, - stop_services + stop_neutron_ha_monitor_daemon ) hooks = Hooks() @@ -77,6 +84,9 @@ def install(): log('Please provide a valid plugin config', level=ERROR) sys.exit(1) + # Legacy HA for Icehouse + update_legacy_ha_files() + @hooks.hook('config-changed') @restart_on_change(restart_map()) @@ -113,11 +123,15 @@ def config_changed(): else: apt_purge('neutron-l3-agent') + # Setup legacy ha configurations + update_legacy_ha_files() + @hooks.hook('upgrade-charm') def upgrade_charm(): install() config_changed() + update_legacy_ha_files(force=True) @hooks.hook('shared-db-relation-joined') @@ -198,6 +212,9 @@ def nm_changed(): ca_crt = b64decode(relation_get('ca_cert')) install_ca_cert(ca_crt) + if config('ha-legacy-mode'): + cache_env_data() + @hooks.hook("cluster-relation-departed") @restart_on_change(restart_map()) @@ -211,7 +228,7 @@ def cluster_departed(): log('Unable to re-assign agent resources for failed nodes with n1kv', level=WARNING) return - if eligible_leader(None): + if not config('ha-legacy-mode') and eligible_leader(None): reassign_agent_resources() CONFIGS.write_all() @@ -220,6 +237,9 @@ def cluster_departed(): @hooks.hook('stop') def stop(): stop_services() + if config('ha-legacy-mode'): + # Cleanup ovs and netns for destroyed units. + cleanup_ovs_netns() @hooks.hook('zeromq-configuration-relation-joined') @@ -262,6 +282,40 @@ def update_nrpe_config(): nrpe_setup.write() +@hooks.hook('ha-relation-joined') +@hooks.hook('ha-relation-changed') +def ha_relation_joined(): + if config('ha-legacy-mode'): + log('ha-relation-changed update_legacy_ha_files') + install_legacy_ha_files() + cache_env_data() + cluster_config = get_hacluster_config(exclude_keys=['vip']) + resources = { + 'res_monitor': 'ocf:canonical:NeutronAgentMon', + } + resource_params = { + 'res_monitor': 'op monitor interval="60s"', + } + clones = { + 'cl_monitor': 'res_monitor meta interleave="true"', + } + + relation_set(corosync_bindiface=cluster_config['ha-bindiface'], + corosync_mcastport=cluster_config['ha-mcastport'], + resources=resources, + resource_params=resource_params, + clones=clones) + + +@hooks.hook('ha-relation-departed') +def ha_relation_destroyed(): + # If e.g. we want to upgrade to Juno and use native Neutron HA support then + # we need to un-corosync-cluster to enable the transition. + if config('ha-legacy-mode'): + stop_neutron_ha_monitor_daemon() + remove_legacy_ha_files() + + if __name__ == '__main__': try: hooks.execute(sys.argv) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index d704dc47..e7813027 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -1,11 +1,18 @@ +import os +import subprocess +from shutil import copy2 from charmhelpers.core.host import ( + mkdir, service_running, service_stop, service_restart, - lsb_release + lsb_release, ) from charmhelpers.core.hookenv import ( log, + DEBUG, + INFO, + ERROR, config, relations_of_type, unit_private_ip, @@ -145,6 +152,22 @@ EARLY_PACKAGES = { N1KV: [] } +LEGACY_HA_TEMPLATE_FILES = 'files' +LEGACY_FILES_MAP = { + 'neutron-ha-monitor.py': { + 'path': '/usr/local/bin/', + 'permissions': 0o755 + }, + 'neutron-ha-monitor.conf': { + 'path': '/var/lib/juju-neutron-ha/', + }, + 'NeutronAgentMon': { + 'path': '/usr/lib/ocf/resource.d/canonical', + 'permissions': 0o755 + }, +} +LEGACY_RES_MAP = ['res_monitor'] + def get_early_packages(): '''Return a list of package for pre-install based on configured plugin''' @@ -163,14 +186,17 @@ def get_packages(): '''Return a list of packages for install based on the configured plugin''' plugin = remap_plugin(config('plugin')) packages = deepcopy(GATEWAY_PKGS[networking_name()][plugin]) - if (get_os_codename_install_source(config('openstack-origin')) - >= 'icehouse' and plugin == 'ovs' - and lsb_release()['DISTRIB_CODENAME'] < 'utopic'): - # NOTE(jamespage) neutron-vpn-agent supercedes l3-agent for icehouse - # but openswan was removed in utopic. - packages.remove('neutron-l3-agent') - packages.append('neutron-vpn-agent') - packages.append('openswan') + source = get_os_codename_install_source(config('openstack-origin')) + if plugin == 'ovs': + if (source >= 'icehouse' and + lsb_release()['DISTRIB_CODENAME'] < 'utopic'): + # NOTE(jamespage) neutron-vpn-agent supercedes l3-agent for + # icehouse but openswan was removed in utopic. + packages.remove('neutron-l3-agent') + packages.append('neutron-vpn-agent') + packages.append('openswan') + if source >= 'kilo': + packages.append('python-neutron-fwaas') return packages @@ -584,6 +610,103 @@ def configure_ovs(): promisc=True) +def copy_file(src, dst, perms=None, force=False): + """Copy file to destination and optionally set permissionss. + + If destination does not exist it will be created. + """ + if not os.path.isdir(dst): + log('Creating directory %s' % dst, level=DEBUG) + mkdir(dst) + + fdst = os.path.join(dst, os.path.basename(src)) + if not os.path.isfile(fdst) or force: + try: + copy2(src, fdst) + if perms: + os.chmod(fdst, perms) + except IOError: + log('Failed to copy file from %s to %s.' % (src, dst), level=ERROR) + raise + + +def remove_file(path): + if not os.path.isfile(path): + log('File %s does not exist.' % path, level=INFO) + return + + try: + os.remove(path) + except IOError: + log('Failed to remove file %s.' % path, level=ERROR) + + +def install_legacy_ha_files(force=False): + for f, p in LEGACY_FILES_MAP.iteritems(): + srcfile = os.path.join(LEGACY_HA_TEMPLATE_FILES, f) + copy_file(srcfile, p['path'], p.get('permissions', None), force=force) + + +def remove_legacy_ha_files(): + for f, p in LEGACY_FILES_MAP.iteritems(): + remove_file(os.path.join(p['path'], f)) + + +def update_legacy_ha_files(force=False): + if config('ha-legacy-mode'): + install_legacy_ha_files(force=force) + else: + remove_legacy_ha_files() + + +def cache_env_data(): + env = NetworkServiceContext()() + if not env: + log('Unable to get NetworkServiceContext at this time', level=ERROR) + return + + no_envrc = False + envrc_f = '/etc/legacy_ha_envrc' + if os.path.isfile(envrc_f): + with open(envrc_f, 'r') as f: + data = f.read() + + data = data.strip().split('\n') + diff = False + for line in data: + k = line.split('=')[0] + v = line.split('=')[1] + if k not in env or v != env[k]: + diff = True + break + else: + no_envrc = True + + if no_envrc or diff: + with open(envrc_f, 'w') as f: + for k, v in env.items(): + f.write(''.join([k, '=', v, '\n'])) + + +def stop_neutron_ha_monitor_daemon(): + try: + cmd = ['pgrep', '-f', 'neutron-ha-monitor.py'] + res = subprocess.check_output(cmd).decode('UTF-8') + pid = res.strip() + if pid: + subprocess.call(['sudo', 'kill', '-9', pid]) + except subprocess.CalledProcessError as e: + log('Faild to kill neutron-ha-monitor daemon, %s' % e, level=ERROR) + + +def cleanup_ovs_netns(): + try: + subprocess.call('neutron-ovs-cleanup') + subprocess.call('neutron-netns-cleanup') + except subprocess.CalledProcessError as e: + log('Faild to cleanup ovs and netns, %s' % e, level=ERROR) + + def get_topics(): # metering_agent topics = [] diff --git a/metadata.yaml b/metadata.yaml index b80a0d9b..8f6b4ff0 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -35,6 +35,9 @@ requires: scope: container neutron-plugin-api: interface: neutron-plugin-api + ha: + interface: hacluster + scope: container peers: cluster: interface: quantum-gateway-ha diff --git a/templates/kilo/fwaas_driver.ini b/templates/kilo/fwaas_driver.ini new file mode 100644 index 00000000..8ce9e542 --- /dev/null +++ b/templates/kilo/fwaas_driver.ini @@ -0,0 +1,7 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[fwaas] +driver = neutron_fwaas.services.firewall.drivers.linux.iptables_fwaas.IptablesFwaasDriver +enabled = True diff --git a/templates/kilo/lbaas_agent.ini b/templates/kilo/lbaas_agent.ini new file mode 100644 index 00000000..b37b7e1d --- /dev/null +++ b/templates/kilo/lbaas_agent.ini @@ -0,0 +1,8 @@ +[DEFAULT] +periodic_interval = 10 +interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver +ovs_use_veth = False +device_driver = neutron_lbaas.services.loadbalancer.drivers.haproxy.namespace_driver.HaproxyNSDriver +[haproxy] +loadbalancer_state_path = $state_path/lbaas +user_group = nogroup diff --git a/templates/kilo/nova.conf b/templates/kilo/nova.conf new file mode 100644 index 00000000..47489f8b --- /dev/null +++ b/templates/kilo/nova.conf @@ -0,0 +1,29 @@ +# kilo +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +logdir=/var/log/nova +state_path=/var/lib/nova +lock_path=/var/lock/nova +root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf +verbose= {{ verbose }} +use_syslog = {{ use_syslog }} +api_paste_config=/etc/nova/api-paste.ini +enabled_apis=metadata +multi_host=True +{% include "parts/database" %} +# Access to message bus +{% include "parts/rabbitmq" %} +# Access to neutron API services +network_api_class=nova.network.neutronv2.api.API +[neutron] +auth_strategy=keystone +url={{ quantum_url }} +admin_tenant_name={{ service_tenant }} +admin_username={{ service_username }} +admin_password={{ service_password }} +admin_auth_url={{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 +service_metadata_proxy=True +metadata_proxy_shared_secret={{ shared_secret }} diff --git a/templates/kilo/vpn_agent.ini b/templates/kilo/vpn_agent.ini new file mode 100644 index 00000000..90874fd0 --- /dev/null +++ b/templates/kilo/vpn_agent.ini @@ -0,0 +1,8 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[vpnagent] +vpn_device_driver=neutron_vpnaas.services.vpn.device_drivers.ipsec.OpenSwanDriver +[ipsec] +ipsec_status_check_interval=60 diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/tests/charmhelpers/__init__.py +++ b/tests/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/__init__.py +++ b/tests/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/amulet/__init__.py +++ b/tests/charmhelpers/contrib/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py index 3d3ef339..367d6b47 100644 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import amulet import os import six diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index d333e63b..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import ConfigParser import io import logging @@ -153,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -171,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/__init__.py +++ b/tests/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment @@ -55,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index 7bbeac5c..06452f36 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -33,8 +33,6 @@ TO_PATCH = [ 'unit_get', 'relation_get', 'install_ca_cert', - 'eligible_leader', - 'reassign_agent_resources', 'get_common_package', 'execd_preinstall', 'lsb_release', @@ -43,6 +41,13 @@ TO_PATCH = [ 'is_relation_made', 'create_sysctl', 'update_nrpe_config', + 'update_legacy_ha_files', + 'install_legacy_ha_files', + 'cache_env_data', + 'get_hacluster_config', + 'remove_legacy_ha_files', + 'cleanup_ovs_netns', + 'stop_neutron_ha_monitor_daemon' ] @@ -247,19 +252,25 @@ class TestQuantumHooks(CharmTestCase): self.test_config.set('plugin', 'nvp') self._call_hook('cluster-relation-departed') self.assertTrue(self.log.called) - self.assertFalse(self.eligible_leader.called) - self.assertFalse(self.reassign_agent_resources.called) - - def test_cluster_departed_ovs_not_leader(self): - self.eligible_leader.return_value = False - self._call_hook('cluster-relation-departed') - self.assertFalse(self.reassign_agent_resources.called) - - def test_cluster_departed_ovs_leader(self): - self.eligible_leader.return_value = True - self._call_hook('cluster-relation-departed') - self.assertTrue(self.reassign_agent_resources.called) def test_stop(self): self._call_hook('stop') self.assertTrue(self.stop_services.called) + + def test_ha_relation_joined(self): + self.test_config.set('ha-legacy-mode', True) + self._call_hook('ha_relation_joined') + self.assertTrue(self.cache_env_data.called) + self.assertTrue(self.get_hacluster_config.called) + self.assertTrue(self.install_legacy_ha_files.called) + + def test_ha_relation_departed(self): + self.test_config.set('ha-legacy-mode', True) + self._call_hook('ha-relation-departed') + self.assertTrue(self.remove_legacy_ha_files.called) + self.assertTrue(self.stop_neutron_ha_monitor_daemon.called) + + def test_quantum_network_service_relation_changed(self): + self.test_config.set('ha-legacy-mode', True) + self._call_hook('quantum-network-service-relation-changed') + self.assertTrue(self.cache_env_data.called) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 64c0c25e..8c319761 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -44,7 +44,9 @@ TO_PATCH = [ 'service_restart', 'remap_plugin', 'is_relation_made', - 'lsb_release' + 'lsb_release', + 'mkdir', + 'copy2' ] @@ -132,6 +134,11 @@ class TestQuantumUtils(CharmTestCase): self.assertTrue('neutron-vpn-agent' in quantum_utils.get_packages()) self.assertFalse('neutron-l3-agent' in quantum_utils.get_packages()) + def test_get_packages_ovs_kilo(self): + self.config.return_value = 'ovs' + self.get_os_codename_install_source.return_value = 'kilo' + self.assertTrue('python-neutron-fwaas' in quantum_utils.get_packages()) + def test_configure_ovs_starts_service_if_required(self): self.config.return_value = 'ovs' self.service_running.return_value = False @@ -360,6 +367,40 @@ class TestQuantumUtils(CharmTestCase): self.get_os_codename_package.return_value = None self.assertEquals(quantum_utils.get_common_package(), 'neutron-common') + def test_copy_file_without_update(self): + src = 'dummy_source_dir/dummy_file' + dst = 'dummy_des_dir' + quantum_utils.copy_file(src, dst) + self.assertTrue(self.mkdir.called) + self.assertTrue(self.copy2.called) + + @patch('quantum_utils.os.path.isfile') + def test_copy_file_with_update(self, _isfile): + src = 'dummy_source_dir/dummy_file' + dst = 'dummy_des_dir' + _isfile.return_value = False + quantum_utils.copy_file(src, dst, force=True) + self.assertTrue(self.mkdir.called) + self.assertTrue(self.copy2.called) + + @patch('quantum_utils.os.remove') + @patch('quantum_utils.os.path.isfile') + def test_remove_file_exists(self, _isfile, _remove): + path = 'dummy_des_dir/dummy_file' + _isfile.return_value = True + quantum_utils.remove_file(path) + self.assertTrue(_remove.called) + self.assertFalse(self.log.called) + + @patch('quantum_utils.os.remove') + @patch('quantum_utils.os.path.isfile') + def test_remove_file_non_exists(self, _isfile, _remove): + path = 'dummy_des_dir/dummy_file' + _isfile.return_value = False + quantum_utils.remove_file(path) + self.assertFalse(_remove.called) + self.assertTrue(self.log.called) + network_context = { 'service_username': 'foo',