diff --git a/anvil/components/helpers/nova.py b/anvil/components/helpers/nova.py index e4f942e8..0c733137 100644 --- a/anvil/components/helpers/nova.py +++ b/anvil/components/helpers/nova.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import psutil +import re import weakref from anvil import cfg @@ -113,6 +115,140 @@ def get_shared_params(ip, protocol, return mp +class ComputeCleaner(object): + def __init__(self, uninstaller): + self.uninstaller = weakref.proxy(uninstaller) + + def clean(self): + virsh = lv.Virsh(self.uninstaller.get_int_option('service_wait_seconds'), self.uninstaller.distro) + virt_driver = canon_virt_driver(self.uninstaller.get_option('virt_driver')) + if virt_driver == 'libvirt': + inst_prefix = self.uninstaller.get_option('instance_name_prefix', default_value='instance-') + libvirt_type = lv.canon_libvirt_type(self.uninstaller.get_option('libvirt_type')) + virsh.clear_domains(libvirt_type, inst_prefix) + + +class NetworkCleaner(object): + def __init__(self, uninstaller): + self.uninstaller = weakref.proxy(uninstaller) + + def _stop_dnsmasq(self): + # Shutdown dnsmasq which is typically used by nova-network + # to provide dhcp leases and since nova currently doesn't + # seem to shut them down itself (why not?) we have to do it for it.. + # + # TODO(harlowja) file a bug to get that fixed... + to_kill = [] + for proc in psutil.process_iter(): + if proc.name.find("dnsmasq") == -1: + continue + cwd = '' + cmdline = '' + with sh.Rooted(True): + cwd = proc.getcwd() + cmdline = proc.cmdline + to_try = False + for t in [cwd, cmdline]: + if t.lower().find("nova") != -1: + to_try = True + if to_try: + to_kill.append(proc.pid) + if len(to_kill): + utils.log_iterable(to_kill, + header="Killing leftover nova dnsmasq processes with process ids", + logger=LOG) + for pid in to_kill: + with sh.Rooted(True): + sh.kill(pid) + + def _clean_iptables(self): + # Nova doesn't seem to cleanup its iptables rules that it + # establishes when it is removed, this is unfortunate as that + # means that when nova is uninstalled it may have just left the + # host machine in a un-useable state... + # + # TODO(harlowja) file a bug to get that fixed... + + def line_matcher(line, start_text): + if not line: + return False + if not line.startswith(start_text): + return False + if line.lower().find("nova") == -1: + return False + return True + + def translate_rule(line, start_search, start_replace): + line = re.sub(r"-c\s+[0-9]*\s+[0-9]*", "", line, re.I) + if not line.startswith(start_search): + return line + return line.replace(start_search, start_replace, 1) + + # Isolate the nova rules + clean_rules = [] + list_cmd = ['iptables', '--list-rules', '--verbose'] + (stdout, _stderr) = sh.execute(list_cmd, run_as_root=True) + for line in stdout.splitlines(): + line = line.strip() + if not line_matcher(line, "-A"): + continue + # Translate it into a delete rule operation + rule = translate_rule(line, "-A", "-D") + if rule: + clean_rules.append(rule) + + # Isolate the nova nat rules + clean_nats = [] + nat_cmd = ['iptables', '--list-rules', '--verbose', '--table', 'nat'] + (stdout, _stderr) = sh.execute(nat_cmd, run_as_root=True) + for line in stdout.splitlines(): + line = line.strip() + if not line_matcher(line, "-A"): + continue + # Translate it into a delete rule operation + rule = translate_rule(line, "-A", "-D") + if rule: + clean_nats.append(rule) + + # Isolate the nova chains + clean_chains = [] + chain_cmd = ['iptables', '--list-rules', '--verbose'] + (stdout, _stderr) = sh.execute(list_cmd, run_as_root=True) + for line in stdout.splitlines(): + if not line_matcher(line, "-N"): + continue + # Translate it into a delete rule operation + rule = translate_rule(line, "-N", "-X") + if rule: + clean_chains.append(rule) + + # Isolate the nova nat chains + clean_nat_chains = [] + nat_chain_cmd = ['iptables', '--list-rules', '--verbose', '--table', 'nat'] + (stdout, _stderr) = sh.execute(list_cmd, run_as_root=True) + for line in stdout.splitlines(): + if not line_matcher(line, "-N"): + continue + # Translate it into a delete rule operation + rule = translate_rule(line, "-N", "-X") + if rule: + clean_nat_chains.append(rule) + + # Now execute them... + for r in clean_rules + clean_chains: + pieces = r.split(None) + pieces = ['iptables'] + pieces + sh.execute(*pieces, run_as_root=True, shell=True) + for r in clean_nats + clean_nat_chains: + pieces = r.split(None) + pieces = ['iptables', '--table', 'nat'] + pieces + sh.execute(*pieces, run_as_root=True, shell=True) + + def clean(self): + self._stop_dnsmasq() + self._clean_iptables() + + # This class has the smarts to build the configuration file based on # various runtime values. A useful reference for figuring out this # is at http://docs.openstack.org/diablo/openstack-compute/admin/content/ch_configuring-openstack-compute.html diff --git a/anvil/components/nova.py b/anvil/components/nova.py index f65c52b1..63ea3d06 100644 --- a/anvil/components/nova.py +++ b/anvil/components/nova.py @@ -24,7 +24,7 @@ except ImportError: from anvil import cfg from anvil import colorizer from anvil import components as comp -from anvil import exceptions +from anvil import exceptions as excp from anvil import log as logging from anvil import shell as sh from anvil import utils @@ -81,9 +81,6 @@ FLOATING_NET_CMDS = [ # Subdirs of the checkout/download BIN_DIR = 'bin' -# This is a special conf -CLEANER_DATA_CONF = 'nova-clean.sh' - class NovaUninstaller(comp.PythonUninstallComponent): def __init__(self, *args, **kargs): @@ -91,26 +88,24 @@ class NovaUninstaller(comp.PythonUninstallComponent): self.virsh = lv.Virsh(self.get_int_option('service_wait_seconds'), self.distro) def pre_uninstall(self): - self._clear_libvirt_domains() - self._clean_it() + if 'compute' in self.subsystems: + self._clean_compute() + if 'network' in self.subsystems: + self._clean_net() - def _clean_it(self): - cleaner_fn = sh.joinpths(self.get_option('component_dir'), 'tools', CLEANER_DATA_CONF) - if sh.isfile(cleaner_fn): - LOG.info("Cleaning up your system by running nova cleaner script: %s", colorizer.quote(cleaner_fn)) - # These environment additions are important - # in that they eventually affect how this script runs - env = { - 'ENABLED_SERVICES': ",".join(self.subsystems.keys()), - } - sh.execute(cleaner_fn, run_as_root=True, env_overrides=env) + def _clean_net(self): + cleaner = nhelper.NetworkCleaner(self) + try: + cleaner.clean() + except excp.AnvilException as e: + LOG.warn("Failed cleaning up nova-networks dirty laundry due to: %s", e) - def _clear_libvirt_domains(self): - virt_driver = nhelper.canon_virt_driver(self.get_option('virt_driver')) - if virt_driver == 'libvirt': - inst_prefix = self.get_option('instance_name_prefix', default_value='instance-') - libvirt_type = lv.canon_libvirt_type(self.get_option('libvirt_type')) - self.virsh.clear_domains(libvirt_type, inst_prefix) + def _clean_compute(self): + cleaner = nhelper.ComputeCleaner(self) + try: + cleaner.clean() + except excp.AnvilException as e: + LOG.warn("Failed cleaning up nova-computes dirty laundry due to: %s", e) class NovaInstaller(comp.PythonInstallComponent): @@ -156,15 +151,6 @@ class NovaInstaller(comp.PythonInstallComponent): if self.get_bool_option('db-sync'): self._setup_db() self._sync_db() - self._setup_cleaner() - - def _setup_cleaner(self): - LOG.info("Configuring cleaner template: %s", colorizer.quote(CLEANER_DATA_CONF)) - (_src_fn, contents) = utils.load_template(self.name, CLEANER_DATA_CONF) - cleaner_fn = sh.joinpths(self.get_option('component_dir'), 'tools', CLEANER_DATA_CONF) - sh.mkdirslist(sh.dirname(cleaner_fn), tracewriter=self.tracewriter) - sh.write_file(cleaner_fn, contents, tracewriter=self.tracewriter) - sh.chmod(cleaner_fn, 0755) def _setup_db(self): dbhelper.drop_db(distro=self.distro, @@ -323,11 +309,11 @@ class NovaRuntime(comp.PythonRuntime): self.virsh.check_virt(virt_type) self.virsh.restart_service() LOG.info("Libvirt virtualization type %s seems to be working and running.", colorizer.quote(virt_type)) - except exceptions.ProcessExecutionError as e: + except excp.ProcessExecutionError as e: msg = ("Libvirt type %r does not seem to be active or configured correctly, " "perhaps you should be using %r instead: %s" % (virt_type, lv.DEF_VIRT_TYPE, e)) - raise exceptions.StartException(msg) + raise excp.StartException(msg) def app_params(self, app_name): params = comp.PythonRuntime.app_params(self, app_name) diff --git a/anvil/shell.py b/anvil/shell.py index 119c7eeb..babf060d 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import errno import getpass import grp import os @@ -27,6 +26,8 @@ import subprocess import sys import time +import psutil # http://code.google.com/p/psutil/wiki/Documentation + from anvil import env from anvil import exceptions as excp from anvil import log as logging @@ -49,6 +50,11 @@ SUDO_UID = env.get_key('SUDO_UID') SUDO_GID = env.get_key('SUDO_GID') +class Process(psutil.Process): + def __str__(self): + return "%s (%s)" % (self.pid, self.name) + + class Rooted(object): def __init__(self, run_as_root): self.root_mode = run_as_root @@ -349,37 +355,39 @@ def explode_path(path): return _explode_path(path)[0] -def _attempt_kill(pid, signal_type, max_try, wait_time): +def _attempt_kill(proc, signal_type, max_try, wait_time): killed = False attempts = 0 for _i in range(0, max_try): + if not proc.is_running(): + killed = True + break try: - LOG.debug("Attempting to kill pid %s" % (pid)) + LOG.debug("Attempting to kill process %s" % (proc)) attempts += 1 - os.kill(pid, signal_type) - LOG.debug("Sleeping for %s seconds before next attempt to kill pid %s" % (wait_time, pid)) + proc.send_signal(signal_type) + LOG.debug("Sleeping for %s seconds before next attempt to kill process %s" % (wait_time, proc)) + sleep(wait_time) + except psutil.error.NoSuchProcess: + killed = True + break + except Exception as e: + LOG.debug("Failed killing %s due to: %s", proc, e) + LOG.debug("Sleeping for %s seconds before next attempt to kill process %s" % (wait_time, proc)) sleep(wait_time) - except OSError as e: - if e.errno == errno.ESRCH: - # Gotcha! - killed = True - break - else: - LOG.debug("Failed killing %s due to: %s", pid, e) - LOG.debug("Sleeping for %s seconds before next attempt to kill pid %s" % (wait_time, pid)) - sleep(wait_time) return (killed, attempts) def kill(pid, max_try=4, wait_time=1): if not is_running(pid) or is_dry_run(): return (True, 0) + proc = Process(pid) # Try the nicer sig-int first... - (killed, i_attempts) = _attempt_kill(pid, signal.SIGINT, int(max_try / 2), wait_time) + (killed, i_attempts) = _attempt_kill(proc, signal.SIGINT, int(max_try / 2), wait_time) if killed: return (True, i_attempts) # Get agressive and try sig-kill.... - (killed, k_attempts) = _attempt_kill(pid, signal.SIGKILL, int(max_try / 2), wait_time) + (killed, k_attempts) = _attempt_kill(proc, signal.SIGKILL, int(max_try / 2), wait_time) if killed: return (True, i_attempts + k_attempts) else: @@ -442,25 +450,7 @@ def fork(program, app_dir, pid_fn, stdout_fn, stderr_fn, *args): def is_running(pid): if is_dry_run(): return True - # TODO(harlowja): this can be done better - # but it will suffice for now.... - # - # Check proc - proc_fn = joinpths("/proc", str(pid)) - if exists(proc_fn): - LOG.debug("By looking at %s we determined %s is still running.", proc_fn, pid) - return True - # Try a slightly more aggressive way... - running = True - try: - os.kill(pid, signal.SIG_DFL) - except OSError as e: - if e.errno == errno.EPERM: - pass - else: - running = False - LOG.debug("By attempting to signal %s we determined it is %s", pid, {True: 'alive', False: 'dead'}[running]) - return running + return Process(pid).is_running() def mkdirslist(path, tracewriter=None, adjust_suids=False): diff --git a/conf/components/glance.yaml b/conf/components/glance.yaml index 93ecc693..11aa14b3 100644 --- a/conf/components/glance.yaml +++ b/conf/components/glance.yaml @@ -41,5 +41,5 @@ keystone: image_cache_dir: "/usr/share/anvil/glance/images" # Used by install section in the specfile -remove_file: /bin/rm -rf %{buildroot}/usr/bin/glance +remove_file: "/bin/rm -rf %{buildroot}/usr/bin/glance" ... diff --git a/conf/templates/nova/nova-clean.sh b/conf/templates/nova/nova-clean.sh deleted file mode 100644 index d5c61679..00000000 --- a/conf/templates/nova/nova-clean.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# This script cleans up the system iptables/services as part of a nova uninstall -# -# It is best effort! -# -# There are other scripts in tools/ that might be able to recover it better (but are distro specific) - -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root!" 1>&2 - exit 1 -fi - -set +o errexit -set -x - -# Clean off networking -if [[ "$ENABLED_SERVICES" =~ "net" ]]; then - - # Ignore any errors from shutting down dnsmasq - service dnsmasq stop - - # The above doesn't always work so this way will just incase - for pid in `ps -elf | grep -i dnsmasq | grep nova | perl -le 'while (<>) { my $pid = (split /\s+/)[3]; print $pid; }'` - do - echo "Killing leftover nova dnsmasq process with process id $pid" - kill -9 $pid - done - - # Delete rules - iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "iptables",$0}' | bash - - # Delete nat rules - iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "iptables -t nat",$0}' | bash - - # Delete chains - iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "iptables",$0}' | bash - - # Delete nat chains - iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "iptables -t nat",$0}' | bash - -fi - -exit 0 diff --git a/tools/pkg-requires b/tools/pkg-requires index 1ba6d6bb..710fa447 100644 --- a/tools/pkg-requires +++ b/tools/pkg-requires @@ -1,13 +1,11 @@ gcc git pylint -python -python-netifaces -# python-pep8 (conflicts with the ones to be installed) +python python-cheetah -# python-pip (conflicts with the ones to be installed) -python-progressbar -PyYAML -python-ordereddict python-iso8601 - +python-netifaces +python-ordereddict +python-progressbar +python-psutil +PyYAML