Convert nova clean into python code
1. Bring in the new dependency psutil to aid in killing and finding processes 2. Translate the actions being done by the nova cleanup script into pure python code
This commit is contained in:
Joshua Harlow
parent
d756c84954
commit
1528b311d7
@ -14,6 +14,8 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import psutil
|
||||
import re
|
||||
import weakref
|
||||
|
||||
from anvil import cfg
|
||||
@ -113,6 +115,140 @@ def get_shared_params(ip, protocol,
|
||||
return mp
|
||||
|
||||
|
||||
class ComputeCleaner(object):
|
||||
def __init__(self, uninstaller):
|
||||
self.uninstaller = weakref.proxy(uninstaller)
|
||||
|
||||
def clean(self):
|
||||
virsh = lv.Virsh(self.uninstaller.get_int_option('service_wait_seconds'), self.uninstaller.distro)
|
||||
virt_driver = canon_virt_driver(self.uninstaller.get_option('virt_driver'))
|
||||
if virt_driver == 'libvirt':
|
||||
inst_prefix = self.uninstaller.get_option('instance_name_prefix', default_value='instance-')
|
||||
libvirt_type = lv.canon_libvirt_type(self.uninstaller.get_option('libvirt_type'))
|
||||
virsh.clear_domains(libvirt_type, inst_prefix)
|
||||
|
||||
|
||||
class NetworkCleaner(object):
|
||||
def __init__(self, uninstaller):
|
||||
self.uninstaller = weakref.proxy(uninstaller)
|
||||
|
||||
def _stop_dnsmasq(self):
|
||||
# Shutdown dnsmasq which is typically used by nova-network
|
||||
# to provide dhcp leases and since nova currently doesn't
|
||||
# seem to shut them down itself (why not?) we have to do it for it..
|
||||
#
|
||||
# TODO(harlowja) file a bug to get that fixed...
|
||||
to_kill = []
|
||||
for proc in psutil.process_iter():
|
||||
if proc.name.find("dnsmasq") == -1:
|
||||
continue
|
||||
cwd = ''
|
||||
cmdline = ''
|
||||
with sh.Rooted(True):
|
||||
cwd = proc.getcwd()
|
||||
cmdline = proc.cmdline
|
||||
to_try = False
|
||||
for t in [cwd, cmdline]:
|
||||
if t.lower().find("nova") != -1:
|
||||
to_try = True
|
||||
if to_try:
|
||||
to_kill.append(proc.pid)
|
||||
if len(to_kill):
|
||||
utils.log_iterable(to_kill,
|
||||
header="Killing leftover nova dnsmasq processes with process ids",
|
||||
logger=LOG)
|
||||
for pid in to_kill:
|
||||
with sh.Rooted(True):
|
||||
sh.kill(pid)
|
||||
|
||||
def _clean_iptables(self):
|
||||
# Nova doesn't seem to cleanup its iptables rules that it
|
||||
# establishes when it is removed, this is unfortunate as that
|
||||
# means that when nova is uninstalled it may have just left the
|
||||
# host machine in a un-useable state...
|
||||
#
|
||||
# TODO(harlowja) file a bug to get that fixed...
|
||||
|
||||
def line_matcher(line, start_text):
|
||||
if not line:
|
||||
return False
|
||||
if not line.startswith(start_text):
|
||||
return False
|
||||
if line.lower().find("nova") == -1:
|
||||
return False
|
||||
return True
|
||||
|
||||
def translate_rule(line, start_search, start_replace):
|
||||
line = re.sub(r"-c\s+[0-9]*\s+[0-9]*", "", line, re.I)
|
||||
if not line.startswith(start_search):
|
||||
return line
|
||||
return line.replace(start_search, start_replace, 1)
|
||||
|
||||
# Isolate the nova rules
|
||||
clean_rules = []
|
||||
list_cmd = ['iptables', '--list-rules', '--verbose']
|
||||
(stdout, _stderr) = sh.execute(list_cmd, run_as_root=True)
|
||||
for line in stdout.splitlines():
|
||||
line = line.strip()
|
||||
if not line_matcher(line, "-A"):
|
||||
continue
|
||||
# Translate it into a delete rule operation
|
||||
rule = translate_rule(line, "-A", "-D")
|
||||
if rule:
|
||||
clean_rules.append(rule)
|
||||
|
||||
# Isolate the nova nat rules
|
||||
clean_nats = []
|
||||
nat_cmd = ['iptables', '--list-rules', '--verbose', '--table', 'nat']
|
||||
(stdout, _stderr) = sh.execute(nat_cmd, run_as_root=True)
|
||||
for line in stdout.splitlines():
|
||||
line = line.strip()
|
||||
if not line_matcher(line, "-A"):
|
||||
continue
|
||||
# Translate it into a delete rule operation
|
||||
rule = translate_rule(line, "-A", "-D")
|
||||
if rule:
|
||||
clean_nats.append(rule)
|
||||
|
||||
# Isolate the nova chains
|
||||
clean_chains = []
|
||||
chain_cmd = ['iptables', '--list-rules', '--verbose']
|
||||
(stdout, _stderr) = sh.execute(list_cmd, run_as_root=True)
|
||||
for line in stdout.splitlines():
|
||||
if not line_matcher(line, "-N"):
|
||||
continue
|
||||
# Translate it into a delete rule operation
|
||||
rule = translate_rule(line, "-N", "-X")
|
||||
if rule:
|
||||
clean_chains.append(rule)
|
||||
|
||||
# Isolate the nova nat chains
|
||||
clean_nat_chains = []
|
||||
nat_chain_cmd = ['iptables', '--list-rules', '--verbose', '--table', 'nat']
|
||||
(stdout, _stderr) = sh.execute(list_cmd, run_as_root=True)
|
||||
for line in stdout.splitlines():
|
||||
if not line_matcher(line, "-N"):
|
||||
continue
|
||||
# Translate it into a delete rule operation
|
||||
rule = translate_rule(line, "-N", "-X")
|
||||
if rule:
|
||||
clean_nat_chains.append(rule)
|
||||
|
||||
# Now execute them...
|
||||
for r in clean_rules + clean_chains:
|
||||
pieces = r.split(None)
|
||||
pieces = ['iptables'] + pieces
|
||||
sh.execute(*pieces, run_as_root=True, shell=True)
|
||||
for r in clean_nats + clean_nat_chains:
|
||||
pieces = r.split(None)
|
||||
pieces = ['iptables', '--table', 'nat'] + pieces
|
||||
sh.execute(*pieces, run_as_root=True, shell=True)
|
||||
|
||||
def clean(self):
|
||||
self._stop_dnsmasq()
|
||||
self._clean_iptables()
|
||||
|
||||
|
||||
# This class has the smarts to build the configuration file based on
|
||||
# various runtime values. A useful reference for figuring out this
|
||||
# is at http://docs.openstack.org/diablo/openstack-compute/admin/content/ch_configuring-openstack-compute.html
|
||||
|
||||
@ -24,7 +24,7 @@ except ImportError:
|
||||
from anvil import cfg
|
||||
from anvil import colorizer
|
||||
from anvil import components as comp
|
||||
from anvil import exceptions
|
||||
from anvil import exceptions as excp
|
||||
from anvil import log as logging
|
||||
from anvil import shell as sh
|
||||
from anvil import utils
|
||||
@ -81,9 +81,6 @@ FLOATING_NET_CMDS = [
|
||||
# Subdirs of the checkout/download
|
||||
BIN_DIR = 'bin'
|
||||
|
||||
# This is a special conf
|
||||
CLEANER_DATA_CONF = 'nova-clean.sh'
|
||||
|
||||
|
||||
class NovaUninstaller(comp.PythonUninstallComponent):
|
||||
def __init__(self, *args, **kargs):
|
||||
@ -91,26 +88,24 @@ class NovaUninstaller(comp.PythonUninstallComponent):
|
||||
self.virsh = lv.Virsh(self.get_int_option('service_wait_seconds'), self.distro)
|
||||
|
||||
def pre_uninstall(self):
|
||||
self._clear_libvirt_domains()
|
||||
self._clean_it()
|
||||
if 'compute' in self.subsystems:
|
||||
self._clean_compute()
|
||||
if 'network' in self.subsystems:
|
||||
self._clean_net()
|
||||
|
||||
def _clean_it(self):
|
||||
cleaner_fn = sh.joinpths(self.get_option('component_dir'), 'tools', CLEANER_DATA_CONF)
|
||||
if sh.isfile(cleaner_fn):
|
||||
LOG.info("Cleaning up your system by running nova cleaner script: %s", colorizer.quote(cleaner_fn))
|
||||
# These environment additions are important
|
||||
# in that they eventually affect how this script runs
|
||||
env = {
|
||||
'ENABLED_SERVICES': ",".join(self.subsystems.keys()),
|
||||
}
|
||||
sh.execute(cleaner_fn, run_as_root=True, env_overrides=env)
|
||||
def _clean_net(self):
|
||||
cleaner = nhelper.NetworkCleaner(self)
|
||||
try:
|
||||
cleaner.clean()
|
||||
except excp.AnvilException as e:
|
||||
LOG.warn("Failed cleaning up nova-networks dirty laundry due to: %s", e)
|
||||
|
||||
def _clear_libvirt_domains(self):
|
||||
virt_driver = nhelper.canon_virt_driver(self.get_option('virt_driver'))
|
||||
if virt_driver == 'libvirt':
|
||||
inst_prefix = self.get_option('instance_name_prefix', default_value='instance-')
|
||||
libvirt_type = lv.canon_libvirt_type(self.get_option('libvirt_type'))
|
||||
self.virsh.clear_domains(libvirt_type, inst_prefix)
|
||||
def _clean_compute(self):
|
||||
cleaner = nhelper.ComputeCleaner(self)
|
||||
try:
|
||||
cleaner.clean()
|
||||
except excp.AnvilException as e:
|
||||
LOG.warn("Failed cleaning up nova-computes dirty laundry due to: %s", e)
|
||||
|
||||
|
||||
class NovaInstaller(comp.PythonInstallComponent):
|
||||
@ -156,15 +151,6 @@ class NovaInstaller(comp.PythonInstallComponent):
|
||||
if self.get_bool_option('db-sync'):
|
||||
self._setup_db()
|
||||
self._sync_db()
|
||||
self._setup_cleaner()
|
||||
|
||||
def _setup_cleaner(self):
|
||||
LOG.info("Configuring cleaner template: %s", colorizer.quote(CLEANER_DATA_CONF))
|
||||
(_src_fn, contents) = utils.load_template(self.name, CLEANER_DATA_CONF)
|
||||
cleaner_fn = sh.joinpths(self.get_option('component_dir'), 'tools', CLEANER_DATA_CONF)
|
||||
sh.mkdirslist(sh.dirname(cleaner_fn), tracewriter=self.tracewriter)
|
||||
sh.write_file(cleaner_fn, contents, tracewriter=self.tracewriter)
|
||||
sh.chmod(cleaner_fn, 0755)
|
||||
|
||||
def _setup_db(self):
|
||||
dbhelper.drop_db(distro=self.distro,
|
||||
@ -323,11 +309,11 @@ class NovaRuntime(comp.PythonRuntime):
|
||||
self.virsh.check_virt(virt_type)
|
||||
self.virsh.restart_service()
|
||||
LOG.info("Libvirt virtualization type %s seems to be working and running.", colorizer.quote(virt_type))
|
||||
except exceptions.ProcessExecutionError as e:
|
||||
except excp.ProcessExecutionError as e:
|
||||
msg = ("Libvirt type %r does not seem to be active or configured correctly, "
|
||||
"perhaps you should be using %r instead: %s" %
|
||||
(virt_type, lv.DEF_VIRT_TYPE, e))
|
||||
raise exceptions.StartException(msg)
|
||||
raise excp.StartException(msg)
|
||||
|
||||
def app_params(self, app_name):
|
||||
params = comp.PythonRuntime.app_params(self, app_name)
|
||||
|
||||
@ -14,7 +14,6 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import errno
|
||||
import getpass
|
||||
import grp
|
||||
import os
|
||||
@ -27,6 +26,8 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil # http://code.google.com/p/psutil/wiki/Documentation
|
||||
|
||||
from anvil import env
|
||||
from anvil import exceptions as excp
|
||||
from anvil import log as logging
|
||||
@ -49,6 +50,11 @@ SUDO_UID = env.get_key('SUDO_UID')
|
||||
SUDO_GID = env.get_key('SUDO_GID')
|
||||
|
||||
|
||||
class Process(psutil.Process):
|
||||
def __str__(self):
|
||||
return "%s (%s)" % (self.pid, self.name)
|
||||
|
||||
|
||||
class Rooted(object):
|
||||
def __init__(self, run_as_root):
|
||||
self.root_mode = run_as_root
|
||||
@ -349,37 +355,39 @@ def explode_path(path):
|
||||
return _explode_path(path)[0]
|
||||
|
||||
|
||||
def _attempt_kill(pid, signal_type, max_try, wait_time):
|
||||
def _attempt_kill(proc, signal_type, max_try, wait_time):
|
||||
killed = False
|
||||
attempts = 0
|
||||
for _i in range(0, max_try):
|
||||
if not proc.is_running():
|
||||
killed = True
|
||||
break
|
||||
try:
|
||||
LOG.debug("Attempting to kill pid %s" % (pid))
|
||||
LOG.debug("Attempting to kill process %s" % (proc))
|
||||
attempts += 1
|
||||
os.kill(pid, signal_type)
|
||||
LOG.debug("Sleeping for %s seconds before next attempt to kill pid %s" % (wait_time, pid))
|
||||
proc.send_signal(signal_type)
|
||||
LOG.debug("Sleeping for %s seconds before next attempt to kill process %s" % (wait_time, proc))
|
||||
sleep(wait_time)
|
||||
except psutil.error.NoSuchProcess:
|
||||
killed = True
|
||||
break
|
||||
except Exception as e:
|
||||
LOG.debug("Failed killing %s due to: %s", proc, e)
|
||||
LOG.debug("Sleeping for %s seconds before next attempt to kill process %s" % (wait_time, proc))
|
||||
sleep(wait_time)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ESRCH:
|
||||
# Gotcha!
|
||||
killed = True
|
||||
break
|
||||
else:
|
||||
LOG.debug("Failed killing %s due to: %s", pid, e)
|
||||
LOG.debug("Sleeping for %s seconds before next attempt to kill pid %s" % (wait_time, pid))
|
||||
sleep(wait_time)
|
||||
return (killed, attempts)
|
||||
|
||||
|
||||
def kill(pid, max_try=4, wait_time=1):
|
||||
if not is_running(pid) or is_dry_run():
|
||||
return (True, 0)
|
||||
proc = Process(pid)
|
||||
# Try the nicer sig-int first...
|
||||
(killed, i_attempts) = _attempt_kill(pid, signal.SIGINT, int(max_try / 2), wait_time)
|
||||
(killed, i_attempts) = _attempt_kill(proc, signal.SIGINT, int(max_try / 2), wait_time)
|
||||
if killed:
|
||||
return (True, i_attempts)
|
||||
# Get agressive and try sig-kill....
|
||||
(killed, k_attempts) = _attempt_kill(pid, signal.SIGKILL, int(max_try / 2), wait_time)
|
||||
(killed, k_attempts) = _attempt_kill(proc, signal.SIGKILL, int(max_try / 2), wait_time)
|
||||
if killed:
|
||||
return (True, i_attempts + k_attempts)
|
||||
else:
|
||||
@ -442,25 +450,7 @@ def fork(program, app_dir, pid_fn, stdout_fn, stderr_fn, *args):
|
||||
def is_running(pid):
|
||||
if is_dry_run():
|
||||
return True
|
||||
# TODO(harlowja): this can be done better
|
||||
# but it will suffice for now....
|
||||
#
|
||||
# Check proc
|
||||
proc_fn = joinpths("/proc", str(pid))
|
||||
if exists(proc_fn):
|
||||
LOG.debug("By looking at %s we determined %s is still running.", proc_fn, pid)
|
||||
return True
|
||||
# Try a slightly more aggressive way...
|
||||
running = True
|
||||
try:
|
||||
os.kill(pid, signal.SIG_DFL)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EPERM:
|
||||
pass
|
||||
else:
|
||||
running = False
|
||||
LOG.debug("By attempting to signal %s we determined it is %s", pid, {True: 'alive', False: 'dead'}[running])
|
||||
return running
|
||||
return Process(pid).is_running()
|
||||
|
||||
|
||||
def mkdirslist(path, tracewriter=None, adjust_suids=False):
|
||||
|
||||
@ -41,5 +41,5 @@ keystone:
|
||||
image_cache_dir: "/usr/share/anvil/glance/images"
|
||||
|
||||
# Used by install section in the specfile
|
||||
remove_file: /bin/rm -rf %{buildroot}/usr/bin/glance
|
||||
remove_file: "/bin/rm -rf %{buildroot}/usr/bin/glance"
|
||||
...
|
||||
|
||||
@ -1,44 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script cleans up the system iptables/services as part of a nova uninstall
|
||||
#
|
||||
# It is best effort!
|
||||
#
|
||||
# There are other scripts in tools/ that might be able to recover it better (but are distro specific)
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script must be run as root!" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set +o errexit
|
||||
set -x
|
||||
|
||||
# Clean off networking
|
||||
if [[ "$ENABLED_SERVICES" =~ "net" ]]; then
|
||||
|
||||
# Ignore any errors from shutting down dnsmasq
|
||||
service dnsmasq stop
|
||||
|
||||
# The above doesn't always work so this way will just incase
|
||||
for pid in `ps -elf | grep -i dnsmasq | grep nova | perl -le 'while (<>) { my $pid = (split /\s+/)[3]; print $pid; }'`
|
||||
do
|
||||
echo "Killing leftover nova dnsmasq process with process id $pid"
|
||||
kill -9 $pid
|
||||
done
|
||||
|
||||
# Delete rules
|
||||
iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "iptables",$0}' | bash
|
||||
|
||||
# Delete nat rules
|
||||
iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "iptables -t nat",$0}' | bash
|
||||
|
||||
# Delete chains
|
||||
iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "iptables",$0}' | bash
|
||||
|
||||
# Delete nat chains
|
||||
iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "iptables -t nat",$0}' | bash
|
||||
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@ -1,13 +1,11 @@
|
||||
gcc
|
||||
git
|
||||
pylint
|
||||
python
|
||||
python-netifaces
|
||||
# python-pep8 (conflicts with the ones to be installed)
|
||||
python
|
||||
python-cheetah
|
||||
# python-pip (conflicts with the ones to be installed)
|
||||
python-progressbar
|
||||
PyYAML
|
||||
python-ordereddict
|
||||
python-iso8601
|
||||
|
||||
python-netifaces
|
||||
python-ordereddict
|
||||
python-progressbar
|
||||
python-psutil
|
||||
PyYAML
|
||||
|
||||
Loading…
Reference in New Issue
Block a user