Pylint cleanups and utils cleanups and moved libvirt cleanup functionality to libvirt file.

This commit is contained in:
Joshua Harlow
2012-02-07 20:56:50 -08:00
parent acabfc802f
commit 972739e77f
18 changed files with 143 additions and 357 deletions

View File

@@ -1,15 +1,20 @@
#!/usr/bin/env bash
#!/bin/bash
# This script cleans up the iptables as part of a nova uninstall
# Eventually it should be moved to python code...
# This was added (so that it dies on errors)
set -o errexit
# Delete rules
iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "sudo iptables",$0}' | bash
# Delete nat rules
iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "sudo iptables -t nat",$0}' | bash
# Delete chains
iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "sudo iptables",$0}' | bash
# Delete nat chains
iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "sudo iptables -t nat",$0}' | bash

View File

@@ -159,7 +159,6 @@ multi_host = ${MULTI_HOST:-0}
# Virtualization settings
virt_driver = ${VIRT_DRIVER:-libvirt}
libvirt_type = ${LIBVIRT_TYPE:-kvm}
# NOT USED connection_type = ${CONNECTION_TYPE:-libvirt}
# What type of image service will be used?
img_service = ${IMG_SERVICE:-nova.image.glance.GlanceImageService}

View File

@@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import ConfigParser
@@ -197,9 +196,9 @@ class IgnoreMissingConfigParser(ConfigParser.RawConfigParser):
value = None
try:
value = ConfigParser.RawConfigParser.get(self, section, option)
except ConfigParser.NoSectionError, e:
except ConfigParser.NoSectionError:
pass
except ConfigParser.NoOptionError, e:
except ConfigParser.NoOptionError:
pass
return value

View File

@@ -56,6 +56,9 @@ class ComponentBase(object):
return list()
return list(deps)
def verify(self):
pass
def warm_configs(self):
pass

View File

@@ -14,8 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from devstack import component as comp
from devstack import exceptions as excp
from devstack import log as logging
@@ -94,7 +92,6 @@ class HorizonInstaller(comp.PythonInstallComponent):
self.horizon_dir = sh.joinpths(self.appdir, ROOT_HORIZON)
self.dash_dir = sh.joinpths(self.appdir, ROOT_DASH)
self.log_dir = sh.joinpths(self.component_root, LOGS_DIR)
self._check_ug()
def _get_download_locations(self):
places = list()
@@ -104,6 +101,9 @@ class HorizonInstaller(comp.PythonInstallComponent):
})
return places
def verify(self):
self._check_ug()
def _get_symlinks(self):
src = self._get_target_config_name(HORIZON_APACHE_CONF)
links = dict()

View File

@@ -21,6 +21,7 @@ import stat
from devstack import cfg
from devstack import component as comp
from devstack import exceptions
from devstack import libvirt as virsh
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
@@ -181,62 +182,34 @@ LIBVIRT_RESTART_CMD = {
settings.UBUNTU11: ['/etc/init.d/libvirt-bin', 'restart'],
}
#this is a special conf
CLEANER_DATA_CONF = 'clean_iptables.sh'
CLEANER_CMD_ROOT = [sh.joinpths("/", "bin", 'bash')]
#pip files that nova requires
REQ_PIPS = ['general.json', 'nova.json']
#used to clear out old domains that may be stuck around
LIBVIRT_PROTOCOL_MAP = {
'qemu': "qemu:///system",
'kvm': "qemu:///system",
}
class NovaUninstaller(comp.PythonUninstallComponent):
def __init__(self, *args, **kargs):
comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)
self.bindir = sh.joinpths(self.appdir, BIN_DIR)
def pre_uninstall(self):
comp.PythonUninstallComponent.pre_uninstall(self)
self._clear_iptables()
self._clear_domains()
self._clear_libvirt_domains()
def _clear_iptables(self):
LOG.info("Cleaning up iptables.")
pth = sh.joinpths('utils', 'clean_iptables.sh')
cmd = [pth]
cmd = CLEANER_CMD_ROOT + [sh.joinpths(self.bindir, CLEANER_DATA_CONF)]
sh.execute(*cmd, run_as_root=True)
def _clear_domains(self):
def _clear_libvirt_domains(self):
libvirt_type = self.cfg.get('nova', 'libvirt_type')
inst_prefix = self.cfg.get('nova', 'instance_name_prefix')
vir_driver = self.cfg.get('nova', 'virt_driver')
if vir_driver == 'libvirt' and inst_prefix and libvirt_type:
#attempt to clear out dead domains
#late import so this is not always required
virt_protocol = LIBVIRT_PROTOCOL_MAP[libvirt_type]
try:
import libvirt
with sh.Rooted(True):
LOG.info("Attempting to clear out leftover libvirt nova domains using protocol %s." % (virt_protocol))
conn = None
try:
conn = libvirt.open(virt_protocol)
except libvirt.libvirtError:
LOG.warn("Could not connect to libvirt using protocol [%s]" % (virt_protocol))
if conn:
try:
definedDomains = conn.listDefinedDomains()
for domain in definedDomains:
if domain.startswith(inst_prefix):
LOG.info("Found old nova domain %s" % (domain))
dom = conn.lookupByName(domain)
LOG.info("Clearing domain id %d running %s" % (dom.ID(), dom.OSType()))
dom.undefine()
except libvirt.libvirtError, e:
LOG.warn("Could not clear out libvirt domains due to [%s]" % (e.message))
except ImportError:
#LOG it?
pass
virt_driver = self.cfg.get('nova', 'virt_driver')
if virt_driver == virsh.VIRT_TYPE:
virsh.clear_libvirt_domains(libvirt_type, inst_prefix)
class NovaInstaller(comp.PythonInstallComponent):
@@ -244,13 +217,17 @@ class NovaInstaller(comp.PythonInstallComponent):
comp.PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
self.bindir = sh.joinpths(self.appdir, BIN_DIR)
self.paste_conf_fn = self._get_target_config_name(PASTE_CONF)
self.volumes_enabled = False
if not self.component_opts or NVOL in self.component_opts:
self.volumes_enabled = True
def _get_pkgs(self):
pkgs = list(REQ_PKGS)
sub_components = self.component_opts or SUBCOMPONENTS
for c in sub_components:
fns = ADD_PKGS.get(c) or []
pkgs.extend(fns)
fns = ADD_PKGS.get(c)
if fns:
pkgs.extend(fns)
return pkgs
def _get_symlinks(self):
@@ -304,12 +281,17 @@ class NovaInstaller(comp.PythonInstallComponent):
self._setup_db()
self._sync_db()
self._setup_network()
self._setup_cleaner()
# check if we need to do the vol subcomponent
if not self.component_opts or NVOL in self.component_opts:
# yes, either no subcomponents were specifically requested or it's
# in the set that was requested
if self.volumes_enabled:
self._setup_vol_groups()
def _setup_cleaner(self):
LOG.info("Configuring cleaner template %s.", CLEANER_DATA_CONF)
(_, contents) = utils.load_template(self.component_name, CLEANER_DATA_CONF)
tgt_fn = sh.joinpths(self.bindir, CLEANER_DATA_CONF)
sh.write_file(tgt_fn, contents)
def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME)

View File

@@ -18,7 +18,6 @@ from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack import utils
from devstack.components import nova

View File

@@ -17,14 +17,8 @@
import os
import re
import subprocess
import sys
from devstack import cfg
from devstack import shell as sh
from devstack import utils
from devstack.progs import common
#general extraction cfg keys
CFG_MAKE = {
'ADMIN_PASSWORD': ('passwords', 'horizon_keystone_admin'),
'MYSQL_PASSWORD': ('passwords', 'sql'),
@@ -110,9 +104,7 @@ def _generate_os_env(fh, cfg):
_write_line("", fh)
def generate_local_rc(fn, cfg=None):
if not cfg:
cfg = common.get_config()
def generate_local_rc(fn, cfg):
with open(fn, "w") as fh:
_write_line('# General stuff', fh)
for (out_name, cfg_data) in CFG_MAKE.items():
@@ -127,9 +119,7 @@ def generate_local_rc(fn, cfg=None):
_generate_os_env(fh, cfg)
def load_local_rc(fn, cfg=None):
if not cfg:
cfg = common.get_config()
def load_local_rc(fn):
am_set = 0
with open(fn, "r") as fh:
for line in fh:

76
devstack/libvirt.py Normal file
View File

@@ -0,0 +1,76 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from devstack import log as logging
from devstack import shell as sh
from devstack import utils
LOG = logging.getLogger('devstack.libvirt')
#http://libvirt.org/uri.html
LIBVIRT_PROTOCOL_MAP = {
'qemu': "qemu:///system",
'kvm': "qemu:///system",
'xen': 'xen:///',
}
VIRT_TYPE = 'libvirt'
VIRT_LIB = VIRT_TYPE
def _get_virt_lib():
return utils.import_module(VIRT_LIB)
def _destroy_domain(conn, dom_name):
libvirt = _get_virt_lib()
if not libvirt or not dom_name:
return
try:
dom = conn.lookupByName(dom_name)
LOG.debug("Destroying domain (%s) (id=%s) running %s" % (dom_name, dom.ID(), dom.OSType()))
dom.destroy()
dom.undefine()
except libvirt.libvirtError, e:
LOG.warn("Could not clear out libvirt domain (%s) due to [%s]" % (dom_name, e.message))
def clear_libvirt_domains(virt_type, inst_prefix):
#late import so that we don't always need this library to be active
#ie if u aren't using libvirt in the first place
libvirt = _get_virt_lib()
virt_protocol = LIBVIRT_PROTOCOL_MAP.get(virt_type)
if not libvirt or not virt_protocol or not inst_prefix:
return
with sh.Rooted(True):
LOG.info("Attempting to clear out leftover libvirt domains using protocol %s." % (virt_protocol))
conn = None
try:
conn = libvirt.open(virt_protocol)
except libvirt.libvirtError:
LOG.warn("Could not connect to libvirt using protocol [%s]" % (virt_protocol))
if conn:
try:
defined_domains = conn.listDefinedDomains()
kill_domains = list()
for domain in defined_domains:
if domain.startswith(inst_prefix):
kill_domains.append(domain)
if kill_domains:
LOG.info("Found %s old domains to destroy (%s)" % (len(kill_domains), ", ".join(kill_domains)))
for domain in kill_domains:
_destroy_domain(conn, domain)
except libvirt.libvirtError, e:
LOG.warn("Could not clear out libvirt domains due to [%s]" % (e.message))

View File

@@ -54,9 +54,8 @@ class TermHandler(logging.Handler):
logging.Handler.__init__(self)
def emit(self, record):
lvl = record.levelno
msg = self.format(record)
if msg:
if msg is not None:
TermHandler.STREAM.write(msg + TermHandler.NL)
if TermHandler.DO_FLUSH:
TermHandler.STREAM.flush()

View File

@@ -43,12 +43,12 @@ def install(pips, distro):
version = pipinfo.get('version')
if version is not None:
pipfull = pipfull + "==" + str(version)
LOG.info("Installing python package (%s)" % (pipfull))
LOG.debug("Installing python package (%s)" % (pipfull))
real_cmd = [root_cmd, 'install']
real_cmd += PIP_INSTALL_CMD_OPTS
options = pipinfo.get('options')
if options is not None:
LOG.info("Using pip options: %s" % (str(options)))
LOG.debug("Using pip options: %s" % (str(options)))
real_cmd += [str(options)]
real_cmd += [pipfull]
sh.execute(*real_cmd, run_as_root=True)
@@ -60,7 +60,8 @@ def uninstall(pips, distro, skip_errors=True):
LOG.info("Uninstalling python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd))
for name in pipnames:
try:
cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [name]
LOG.debug("Uninstall python package (%s)" % (name))
cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [str(name)]
sh.execute(*cmd, run_as_root=True)
except excp.ProcessExecutionError:
if skip_errors:

View File

@@ -16,7 +16,6 @@
import time
from devstack import cfg
from devstack import date
from devstack import env_rc
from devstack import exceptions as excp
@@ -85,7 +84,7 @@ def _pre_run(action_name, root_dir, pkg_manager, config, component_order, instan
try:
if sh.isfile(rc_fn):
LOG.info("Attempting to load rc file at [%s] which has your environment settings." % (rc_fn))
am_loaded = env_rc.load_local_rc(rc_fn, config)
am_loaded = env_rc.load_local_rc(rc_fn)
loaded_env = True
LOG.info("Loaded [%s] settings from rc file [%s]" % (am_loaded, rc_fn))
except IOError:
@@ -93,9 +92,17 @@ def _pre_run(action_name, root_dir, pkg_manager, config, component_order, instan
if action_name == settings.INSTALL:
if root_dir:
sh.mkdir(root_dir)
LOG.info("Warming up your component configurations (ie so you won't be prompted later)")
LOG.info("Verifying that the components are ready to rock-n-roll.")
all_instances = instances[0]
prerequisite_instances = instances[1]
for component in component_order:
base_inst = all_instances.get(component)
if base_inst:
base_inst.verify()
pre_inst = prerequisite_instances.get(component)
if pre_inst:
pre_inst.verify()
LOG.info("Warming up your component configurations (ie so you won't be prompted later)")
for component in component_order:
base_inst = all_instances.get(component)
if base_inst:
@@ -103,7 +110,6 @@ def _pre_run(action_name, root_dir, pkg_manager, config, component_order, instan
pre_inst = prerequisite_instances.get(component)
if pre_inst:
pre_inst.warm_configs()
LOG.info("Your component configurations should now be nice and warm!")
if action_name in _RC_FILE_MAKE_ACTIONS and not loaded_env:
_gen_localrc(config, rc_fn)

View File

@@ -20,7 +20,6 @@ import os.path
import pwd
import shutil
import subprocess
import tempfile
import fileinput
from devstack import env
@@ -67,7 +66,6 @@ def execute(*cmd, **kwargs):
shell = kwargs.pop('shell', False)
execute_cmd = list()
for c in cmd:
execute_cmd.append(str(c))

View File

@@ -23,6 +23,7 @@ import os
import platform
import random
import re
import sys
import termcolor
from devstack import exceptions as excp
@@ -99,6 +100,14 @@ def to_bytes(text):
return byte_val
def import_module(module_name):
try:
__import__(module_name)
return sys.modules.get(module_name, None)
except ImportError:
return None
def load_json(fn):
data = sh.load_file(fn)
lines = data.splitlines()

View File

@@ -1,15 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

View File

@@ -1,103 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import optparse
import os
import sys
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
sys.path.insert(0, POSSIBLE_TOPDIR)
from devstack import settings
from devstack import utils
from devstack.progs import common
PROG_NAME = "Dep. Lister"
DEF_ACTION = settings.INSTALL
def _print_deps(component, deps):
print(" + " + utils.color_text(component, "blue", True))
if deps:
for d in sorted(deps):
print(" |")
print(" ------> %s" % (d))
def _clean_action(action):
if not action:
return DEF_ACTION
action = action.strip.lower()
if action not in settings.ACTIONS:
return DEF_ACTION
return action
def _run_dep_comps(args, rep, maxlen):
components = utils.parse_components(args.get("components"))
if not components:
components = dict()
for c in settings.COMPONENT_NAMES:
components[c] = list()
header = utils.center_text("Dependencies (defaulted)", rep, maxlen)
else:
header = utils.center_text("Dependencies", rep, maxlen)
print(header)
action = _clean_action(args.pop("action"))
msg = "For action %s" % (action)
print(utils.center_text(msg, rep, maxlen))
all_deps = common.get_components_deps(action, components)
for c in sorted(all_deps.keys()):
_print_deps(c, all_deps.get(c))
def run(args):
(rep, maxlen) = utils.welcome(PROG_NAME)
_run_dep_comps(args, rep, maxlen)
return True
def main():
parser = optparse.OptionParser()
known_components = sorted(settings.COMPONENT_NAMES)
components = "(" + ", ".join(known_components) + ")"
parser.add_option("-c", "--component",
action="append",
dest="component",
help="openstack component, ie %s" % (components))
known_actions = sorted(settings.ACTIONS)
actions = "(" + ", ".join(known_actions) + ")"
parser.add_option("-a", "--action",
action="store",
type="string",
dest="action",
metavar="ACTION",
help="action to perform, ie %s" % (actions))
(options, args) = parser.parse_args()
opts = dict()
opts['components'] = options.component
opts['action'] = options.action
result = run(opts)
if not result:
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,109 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import optparse
import os
import re
import sys
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
sys.path.insert(0, POSSIBLE_TOPDIR)
from devstack import settings
from devstack import utils
from devstack.components import db
from devstack.components import glance
from devstack.components import horizon
from devstack.components import keystone
from devstack.components import keystone_client
from devstack.components import melange
from devstack.components import melange_client
from devstack.components import nova
from devstack.components import nova_client
from devstack.components import novnc
from devstack.components import quantum
from devstack.components import quantum_client
from devstack.components import rabbit
from devstack.components import swift
from devstack.components import swift_keystone
PROG_NAME = "Describer"
#this determines how descriptions for components are found
_DESCR_MAP = {
settings.DB: db.describe,
settings.GLANCE: glance.describe,
settings.HORIZON: horizon.describe,
settings.KEYSTONE: keystone.describe,
settings.KEYSTONE_CLIENT: keystone_client.describe,
settings.NOVA: nova.describe,
settings.NOVA_CLIENT: nova_client.describe,
settings.QUANTUM: quantum.describe,
settings.RABBIT: rabbit.describe,
settings.SWIFT: swift.describe,
settings.SWIFT_KEYSTONE: swift_keystone.describe,
settings.NOVNC: novnc.describe,
settings.QUANTUM_CLIENT: quantum_client.describe,
settings.MELANGE: melange.describe,
settings.MELANGE_CLIENT: melange_client.describe,
}
def _run_describe_comps(args, rep, maxlen):
components = utils.parse_components(args.get("components"))
if not components:
components = dict()
for c in settings.COMPONENT_NAMES:
components[c] = list()
header = utils.center_text("Descriptions (defaulted)", rep, maxlen)
else:
header = utils.center_text("Descriptions", rep, maxlen)
print(header)
c_keys = sorted(components.keys())
for c in c_keys:
print("Name: " + utils.color_text(c, "blue", True))
describer = _DESCR_MAP.get(c)
print(describer(components.get(c)))
def run(args):
(rep, maxlen) = utils.welcome(PROG_NAME)
_run_describe_comps(args, rep, maxlen)
return True
def main():
parser = optparse.OptionParser()
known_components = sorted(settings.COMPONENT_NAMES)
components = "(" + ", ".join(known_components) + ")"
parser.add_option("-c", "--component",
action="append",
dest="component",
help="openstack component, ie %s" % (components))
(options, args) = parser.parse_args()
opts = dict()
opts['components'] = options.component
result = run(opts)
if not result:
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,53 +0,0 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import optparse
import os
import sys
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
sys.path.insert(0, POSSIBLE_TOPDIR)
from devstack import env_rc
from devstack import utils
from devstack import shell as sh
PROG_NAME = "Env. rc file generator"
DEF_FN = 'openstackrc'
def main():
opts = optparse.OptionParser()
opts.add_option("-o", "--output", dest="filename",
help="write output to FILE", metavar="FILE")
(options, args) = opts.parse_args()
utils.welcome(PROG_NAME)
fn = options.filename
if not fn:
fn = DEF_FN
env_rc.generate_local_rc(fn)
print("Check file \"%s\" for your environment configuration." \
% (sh.abspth(fn)))
return 0
if __name__ == "__main__":
sys.exit(main())