1655 lines
52 KiB
Python
1655 lines
52 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright 2010 United States Government as represented by the
|
|
# Administrator of the National Aeronautics and Space Administration.
|
|
# Copyright 2011 Justin Santa Barbara
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Utilities and helper functions."""
|
|
|
|
import contextlib
|
|
import datetime
|
|
import errno
|
|
import functools
|
|
import hashlib
|
|
import inspect
|
|
import itertools
|
|
import json
|
|
import os
|
|
import pyclbr
|
|
import random
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import socket
|
|
import struct
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import types
|
|
import uuid
|
|
import warnings
|
|
from xml.sax import saxutils
|
|
|
|
from eventlet import corolocal
|
|
from eventlet import event
|
|
from eventlet import greenthread
|
|
from eventlet import semaphore
|
|
from eventlet.green import subprocess
|
|
import iso8601
|
|
import lockfile
|
|
import netaddr
|
|
|
|
from nova import exception
|
|
from nova import flags
|
|
from nova import log as logging
|
|
from nova.openstack.common import cfg
|
|
from nova.openstack.common import excutils
|
|
from nova.openstack.common import importutils
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
ISO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
|
PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
|
|
FLAGS = flags.FLAGS
|
|
|
|
FLAGS.register_opt(
|
|
cfg.BoolOpt('disable_process_locking', default=False,
|
|
help='Whether to disable inter-process locks'))
|
|
|
|
|
|
def find_config(config_path):
|
|
"""Find a configuration file using the given hint.
|
|
|
|
:param config_path: Full or relative path to the config.
|
|
:returns: Full path of the config, if it exists.
|
|
:raises: `nova.exception.ConfigNotFound`
|
|
|
|
"""
|
|
possible_locations = [
|
|
config_path,
|
|
os.path.join(FLAGS.state_path, "etc", "nova", config_path),
|
|
os.path.join(FLAGS.state_path, "etc", config_path),
|
|
os.path.join(FLAGS.state_path, config_path),
|
|
"/etc/nova/%s" % config_path,
|
|
]
|
|
|
|
for path in possible_locations:
|
|
if os.path.exists(path):
|
|
return os.path.abspath(path)
|
|
|
|
raise exception.ConfigNotFound(path=os.path.abspath(config_path))
|
|
|
|
|
|
def vpn_ping(address, port, timeout=0.05, session_id=None):
|
|
"""Sends a vpn negotiation packet and returns the server session.
|
|
|
|
Returns False on a failure. Basic packet structure is below.
|
|
|
|
Client packet (14 bytes)::
|
|
|
|
0 1 8 9 13
|
|
+-+--------+-----+
|
|
|x| cli_id |?????|
|
|
+-+--------+-----+
|
|
x = packet identifier 0x38
|
|
cli_id = 64 bit identifier
|
|
? = unknown, probably flags/padding
|
|
|
|
Server packet (26 bytes)::
|
|
|
|
0 1 8 9 13 14 21 2225
|
|
+-+--------+-----+--------+----+
|
|
|x| srv_id |?????| cli_id |????|
|
|
+-+--------+-----+--------+----+
|
|
x = packet identifier 0x40
|
|
cli_id = 64 bit identifier
|
|
? = unknown, probably flags/padding
|
|
bit 9 was 1 and the rest were 0 in testing
|
|
|
|
"""
|
|
if session_id is None:
|
|
session_id = random.randint(0, 0xffffffffffffffff)
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
data = struct.pack('!BQxxxxx', 0x38, session_id)
|
|
sock.sendto(data, (address, port))
|
|
sock.settimeout(timeout)
|
|
try:
|
|
received = sock.recv(2048)
|
|
except socket.timeout:
|
|
return False
|
|
finally:
|
|
sock.close()
|
|
fmt = '!BQxxxxxQxxxx'
|
|
if len(received) != struct.calcsize(fmt):
|
|
print struct.calcsize(fmt)
|
|
return False
|
|
(identifier, server_sess, client_sess) = struct.unpack(fmt, received)
|
|
if identifier == 0x40 and client_sess == session_id:
|
|
return server_sess
|
|
|
|
|
|
def fetchfile(url, target):
|
|
LOG.debug(_('Fetching %s') % url)
|
|
execute('curl', '--fail', url, '-o', target)
|
|
|
|
|
|
def execute(*cmd, **kwargs):
|
|
"""Helper method to execute command with optional retry.
|
|
|
|
If you add a run_as_root=True command, don't forget to add the
|
|
corresponding filter to nova.rootwrap !
|
|
|
|
:param cmd: Passed to subprocess.Popen.
|
|
:param process_input: Send to opened process.
|
|
:param check_exit_code: Single bool, int, or list of allowed exit
|
|
codes. Defaults to [0]. Raise
|
|
exception.ProcessExecutionError unless
|
|
program exits with one of these code.
|
|
:param delay_on_retry: True | False. Defaults to True. If set to
|
|
True, wait a short amount of time
|
|
before retrying.
|
|
:param attempts: How many times to retry cmd.
|
|
:param run_as_root: True | False. Defaults to False. If set to True,
|
|
the command is prefixed by the command specified
|
|
in the root_helper FLAG.
|
|
|
|
:raises exception.Error: on receiving unknown arguments
|
|
:raises exception.ProcessExecutionError:
|
|
|
|
:returns: a tuple, (stdout, stderr) from the spawned process, or None if
|
|
the command fails.
|
|
"""
|
|
|
|
process_input = kwargs.pop('process_input', None)
|
|
check_exit_code = kwargs.pop('check_exit_code', [0])
|
|
ignore_exit_code = False
|
|
if isinstance(check_exit_code, bool):
|
|
ignore_exit_code = not check_exit_code
|
|
check_exit_code = [0]
|
|
elif isinstance(check_exit_code, int):
|
|
check_exit_code = [check_exit_code]
|
|
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
|
attempts = kwargs.pop('attempts', 1)
|
|
run_as_root = kwargs.pop('run_as_root', False)
|
|
shell = kwargs.pop('shell', False)
|
|
|
|
if len(kwargs):
|
|
raise exception.Error(_('Got unknown keyword args '
|
|
'to utils.execute: %r') % kwargs)
|
|
|
|
if run_as_root:
|
|
cmd = shlex.split(FLAGS.root_helper) + list(cmd)
|
|
cmd = map(str, cmd)
|
|
|
|
while attempts > 0:
|
|
attempts -= 1
|
|
try:
|
|
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
|
|
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
|
obj = subprocess.Popen(cmd,
|
|
stdin=_PIPE,
|
|
stdout=_PIPE,
|
|
stderr=_PIPE,
|
|
close_fds=True,
|
|
shell=shell)
|
|
result = None
|
|
if process_input is not None:
|
|
result = obj.communicate(process_input)
|
|
else:
|
|
result = obj.communicate()
|
|
obj.stdin.close() # pylint: disable=E1101
|
|
_returncode = obj.returncode # pylint: disable=E1101
|
|
if _returncode:
|
|
LOG.debug(_('Result was %s') % _returncode)
|
|
if not ignore_exit_code and _returncode not in check_exit_code:
|
|
(stdout, stderr) = result
|
|
raise exception.ProcessExecutionError(
|
|
exit_code=_returncode,
|
|
stdout=stdout,
|
|
stderr=stderr,
|
|
cmd=' '.join(cmd))
|
|
return result
|
|
except exception.ProcessExecutionError:
|
|
if not attempts:
|
|
raise
|
|
else:
|
|
LOG.debug(_('%r failed. Retrying.'), cmd)
|
|
if delay_on_retry:
|
|
greenthread.sleep(random.randint(20, 200) / 100.0)
|
|
finally:
|
|
# NOTE(termie): this appears to be necessary to let the subprocess
|
|
# call clean something up in between calls, without
|
|
# it two execute calls in a row hangs the second one
|
|
greenthread.sleep(0)
|
|
|
|
|
|
def trycmd(*args, **kwargs):
|
|
"""
|
|
A wrapper around execute() to more easily handle warnings and errors.
|
|
|
|
Returns an (out, err) tuple of strings containing the output of
|
|
the command's stdout and stderr. If 'err' is not empty then the
|
|
command can be considered to have failed.
|
|
|
|
:discard_warnings True | False. Defaults to False. If set to True,
|
|
then for succeeding commands, stderr is cleared
|
|
|
|
"""
|
|
discard_warnings = kwargs.pop('discard_warnings', False)
|
|
|
|
try:
|
|
out, err = execute(*args, **kwargs)
|
|
failed = False
|
|
except exception.ProcessExecutionError, exn:
|
|
out, err = '', str(exn)
|
|
LOG.debug(err)
|
|
failed = True
|
|
|
|
if not failed and discard_warnings and err:
|
|
# Handle commands that output to stderr but otherwise succeed
|
|
LOG.debug(err)
|
|
err = ''
|
|
|
|
return out, err
|
|
|
|
|
|
def ssh_execute(ssh, cmd, process_input=None,
|
|
addl_env=None, check_exit_code=True):
|
|
LOG.debug(_('Running cmd (SSH): %s'), ' '.join(cmd))
|
|
if addl_env:
|
|
raise exception.Error(_('Environment not supported over SSH'))
|
|
|
|
if process_input:
|
|
# This is (probably) fixable if we need it...
|
|
raise exception.Error(_('process_input not supported over SSH'))
|
|
|
|
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
|
|
channel = stdout_stream.channel
|
|
|
|
#stdin.write('process_input would go here')
|
|
#stdin.flush()
|
|
|
|
# NOTE(justinsb): This seems suspicious...
|
|
# ...other SSH clients have buffering issues with this approach
|
|
stdout = stdout_stream.read()
|
|
stderr = stderr_stream.read()
|
|
stdin_stream.close()
|
|
|
|
exit_status = channel.recv_exit_status()
|
|
|
|
# exit_status == -1 if no exit code was returned
|
|
if exit_status != -1:
|
|
LOG.debug(_('Result was %s') % exit_status)
|
|
if check_exit_code and exit_status != 0:
|
|
raise exception.ProcessExecutionError(exit_code=exit_status,
|
|
stdout=stdout,
|
|
stderr=stderr,
|
|
cmd=' '.join(cmd))
|
|
|
|
return (stdout, stderr)
|
|
|
|
|
|
def novadir():
|
|
import nova
|
|
return os.path.abspath(nova.__file__).split('nova/__init__.py')[0]
|
|
|
|
|
|
def default_flagfile(filename='nova.conf', args=None):
|
|
if args is None:
|
|
args = sys.argv
|
|
for arg in args:
|
|
if arg.find('flagfile') != -1:
|
|
return arg[arg.index('flagfile') + len('flagfile') + 1:]
|
|
else:
|
|
if not os.path.isabs(filename):
|
|
# turn relative filename into an absolute path
|
|
script_dir = os.path.dirname(inspect.stack()[-1][1])
|
|
filename = os.path.abspath(os.path.join(script_dir, filename))
|
|
if not os.path.exists(filename):
|
|
filename = "./nova.conf"
|
|
if not os.path.exists(filename):
|
|
filename = '/etc/nova/nova.conf'
|
|
if os.path.exists(filename):
|
|
flagfile = '--flagfile=%s' % filename
|
|
args.insert(1, flagfile)
|
|
return filename
|
|
|
|
|
|
def debug(arg):
|
|
LOG.debug(_('debug in callback: %s'), arg)
|
|
return arg
|
|
|
|
|
|
def generate_uid(topic, size=8):
|
|
characters = '01234567890abcdefghijklmnopqrstuvwxyz'
|
|
choices = [random.choice(characters) for x in xrange(size)]
|
|
return '%s-%s' % (topic, ''.join(choices))
|
|
|
|
|
|
# Default symbols to use for passwords. Avoids visually confusing characters.
|
|
# ~6 bits per symbol
|
|
DEFAULT_PASSWORD_SYMBOLS = ('23456789', # Removed: 0,1
|
|
'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O
|
|
'abcdefghijkmnopqrstuvwxyz') # Removed: l
|
|
|
|
|
|
# ~5 bits per symbol
|
|
EASIER_PASSWORD_SYMBOLS = ('23456789', # Removed: 0, 1
|
|
'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O
|
|
|
|
|
|
def last_completed_audit_period(unit=None):
|
|
"""This method gives you the most recently *completed* audit period.
|
|
|
|
arguments:
|
|
units: string, one of 'hour', 'day', 'month', 'year'
|
|
Periods normally begin at the beginning (UTC) of the
|
|
period unit (So a 'day' period begins at midnight UTC,
|
|
a 'month' unit on the 1st, a 'year' on Jan, 1)
|
|
unit string may be appended with an optional offset
|
|
like so: 'day@18' This will begin the period at 18:00
|
|
UTC. 'month@15' starts a monthly period on the 15th,
|
|
and year@3 begins a yearly one on March 1st.
|
|
|
|
|
|
returns: 2 tuple of datetimes (begin, end)
|
|
The begin timestamp of this audit period is the same as the
|
|
end of the previous."""
|
|
if not unit:
|
|
unit = FLAGS.instance_usage_audit_period
|
|
|
|
offset = 0
|
|
if '@' in unit:
|
|
unit, offset = unit.split("@", 1)
|
|
offset = int(offset)
|
|
|
|
rightnow = utcnow()
|
|
if unit not in ('month', 'day', 'year', 'hour'):
|
|
raise ValueError('Time period must be hour, day, month or year')
|
|
if unit == 'month':
|
|
if offset == 0:
|
|
offset = 1
|
|
end = datetime.datetime(day=offset,
|
|
month=rightnow.month,
|
|
year=rightnow.year)
|
|
if end >= rightnow:
|
|
year = rightnow.year
|
|
if 1 >= rightnow.month:
|
|
year -= 1
|
|
month = 12 + (rightnow.month - 1)
|
|
else:
|
|
month = rightnow.month - 1
|
|
end = datetime.datetime(day=offset,
|
|
month=month,
|
|
year=year)
|
|
year = end.year
|
|
if 1 >= end.month:
|
|
year -= 1
|
|
month = 12 + (end.month - 1)
|
|
else:
|
|
month = end.month - 1
|
|
begin = datetime.datetime(day=offset, month=month, year=year)
|
|
|
|
elif unit == 'year':
|
|
if offset == 0:
|
|
offset = 1
|
|
end = datetime.datetime(day=1, month=offset, year=rightnow.year)
|
|
if end >= rightnow:
|
|
end = datetime.datetime(day=1,
|
|
month=offset,
|
|
year=rightnow.year - 1)
|
|
begin = datetime.datetime(day=1,
|
|
month=offset,
|
|
year=rightnow.year - 2)
|
|
else:
|
|
begin = datetime.datetime(day=1,
|
|
month=offset,
|
|
year=rightnow.year - 1)
|
|
|
|
elif unit == 'day':
|
|
end = datetime.datetime(hour=offset,
|
|
day=rightnow.day,
|
|
month=rightnow.month,
|
|
year=rightnow.year)
|
|
if end >= rightnow:
|
|
end = end - datetime.timedelta(days=1)
|
|
begin = end - datetime.timedelta(days=1)
|
|
|
|
elif unit == 'hour':
|
|
end = rightnow.replace(minute=offset, second=0, microsecond=0)
|
|
if end >= rightnow:
|
|
end = end - datetime.timedelta(hours=1)
|
|
begin = end - datetime.timedelta(hours=1)
|
|
|
|
return (begin, end)
|
|
|
|
|
|
def generate_password(length=20, symbolgroups=DEFAULT_PASSWORD_SYMBOLS):
|
|
"""Generate a random password from the supplied symbol groups.
|
|
|
|
At least one symbol from each group will be included. Unpredictable
|
|
results if length is less than the number of symbol groups.
|
|
|
|
Believed to be reasonably secure (with a reasonable password length!)
|
|
|
|
"""
|
|
r = random.SystemRandom()
|
|
|
|
# NOTE(jerdfelt): Some password policies require at least one character
|
|
# from each group of symbols, so start off with one random character
|
|
# from each symbol group
|
|
password = [r.choice(s) for s in symbolgroups]
|
|
# If length < len(symbolgroups), the leading characters will only
|
|
# be from the first length groups. Try our best to not be predictable
|
|
# by shuffling and then truncating.
|
|
r.shuffle(password)
|
|
password = password[:length]
|
|
length -= len(password)
|
|
|
|
# then fill with random characters from all symbol groups
|
|
symbols = ''.join(symbolgroups)
|
|
password.extend([r.choice(symbols) for _i in xrange(length)])
|
|
|
|
# finally shuffle to ensure first x characters aren't from a
|
|
# predictable group
|
|
r.shuffle(password)
|
|
|
|
return ''.join(password)
|
|
|
|
|
|
def last_octet(address):
|
|
return int(address.split('.')[-1])
|
|
|
|
|
|
def get_my_linklocal(interface):
|
|
try:
|
|
if_str = execute('ip', '-f', 'inet6', '-o', 'addr', 'show', interface)
|
|
condition = '\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link'
|
|
links = [re.search(condition, x) for x in if_str[0].split('\n')]
|
|
address = [w.group(1) for w in links if w is not None]
|
|
if address[0] is not None:
|
|
return address[0]
|
|
else:
|
|
raise exception.Error(_('Link Local address is not found.:%s')
|
|
% if_str)
|
|
except Exception as ex:
|
|
raise exception.Error(_("Couldn't get Link Local IP of %(interface)s"
|
|
" :%(ex)s") % locals())
|
|
|
|
|
|
def utcnow():
|
|
"""Overridable version of utils.utcnow."""
|
|
if utcnow.override_time:
|
|
return utcnow.override_time
|
|
return datetime.datetime.utcnow()
|
|
|
|
|
|
utcnow.override_time = None
|
|
|
|
|
|
def is_older_than(before, seconds):
|
|
"""Return True if before is older than seconds."""
|
|
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
|
|
|
|
|
def utcnow_ts():
|
|
"""Timestamp version of our utcnow function."""
|
|
return time.mktime(utcnow().timetuple())
|
|
|
|
|
|
def set_time_override(override_time=datetime.datetime.utcnow()):
|
|
"""Override utils.utcnow to return a constant time."""
|
|
utcnow.override_time = override_time
|
|
|
|
|
|
def advance_time_delta(timedelta):
|
|
"""Advance overriden time using a datetime.timedelta."""
|
|
assert(not utcnow.override_time is None)
|
|
utcnow.override_time += timedelta
|
|
|
|
|
|
def advance_time_seconds(seconds):
|
|
"""Advance overriden time by seconds."""
|
|
advance_time_delta(datetime.timedelta(0, seconds))
|
|
|
|
|
|
def clear_time_override():
|
|
"""Remove the overridden time."""
|
|
utcnow.override_time = None
|
|
|
|
|
|
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
|
"""Returns formatted utcnow."""
|
|
if not at:
|
|
at = utcnow()
|
|
return at.strftime(fmt)
|
|
|
|
|
|
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
|
"""Turn a formatted time back into a datetime."""
|
|
return datetime.datetime.strptime(timestr, fmt)
|
|
|
|
|
|
def isotime(at=None):
|
|
"""Stringify time in ISO 8601 format"""
|
|
if not at:
|
|
at = datetime.datetime.utcnow()
|
|
str = at.strftime(ISO_TIME_FORMAT)
|
|
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
|
str += ('Z' if tz == 'UTC' else tz)
|
|
return str
|
|
|
|
|
|
def parse_isotime(timestr):
|
|
"""Turn an iso formatted time back into a datetime."""
|
|
try:
|
|
return iso8601.parse_date(timestr)
|
|
except (iso8601.ParseError, TypeError) as e:
|
|
raise ValueError(e.message)
|
|
|
|
|
|
def normalize_time(timestamp):
|
|
"""Normalize time in arbitrary timezone to UTC"""
|
|
offset = timestamp.utcoffset()
|
|
return timestamp.replace(tzinfo=None) - offset if offset else timestamp
|
|
|
|
|
|
def parse_mailmap(mailmap='.mailmap'):
|
|
mapping = {}
|
|
if os.path.exists(mailmap):
|
|
fp = open(mailmap, 'r')
|
|
for l in fp:
|
|
l = l.strip()
|
|
if not l.startswith('#') and ' ' in l:
|
|
canonical_email, alias = l.split(' ')
|
|
mapping[alias.lower()] = canonical_email.lower()
|
|
return mapping
|
|
|
|
|
|
def str_dict_replace(s, mapping):
|
|
for s1, s2 in mapping.iteritems():
|
|
s = s.replace(s1, s2)
|
|
return s
|
|
|
|
|
|
class LazyPluggable(object):
|
|
"""A pluggable backend loaded lazily based on some value."""
|
|
|
|
def __init__(self, pivot, **backends):
|
|
self.__backends = backends
|
|
self.__pivot = pivot
|
|
self.__backend = None
|
|
|
|
def __get_backend(self):
|
|
if not self.__backend:
|
|
backend_name = FLAGS[self.__pivot]
|
|
if backend_name not in self.__backends:
|
|
raise exception.Error(_('Invalid backend: %s') % backend_name)
|
|
|
|
backend = self.__backends[backend_name]
|
|
if isinstance(backend, tuple):
|
|
name = backend[0]
|
|
fromlist = backend[1]
|
|
else:
|
|
name = backend
|
|
fromlist = backend
|
|
|
|
self.__backend = __import__(name, None, None, fromlist)
|
|
LOG.debug(_('backend %s'), self.__backend)
|
|
return self.__backend
|
|
|
|
def __getattr__(self, key):
|
|
backend = self.__get_backend()
|
|
return getattr(backend, key)
|
|
|
|
|
|
class LoopingCallDone(Exception):
|
|
"""Exception to break out and stop a LoopingCall.
|
|
|
|
The poll-function passed to LoopingCall can raise this exception to
|
|
break out of the loop normally. This is somewhat analogous to
|
|
StopIteration.
|
|
|
|
An optional return-value can be included as the argument to the exception;
|
|
this return-value will be returned by LoopingCall.wait()
|
|
|
|
"""
|
|
|
|
def __init__(self, retvalue=True):
|
|
""":param retvalue: Value that LoopingCall.wait() should return."""
|
|
self.retvalue = retvalue
|
|
|
|
|
|
class LoopingCall(object):
|
|
def __init__(self, f=None, *args, **kw):
|
|
self.args = args
|
|
self.kw = kw
|
|
self.f = f
|
|
self._running = False
|
|
|
|
def start(self, interval, initial_delay=None):
|
|
self._running = True
|
|
done = event.Event()
|
|
|
|
def _inner():
|
|
if initial_delay:
|
|
greenthread.sleep(initial_delay)
|
|
|
|
try:
|
|
while self._running:
|
|
self.f(*self.args, **self.kw)
|
|
if not self._running:
|
|
break
|
|
greenthread.sleep(interval)
|
|
except LoopingCallDone, e:
|
|
self.stop()
|
|
done.send(e.retvalue)
|
|
except Exception:
|
|
LOG.exception(_('in looping call'))
|
|
done.send_exception(*sys.exc_info())
|
|
return
|
|
else:
|
|
done.send(True)
|
|
|
|
self.done = done
|
|
|
|
greenthread.spawn(_inner)
|
|
return self.done
|
|
|
|
def stop(self):
|
|
self._running = False
|
|
|
|
def wait(self):
|
|
return self.done.wait()
|
|
|
|
|
|
def xhtml_escape(value):
|
|
"""Escapes a string so it is valid within XML or XHTML.
|
|
|
|
"""
|
|
return saxutils.escape(value, {'"': '"', "'": '''})
|
|
|
|
|
|
def utf8(value):
|
|
"""Try to turn a string into utf-8 if possible.
|
|
|
|
Code is directly from the utf8 function in
|
|
http://github.com/facebook/tornado/blob/master/tornado/escape.py
|
|
|
|
"""
|
|
if isinstance(value, unicode):
|
|
return value.encode('utf-8')
|
|
assert isinstance(value, str)
|
|
return value
|
|
|
|
|
|
def to_primitive(value, convert_instances=False, level=0):
|
|
"""Convert a complex object into primitives.
|
|
|
|
Handy for JSON serialization. We can optionally handle instances,
|
|
but since this is a recursive function, we could have cyclical
|
|
data structures.
|
|
|
|
To handle cyclical data structures we could track the actual objects
|
|
visited in a set, but not all objects are hashable. Instead we just
|
|
track the depth of the object inspections and don't go too deep.
|
|
|
|
Therefore, convert_instances=True is lossy ... be aware.
|
|
|
|
"""
|
|
nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
|
|
inspect.isfunction, inspect.isgeneratorfunction,
|
|
inspect.isgenerator, inspect.istraceback, inspect.isframe,
|
|
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
|
|
inspect.isabstract]
|
|
for test in nasty:
|
|
if test(value):
|
|
return unicode(value)
|
|
|
|
# value of itertools.count doesn't get caught by inspects
|
|
# above and results in infinite loop when list(value) is called.
|
|
if type(value) == itertools.count:
|
|
return unicode(value)
|
|
|
|
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
|
|
# tests that raise an exception in a mocked method that
|
|
# has a @wrap_exception with a notifier will fail. If
|
|
# we up the dependency to 0.5.4 (when it is released) we
|
|
# can remove this workaround.
|
|
if getattr(value, '__module__', None) == 'mox':
|
|
return 'mock'
|
|
|
|
if level > 3:
|
|
return '?'
|
|
|
|
# The try block may not be necessary after the class check above,
|
|
# but just in case ...
|
|
try:
|
|
if isinstance(value, (list, tuple)):
|
|
o = []
|
|
for v in value:
|
|
o.append(to_primitive(v, convert_instances=convert_instances,
|
|
level=level))
|
|
return o
|
|
elif isinstance(value, dict):
|
|
o = {}
|
|
for k, v in value.iteritems():
|
|
o[k] = to_primitive(v, convert_instances=convert_instances,
|
|
level=level)
|
|
return o
|
|
elif isinstance(value, datetime.datetime):
|
|
return str(value)
|
|
elif hasattr(value, 'iteritems'):
|
|
return to_primitive(dict(value.iteritems()),
|
|
convert_instances=convert_instances,
|
|
level=level)
|
|
elif hasattr(value, '__iter__'):
|
|
return to_primitive(list(value), level)
|
|
elif convert_instances and hasattr(value, '__dict__'):
|
|
# Likely an instance of something. Watch for cycles.
|
|
# Ignore class member vars.
|
|
return to_primitive(value.__dict__,
|
|
convert_instances=convert_instances,
|
|
level=level + 1)
|
|
else:
|
|
return value
|
|
except TypeError, e:
|
|
# Class objects are tricky since they may define something like
|
|
# __iter__ defined but it isn't callable as list().
|
|
return unicode(value)
|
|
|
|
|
|
def dumps(value):
|
|
try:
|
|
return json.dumps(value)
|
|
except TypeError:
|
|
pass
|
|
return json.dumps(to_primitive(value))
|
|
|
|
|
|
def loads(s):
|
|
return json.loads(s)
|
|
|
|
|
|
try:
|
|
import anyjson
|
|
except ImportError:
|
|
pass
|
|
else:
|
|
anyjson._modules.append(("nova.utils", "dumps", TypeError,
|
|
"loads", ValueError))
|
|
anyjson.force_implementation("nova.utils")
|
|
|
|
|
|
class GreenLockFile(lockfile.FileLock):
|
|
"""Implementation of lockfile that allows for a lock per greenthread.
|
|
|
|
Simply implements lockfile:LockBase init with an addiontall suffix
|
|
on the unique name of the greenthread identifier
|
|
"""
|
|
def __init__(self, path, threaded=True):
|
|
self.path = path
|
|
self.lock_file = os.path.abspath(path) + ".lock"
|
|
self.hostname = socket.gethostname()
|
|
self.pid = os.getpid()
|
|
if threaded:
|
|
t = threading.current_thread()
|
|
# Thread objects in Python 2.4 and earlier do not have ident
|
|
# attrs. Worm around that.
|
|
ident = getattr(t, "ident", hash(t)) or hash(t)
|
|
gident = corolocal.get_ident()
|
|
self.tname = "-%x-%x" % (ident & 0xffffffff, gident & 0xffffffff)
|
|
else:
|
|
self.tname = ""
|
|
dirname = os.path.dirname(self.lock_file)
|
|
self.unique_name = os.path.join(dirname,
|
|
"%s%s.%s" % (self.hostname,
|
|
self.tname,
|
|
self.pid))
|
|
|
|
|
|
_semaphores = {}
|
|
|
|
|
|
def synchronized(name, external=False):
|
|
"""Synchronization decorator.
|
|
|
|
Decorating a method like so::
|
|
|
|
@synchronized('mylock')
|
|
def foo(self, *args):
|
|
...
|
|
|
|
ensures that only one thread will execute the bar method at a time.
|
|
|
|
Different methods can share the same lock::
|
|
|
|
@synchronized('mylock')
|
|
def foo(self, *args):
|
|
...
|
|
|
|
@synchronized('mylock')
|
|
def bar(self, *args):
|
|
...
|
|
|
|
This way only one of either foo or bar can be executing at a time.
|
|
|
|
The external keyword argument denotes whether this lock should work across
|
|
multiple processes. This means that if two different workers both run a
|
|
a method decorated with @synchronized('mylock', external=True), only one
|
|
of them will execute at a time.
|
|
|
|
Important limitation: you can only have one external lock running per
|
|
thread at a time. For example the following will fail:
|
|
|
|
@utils.synchronized('testlock1', external=True)
|
|
def outer_lock():
|
|
|
|
@utils.synchronized('testlock2', external=True)
|
|
def inner_lock():
|
|
pass
|
|
inner_lock()
|
|
|
|
outer_lock()
|
|
|
|
"""
|
|
|
|
def wrap(f):
|
|
@functools.wraps(f)
|
|
def inner(*args, **kwargs):
|
|
# NOTE(soren): If we ever go natively threaded, this will be racy.
|
|
# See http://stackoverflow.com/questions/5390569/dyn
|
|
# amically-allocating-and-destroying-mutexes
|
|
if name not in _semaphores:
|
|
_semaphores[name] = semaphore.Semaphore()
|
|
sem = _semaphores[name]
|
|
LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method '
|
|
'"%(method)s"...'), {'lock': name,
|
|
'method': f.__name__})
|
|
with sem:
|
|
LOG.debug(_('Got semaphore "%(lock)s" for method '
|
|
'"%(method)s"...'), {'lock': name,
|
|
'method': f.__name__})
|
|
if external and not FLAGS.disable_process_locking:
|
|
LOG.debug(_('Attempting to grab file lock "%(lock)s" for '
|
|
'method "%(method)s"...'),
|
|
{'lock': name, 'method': f.__name__})
|
|
lock_file_path = os.path.join(FLAGS.lock_path,
|
|
'nova-%s' % name)
|
|
lock = GreenLockFile(lock_file_path)
|
|
with lock:
|
|
LOG.debug(_('Got file lock "%(lock)s" for '
|
|
'method "%(method)s"...'),
|
|
{'lock': name, 'method': f.__name__})
|
|
retval = f(*args, **kwargs)
|
|
else:
|
|
retval = f(*args, **kwargs)
|
|
|
|
# If no-one else is waiting for it, delete it.
|
|
# See note about possible raciness above.
|
|
if not sem.balance < 1:
|
|
del _semaphores[name]
|
|
|
|
return retval
|
|
return inner
|
|
return wrap
|
|
|
|
|
|
def cleanup_file_locks():
|
|
"""clean up stale locks left behind by process failures
|
|
|
|
The lockfile module, used by @synchronized, can leave stale lockfiles
|
|
behind after process failure. These locks can cause process hangs
|
|
at startup, when a process deadlocks on a lock which will never
|
|
be unlocked.
|
|
|
|
Intended to be called at service startup.
|
|
|
|
"""
|
|
|
|
# NOTE(mikeyp) this routine incorporates some internal knowledge
|
|
# from the lockfile module, and this logic really
|
|
# should be part of that module.
|
|
#
|
|
# cleanup logic:
|
|
# 1) look for the lockfile modules's 'sentinel' files, of the form
|
|
# hostname.[thread-.*]-pid, extract the pid.
|
|
# if pid doesn't match a running process, delete the file since
|
|
# it's from a dead process.
|
|
# 2) check for the actual lockfiles. if lockfile exists with linkcount
|
|
# of 1, it's bogus, so delete it. A link count >= 2 indicates that
|
|
# there are probably sentinels still linked to it from active
|
|
# processes. This check isn't perfect, but there is no way to
|
|
# reliably tell which sentinels refer to which lock in the
|
|
# lockfile implementation.
|
|
|
|
if FLAGS.disable_process_locking:
|
|
return
|
|
|
|
hostname = socket.gethostname()
|
|
sentinel_re = hostname + r'\..*-(\d+$)'
|
|
lockfile_re = r'nova-.*\.lock'
|
|
files = os.listdir(FLAGS.lock_path)
|
|
|
|
# cleanup sentinels
|
|
for filename in files:
|
|
match = re.match(sentinel_re, filename)
|
|
if match is None:
|
|
continue
|
|
pid = match.group(1)
|
|
LOG.debug(_('Found sentinel %(filename)s for pid %(pid)s'),
|
|
{'filename': filename, 'pid': pid})
|
|
try:
|
|
os.kill(int(pid), 0)
|
|
except OSError, e:
|
|
# PID wasn't found
|
|
delete_if_exists(os.path.join(FLAGS.lock_path, filename))
|
|
LOG.debug(_('Cleaned sentinel %(filename)s for pid %(pid)s'),
|
|
{'filename': filename, 'pid': pid})
|
|
|
|
# cleanup lock files
|
|
for filename in files:
|
|
match = re.match(lockfile_re, filename)
|
|
if match is None:
|
|
continue
|
|
try:
|
|
stat_info = os.stat(os.path.join(FLAGS.lock_path, filename))
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
continue
|
|
else:
|
|
raise
|
|
LOG.debug(_('Found lockfile %(file)s with link count %(count)d'),
|
|
{'file': filename, 'count': stat_info.st_nlink})
|
|
if stat_info.st_nlink == 1:
|
|
delete_if_exists(os.path.join(FLAGS.lock_path, filename))
|
|
LOG.debug(_('Cleaned lockfile %(file)s with link count %(count)d'),
|
|
{'file': filename, 'count': stat_info.st_nlink})
|
|
|
|
|
|
def delete_if_exists(pathname):
|
|
"""delete a file, but ignore file not found error"""
|
|
|
|
try:
|
|
os.unlink(pathname)
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
return
|
|
else:
|
|
raise
|
|
|
|
|
|
def get_from_path(items, path):
|
|
"""Returns a list of items matching the specified path.
|
|
|
|
Takes an XPath-like expression e.g. prop1/prop2/prop3, and for each item
|
|
in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the
|
|
intermediate results are lists it will treat each list item individually.
|
|
A 'None' in items or any child expressions will be ignored, this function
|
|
will not throw because of None (anywhere) in items. The returned list
|
|
will contain no None values.
|
|
|
|
"""
|
|
if path is None:
|
|
raise exception.Error('Invalid mini_xpath')
|
|
|
|
(first_token, sep, remainder) = path.partition('/')
|
|
|
|
if first_token == '':
|
|
raise exception.Error('Invalid mini_xpath')
|
|
|
|
results = []
|
|
|
|
if items is None:
|
|
return results
|
|
|
|
if not isinstance(items, list):
|
|
# Wrap single objects in a list
|
|
items = [items]
|
|
|
|
for item in items:
|
|
if item is None:
|
|
continue
|
|
get_method = getattr(item, 'get', None)
|
|
if get_method is None:
|
|
continue
|
|
child = get_method(first_token)
|
|
if child is None:
|
|
continue
|
|
if isinstance(child, list):
|
|
# Flatten intermediate lists
|
|
for x in child:
|
|
results.append(x)
|
|
else:
|
|
results.append(child)
|
|
|
|
if not sep:
|
|
# No more tokens
|
|
return results
|
|
else:
|
|
return get_from_path(results, remainder)
|
|
|
|
|
|
def flatten_dict(dict_, flattened=None):
|
|
"""Recursively flatten a nested dictionary."""
|
|
flattened = flattened or {}
|
|
for key, value in dict_.iteritems():
|
|
if hasattr(value, 'iteritems'):
|
|
flatten_dict(value, flattened)
|
|
else:
|
|
flattened[key] = value
|
|
return flattened
|
|
|
|
|
|
def partition_dict(dict_, keys):
|
|
"""Return two dicts, one with `keys` the other with everything else."""
|
|
intersection = {}
|
|
difference = {}
|
|
for key, value in dict_.iteritems():
|
|
if key in keys:
|
|
intersection[key] = value
|
|
else:
|
|
difference[key] = value
|
|
return intersection, difference
|
|
|
|
|
|
def map_dict_keys(dict_, key_map):
|
|
"""Return a dict in which the dictionaries keys are mapped to new keys."""
|
|
mapped = {}
|
|
for key, value in dict_.iteritems():
|
|
mapped_key = key_map[key] if key in key_map else key
|
|
mapped[mapped_key] = value
|
|
return mapped
|
|
|
|
|
|
def subset_dict(dict_, keys):
|
|
"""Return a dict that only contains a subset of keys."""
|
|
subset = partition_dict(dict_, keys)[0]
|
|
return subset
|
|
|
|
|
|
def check_isinstance(obj, cls):
|
|
"""Checks that obj is of type cls, and lets PyLint infer types."""
|
|
if isinstance(obj, cls):
|
|
return obj
|
|
raise Exception(_('Expected object of type: %s') % (str(cls)))
|
|
# TODO(justinsb): Can we make this better??
|
|
return cls() # Ugly PyLint hack
|
|
|
|
|
|
def parse_server_string(server_str):
|
|
"""
|
|
Parses the given server_string and returns a list of host and port.
|
|
If it's not a combination of host part and port, the port element
|
|
is a null string. If the input is invalid expression, return a null
|
|
list.
|
|
"""
|
|
try:
|
|
# First of all, exclude pure IPv6 address (w/o port).
|
|
if netaddr.valid_ipv6(server_str):
|
|
return (server_str, '')
|
|
|
|
# Next, check if this is IPv6 address with a port number combination.
|
|
if server_str.find("]:") != -1:
|
|
(address, port) = server_str.replace('[', '', 1).split(']:')
|
|
return (address, port)
|
|
|
|
# Third, check if this is a combination of an address and a port
|
|
if server_str.find(':') == -1:
|
|
return (server_str, '')
|
|
|
|
# This must be a combination of an address and a port
|
|
(address, port) = server_str.split(':')
|
|
return (address, port)
|
|
|
|
except Exception:
|
|
LOG.debug(_('Invalid server_string: %s'), server_str)
|
|
return ('', '')
|
|
|
|
|
|
def gen_uuid():
|
|
return uuid.uuid4()
|
|
|
|
|
|
def is_uuid_like(val):
|
|
"""For our purposes, a UUID is a string in canonical form:
|
|
|
|
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
|
|
"""
|
|
try:
|
|
uuid.UUID(val)
|
|
return True
|
|
except (TypeError, ValueError, AttributeError):
|
|
return False
|
|
|
|
|
|
def bool_from_str(val):
|
|
"""Convert a string representation of a bool into a bool value"""
|
|
|
|
if not val:
|
|
return False
|
|
try:
|
|
return True if int(val) else False
|
|
except ValueError:
|
|
return val.lower() == 'true'
|
|
|
|
|
|
def is_valid_ipv4(address):
|
|
"""valid the address strictly as per format xxx.xxx.xxx.xxx.
|
|
where xxx is a value between 0 and 255.
|
|
"""
|
|
parts = address.split(".")
|
|
if len(parts) != 4:
|
|
return False
|
|
for item in parts:
|
|
try:
|
|
if not 0 <= int(item) <= 255:
|
|
return False
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def is_valid_cidr(address):
|
|
"""Check if the provided ipv4 or ipv6 address is a valid
|
|
CIDR address or not"""
|
|
try:
|
|
# Validate the correct CIDR Address
|
|
netaddr.IPNetwork(address)
|
|
except netaddr.core.AddrFormatError:
|
|
return False
|
|
except UnboundLocalError:
|
|
# NOTE(MotoKen): work around bug in netaddr 0.7.5 (see detail in
|
|
# https://github.com/drkjam/netaddr/issues/2)
|
|
return False
|
|
|
|
# Prior validation partially verify /xx part
|
|
# Verify it here
|
|
ip_segment = address.split('/')
|
|
|
|
if (len(ip_segment) <= 1 or
|
|
ip_segment[1] == ''):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def monkey_patch():
|
|
""" If the Flags.monkey_patch set as True,
|
|
this function patches a decorator
|
|
for all functions in specified modules.
|
|
You can set decorators for each modules
|
|
using FLAGS.monkey_patch_modules.
|
|
The format is "Module path:Decorator function".
|
|
Example: 'nova.api.ec2.cloud:nova.notifier.api.notify_decorator'
|
|
|
|
Parameters of the decorator is as follows.
|
|
(See nova.notifier.api.notify_decorator)
|
|
|
|
name - name of the function
|
|
function - object of the function
|
|
"""
|
|
# If FLAGS.monkey_patch is not True, this function do nothing.
|
|
if not FLAGS.monkey_patch:
|
|
return
|
|
# Get list of modules and decorators
|
|
for module_and_decorator in FLAGS.monkey_patch_modules:
|
|
module, decorator_name = module_and_decorator.split(':')
|
|
# import decorator function
|
|
decorator = importutils.import_class(decorator_name)
|
|
__import__(module)
|
|
# Retrieve module information using pyclbr
|
|
module_data = pyclbr.readmodule_ex(module)
|
|
for key in module_data.keys():
|
|
# set the decorator for the class methods
|
|
if isinstance(module_data[key], pyclbr.Class):
|
|
clz = importutils.import_class("%s.%s" % (module, key))
|
|
for method, func in inspect.getmembers(clz, inspect.ismethod):
|
|
setattr(clz, method,
|
|
decorator("%s.%s.%s" % (module, key, method), func))
|
|
# set the decorator for the function
|
|
if isinstance(module_data[key], pyclbr.Function):
|
|
func = importutils.import_class("%s.%s" % (module, key))
|
|
setattr(sys.modules[module], key,
|
|
decorator("%s.%s" % (module, key), func))
|
|
|
|
|
|
def convert_to_list_dict(lst, label):
|
|
"""Convert a value or list into a list of dicts"""
|
|
if not lst:
|
|
return None
|
|
if not isinstance(lst, list):
|
|
lst = [lst]
|
|
return [{label: x} for x in lst]
|
|
|
|
|
|
def timefunc(func):
|
|
"""Decorator that logs how long a particular function took to execute"""
|
|
@functools.wraps(func)
|
|
def inner(*args, **kwargs):
|
|
start_time = time.time()
|
|
try:
|
|
return func(*args, **kwargs)
|
|
finally:
|
|
total_time = time.time() - start_time
|
|
LOG.debug(_("timefunc: '%(name)s' took %(total_time).2f secs") %
|
|
dict(name=func.__name__, total_time=total_time))
|
|
return inner
|
|
|
|
|
|
def generate_glance_url():
|
|
"""Generate the URL to glance."""
|
|
# TODO(jk0): This will eventually need to take SSL into consideration
|
|
# when supported in glance.
|
|
return "http://%s:%d" % (FLAGS.glance_host, FLAGS.glance_port)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def logging_error(message):
|
|
"""Catches exception, write message to the log, re-raise.
|
|
This is a common refinement of save_and_reraise that writes a specific
|
|
message to the log.
|
|
"""
|
|
try:
|
|
yield
|
|
except Exception as error:
|
|
with excutils.save_and_reraise_exception():
|
|
LOG.exception(message)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def remove_path_on_error(path):
|
|
"""Protect code that wants to operate on PATH atomically.
|
|
Any exception will cause PATH to be removed.
|
|
"""
|
|
try:
|
|
yield
|
|
except Exception:
|
|
with excutils.save_and_reraise_exception():
|
|
delete_if_exists(path)
|
|
|
|
|
|
def make_dev_path(dev, partition=None, base='/dev'):
|
|
"""Return a path to a particular device.
|
|
|
|
>>> make_dev_path('xvdc')
|
|
/dev/xvdc
|
|
|
|
>>> make_dev_path('xvdc', 1)
|
|
/dev/xvdc1
|
|
"""
|
|
path = os.path.join(base, dev)
|
|
if partition:
|
|
path += str(partition)
|
|
return path
|
|
|
|
|
|
def total_seconds(td):
|
|
"""Local total_seconds implementation for compatibility with python 2.6"""
|
|
if hasattr(td, 'total_seconds'):
|
|
return td.total_seconds()
|
|
else:
|
|
return ((td.days * 86400 + td.seconds) * 10 ** 6 +
|
|
td.microseconds) / 10.0 ** 6
|
|
|
|
|
|
def sanitize_hostname(hostname):
|
|
"""Return a hostname which conforms to RFC-952 and RFC-1123 specs."""
|
|
if isinstance(hostname, unicode):
|
|
hostname = hostname.encode('latin-1', 'ignore')
|
|
|
|
hostname = re.sub('[ _]', '-', hostname)
|
|
hostname = re.sub('[^\w.-]+', '', hostname)
|
|
hostname = hostname.lower()
|
|
hostname = hostname.strip('.-')
|
|
|
|
return hostname
|
|
|
|
|
|
def read_cached_file(filename, cache_info, reload_func=None):
|
|
"""Read from a file if it has been modified.
|
|
|
|
:param cache_info: dictionary to hold opaque cache.
|
|
:param reload_func: optional function to be called with data when
|
|
file is reloaded due to a modification.
|
|
|
|
:returns: data from file
|
|
|
|
"""
|
|
mtime = os.path.getmtime(filename)
|
|
if not cache_info or mtime != cache_info.get('mtime'):
|
|
LOG.debug(_("Reloading cached file %s") % filename)
|
|
with open(filename) as fap:
|
|
cache_info['data'] = fap.read()
|
|
cache_info['mtime'] = mtime
|
|
if reload_func:
|
|
reload_func(cache_info['data'])
|
|
return cache_info['data']
|
|
|
|
|
|
def hash_file(file_like_object):
|
|
"""Generate a hash for the contents of a file."""
|
|
checksum = hashlib.sha1()
|
|
any(map(checksum.update, iter(lambda: file_like_object.read(32768), '')))
|
|
return checksum.hexdigest()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def temporary_mutation(obj, **kwargs):
|
|
"""Temporarily set the attr on a particular object to a given value then
|
|
revert when finished.
|
|
|
|
One use of this is to temporarily set the read_deleted flag on a context
|
|
object:
|
|
|
|
with temporary_mutation(context, read_deleted="yes"):
|
|
do_something_that_needed_deleted_objects()
|
|
"""
|
|
NOT_PRESENT = object()
|
|
|
|
old_values = {}
|
|
for attr, new_value in kwargs.items():
|
|
old_values[attr] = getattr(obj, attr, NOT_PRESENT)
|
|
setattr(obj, attr, new_value)
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
for attr, old_value in old_values.items():
|
|
if old_value is NOT_PRESENT:
|
|
del obj[attr]
|
|
else:
|
|
setattr(obj, attr, old_value)
|
|
|
|
|
|
def warn_deprecated_class(cls, msg):
|
|
"""
|
|
Issues a warning to indicate that the given class is deprecated.
|
|
If a message is given, it is appended to the deprecation warning.
|
|
"""
|
|
|
|
fullname = '%s.%s' % (cls.__module__, cls.__name__)
|
|
if msg:
|
|
fullmsg = _("Class %(fullname)s is deprecated: %(msg)s")
|
|
else:
|
|
fullmsg = _("Class %(fullname)s is deprecated")
|
|
|
|
# Issue the warning
|
|
warnings.warn(fullmsg % locals(), DeprecationWarning, stacklevel=3)
|
|
|
|
|
|
def warn_deprecated_function(func, msg):
|
|
"""
|
|
Issues a warning to indicate that the given function is
|
|
deprecated. If a message is given, it is appended to the
|
|
deprecation warning.
|
|
"""
|
|
|
|
name = func.__name__
|
|
|
|
# Find the function's definition
|
|
sourcefile = inspect.getsourcefile(func)
|
|
|
|
# Find the line number, if possible
|
|
if inspect.ismethod(func):
|
|
code = func.im_func.func_code
|
|
else:
|
|
code = func.func_code
|
|
lineno = getattr(code, 'co_firstlineno', None)
|
|
|
|
if lineno is None:
|
|
location = sourcefile
|
|
else:
|
|
location = "%s:%d" % (sourcefile, lineno)
|
|
|
|
# Build up the message
|
|
if msg:
|
|
fullmsg = _("Function %(name)s in %(location)s is deprecated: %(msg)s")
|
|
else:
|
|
fullmsg = _("Function %(name)s in %(location)s is deprecated")
|
|
|
|
# Issue the warning
|
|
warnings.warn(fullmsg % locals(), DeprecationWarning, stacklevel=3)
|
|
|
|
|
|
def _stubout(klass, message):
|
|
"""
|
|
Scans a class and generates wrapping stubs for __new__() and every
|
|
class and static method. Returns a dictionary which can be passed
|
|
to type() to generate a wrapping class.
|
|
"""
|
|
|
|
overrides = {}
|
|
|
|
def makestub_class(name, func):
|
|
"""
|
|
Create a stub for wrapping class methods.
|
|
"""
|
|
|
|
def stub(cls, *args, **kwargs):
|
|
warn_deprecated_class(klass, message)
|
|
return func(*args, **kwargs)
|
|
|
|
# Overwrite the stub's name
|
|
stub.__name__ = name
|
|
stub.func_name = name
|
|
|
|
return classmethod(stub)
|
|
|
|
def makestub_static(name, func):
|
|
"""
|
|
Create a stub for wrapping static methods.
|
|
"""
|
|
|
|
def stub(*args, **kwargs):
|
|
warn_deprecated_class(klass, message)
|
|
return func(*args, **kwargs)
|
|
|
|
# Overwrite the stub's name
|
|
stub.__name__ = name
|
|
stub.func_name = name
|
|
|
|
return staticmethod(stub)
|
|
|
|
for name, kind, _klass, _obj in inspect.classify_class_attrs(klass):
|
|
# We're only interested in __new__(), class methods, and
|
|
# static methods...
|
|
if (name != '__new__' and
|
|
kind not in ('class method', 'static method')):
|
|
continue
|
|
|
|
# Get the function...
|
|
func = getattr(klass, name)
|
|
|
|
# Override it in the class
|
|
if kind == 'class method':
|
|
stub = makestub_class(name, func)
|
|
elif kind == 'static method' or name == '__new__':
|
|
stub = makestub_static(name, func)
|
|
|
|
# Save it in the overrides dictionary...
|
|
overrides[name] = stub
|
|
|
|
# Apply the overrides
|
|
for name, stub in overrides.items():
|
|
setattr(klass, name, stub)
|
|
|
|
|
|
def deprecated(message=''):
|
|
"""
|
|
Marks a function, class, or method as being deprecated. For
|
|
functions and methods, emits a warning each time the function or
|
|
method is called. For classes, generates a new subclass which
|
|
will emit a warning each time the class is instantiated, or each
|
|
time any class or static method is called.
|
|
|
|
If a message is passed to the decorator, that message will be
|
|
appended to the emitted warning. This may be used to suggest an
|
|
alternate way of achieving the desired effect, or to explain why
|
|
the function, class, or method is deprecated.
|
|
"""
|
|
|
|
def decorator(f_or_c):
|
|
# Make sure we can deprecate it...
|
|
if not callable(f_or_c) or isinstance(f_or_c, types.ClassType):
|
|
warnings.warn("Cannot mark object %r as deprecated" % f_or_c,
|
|
DeprecationWarning, stacklevel=2)
|
|
return f_or_c
|
|
|
|
# If we're deprecating a class, create a subclass of it and
|
|
# stub out all the class and static methods
|
|
if inspect.isclass(f_or_c):
|
|
klass = f_or_c
|
|
_stubout(klass, message)
|
|
return klass
|
|
|
|
# OK, it's a function; use a traditional wrapper...
|
|
func = f_or_c
|
|
|
|
@functools.wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
warn_deprecated_function(func, message)
|
|
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapper
|
|
return decorator
|
|
|
|
|
|
def _showwarning(message, category, filename, lineno, file=None, line=None):
|
|
"""
|
|
Redirect warnings into logging.
|
|
"""
|
|
|
|
fmtmsg = warnings.formatwarning(message, category, filename, lineno, line)
|
|
LOG.warning(fmtmsg)
|
|
|
|
|
|
# Install our warnings handler
|
|
warnings.showwarning = _showwarning
|
|
|
|
|
|
def service_is_up(service):
|
|
"""Check whether a service is up based on last heartbeat."""
|
|
last_heartbeat = service['updated_at'] or service['created_at']
|
|
# Timestamps in DB are UTC.
|
|
elapsed = total_seconds(utcnow() - last_heartbeat)
|
|
return abs(elapsed) <= FLAGS.service_down_time
|
|
|
|
|
|
def generate_mac_address():
|
|
"""Generate an Ethernet MAC address."""
|
|
# NOTE(vish): We would prefer to use 0xfe here to ensure that linux
|
|
# bridge mac addresses don't change, but it appears to
|
|
# conflict with libvirt, so we use the next highest octet
|
|
# that has the unicast and locally administered bits set
|
|
# properly: 0xfa.
|
|
# Discussion: https://bugs.launchpad.net/nova/+bug/921838
|
|
mac = [0xfa, 0x16, 0x3e,
|
|
random.randint(0x00, 0x7f),
|
|
random.randint(0x00, 0xff),
|
|
random.randint(0x00, 0xff)]
|
|
return ':'.join(map(lambda x: "%02x" % x, mac))
|
|
|
|
|
|
def read_file_as_root(file_path):
|
|
"""Secure helper to read file as root."""
|
|
try:
|
|
out, _err = execute('cat', file_path, run_as_root=True)
|
|
return out
|
|
except exception.ProcessExecutionError:
|
|
raise exception.FileNotFound(file_path=file_path)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def temporary_chown(path, owner_uid=None):
|
|
"""Temporarily chown a path.
|
|
|
|
:params owner_uid: UID of temporary owner (defaults to current user)
|
|
"""
|
|
if owner_uid is None:
|
|
owner_uid = os.getuid()
|
|
|
|
orig_uid = os.stat(path).st_uid
|
|
|
|
if orig_uid != owner_uid:
|
|
execute('chown', owner_uid, path, run_as_root=True)
|
|
try:
|
|
yield
|
|
finally:
|
|
if orig_uid != owner_uid:
|
|
execute('chown', orig_uid, path, run_as_root=True)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def tempdir(**kwargs):
|
|
tmpdir = tempfile.mkdtemp(**kwargs)
|
|
try:
|
|
yield tmpdir
|
|
finally:
|
|
try:
|
|
shutil.rmtree(tmpdir)
|
|
except OSError, e:
|
|
LOG.debug(_('Could not remove tmpdir: %s'), str(e))
|
|
|
|
|
|
def strcmp_const_time(s1, s2):
|
|
"""Constant-time string comparison.
|
|
|
|
:params s1: the first string
|
|
:params s2: the second string
|
|
|
|
:return: True if the strings are equal.
|
|
|
|
This function takes two strings and compares them. It is intended to be
|
|
used when doing a comparison for authentication purposes to help guard
|
|
against timing attacks.
|
|
"""
|
|
if len(s1) != len(s2):
|
|
return False
|
|
result = 0
|
|
for (a, b) in zip(s1, s2):
|
|
result |= ord(a) ^ ord(b)
|
|
return result == 0
|
|
|
|
|
|
class UndoManager(object):
|
|
"""Provides a mechanism to facilitate rolling back a series of actions
|
|
when an exception is raised.
|
|
"""
|
|
def __init__(self):
|
|
self.undo_stack = []
|
|
|
|
def undo_with(self, undo_func):
|
|
self.undo_stack.append(undo_func)
|
|
|
|
def _rollback(self):
|
|
for undo_func in reversed(self.undo_stack):
|
|
undo_func()
|
|
|
|
def rollback_and_reraise(self, msg=None, **kwargs):
|
|
"""Rollback a series of actions then re-raise the exception.
|
|
|
|
.. note:: (sirp) This should only be called within an
|
|
exception handler.
|
|
"""
|
|
with excutils.save_and_reraise_exception():
|
|
if msg:
|
|
LOG.exception(msg, **kwargs)
|
|
|
|
self._rollback()
|