448 lines
14 KiB
Python
448 lines
14 KiB
Python
# vi: ts=4 expandtab
|
|
#
|
|
# Copyright (C) 2012 Canonical Ltd.
|
|
# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
|
|
# Copyright (C) 2012 Yahoo! Inc.
|
|
#
|
|
# Author: Scott Moser <scott.moser@canonical.com>
|
|
# Author: Juerg Haefliger <juerg.haefliger@hp.com>
|
|
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 3, as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from time import time
|
|
|
|
import contextlib
|
|
import io
|
|
import os
|
|
|
|
from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser)
|
|
|
|
from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
|
|
CFG_ENV_NAME)
|
|
|
|
from cloudinit import log as logging
|
|
from cloudinit import type_utils
|
|
from cloudinit import util
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class LockFailure(Exception):
|
|
pass
|
|
|
|
|
|
class DummyLock(object):
|
|
pass
|
|
|
|
|
|
class DummySemaphores(object):
|
|
def __init__(self):
|
|
pass
|
|
|
|
@contextlib.contextmanager
|
|
def lock(self, _name, _freq, _clear_on_fail=False):
|
|
yield DummyLock()
|
|
|
|
def has_run(self, _name, _freq):
|
|
return False
|
|
|
|
def clear(self, _name, _freq):
|
|
return True
|
|
|
|
def clear_all(self):
|
|
pass
|
|
|
|
|
|
class FileLock(object):
|
|
def __init__(self, fn):
|
|
self.fn = fn
|
|
|
|
def __str__(self):
|
|
return "<%s using file %r>" % (type_utils.obj_name(self), self.fn)
|
|
|
|
|
|
def canon_sem_name(name):
|
|
return name.replace("-", "_")
|
|
|
|
|
|
class FileSemaphores(object):
|
|
def __init__(self, sem_path):
|
|
self.sem_path = sem_path
|
|
|
|
@contextlib.contextmanager
|
|
def lock(self, name, freq, clear_on_fail=False):
|
|
name = canon_sem_name(name)
|
|
try:
|
|
yield self._acquire(name, freq)
|
|
except:
|
|
if clear_on_fail:
|
|
self.clear(name, freq)
|
|
raise
|
|
|
|
def clear(self, name, freq):
|
|
name = canon_sem_name(name)
|
|
sem_file = self._get_path(name, freq)
|
|
try:
|
|
util.del_file(sem_file)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed deleting semaphore %s", sem_file)
|
|
return False
|
|
return True
|
|
|
|
def clear_all(self):
|
|
try:
|
|
util.del_dir(self.sem_path)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed deleting semaphore directory %s",
|
|
self.sem_path)
|
|
|
|
def _acquire(self, name, freq):
|
|
# Check again if its been already gotten
|
|
if self.has_run(name, freq):
|
|
return None
|
|
# This is a race condition since nothing atomic is happening
|
|
# here, but this should be ok due to the nature of when
|
|
# and where cloud-init runs... (file writing is not a lock...)
|
|
sem_file = self._get_path(name, freq)
|
|
contents = "%s: %s\n" % (os.getpid(), time())
|
|
try:
|
|
util.write_file(sem_file, contents)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed writing semaphore file %s", sem_file)
|
|
return None
|
|
return FileLock(sem_file)
|
|
|
|
def has_run(self, name, freq):
|
|
if not freq or freq == PER_ALWAYS:
|
|
return False
|
|
|
|
cname = canon_sem_name(name)
|
|
sem_file = self._get_path(cname, freq)
|
|
# This isn't really a good atomic check
|
|
# but it suffices for where and when cloudinit runs
|
|
if os.path.exists(sem_file):
|
|
return True
|
|
|
|
# this case could happen if the migrator module hadn't run yet
|
|
# but the item had run before we did canon_sem_name.
|
|
if cname != name and os.path.exists(self._get_path(name, freq)):
|
|
LOG.warn("%s has run without canonicalized name [%s].\n"
|
|
"likely the migrator has not yet run. It will run next boot.\n"
|
|
"run manually with: cloud-init single --name=migrator"
|
|
% (name, cname))
|
|
return True
|
|
|
|
return False
|
|
|
|
def _get_path(self, name, freq):
|
|
sem_path = self.sem_path
|
|
if not freq or freq == PER_INSTANCE:
|
|
return os.path.join(sem_path, name)
|
|
else:
|
|
return os.path.join(sem_path, "%s.%s" % (name, freq))
|
|
|
|
|
|
class Runners(object):
|
|
def __init__(self, paths):
|
|
self.paths = paths
|
|
self.sems = {}
|
|
|
|
def _get_sem(self, freq):
|
|
if freq == PER_ALWAYS or not freq:
|
|
return None
|
|
sem_path = None
|
|
if freq == PER_INSTANCE:
|
|
# This may not exist,
|
|
# so thats why we still check for none
|
|
# below if say the paths object
|
|
# doesn't have a datasource that can
|
|
# provide this instance path...
|
|
sem_path = self.paths.get_ipath("sem")
|
|
elif freq == PER_ONCE:
|
|
sem_path = self.paths.get_cpath("sem")
|
|
if not sem_path:
|
|
return None
|
|
if sem_path not in self.sems:
|
|
self.sems[sem_path] = FileSemaphores(sem_path)
|
|
return self.sems[sem_path]
|
|
|
|
def run(self, name, functor, args, freq=None, clear_on_fail=False):
|
|
sem = self._get_sem(freq)
|
|
if not sem:
|
|
sem = DummySemaphores()
|
|
if not args:
|
|
args = []
|
|
if sem.has_run(name, freq):
|
|
LOG.debug("%s already ran (freq=%s)", name, freq)
|
|
return (False, None)
|
|
with sem.lock(name, freq, clear_on_fail) as lk:
|
|
if not lk:
|
|
raise LockFailure("Failed to acquire lock for %s" % name)
|
|
else:
|
|
LOG.debug("Running %s using lock (%s)", name, lk)
|
|
if isinstance(args, (dict)):
|
|
results = functor(**args)
|
|
else:
|
|
results = functor(*args)
|
|
return (True, results)
|
|
|
|
|
|
class ConfigMerger(object):
|
|
def __init__(self, paths=None, datasource=None,
|
|
additional_fns=None, base_cfg=None):
|
|
self._paths = paths
|
|
self._ds = datasource
|
|
self._fns = additional_fns
|
|
self._base_cfg = base_cfg
|
|
# Created on first use
|
|
self._cfg = None
|
|
|
|
def _get_datasource_configs(self):
|
|
d_cfgs = []
|
|
if self._ds:
|
|
try:
|
|
ds_cfg = self._ds.get_config_obj()
|
|
if ds_cfg and isinstance(ds_cfg, (dict)):
|
|
d_cfgs.append(ds_cfg)
|
|
except:
|
|
util.logexc(LOG, "Failed loading of datasource config object "
|
|
"from %s", self._ds)
|
|
return d_cfgs
|
|
|
|
def _get_env_configs(self):
|
|
e_cfgs = []
|
|
if CFG_ENV_NAME in os.environ:
|
|
e_fn = os.environ[CFG_ENV_NAME]
|
|
try:
|
|
e_cfgs.append(util.read_conf(e_fn))
|
|
except:
|
|
util.logexc(LOG, 'Failed loading of env. config from %s',
|
|
e_fn)
|
|
return e_cfgs
|
|
|
|
def _get_instance_configs(self):
|
|
i_cfgs = []
|
|
# If cloud-config was written, pick it up as
|
|
# a configuration file to use when running...
|
|
if not self._paths:
|
|
return i_cfgs
|
|
cc_fn = self._paths.get_ipath_cur('cloud_config')
|
|
if cc_fn and os.path.isfile(cc_fn):
|
|
try:
|
|
i_cfgs.append(util.read_conf(cc_fn))
|
|
except:
|
|
util.logexc(LOG, 'Failed loading of cloud-config from %s',
|
|
cc_fn)
|
|
return i_cfgs
|
|
|
|
def _read_cfg(self):
|
|
# Input config files override
|
|
# env config files which
|
|
# override instance configs
|
|
# which override datasource
|
|
# configs which override
|
|
# base configuration
|
|
cfgs = []
|
|
if self._fns:
|
|
for c_fn in self._fns:
|
|
try:
|
|
cfgs.append(util.read_conf(c_fn))
|
|
except:
|
|
util.logexc(LOG, "Failed loading of configuration from %s",
|
|
c_fn)
|
|
|
|
cfgs.extend(self._get_env_configs())
|
|
cfgs.extend(self._get_instance_configs())
|
|
cfgs.extend(self._get_datasource_configs())
|
|
if self._base_cfg:
|
|
cfgs.append(self._base_cfg)
|
|
return util.mergemanydict(cfgs)
|
|
|
|
@property
|
|
def cfg(self):
|
|
# None check to avoid empty case causing re-reading
|
|
if self._cfg is None:
|
|
self._cfg = self._read_cfg()
|
|
return self._cfg
|
|
|
|
|
|
class ContentHandlers(object):
|
|
|
|
def __init__(self):
|
|
self.registered = {}
|
|
|
|
def __contains__(self, item):
|
|
return self.is_registered(item)
|
|
|
|
def __getitem__(self, key):
|
|
return self._get_handler(key)
|
|
|
|
def is_registered(self, content_type):
|
|
return content_type in self.registered
|
|
|
|
def register(self, mod):
|
|
types = set()
|
|
for t in mod.list_types():
|
|
self.registered[t] = mod
|
|
types.add(t)
|
|
return types
|
|
|
|
def _get_handler(self, content_type):
|
|
return self.registered[content_type]
|
|
|
|
def items(self):
|
|
return self.registered.items()
|
|
|
|
def iteritems(self):
|
|
return self.registered.iteritems()
|
|
|
|
def register_defaults(self, defs):
|
|
registered = set()
|
|
for mod in defs:
|
|
for t in mod.list_types():
|
|
if not self.is_registered(t):
|
|
self.registered[t] = mod
|
|
registered.add(t)
|
|
return registered
|
|
|
|
|
|
class Paths(object):
|
|
def __init__(self, path_cfgs, ds=None):
|
|
self.cfgs = path_cfgs
|
|
# Populate all the initial paths
|
|
self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
|
|
self.instance_link = os.path.join(self.cloud_dir, 'instance')
|
|
self.boot_finished = os.path.join(self.instance_link, "boot-finished")
|
|
self.upstart_conf_d = path_cfgs.get('upstart_dir')
|
|
self.seed_dir = os.path.join(self.cloud_dir, 'seed')
|
|
# This one isn't joined, since it should just be read-only
|
|
template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/')
|
|
self.template_tpl = os.path.join(template_dir, '%s.tmpl')
|
|
self.lookups = {
|
|
"handlers": "handlers",
|
|
"scripts": "scripts",
|
|
"sem": "sem",
|
|
"boothooks": "boothooks",
|
|
"userdata_raw": "user-data.txt",
|
|
"userdata": "user-data.txt.i",
|
|
"obj_pkl": "obj.pkl",
|
|
"cloud_config": "cloud-config.txt",
|
|
"data": "data",
|
|
}
|
|
# Set when a datasource becomes active
|
|
self.datasource = ds
|
|
|
|
# get_ipath_cur: get the current instance path for an item
|
|
def get_ipath_cur(self, name=None):
|
|
ipath = self.instance_link
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
ipath = os.path.join(ipath, add_on)
|
|
return ipath
|
|
|
|
# get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
|
|
# for a name in dirmap
|
|
def get_cpath(self, name=None):
|
|
cpath = self.cloud_dir
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
cpath = os.path.join(cpath, add_on)
|
|
return cpath
|
|
|
|
# _get_ipath : get the instance path for a name in pathmap
|
|
# (/var/lib/cloud/instances/<instance>/<name>)
|
|
def _get_ipath(self, name=None):
|
|
if not self.datasource:
|
|
return None
|
|
iid = self.datasource.get_instance_id()
|
|
if iid is None:
|
|
return None
|
|
ipath = os.path.join(self.cloud_dir, 'instances', str(iid))
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
ipath = os.path.join(ipath, add_on)
|
|
return ipath
|
|
|
|
# get_ipath : get the instance path for a name in pathmap
|
|
# (/var/lib/cloud/instances/<instance>/<name>)
|
|
# returns None + warns if no active datasource....
|
|
def get_ipath(self, name=None):
|
|
ipath = self._get_ipath(name)
|
|
if not ipath:
|
|
LOG.warn(("No per instance data available, "
|
|
"is there an datasource/iid set?"))
|
|
return None
|
|
else:
|
|
return ipath
|
|
|
|
|
|
# This config parser will not throw when sections don't exist
|
|
# and you are setting values on those sections which is useful
|
|
# when writing to new options that may not have corresponding
|
|
# sections. Also it can default other values when doing gets
|
|
# so that if those sections/options do not exist you will
|
|
# get a default instead of an error. Another useful case where
|
|
# you can avoid catching exceptions that you typically don't
|
|
# care about...
|
|
|
|
class DefaultingConfigParser(RawConfigParser):
|
|
DEF_INT = 0
|
|
DEF_FLOAT = 0.0
|
|
DEF_BOOLEAN = False
|
|
DEF_BASE = None
|
|
|
|
def get(self, section, option):
|
|
value = self.DEF_BASE
|
|
try:
|
|
value = RawConfigParser.get(self, section, option)
|
|
except NoSectionError:
|
|
pass
|
|
except NoOptionError:
|
|
pass
|
|
return value
|
|
|
|
def set(self, section, option, value=None):
|
|
if not self.has_section(section) and section.lower() != 'default':
|
|
self.add_section(section)
|
|
RawConfigParser.set(self, section, option, value)
|
|
|
|
def remove_option(self, section, option):
|
|
if self.has_option(section, option):
|
|
RawConfigParser.remove_option(self, section, option)
|
|
|
|
def getboolean(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_BOOLEAN
|
|
return RawConfigParser.getboolean(self, section, option)
|
|
|
|
def getfloat(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_FLOAT
|
|
return RawConfigParser.getfloat(self, section, option)
|
|
|
|
def getint(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_INT
|
|
return RawConfigParser.getint(self, section, option)
|
|
|
|
def stringify(self, header=None):
|
|
contents = ''
|
|
with io.BytesIO() as outputstream:
|
|
self.write(outputstream)
|
|
outputstream.flush()
|
|
contents = outputstream.getvalue()
|
|
if header:
|
|
contents = "\n".join([header, contents])
|
|
return contents
|