563 lines
22 KiB
Python
563 lines
22 KiB
Python
# Copyright 2015 Tesora Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import abc
|
|
import os
|
|
import re
|
|
|
|
from oslo_log import log as logging
|
|
|
|
from trove.guestagent.common import guestagent_utils
|
|
from trove.guestagent.common import operating_system
|
|
from trove.guestagent.common.operating_system import FileMode
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class ConfigurationManager(object):
|
|
"""
|
|
ConfigurationManager is responsible for management of
|
|
datastore configuration.
|
|
Its base functionality includes reading and writing configuration files.
|
|
It is responsible for validating user inputs and requests.
|
|
When supplied an override strategy it allows the user to manage
|
|
configuration overrides as well.
|
|
"""
|
|
|
|
# Configuration group names. The names determine the order in which the
|
|
# groups get applied. System groups are divided into two camps; pre-user
|
|
# and post-user. In general system overrides will get applied over the
|
|
# user group, unless specified otherwise (i.e. SYSTEM_POST_USER_GROUP
|
|
# will be used).
|
|
SYSTEM_PRE_USER_GROUP = '10-system'
|
|
USER_GROUP = '20-user'
|
|
SYSTEM_POST_USER_GROUP = '50-system'
|
|
|
|
DEFAULT_STRATEGY_OVERRIDES_SUB_DIR = 'overrides'
|
|
DEFAULT_CHANGE_ID = 'common'
|
|
|
|
def __init__(self, base_config_path, owner, group, codec,
|
|
requires_root=False, override_strategy=None):
|
|
"""
|
|
:param base_config_path Path to the configuration file.
|
|
:type base_config_path string
|
|
|
|
:param owner Owner of the configuration files.
|
|
:type owner string
|
|
|
|
:param group Group of the configuration files.
|
|
:type group string
|
|
|
|
:param codec Codec for reading/writing of the particular
|
|
configuration format.
|
|
:type codec StreamCodec
|
|
|
|
:param requires_root Whether the manager requires superuser
|
|
privileges.
|
|
:type requires_root boolean
|
|
|
|
:param override_strategy Strategy used to manage configuration
|
|
overrides (e.g. ImportOverrideStrategy).
|
|
Defaults to OneFileOverrideStrategy
|
|
if None. This strategy should be
|
|
compatible with very much any datastore.
|
|
It is recommended each datastore defines
|
|
its strategy explicitly to avoid upgrade
|
|
compatibility issues in case the default
|
|
implementation changes in the future.
|
|
:type override_strategy ConfigurationOverrideStrategy
|
|
"""
|
|
base_config_dir = os.path.dirname(base_config_path)
|
|
operating_system.ensure_directory(
|
|
base_config_dir, user=owner, group=group, force=True, as_root=True
|
|
)
|
|
|
|
self._base_config_path = base_config_path
|
|
self._owner = owner
|
|
self._group = group
|
|
self._codec = codec
|
|
self._requires_root = requires_root
|
|
self._value_cache = None
|
|
|
|
if not override_strategy:
|
|
# Use OneFile strategy by default. Store the revisions in a
|
|
# sub-directory at the location of the configuration file.
|
|
revision_dir = guestagent_utils.build_file_path(
|
|
os.path.dirname(base_config_path),
|
|
self.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
|
|
self._override_strategy = OneFileOverrideStrategy(revision_dir)
|
|
else:
|
|
self._override_strategy = override_strategy
|
|
|
|
self._override_strategy.configure(
|
|
base_config_path, owner, group, codec, requires_root)
|
|
|
|
def get_value(self, key, section=None, default=None):
|
|
"""Return the current value at a given key or 'default'.
|
|
"""
|
|
if self._value_cache is None:
|
|
self.refresh_cache()
|
|
|
|
if section:
|
|
return self._value_cache.get(section, {}).get(key, default)
|
|
|
|
return self._value_cache.get(key, default)
|
|
|
|
def parse_configuration(self):
|
|
"""Read contents of the configuration file (applying overrides if any)
|
|
and parse it into a dict.
|
|
|
|
:returns: Configuration file as a Python dict.
|
|
"""
|
|
|
|
try:
|
|
base_options = operating_system.read_file(
|
|
self._base_config_path, codec=self._codec,
|
|
as_root=self._requires_root)
|
|
except Exception:
|
|
LOG.warning('File %s not found', self._base_config_path)
|
|
return None
|
|
|
|
updates = self._override_strategy.parse_updates()
|
|
guestagent_utils.update_dict(updates, base_options)
|
|
|
|
return base_options
|
|
|
|
def reset_configuration(self, options, remove_overrides=False):
|
|
"""Write given contents to the base configuration file.
|
|
|
|
Remove all existing overrides (both system and user) as required.
|
|
|
|
:param options: Contents of the configuration file (string or dict).
|
|
:param remove_overrides: Remove the overrides or not.
|
|
"""
|
|
if isinstance(options, dict):
|
|
# Serialize a dict of options for writing.
|
|
self.reset_configuration(self._codec.serialize(options),
|
|
remove_overrides=remove_overrides)
|
|
else:
|
|
if remove_overrides:
|
|
self._override_strategy.remove(self.USER_GROUP)
|
|
self._override_strategy.remove(self.SYSTEM_PRE_USER_GROUP)
|
|
self._override_strategy.remove(self.SYSTEM_POST_USER_GROUP)
|
|
|
|
operating_system.write_file(
|
|
self._base_config_path, options, as_root=self._requires_root)
|
|
operating_system.chown(
|
|
self._base_config_path, self._owner, self._group,
|
|
as_root=self._requires_root)
|
|
operating_system.chmod(
|
|
self._base_config_path, FileMode.ADD_READ_ALL,
|
|
as_root=self._requires_root)
|
|
|
|
self.refresh_cache()
|
|
|
|
def has_system_override(self, change_id):
|
|
"""Return whether a given 'system' change exists.
|
|
"""
|
|
return (self._override_strategy.exists(self.SYSTEM_POST_USER_GROUP,
|
|
change_id) or
|
|
self._override_strategy.exists(self.SYSTEM_PRE_USER_GROUP,
|
|
change_id))
|
|
|
|
def apply_system_override(self, options, change_id=DEFAULT_CHANGE_ID,
|
|
pre_user=False):
|
|
"""Apply a 'system' change to the configuration.
|
|
|
|
System overrides are always applied after all user changes so that
|
|
they override any user-defined setting.
|
|
|
|
:param options Configuration changes.
|
|
:type options string or dict
|
|
"""
|
|
group_name = (
|
|
self.SYSTEM_PRE_USER_GROUP if pre_user else
|
|
self.SYSTEM_POST_USER_GROUP)
|
|
self._apply_override(group_name, change_id, options)
|
|
|
|
def apply_user_override(self, options, change_id=DEFAULT_CHANGE_ID):
|
|
"""Apply a 'user' change to the configuration.
|
|
|
|
The 'system' values will be re-applied over this override.
|
|
|
|
:param options Configuration changes.
|
|
:type options string or dict
|
|
"""
|
|
self._apply_override(self.USER_GROUP, change_id, options)
|
|
|
|
def get_user_override(self, change_id=DEFAULT_CHANGE_ID):
|
|
"""Get the user overrides"""
|
|
return self._override_strategy.get(self.USER_GROUP, change_id)
|
|
|
|
def _apply_override(self, group_name, change_id, options):
|
|
if not isinstance(options, dict):
|
|
# Deserialize the options into a dict if not already.
|
|
self._apply_override(
|
|
group_name, change_id, self._codec.deserialize(options))
|
|
else:
|
|
self._override_strategy.apply(group_name, change_id, options)
|
|
self.refresh_cache()
|
|
|
|
def remove_system_override(self, change_id=DEFAULT_CHANGE_ID):
|
|
"""Revert a 'system' configuration change.
|
|
"""
|
|
self._remove_override(self.SYSTEM_POST_USER_GROUP, change_id)
|
|
self._remove_override(self.SYSTEM_PRE_USER_GROUP, change_id)
|
|
|
|
def remove_user_override(self, change_id=DEFAULT_CHANGE_ID):
|
|
"""Revert a 'user' configuration change.
|
|
"""
|
|
self._remove_override(self.USER_GROUP, change_id)
|
|
|
|
def _remove_override(self, group_name, change_id):
|
|
self._override_strategy.remove(group_name, change_id)
|
|
self.refresh_cache()
|
|
|
|
def refresh_cache(self):
|
|
self._value_cache = self.parse_configuration()
|
|
|
|
|
|
class ConfigurationOverrideStrategy(object, metaclass=abc.ABCMeta):
|
|
"""ConfigurationOverrideStrategy handles configuration files.
|
|
The strategy provides functionality to enumerate, apply and remove
|
|
configuration overrides.
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def configure(self, *args, **kwargs):
|
|
"""Configure this strategy.
|
|
A strategy needs to be configured before it can be used.
|
|
It would typically be configured by the ConfigurationManager.
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def exists(self, group_name, change_id):
|
|
"""Return whether a given revision exists.
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def apply(self, group_name, change_id, options):
|
|
"""Apply given options on the most current configuration revision.
|
|
Update if a file with the same id already exists.
|
|
|
|
:param group_name The group the override belongs to.
|
|
:type group_name string
|
|
|
|
:param change_id The name of the override within the group.
|
|
:type change_id string
|
|
|
|
:param options Configuration changes.
|
|
:type options dict
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def remove(self, group_name, change_id=None):
|
|
"""Rollback a given configuration override.
|
|
Remove the whole group if 'change_id' is None.
|
|
|
|
:param group_name The group the override belongs to.
|
|
:type group_name string
|
|
|
|
:param change_id The name of the override within the group.
|
|
:type change_id string
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def get(self, group_name, change_id=None):
|
|
"""Return the contents of a given configuration override
|
|
|
|
:param group_name The group the override belongs to.
|
|
:type group_name string
|
|
|
|
:param change_id The name of the override within the group.
|
|
:type change_id string
|
|
"""
|
|
|
|
def parse_updates(self):
|
|
"""Return all updates applied to the base revision as a single dict.
|
|
Return an empty dict if the base file is always the most current
|
|
version of configuration.
|
|
|
|
:returns: Updates to the base revision as a Python dict.
|
|
"""
|
|
return {}
|
|
|
|
|
|
class ImportOverrideStrategy(ConfigurationOverrideStrategy):
|
|
"""Import strategy keeps overrides in separate files that get imported
|
|
into the base configuration file which never changes itself.
|
|
An override file is simply deleted when the override is removed.
|
|
|
|
We keep two sets of override files in a separate directory.
|
|
- User overrides - configuration overrides applied by the user via the
|
|
Trove API.
|
|
- System overrides - 'internal' configuration changes applied by the
|
|
guestagent.
|
|
|
|
The name format of override files is: '<set prefix>-<n>-<group name>.<ext>'
|
|
where 'set prefix' is to used to order user/system sets,
|
|
'n' is an index used to keep track of the order in which overrides
|
|
within their set got applied.
|
|
"""
|
|
|
|
FILE_NAME_PATTERN = r'%s-([0-9]+)-%s\.%s$'
|
|
|
|
def __init__(self, revision_dir, revision_ext):
|
|
"""
|
|
:param revision_dir Path to the directory for import files.
|
|
:type revision_dir string
|
|
|
|
:param revision_ext Extension of revision files.
|
|
:type revision_ext string
|
|
"""
|
|
self._revision_dir = revision_dir
|
|
self._revision_ext = revision_ext
|
|
|
|
def configure(self, base_config_path, owner, group, codec, requires_root):
|
|
"""
|
|
:param base_config_path Path to the configuration file.
|
|
:type base_config_path string
|
|
|
|
:param owner Owner of the configuration and
|
|
revision files.
|
|
:type owner string
|
|
|
|
:param group Group of the configuration and
|
|
revision files.
|
|
:type group string
|
|
|
|
:param codec Codec for reading/writing of the particular
|
|
configuration format.
|
|
:type codec StreamCodec
|
|
|
|
:param requires_root Whether the strategy requires superuser
|
|
privileges.
|
|
:type requires_root boolean
|
|
"""
|
|
self._base_config_path = base_config_path
|
|
self._owner = owner
|
|
self._group = group
|
|
self._codec = codec
|
|
self._requires_root = requires_root
|
|
|
|
self._initialize_import_directory()
|
|
|
|
def exists(self, group_name, change_id):
|
|
return self._find_revision_file(group_name, change_id) is not None
|
|
|
|
def apply(self, group_name, change_id, options):
|
|
self._initialize_import_directory()
|
|
revision_file = self._find_revision_file(group_name, change_id)
|
|
if revision_file is None:
|
|
# Create a new file.
|
|
last_revision_index = self._get_last_file_index(group_name)
|
|
revision_file = guestagent_utils.build_file_path(
|
|
self._revision_dir,
|
|
'%s-%03d-%s' % (group_name, last_revision_index + 1,
|
|
change_id),
|
|
self._revision_ext)
|
|
else:
|
|
# Update the existing file.
|
|
current = operating_system.read_file(
|
|
revision_file, codec=self._codec, as_root=self._requires_root)
|
|
options = guestagent_utils.update_dict(options, current)
|
|
|
|
operating_system.write_file(
|
|
revision_file, options, codec=self._codec,
|
|
as_root=self._requires_root)
|
|
operating_system.chown(
|
|
revision_file, self._owner, self._group,
|
|
as_root=self._requires_root)
|
|
operating_system.chmod(
|
|
revision_file, FileMode.ADD_READ_ALL, as_root=self._requires_root)
|
|
|
|
def _initialize_import_directory(self):
|
|
"""Lazy-initialize the directory for imported revision files.
|
|
"""
|
|
if not os.path.exists(self._revision_dir):
|
|
operating_system.ensure_directory(
|
|
self._revision_dir, user=self._owner, group=self._group,
|
|
force=True, as_root=self._requires_root)
|
|
|
|
def remove(self, group_name, change_id=None):
|
|
removed = set()
|
|
if change_id:
|
|
# Remove a given file.
|
|
revision_file = self._find_revision_file(group_name, change_id)
|
|
if revision_file:
|
|
removed.add(revision_file)
|
|
else:
|
|
# Remove the entire group.
|
|
removed = self._collect_revision_files(group_name)
|
|
|
|
for path in removed:
|
|
operating_system.remove(path, force=True,
|
|
as_root=self._requires_root)
|
|
|
|
def get(self, group_name, change_id):
|
|
revision_file = self._find_revision_file(group_name, change_id)
|
|
|
|
return operating_system.read_file(revision_file,
|
|
codec=self._codec,
|
|
as_root=self._requires_root)
|
|
|
|
def parse_updates(self):
|
|
parsed_options = {}
|
|
for path in self._collect_revision_files():
|
|
options = operating_system.read_file(path, codec=self._codec,
|
|
as_root=self._requires_root)
|
|
guestagent_utils.update_dict(options, parsed_options)
|
|
|
|
LOG.debug(f"Parsed overrides options: {parsed_options}")
|
|
return parsed_options
|
|
|
|
@property
|
|
def has_revisions(self):
|
|
"""Return True if there currently are any revision files.
|
|
"""
|
|
return (operating_system.exists(
|
|
self._revision_dir, is_directory=True,
|
|
as_root=self._requires_root) and
|
|
(len(self._collect_revision_files()) > 0))
|
|
|
|
def _get_last_file_index(self, group_name):
|
|
"""Get the index of the most current file in a given group.
|
|
"""
|
|
current_files = self._collect_revision_files(group_name)
|
|
if current_files:
|
|
name_pattern = self._build_rev_name_pattern(group_name=group_name)
|
|
last_file_name = os.path.basename(current_files[-1])
|
|
last_index_match = re.match(name_pattern, last_file_name)
|
|
if last_index_match:
|
|
return int(last_index_match.group(1))
|
|
|
|
return 0
|
|
|
|
def _collect_revision_files(self, group_name='.+'):
|
|
"""Collect and return a sorted list of paths to existing revision
|
|
files. The files should be sorted in the same order in which
|
|
they were applied.
|
|
"""
|
|
name_pattern = self._build_rev_name_pattern(group_name=group_name)
|
|
return sorted(operating_system.list_files_in_directory(
|
|
self._revision_dir, recursive=True, pattern=name_pattern,
|
|
as_root=self._requires_root))
|
|
|
|
def _find_revision_file(self, group_name, change_id):
|
|
name_pattern = self._build_rev_name_pattern(group_name, change_id)
|
|
found = operating_system.list_files_in_directory(
|
|
self._revision_dir, recursive=True, pattern=name_pattern,
|
|
as_root=self._requires_root)
|
|
return next(iter(found), None)
|
|
|
|
def _build_rev_name_pattern(self, group_name='.+', change_id='.+'):
|
|
return self.FILE_NAME_PATTERN % (group_name, change_id,
|
|
self._revision_ext)
|
|
|
|
|
|
class OneFileOverrideStrategy(ConfigurationOverrideStrategy):
|
|
"""This is a strategy for datastores that do not support multiple
|
|
configuration files.
|
|
It uses the Import Strategy to keep the overrides internally.
|
|
When an override is applied or removed a new configuration file is
|
|
generated by applying all changes on a saved-off base revision.
|
|
"""
|
|
|
|
BASE_REVISION_NAME = 'base'
|
|
REVISION_EXT = 'rev'
|
|
|
|
def __init__(self, revision_dir):
|
|
"""
|
|
:param revision_dir Path to the directory for import files.
|
|
:type revision_dir string
|
|
"""
|
|
self._revision_dir = revision_dir
|
|
self._import_strategy = ImportOverrideStrategy(revision_dir,
|
|
self.REVISION_EXT)
|
|
|
|
def configure(self, base_config_path, owner, group, codec, requires_root):
|
|
"""
|
|
:param base_config_path Path to the configuration file.
|
|
:type base_config_path string
|
|
|
|
:param owner Owner of the configuration and
|
|
revision files.
|
|
:type owner string
|
|
|
|
:param group Group of the configuration and
|
|
revision files.
|
|
:type group string
|
|
|
|
:param codec Codec for reading/writing of the particular
|
|
configuration format.
|
|
:type codec StreamCodec
|
|
|
|
:param requires_root Whether the strategy requires superuser
|
|
privileges.
|
|
:type requires_root boolean
|
|
"""
|
|
self._base_config_path = base_config_path
|
|
self._owner = owner
|
|
self._group = group
|
|
self._codec = codec
|
|
self._requires_root = requires_root
|
|
self._base_revision_file = guestagent_utils.build_file_path(
|
|
self._revision_dir, self.BASE_REVISION_NAME, self.REVISION_EXT)
|
|
|
|
self._import_strategy.configure(
|
|
base_config_path, owner, group, codec, requires_root)
|
|
|
|
def exists(self, group_name, change_id):
|
|
return self._import_strategy.exists(group_name, change_id)
|
|
|
|
def apply(self, group_name, change_id, options):
|
|
self._import_strategy.apply(group_name, change_id, options)
|
|
self._regenerate_base_configuration()
|
|
|
|
def remove(self, group_name, change_id=None):
|
|
if self._import_strategy.has_revisions:
|
|
self._import_strategy.remove(group_name, change_id=change_id)
|
|
self._regenerate_base_configuration()
|
|
if not self._import_strategy.has_revisions:
|
|
# The base revision file is no longer needed if there are no
|
|
# overrides. It will be regenerated based on the current
|
|
# configuration file on the first 'apply()'.
|
|
operating_system.remove(self._base_revision_file, force=True,
|
|
as_root=self._requires_root)
|
|
|
|
def get(self, group_name, change_id):
|
|
return self._import_strategy.get(group_name, change_id)
|
|
|
|
def _regenerate_base_configuration(self):
|
|
"""Gather all configuration changes and apply them in order on the base
|
|
revision. Write the results to the configuration file.
|
|
"""
|
|
|
|
if not os.path.exists(self._base_revision_file):
|
|
# Initialize the file with the current configuration contents if it
|
|
# does not exist.
|
|
operating_system.copy(
|
|
self._base_config_path, self._base_revision_file,
|
|
force=True, preserve=True, as_root=self._requires_root)
|
|
|
|
base_revision = operating_system.read_file(
|
|
self._base_revision_file, codec=self._codec,
|
|
as_root=self._requires_root)
|
|
changes = self._import_strategy.parse_updates()
|
|
updated_revision = guestagent_utils.update_dict(changes, base_revision)
|
|
operating_system.write_file(
|
|
self._base_config_path, updated_revision, codec=self._codec,
|
|
as_root=self._requires_root)
|