Inventory filesystem refactor load funcs moved
The last uses of the builtin os module are removed from generate as the load_environment and load_user_configuration functions are moved, along with the _extra_config method they rely upon. This also moves the last use of the yaml module. These functions are not refactored to make use of the other funcs available in the filesystem package, and as a result there is still an undesirable use of the dir_find func in the generate script. These will all be cleaned up in a following patch to keep this easier to review. An unused left-over function is removed now that there are no merge conflicts introduced by it. A debug logging message is formatted consistent to other messages; the only impact in this case is applying consistency in log message formatting since the formatting was only occuring when the log level is activated already. Parent-Id: I4c7b45f16f8ffecb91bf509daeca47b2c2450681 Change-Id: I577cdbf4aadfcce846412edd7e2a394c257c0243
This commit is contained in:
Steve Lewis
Steve Lewis (stevelle)
parent
d82d5457b6
commit
ee33b159c1
|
|
@ -22,6 +22,9 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import tarfile
|
||||
import yaml
|
||||
|
||||
import dictutils as du
|
||||
|
||||
logger = logging.getLogger('osa-inventory')
|
||||
|
||||
|
|
@ -105,6 +108,23 @@ def dir_find(preferred_path=None, suffix=None, raise_if_missing=True):
|
|||
return False
|
||||
|
||||
|
||||
def _extra_config(user_defined_config, base_dir):
|
||||
"""Discover new items in any extra directories and add the new values.
|
||||
|
||||
:param user_defined_config: ``dict``
|
||||
:param base_dir: ``str``
|
||||
"""
|
||||
for root_dir, _, files in os.walk(base_dir):
|
||||
for name in files:
|
||||
if name.endswith(('.yml', '.yaml')):
|
||||
with open(os.path.join(root_dir, name), 'rb') as f:
|
||||
du.merge_dict(
|
||||
user_defined_config,
|
||||
yaml.safe_load(f.read()) or {}
|
||||
)
|
||||
logger.debug("Merged overrides from file {}".format(name))
|
||||
|
||||
|
||||
def _make_backup(backup_path, source_file_path):
|
||||
""" Create a backup of all previous inventory files as a tar archive
|
||||
|
||||
|
|
@ -217,3 +237,49 @@ def save_inventory(inventory_json, save_path):
|
|||
with open(inventory_file, 'wb') as f:
|
||||
f.write(inventory_json)
|
||||
logger.info("Inventory written")
|
||||
|
||||
|
||||
def load_environment(config_path, environment):
|
||||
"""Create an environment dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the environment files are kept
|
||||
:param environment: ``dict`` dictionary to populate with environment data
|
||||
"""
|
||||
|
||||
# Load all YAML files found in the env.d directory
|
||||
env_plugins = dir_find(config_path, 'env.d', raise_if_missing=False)
|
||||
|
||||
if env_plugins is not False:
|
||||
_extra_config(user_defined_config=environment, base_dir=env_plugins)
|
||||
logger.debug("Loaded environment from {}".format(config_path))
|
||||
return environment
|
||||
|
||||
|
||||
def load_user_configuration(config_path):
|
||||
"""Create a user configuration dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the configuration files are kept
|
||||
"""
|
||||
|
||||
user_defined_config = dict()
|
||||
|
||||
# Load the user defined configuration file
|
||||
user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
|
||||
if os.path.isfile(user_config_file):
|
||||
with open(user_config_file, 'rb') as f:
|
||||
user_defined_config.update(yaml.safe_load(f.read()) or {})
|
||||
|
||||
# Load anything in a conf.d directory if found
|
||||
base_dir = dir_find(config_path, 'conf.d', raise_if_missing=False)
|
||||
if base_dir is not False:
|
||||
_extra_config(user_defined_config, base_dir)
|
||||
|
||||
# Exit if no user_config was found and loaded
|
||||
if not user_defined_config:
|
||||
raise SystemExit(
|
||||
'No user config loaded\n'
|
||||
'No openstack_user_config files are available in either \n{}'
|
||||
'\nor \n{}/conf.d directory'.format(config_path, config_path)
|
||||
)
|
||||
logger.debug("User configuration loaded from: {}".format(user_config_file))
|
||||
return user_defined_config
|
||||
|
|
|
|||
110
lib/generate.py
110
lib/generate.py
|
|
@ -19,10 +19,8 @@ import ip
|
|||
import json
|
||||
import logging
|
||||
import netaddr
|
||||
import os
|
||||
import uuid
|
||||
import warnings
|
||||
import yaml
|
||||
|
||||
import dictutils as du
|
||||
import filesystem as filesys
|
||||
|
|
@ -730,34 +728,6 @@ def container_skel_load(container_skel, inventory, config):
|
|||
)
|
||||
|
||||
|
||||
def find_config_path(user_config_path=None):
|
||||
# TODO(stevelle) REMOVE THIS FUNCTION: after other changes in chain
|
||||
# prevents the clean removal right now
|
||||
"""Return the path to the user configuration files.
|
||||
|
||||
If no directory is found the system will exit.
|
||||
|
||||
The lookup will be done in the following directories:
|
||||
|
||||
* user_config_path
|
||||
* ``/etc/openstack_deploy/``
|
||||
|
||||
:param user_config_path: ``str`` Location to look in FIRST for a file
|
||||
"""
|
||||
path_check = [
|
||||
os.path.join('/etc', 'openstack_deploy'),
|
||||
]
|
||||
|
||||
if user_config_path is not None:
|
||||
path_check.insert(0, os.path.expanduser(user_config_path))
|
||||
|
||||
for f in path_check:
|
||||
if os.path.isdir(f):
|
||||
return f
|
||||
else:
|
||||
raise SystemExit('No config found at: {}'.format(path_check))
|
||||
|
||||
|
||||
def _ensure_inventory_uptodate(inventory, container_skel):
|
||||
"""Update inventory if needed.
|
||||
|
||||
|
|
@ -823,23 +793,6 @@ def _parse_global_variables(user_cidr, inventory, user_defined_config):
|
|||
del inventory['all']['vars'][key]
|
||||
|
||||
|
||||
def _extra_config(user_defined_config, base_dir):
|
||||
"""Discover new items in any extra directories and add the new values.
|
||||
|
||||
:param user_defined_config: ``dict``
|
||||
:param base_dir: ``str``
|
||||
"""
|
||||
for root_dir, _, files in os.walk(base_dir):
|
||||
for name in files:
|
||||
if name.endswith(('.yml', '.yaml')):
|
||||
with open(os.path.join(root_dir, name), 'rb') as f:
|
||||
du.merge_dict(
|
||||
user_defined_config,
|
||||
yaml.safe_load(f.read()) or {}
|
||||
)
|
||||
logger.debug("Merged overrides from file %s", name)
|
||||
|
||||
|
||||
def _check_same_ip_to_multiple_host(config):
|
||||
"""Check for IPs assigned to multiple hosts
|
||||
|
||||
|
|
@ -988,51 +941,11 @@ def _collect_hostnames(dynamic_inventory):
|
|||
return hostnames_ips
|
||||
|
||||
|
||||
def load_environment(config_path, environment):
|
||||
"""Create an environment dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the environment files are kept
|
||||
:param environment: ``dict`` dictionary to populate with environment data
|
||||
"""
|
||||
|
||||
# Load all YAML files found in the env.d directory
|
||||
env_plugins = filesys.dir_find(config_path, 'env.d',
|
||||
raise_if_missing=False)
|
||||
|
||||
if env_plugins is not False:
|
||||
_extra_config(user_defined_config=environment, base_dir=env_plugins)
|
||||
logger.debug("Loaded environment from %s", config_path)
|
||||
return environment
|
||||
|
||||
|
||||
def load_user_configuration(config_path):
|
||||
"""Create a user configuration dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the configuration files are kept
|
||||
"""
|
||||
|
||||
user_defined_config = dict()
|
||||
|
||||
# Load the user defined configuration file
|
||||
user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
|
||||
if os.path.isfile(user_config_file):
|
||||
with open(user_config_file, 'rb') as f:
|
||||
user_defined_config.update(yaml.safe_load(f.read()) or {})
|
||||
|
||||
# Load anything in a conf.d directory if found
|
||||
base_dir = filesys.dir_find(config_path, 'conf.d', raise_if_missing=False)
|
||||
if base_dir is not False:
|
||||
_extra_config(user_defined_config, base_dir)
|
||||
|
||||
# Exit if no user_config was found and loaded
|
||||
if not user_defined_config:
|
||||
raise SystemExit(
|
||||
'No user config loaded\n'
|
||||
'No openstack_user_config files are available in either \n{}'
|
||||
'\nor \n{}/conf.d directory'.format(config_path, config_path)
|
||||
)
|
||||
logger.debug("User configuration loaded from: %s", user_config_file)
|
||||
return user_defined_config
|
||||
def _prepare_debug_logger():
|
||||
log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
|
||||
logging.basicConfig(format=log_fmt, filename='inventory.log')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Beginning new inventory run")
|
||||
|
||||
|
||||
def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
||||
|
|
@ -1047,18 +960,15 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
|||
:param environment: ``str`` Directory containing the base env.d
|
||||
"""
|
||||
if debug:
|
||||
log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
|
||||
logging.basicConfig(format=log_fmt, filename='inventory.log')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Beginning new inventory run")
|
||||
_prepare_debug_logger()
|
||||
|
||||
# Get the path to the user configuration files
|
||||
config_path = filesys.dir_find(preferred_path=config)
|
||||
|
||||
user_defined_config = load_user_configuration(config_path)
|
||||
user_defined_config = filesys.load_user_configuration(config_path)
|
||||
base_env_dir = environment
|
||||
base_env = load_environment(base_env_dir, {})
|
||||
environment = load_environment(config_path, base_env)
|
||||
base_env = filesys.load_environment(base_env_dir, {})
|
||||
environment = filesys.load_environment(config_path, base_env)
|
||||
|
||||
# Load existing inventory file if found
|
||||
dynamic_inventory = filesys.load_inventory(config_path, INVENTORY_SKEL)
|
||||
|
|
@ -1129,7 +1039,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
|||
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
num_hosts = len(dynamic_inventory['_meta']['hostvars'])
|
||||
logger.debug("%d hosts found." % num_hosts)
|
||||
logger.debug("%d hosts found.", num_hosts)
|
||||
|
||||
# Save new dynamic inventory
|
||||
filesys.save_inventory(dynamic_inventory_json, config_path)
|
||||
|
|
|
|||
|
|
@ -426,7 +426,7 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
|
|||
class TestUserConfiguration(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.longMessage = True
|
||||
self.loaded_user_configuration = di.load_user_configuration(TARGET_DIR)
|
||||
self.loaded_user_configuration = fs.load_user_configuration(TARGET_DIR)
|
||||
|
||||
def test_loading_user_configuration(self):
|
||||
"""Test that the user configuration can be loaded"""
|
||||
|
|
@ -436,7 +436,7 @@ class TestUserConfiguration(unittest.TestCase):
|
|||
class TestEnvironments(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.longMessage = True
|
||||
self.loaded_environment = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.loaded_environment = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
def test_loading_environment(self):
|
||||
"""Test that the environment can be loaded"""
|
||||
|
|
@ -458,14 +458,14 @@ class TestIps(unittest.TestCase):
|
|||
# Allow custom assertion errors.
|
||||
self.longMessage = True
|
||||
|
||||
@mock.patch('generate.load_environment')
|
||||
@mock.patch('generate.load_user_configuration')
|
||||
@mock.patch('filesystem.load_environment')
|
||||
@mock.patch('filesystem.load_user_configuration')
|
||||
def test_duplicates(self, mock_load_config, mock_load_env):
|
||||
"""Test that no duplicate IPs are made on any network."""
|
||||
|
||||
# Grab our values read from the file system just once.
|
||||
mock_load_config.return_value = get_config()
|
||||
mock_load_env.return_value = di.load_environment(BASE_ENV_DIR, {})
|
||||
mock_load_env.return_value = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
|
||||
|
|
@ -843,7 +843,7 @@ class TestGlobalOverridesConfigDeletion(TestConfigCheckBase):
|
|||
|
||||
class TestEnsureInventoryUptoDate(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
# Copy because we manipulate the structure in each test;
|
||||
# not copying would modify the global var in the target code
|
||||
self.inv = copy.deepcopy(di.INVENTORY_SKEL)
|
||||
|
|
@ -900,7 +900,7 @@ class TestEnsureInventoryUptoDate(unittest.TestCase):
|
|||
|
||||
class OverridingEnvBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.base_env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.base_env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
# Use the cinder configuration as our sample for override testing
|
||||
with open(path.join(BASE_ENV_DIR, 'env.d', 'cinder.yml'), 'r') as f:
|
||||
|
|
@ -926,7 +926,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
|||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
self.assertFalse(test_vol['properties']['is_metal'])
|
||||
|
|
@ -942,7 +942,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
|||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
|
|
@ -954,7 +954,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
|||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
|
|
@ -966,7 +966,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
|||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
|
|
@ -978,7 +978,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
|||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
|
|
@ -994,7 +994,7 @@ class TestOverridingEnvIntegration(OverridingEnvBase):
|
|||
self.inv = fs.load_inventory(TARGET_DIR, di.INVENTORY_SKEL)
|
||||
|
||||
def skel_setup(self):
|
||||
self.environment = di.load_environment(TARGET_DIR, self.base_env)
|
||||
self.environment = fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
di.skel_setup(self.environment, self.inv)
|
||||
|
||||
|
|
@ -1173,7 +1173,7 @@ class TestLxcHosts(TestConfigCheckBase):
|
|||
|
||||
class TestConfigMatchesEnvironment(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
def test_matching_keys(self):
|
||||
config = get_config()
|
||||
|
|
|
|||
Loading…
Reference in New Issue