diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py index 7d10772e22..a2d875a47a 100644 --- a/docker/base/set_configs.py +++ b/docker/base/set_configs.py @@ -55,25 +55,169 @@ class UserNotFound(ExitingException): pass -class DifferentPermissions(ExitingException): +class ConfigFileBadState(ExitingException): pass -class DifferentUser(ExitingException): - pass +class ConfigFile(object): + def __init__(self, source, dest, owner, perm, optional=False): + self.source = source + self.dest = dest + self.owner = owner + self.perm = perm + self.optional = optional -class RequiredFileNotFound(ExitingException): - pass + def __str__(self): + return ''.format(self.source, + self.dest) + def _copy_dir(self, source, dest): + LOG.info('Copying dir from %s to %s', source, dest) + shutil.copytree(source, dest) + for root, dirs, files in os.walk(dest): + for dir_ in dirs: + self._set_permission(os.path.join(root, dir_)) + for file_ in files: + self._set_permission(os.path.join(root, file_)) -class ContentDifferent(ExitingException): - pass + def _delete_path(self, path): + if not os.path.exists(path): + return + if os.path.isdir(path): + LOG.info('Deleting dir %s', path) + shutil.rmtree(path) + else: + LOG.info('Deleting file %s', path) + os.remove(path) + + def _create_parent_dirs(self, path): + parent_path = os.path.dirname(path) + if not os.path.exists(parent_path): + os.makedirs(parent_path) + + def _copy_file(self, source, dest): + # dest endswith / means copy the to folder + LOG.info('Coping file from %s to %s', source, dest) + shutil.copy(source, dest) + self._set_permission(dest) + + def _set_permission(self, path): + user = pwd.getpwnam(self.owner) + uid, gid = user.pw_uid, user.pw_gid + LOG.info('Setting file %s owner to %s:%s', path, + self.owner, self.owner) + os.chown(path, uid, gid) + # NOTE(Jeffrey4l): py3 need '0oXXX' format for octal literals, and py2 + # support such format too. + perm = self.perm + if len(perm) == 4 and perm[1] != 'o': + perm = ''.join([perm[:1], 'o', perm[1:]]) + perm = int(perm, base=0) + LOG.info('Setting file %s permission to %s', path, self.perm) + os.chmod(path, perm) + + def copy(self): + + sources = glob.glob(self.source) + + if not self.optional and not sources: + raise MissingRequiredSource('%s file is not found' % self.source) + # skip when there is no sources and optional + elif self.optional and not sources: + return + + for source in sources: + dest = self.dest + # dest endswith / means copy the into folder, + # otherwise means copy the source to dest + if dest.endswith(os.sep): + dest = os.path.join(dest, os.path.basename(source)) + self._delete_path(dest) + self._create_parent_dirs(dest) + if os.path.isdir(source): + self._copy_dir(source, dest) + else: + self._copy_file(source, dest) + + def _cmp_file(self, source, dest): + # check exsit + if (os.path.exists(source) and + not self.optional and + not os.path.exists(dest)): + return False + # check content + with open(source) as f1, open(dest) as f2: + if f1.read() != f2.read(): + LOG.error('The content of source file(%s) and' + ' dest file(%s) are not equal.', source, dest) + return False + # check perm + file_stat = os.stat(dest) + actual_perm = oct(file_stat.st_mode)[-4:] + if self.perm != actual_perm: + LOG.error('Dest file does not have expected perm: %s, actual: %s', + self.perm, actual_perm) + return False + # check owner + actual_user = pwd.getpwuid(file_stat.st_uid) + if actual_user.pw_name != self.owner: + LOG.error('Dest file does not have expected user: %s,' + ' actual: %s ', self.owner, actual_user.pw_name) + return False + actual_group = grp.getgrgid(file_stat.st_gid) + if actual_group.gr_name != self.owner: + LOG.error('Dest file does not have expected group: %s,' + ' actual: %s ', self.owner, actual_group.gr_name) + return False + return True + + def _cmp_dir(self, source, dest): + for root, dirs, files in os.walk(source): + for dir_ in dirs: + full_path = os.path.join(root, dir_) + dest_full_path = os.path.join(dest, os.path.relpath(source, + full_path)) + dir_stat = os.stat(dest_full_path) + actual_perm = oct(dir_stat.st_mode)[-4:] + if self.perm != actual_perm: + LOG.error('Dest dir does not have expected perm: %s,' + ' acutal %s', self.perm, actual_perm) + return False + for file_ in files: + full_path = os.path.join(root, file_) + dest_full_path = os.path.join(dest, os.path.relpath(source, + full_path)) + if not self._cmp_file(full_path, dest_full_path): + return False + return True + + def check(self): + bad_state_files = [] + sources = glob.glob(self.source) + + if not sources and not self.optional: + raise MissingRequiredSource('%s file is not found' % self.source) + elif self.optional and not sources: + return + + for source in sources: + dest = self.dest + # dest endswith / means copy the into folder, + # otherwise means copy the source to dest + if dest.endswith(os.sep): + dest = os.path.join(dest, os.path.basename(source)) + if os.path.isdir(source) and not self._cmp_dir(source, dest): + bad_state_files.append(source) + elif not self._cmp_file(source, dest): + bad_state_files.append(source) + if len(bad_state_files) != 0: + msg = 'Following files are in bad state: %s' % bad_state_files + raise ConfigFileBadState(msg) def validate_config(config): - config_files_required_keys = {'source', 'dest', 'owner', 'perm'} - permissions_required_keys = {'path', 'owner'} + required_keys = {'source', 'dest', 'owner', 'perm'} if 'command' not in config: raise InvalidConfig('Config is missing required "command" key') @@ -81,12 +225,9 @@ def validate_config(config): # Validate config sections for data in config.get('config_files', list()): # Verify required keys exist. - if not data.viewkeys() >= config_files_required_keys: - raise InvalidConfig( - "Config is missing required keys: %s" % data) - for data in config.get('permissions', list()): - if not data.viewkeys() >= permissions_required_keys: - raise InvalidConfig("Config is missing required keys: %s" % data) + if not data.viewkeys() >= required_keys: + message = 'Config is missing required keys: %s' % required_keys + raise InvalidConfig(message) def validate_source(data): @@ -105,45 +246,6 @@ def validate_source(data): return True -def copy_files(data): - dest = data.get('dest') - source = data.get('source') - - if os.path.exists(dest): - LOG.info("Removing existing destination: %s", dest) - if os.path.isdir(dest): - shutil.rmtree(dest) - else: - os.remove(dest) - - if os.path.isdir(source): - source_path = source - dest_path = dest - else: - source_path = os.path.dirname(source) - dest_path = os.path.dirname(dest) - - if not os.path.exists(dest_path): - LOG.info("Creating dest parent directory: %s", dest_path) - os.makedirs(dest_path) - - if source != source_path: - # Source is file - for file in glob.glob(source): - LOG.info("Copying %s to %s", file, dest) - shutil.copy(file, dest) - else: - # Source is a directory - for src in os.listdir(source_path): - LOG.info("Copying %s to %s", - os.path.join(source_path, src), dest_path) - - if os.path.isdir(os.path.join(source_path, src)): - shutil.copytree(os.path.join(source_path, src), dest_path) - else: - shutil.copy(os.path.join(source_path, src), dest_path) - - def set_permissions(data): def set_perms(file_, uid, guid, perm): LOG.info("Setting permissions for %s", file_) @@ -222,9 +324,8 @@ def copy_config(config): if 'config_files' in config: LOG.info('Copying service configuration files') for data in config['config_files']: - if validate_source(data): - copy_files(data) - set_permissions(data) + config_file = ConfigFile(**data) + config_file.copy() else: LOG.debug('No files to copy found in config') @@ -265,11 +366,9 @@ def handle_permissions(config): set_perms(os.path.join(root, file_), uid, gid) -def execute_config_strategy(): +def execute_config_strategy(config): config_strategy = os.environ.get("KOLLA_CONFIG_STRATEGY") LOG.info("Kolla config strategy set to: %s", config_strategy) - config = load_config() - if config_strategy == "COPY_ALWAYS": copy_config(config) handle_permissions(config) @@ -286,42 +385,10 @@ def execute_config_strategy(): raise InvalidConfig('KOLLA_CONFIG_STRATEGY is not set properly') -def execute_config_check(): - config = load_config() - for config_file in config.get('config_files', {}): - source = config_file.get('source') - dest = config_file.get('dest') - perm = config_file.get('perm') - owner = config_file.get('owner') - optional = config_file.get('optional', False) - if not os.path.exists(dest): - if optional: - LOG.info('Dest file does not exist, but is optional: %s', - dest) - return - else: - raise RequiredFileNotFound( - 'Dest file does not exist and is: %s' % dest) - # check content - with open(source) as fp1, open(dest) as fp2: - if fp1.read() != fp2.read(): - raise ContentDifferent( - 'The content of source file(%s) and' - ' dest file(%s) are not equal.' % (source, dest)) - # check perm - file_stat = os.stat(dest) - actual_perm = oct(file_stat.st_mode)[-4:] - if perm != actual_perm: - raise DifferentPermissions( - 'Dest file does not have expected perm: %s, actual: %s' - % (perm, actual_perm)) - # check owner - actual_user = pwd.getpwuid(file_stat.st_uid) - if actual_user.pw_name != owner: - raise DifferentUser( - 'Dest file does not have expected user: %s, actual: %s ' - % (owner, actual_user.pw_name)) - LOG.info('The config files are in the expected state') +def execute_config_check(config): + for data in config['config_files']: + config_file = ConfigFile(**data) + config_file.check() def main(): @@ -330,12 +397,13 @@ def main(): action='store_true', required=False, help='Check whether the configs changed') - conf = parser.parse_args() + args = parser.parse_args() + config = load_config() - if conf.check: - execute_config_check() + if args.check: + execute_config_check(config) else: - execute_config_strategy() + execute_config_strategy(config) if __name__ == "__main__": diff --git a/tests/test_set_config.py b/tests/test_set_config.py index 3e946c7979..e67db9bc24 100644 --- a/tests/test_set_config.py +++ b/tests/test_set_config.py @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections +import copy import imp import json import mock @@ -64,3 +66,212 @@ class LoadFromEnv(base.BaseTestCase): mock.call().write(u'/bin/true'), mock.call().__exit__(None, None, None)], mo.mock_calls) + +FAKE_CONFIG_FILES = [ + set_configs.ConfigFile( + '/var/lib/kolla/config_files/bar.conf', + '/foo/bar.conf', 'user1', '0644') +] + +FAKE_CONFIG_FILE = FAKE_CONFIG_FILES[0] + + +class ConfigFileTest(base.BaseTestCase): + + @mock.patch('os.path.exists', return_value=False) + def test_delete_path_not_exists(self, mock_exists): + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + config_file._delete_path(config_file.dest) + + mock_exists.assert_called_with(config_file.dest) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('os.path.isdir', return_value=True) + @mock.patch('shutil.rmtree') + def test_delete_path_exist_dir(self, + mock_rmtree, + mock_isdir, + mock_exists): + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + config_file._delete_path(config_file.dest) + + mock_exists.assert_called_with(config_file.dest) + mock_isdir.assert_called_with(config_file.dest) + mock_rmtree.assert_called_with(config_file.dest) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('os.path.isdir', return_value=False) + @mock.patch('os.remove') + def test_delete_path_exist_file(self, + mock_remove, + mock_isdir, + mock_exists): + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + config_file._delete_path(config_file.dest) + + mock_exists.assert_called_with(config_file.dest) + mock_isdir.assert_called_with(config_file.dest) + mock_remove.assert_called_with(config_file.dest) + + @mock.patch('os.chmod') + @mock.patch('os.chown') + @mock.patch('pwd.getpwnam') + def test_set_permission(self, + mock_getpwnam, + mock_chown, + mock_chmod): + + User = collections.namedtuple('User', ['pw_uid', 'pw_gid']) + user = User(3333, 4444) + mock_getpwnam.return_value = user + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + config_file._set_permission(config_file.dest) + + mock_getpwnam.assert_called_with(config_file.owner) + mock_chown.assert_called_with(config_file.dest, 3333, 4444) + mock_chmod.assert_called_with(config_file.dest, 420) + + @mock.patch('glob.glob', return_value=[]) + def test_copy_no_source_not_optional(self, + mock_glob): + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + + self.assertRaises(set_configs.MissingRequiredSource, + config_file.copy) + + @mock.patch('glob.glob', return_value=[]) + def test_copy_no_source_optional(self, mock_glob): + + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + config_file.optional = True + + config_file.copy() + + mock_glob.assert_called_with(config_file.source) + + @mock.patch.object(set_configs.ConfigFile, '_copy_file') + @mock.patch('os.path.isdir', return_value=False) + @mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs') + @mock.patch.object(set_configs.ConfigFile, '_delete_path') + @mock.patch('glob.glob') + def test_copy_one_source_file(self, mock_glob, mock_delete_path, + mock_create_parent_dirs, mock_isdir, + mock_copy_file): + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + + mock_glob.return_value = [config_file.source] + + config_file.copy() + + mock_glob.assert_called_with(config_file.source) + mock_delete_path.assert_called_with(config_file.dest) + mock_create_parent_dirs.assert_called_with(config_file.dest) + mock_isdir.assert_called_with(config_file.source) + mock_copy_file.assert_called_with(config_file.source, + config_file.dest) + + @mock.patch.object(set_configs.ConfigFile, '_copy_dir') + @mock.patch('os.path.isdir', return_value=True) + @mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs') + @mock.patch.object(set_configs.ConfigFile, '_delete_path') + @mock.patch('glob.glob') + def test_copy_one_source_dir(self, mock_glob, mock_delete_path, + mock_create_parent_dirs, mock_isdir, + mock_copy_dir): + config_file = copy.deepcopy(FAKE_CONFIG_FILE) + + mock_glob.return_value = [config_file.source] + + config_file.copy() + + mock_glob.assert_called_with(config_file.source) + mock_delete_path.assert_called_with(config_file.dest) + mock_create_parent_dirs.assert_called_with(config_file.dest) + mock_isdir.assert_called_with(config_file.source) + mock_copy_dir.assert_called_with(config_file.source, + config_file.dest) + + @mock.patch.object(set_configs.ConfigFile, '_copy_file') + @mock.patch('os.path.isdir', return_value=False) + @mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs') + @mock.patch.object(set_configs.ConfigFile, '_delete_path') + @mock.patch('glob.glob') + def test_copy_glob_source_file(self, mock_glob, mock_delete_path, + mock_create_parent_dirs, mock_isdir, + mock_copy_file): + config_file = set_configs.ConfigFile( + '/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644') + + mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf', + '/var/lib/kolla/config_files/bar.yml'] + + config_file.copy() + + mock_glob.assert_called_with(config_file.source) + + self.assertEqual(mock_delete_path.mock_calls, + [mock.call('/foo/bar.conf'), + mock.call('/foo/bar.yml')]) + self.assertEqual(mock_create_parent_dirs.mock_calls, + [mock.call('/foo/bar.conf'), + mock.call('/foo/bar.yml')]) + self.assertEqual(mock_isdir.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml')]) + self.assertEqual(mock_copy_file.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf', + '/foo/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml', + '/foo/bar.yml')]) + + @mock.patch.object(set_configs.ConfigFile, '_cmp_file') + @mock.patch('os.path.isdir', return_value=False) + @mock.patch('glob.glob') + def test_check_glob_source_file(self, mock_glob, mock_isdir, + mock_cmp_file): + config_file = set_configs.ConfigFile( + '/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644') + + mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf', + '/var/lib/kolla/config_files/bar.yml'] + mock_cmp_file.return_value = True + + config_file.check() + + self.assertEqual(mock_isdir.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml')]) + self.assertEqual(mock_cmp_file.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf', + '/foo/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml', + '/foo/bar.yml')]) + + @mock.patch.object(set_configs.ConfigFile, '_cmp_file') + @mock.patch('os.path.isdir', return_value=False) + @mock.patch('glob.glob') + def test_check_glob_source_file_no_equal(self, mock_glob, mock_isdir, + mock_cmp_file): + config_file = set_configs.ConfigFile( + '/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644') + + mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf', + '/var/lib/kolla/config_files/bar.yml'] + mock_cmp_file.side_effect = [True, False] + + self.assertRaises(set_configs.ConfigFileBadState, + config_file.check) + + self.assertEqual(mock_isdir.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml')]) + self.assertEqual(mock_cmp_file.mock_calls, + [mock.call('/var/lib/kolla/config_files/bar.conf', + '/foo/bar.conf'), + mock.call('/var/lib/kolla/config_files/bar.yml', + '/foo/bar.yml')])