openstack/kolla
Code Issues Proposed changes

Merge "Fix the set config error"

Browse Source
This commit is contained in:
Jenkins 2017-01-23 15:27:37 +00:00 committed by Gerrit Code Review
commit ad4d2a433a
2 changed files with 379 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

268
docker/base/set_configs.py
View File

@ -55,25 +55,169 @@ class UserNotFound(ExitingException):
pass
class DifferentPermissions(ExitingException):
class ConfigFileBadState(ExitingException):
pass
class DifferentUser(ExitingException):
pass
class ConfigFile(object):
def __init__(self, source, dest, owner, perm, optional=False):
self.source = source
self.dest = dest
self.owner = owner
self.perm = perm
self.optional = optional
class RequiredFileNotFound(ExitingException):
pass
def __str__(self):
return '<ConfigFile source:"{}" dest:"{}">'.format(self.source,
self.dest)
def _copy_dir(self, source, dest):
LOG.info('Copying dir from %s to %s', source, dest)
shutil.copytree(source, dest)
for root, dirs, files in os.walk(dest):
for dir_ in dirs:
self._set_permission(os.path.join(root, dir_))
for file_ in files:
self._set_permission(os.path.join(root, file_))
class ContentDifferent(ExitingException):
pass
def _delete_path(self, path):
if not os.path.exists(path):
return
if os.path.isdir(path):
LOG.info('Deleting dir %s', path)
shutil.rmtree(path)
else:
LOG.info('Deleting file %s', path)
os.remove(path)
def _create_parent_dirs(self, path):
parent_path = os.path.dirname(path)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
def _copy_file(self, source, dest):
# dest endswith / means copy the <source> to <dest> folder
LOG.info('Coping file from %s to %s', source, dest)
shutil.copy(source, dest)
self._set_permission(dest)
def _set_permission(self, path):
user = pwd.getpwnam(self.owner)
uid, gid = user.pw_uid, user.pw_gid
LOG.info('Setting file %s owner to %s:%s', path,
self.owner, self.owner)
os.chown(path, uid, gid)
# NOTE(Jeffrey4l): py3 need '0oXXX' format for octal literals, and py2
# support such format too.
perm = self.perm
if len(perm) == 4 and perm[1] != 'o':
perm = ''.join([perm[:1], 'o', perm[1:]])
perm = int(perm, base=0)
LOG.info('Setting file %s permission to %s', path, self.perm)
os.chmod(path, perm)
def copy(self):
sources = glob.glob(self.source)
if not self.optional and not sources:
raise MissingRequiredSource('%s file is not found' % self.source)
# skip when there is no sources and optional
elif self.optional and not sources:
return
for source in sources:
dest = self.dest
# dest endswith / means copy the <source> into <dest> folder,
# otherwise means copy the source to dest
if dest.endswith(os.sep):
dest = os.path.join(dest, os.path.basename(source))
self._delete_path(dest)
self._create_parent_dirs(dest)
if os.path.isdir(source):
self._copy_dir(source, dest)
else:
self._copy_file(source, dest)
def _cmp_file(self, source, dest):
# check exsit
if (os.path.exists(source) and
not self.optional and
not os.path.exists(dest)):
return False
# check content
with open(source) as f1, open(dest) as f2:
if f1.read() != f2.read():
LOG.error('The content of source file(%s) and'
' dest file(%s) are not equal.', source, dest)
return False
# check perm
file_stat = os.stat(dest)
actual_perm = oct(file_stat.st_mode)[-4:]
if self.perm != actual_perm:
LOG.error('Dest file does not have expected perm: %s, actual: %s',
self.perm, actual_perm)
return False
# check owner
actual_user = pwd.getpwuid(file_stat.st_uid)
if actual_user.pw_name != self.owner:
LOG.error('Dest file does not have expected user: %s,'
' actual: %s ', self.owner, actual_user.pw_name)
return False
actual_group = grp.getgrgid(file_stat.st_gid)
if actual_group.gr_name != self.owner:
LOG.error('Dest file does not have expected group: %s,'
' actual: %s ', self.owner, actual_group.gr_name)
return False
return True
def _cmp_dir(self, source, dest):
for root, dirs, files in os.walk(source):
for dir_ in dirs:
full_path = os.path.join(root, dir_)
dest_full_path = os.path.join(dest, os.path.relpath(source,
full_path))
dir_stat = os.stat(dest_full_path)
actual_perm = oct(dir_stat.st_mode)[-4:]
if self.perm != actual_perm:
LOG.error('Dest dir does not have expected perm: %s,'
' acutal %s', self.perm, actual_perm)
return False
for file_ in files:
full_path = os.path.join(root, file_)
dest_full_path = os.path.join(dest, os.path.relpath(source,
full_path))
if not self._cmp_file(full_path, dest_full_path):
return False
return True
def check(self):
bad_state_files = []
sources = glob.glob(self.source)
if not sources and not self.optional:
raise MissingRequiredSource('%s file is not found' % self.source)
elif self.optional and not sources:
return
for source in sources:
dest = self.dest
# dest endswith / means copy the <source> into <dest> folder,
# otherwise means copy the source to dest
if dest.endswith(os.sep):
dest = os.path.join(dest, os.path.basename(source))
if os.path.isdir(source) and not self._cmp_dir(source, dest):
bad_state_files.append(source)
elif not self._cmp_file(source, dest):
bad_state_files.append(source)
if len(bad_state_files) != 0:
msg = 'Following files are in bad state: %s' % bad_state_files
raise ConfigFileBadState(msg)
def validate_config(config):
config_files_required_keys = {'source', 'dest', 'owner', 'perm'}
permissions_required_keys = {'path', 'owner'}
required_keys = {'source', 'dest', 'owner', 'perm'}
if 'command' not in config:
raise InvalidConfig('Config is missing required "command" key')
@ -81,12 +225,9 @@ def validate_config(config):
# Validate config sections
for data in config.get('config_files', list()):
# Verify required keys exist.
if not data.viewkeys() >= config_files_required_keys:
raise InvalidConfig(
"Config is missing required keys: %s" % data)
for data in config.get('permissions', list()):
if not data.viewkeys() >= permissions_required_keys:
raise InvalidConfig("Config is missing required keys: %s" % data)
if not data.viewkeys() >= required_keys:
message = 'Config is missing required keys: %s' % required_keys
raise InvalidConfig(message)
def validate_source(data):
@ -105,45 +246,6 @@ def validate_source(data):
return True
def copy_files(data):
dest = data.get('dest')
source = data.get('source')
if os.path.exists(dest):
LOG.info("Removing existing destination: %s", dest)
if os.path.isdir(dest):
shutil.rmtree(dest)
else:
os.remove(dest)
if os.path.isdir(source):
source_path = source
dest_path = dest
else:
source_path = os.path.dirname(source)
dest_path = os.path.dirname(dest)
if not os.path.exists(dest_path):
LOG.info("Creating dest parent directory: %s", dest_path)
os.makedirs(dest_path)
if source != source_path:
# Source is file
for file in glob.glob(source):
LOG.info("Copying %s to %s", file, dest)
shutil.copy(file, dest)
else:
# Source is a directory
for src in os.listdir(source_path):
LOG.info("Copying %s to %s",
os.path.join(source_path, src), dest_path)
if os.path.isdir(os.path.join(source_path, src)):
shutil.copytree(os.path.join(source_path, src), dest_path)
else:
shutil.copy(os.path.join(source_path, src), dest_path)
def set_permissions(data):
def set_perms(file_, uid, guid, perm):
LOG.info("Setting permissions for %s", file_)
@ -222,9 +324,8 @@ def copy_config(config):
if 'config_files' in config:
LOG.info('Copying service configuration files')
for data in config['config_files']:
if validate_source(data):
copy_files(data)
set_permissions(data)
config_file = ConfigFile(**data)
config_file.copy()
else:
LOG.debug('No files to copy found in config')
@ -265,11 +366,9 @@ def handle_permissions(config):
set_perms(os.path.join(root, file_), uid, gid)
def execute_config_strategy():
def execute_config_strategy(config):
config_strategy = os.environ.get("KOLLA_CONFIG_STRATEGY")
LOG.info("Kolla config strategy set to: %s", config_strategy)
config = load_config()
if config_strategy == "COPY_ALWAYS":
copy_config(config)
handle_permissions(config)
@ -286,42 +385,10 @@ def execute_config_strategy():
raise InvalidConfig('KOLLA_CONFIG_STRATEGY is not set properly')
def execute_config_check():
config = load_config()
for config_file in config.get('config_files', {}):
source = config_file.get('source')
dest = config_file.get('dest')
perm = config_file.get('perm')
owner = config_file.get('owner')
optional = config_file.get('optional', False)
if not os.path.exists(dest):
if optional:
LOG.info('Dest file does not exist, but is optional: %s',
dest)
return
else:
raise RequiredFileNotFound(
'Dest file does not exist and is: %s' % dest)
# check content
with open(source) as fp1, open(dest) as fp2:
if fp1.read() != fp2.read():
raise ContentDifferent(
'The content of source file(%s) and'
' dest file(%s) are not equal.' % (source, dest))
# check perm
file_stat = os.stat(dest)
actual_perm = oct(file_stat.st_mode)[-4:]
if perm != actual_perm:
raise DifferentPermissions(
'Dest file does not have expected perm: %s, actual: %s'
% (perm, actual_perm))
# check owner
actual_user = pwd.getpwuid(file_stat.st_uid)
if actual_user.pw_name != owner:
raise DifferentUser(
'Dest file does not have expected user: %s, actual: %s '
% (owner, actual_user.pw_name))
LOG.info('The config files are in the expected state')
def execute_config_check(config):
for data in config['config_files']:
config_file = ConfigFile(**data)
config_file.check()
def main():
@ -330,12 +397,13 @@ def main():
action='store_true',
required=False,
help='Check whether the configs changed')
conf = parser.parse_args()
args = parser.parse_args()
config = load_config()
if conf.check:
execute_config_check()
if args.check:
execute_config_check(config)
else:
execute_config_strategy()
execute_config_strategy(config)
if __name__ == "__main__":

211
tests/test_set_config.py
View File

@ -10,6 +10,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import copy
import imp
import json
import mock
@ -64,3 +66,212 @@ class LoadFromEnv(base.BaseTestCase):
mock.call().write(u'/bin/true'),
mock.call().__exit__(None, None, None)],
mo.mock_calls)
FAKE_CONFIG_FILES = [
set_configs.ConfigFile(
'/var/lib/kolla/config_files/bar.conf',
'/foo/bar.conf', 'user1', '0644')
]
FAKE_CONFIG_FILE = FAKE_CONFIG_FILES[0]
class ConfigFileTest(base.BaseTestCase):
@mock.patch('os.path.exists', return_value=False)
def test_delete_path_not_exists(self, mock_exists):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
config_file._delete_path(config_file.dest)
mock_exists.assert_called_with(config_file.dest)
@mock.patch('os.path.exists', return_value=True)
@mock.patch('os.path.isdir', return_value=True)
@mock.patch('shutil.rmtree')
def test_delete_path_exist_dir(self,
mock_rmtree,
mock_isdir,
mock_exists):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
config_file._delete_path(config_file.dest)
mock_exists.assert_called_with(config_file.dest)
mock_isdir.assert_called_with(config_file.dest)
mock_rmtree.assert_called_with(config_file.dest)
@mock.patch('os.path.exists', return_value=True)
@mock.patch('os.path.isdir', return_value=False)
@mock.patch('os.remove')
def test_delete_path_exist_file(self,
mock_remove,
mock_isdir,
mock_exists):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
config_file._delete_path(config_file.dest)
mock_exists.assert_called_with(config_file.dest)
mock_isdir.assert_called_with(config_file.dest)
mock_remove.assert_called_with(config_file.dest)
@mock.patch('os.chmod')
@mock.patch('os.chown')
@mock.patch('pwd.getpwnam')
def test_set_permission(self,
mock_getpwnam,
mock_chown,
mock_chmod):
User = collections.namedtuple('User', ['pw_uid', 'pw_gid'])
user = User(3333, 4444)
mock_getpwnam.return_value = user
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
config_file._set_permission(config_file.dest)
mock_getpwnam.assert_called_with(config_file.owner)
mock_chown.assert_called_with(config_file.dest, 3333, 4444)
mock_chmod.assert_called_with(config_file.dest, 420)
@mock.patch('glob.glob', return_value=[])
def test_copy_no_source_not_optional(self,
mock_glob):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
self.assertRaises(set_configs.MissingRequiredSource,
config_file.copy)
@mock.patch('glob.glob', return_value=[])
def test_copy_no_source_optional(self, mock_glob):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
config_file.optional = True
config_file.copy()
mock_glob.assert_called_with(config_file.source)
@mock.patch.object(set_configs.ConfigFile, '_copy_file')
@mock.patch('os.path.isdir', return_value=False)
@mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs')
@mock.patch.object(set_configs.ConfigFile, '_delete_path')
@mock.patch('glob.glob')
def test_copy_one_source_file(self, mock_glob, mock_delete_path,
mock_create_parent_dirs, mock_isdir,
mock_copy_file):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
mock_glob.return_value = [config_file.source]
config_file.copy()
mock_glob.assert_called_with(config_file.source)
mock_delete_path.assert_called_with(config_file.dest)
mock_create_parent_dirs.assert_called_with(config_file.dest)
mock_isdir.assert_called_with(config_file.source)
mock_copy_file.assert_called_with(config_file.source,
config_file.dest)
@mock.patch.object(set_configs.ConfigFile, '_copy_dir')
@mock.patch('os.path.isdir', return_value=True)
@mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs')
@mock.patch.object(set_configs.ConfigFile, '_delete_path')
@mock.patch('glob.glob')
def test_copy_one_source_dir(self, mock_glob, mock_delete_path,
mock_create_parent_dirs, mock_isdir,
mock_copy_dir):
config_file = copy.deepcopy(FAKE_CONFIG_FILE)
mock_glob.return_value = [config_file.source]
config_file.copy()
mock_glob.assert_called_with(config_file.source)
mock_delete_path.assert_called_with(config_file.dest)
mock_create_parent_dirs.assert_called_with(config_file.dest)
mock_isdir.assert_called_with(config_file.source)
mock_copy_dir.assert_called_with(config_file.source,
config_file.dest)
@mock.patch.object(set_configs.ConfigFile, '_copy_file')
@mock.patch('os.path.isdir', return_value=False)
@mock.patch.object(set_configs.ConfigFile, '_create_parent_dirs')
@mock.patch.object(set_configs.ConfigFile, '_delete_path')
@mock.patch('glob.glob')
def test_copy_glob_source_file(self, mock_glob, mock_delete_path,
mock_create_parent_dirs, mock_isdir,
mock_copy_file):
config_file = set_configs.ConfigFile(
'/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644')
mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf',
'/var/lib/kolla/config_files/bar.yml']
config_file.copy()
mock_glob.assert_called_with(config_file.source)
self.assertEqual(mock_delete_path.mock_calls,
[mock.call('/foo/bar.conf'),
mock.call('/foo/bar.yml')])
self.assertEqual(mock_create_parent_dirs.mock_calls,
[mock.call('/foo/bar.conf'),
mock.call('/foo/bar.yml')])
self.assertEqual(mock_isdir.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml')])
self.assertEqual(mock_copy_file.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf',
'/foo/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml',
'/foo/bar.yml')])
@mock.patch.object(set_configs.ConfigFile, '_cmp_file')
@mock.patch('os.path.isdir', return_value=False)
@mock.patch('glob.glob')
def test_check_glob_source_file(self, mock_glob, mock_isdir,
mock_cmp_file):
config_file = set_configs.ConfigFile(
'/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644')
mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf',
'/var/lib/kolla/config_files/bar.yml']
mock_cmp_file.return_value = True
config_file.check()
self.assertEqual(mock_isdir.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml')])
self.assertEqual(mock_cmp_file.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf',
'/foo/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml',
'/foo/bar.yml')])
@mock.patch.object(set_configs.ConfigFile, '_cmp_file')
@mock.patch('os.path.isdir', return_value=False)
@mock.patch('glob.glob')
def test_check_glob_source_file_no_equal(self, mock_glob, mock_isdir,
mock_cmp_file):
config_file = set_configs.ConfigFile(
'/var/lib/kolla/config_files/bar.*', '/foo/', 'user1', '0644')
mock_glob.return_value = ['/var/lib/kolla/config_files/bar.conf',
'/var/lib/kolla/config_files/bar.yml']
mock_cmp_file.side_effect = [True, False]
self.assertRaises(set_configs.ConfigFileBadState,
config_file.check)
self.assertEqual(mock_isdir.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml')])
self.assertEqual(mock_cmp_file.mock_calls,
[mock.call('/var/lib/kolla/config_files/bar.conf',
'/foo/bar.conf'),
mock.call('/var/lib/kolla/config_files/bar.yml',
'/foo/bar.yml')])