Avoid failing on deleted file

Avoid failing on deleted file as sometimes file might get
deleted while the script run.  Log the exception instead for
troubleshooting purposes.

Conflicts:
  docker_config_scripts/nova_statedir_ownership.py
  docker_config_scripts/tests/test_nova_statedir_ownership.py
  Manual merge as Id5503ed274bd5dc0c5365cc994de7e5cdcbc2fb6 has not been
  backported to stable/queens
  Partial merge of Iaf0299983d3a3fe48e3beb8f47bd33c21deb4972 for flake8

Change-Id: I733cec2b34ef0bd0780ba5b0520127b911505e1b
(cherry picked from commit 6c3c8b41de)
(cherry picked from commit eddbc4b2c4)
(cherry picked from commit a549491bd4)
This commit is contained in:
David Hill 2020-06-05 14:12:31 -04:00 committed by Oliver Walsh
parent 20929f9832
commit 5f1640b352
5 changed files with 135 additions and 66 deletions

View File

@ -28,13 +28,20 @@ class PathManager(object):
"""Helper class to manipulate ownership of a given path""" """Helper class to manipulate ownership of a given path"""
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
self.uid = None
self.gid = None
self.is_dir = None
self._update() self._update()
def _update(self): def _update(self):
statinfo = os.stat(self.path) try:
self.is_dir = stat.S_ISDIR(statinfo.st_mode) statinfo = os.stat(self.path)
self.uid = statinfo.st_uid self.is_dir = stat.S_ISDIR(statinfo.st_mode)
self.gid = statinfo.st_gid self.uid = statinfo.st_uid
self.gid = statinfo.st_gid
except Exception:
LOG.exception('Could not update metadata for %s', self.path)
raise
def __str__(self): def __str__(self):
return "uid: {} gid: {} path: {}{}".format( return "uid: {} gid: {} path: {}{}".format(
@ -70,6 +77,7 @@ class PathManager(object):
except Exception: except Exception:
LOG.exception('Could not change ownership of %s: ', LOG.exception('Could not change ownership of %s: ',
self.path) self.path)
raise
else: else:
LOG.info('Ownership of %s already %d:%d', LOG.info('Ownership of %s already %d:%d',
self.path, self.path,
@ -129,21 +137,28 @@ class NovaStatedirOwnershipManager(object):
if pathname == self.upgrade_marker_path: if pathname == self.upgrade_marker_path:
continue continue
pathinfo = PathManager(pathname) try:
LOG.info("Checking %s", pathinfo) pathinfo = PathManager(pathname)
if pathinfo.is_dir: LOG.info("Checking %s", pathinfo)
# Always chown the directories if pathinfo.is_dir:
pathinfo.chown(self.target_uid, self.target_gid) # Always chown the directories
self._walk(pathname) pathinfo.chown(self.target_uid, self.target_gid)
elif self.id_change: self._walk(pathname)
# Only chown files if it's an upgrade and the file is owned by elif self.id_change:
# the host nova uid/gid # Only chown files if it's an upgrade and the file is owned by
pathinfo.chown( # the host nova uid/gid
self.target_uid if pathinfo.uid == self.previous_uid pathinfo.chown(
else pathinfo.uid, self.target_uid if pathinfo.uid == self.previous_uid
self.target_gid if pathinfo.gid == self.previous_gid else pathinfo.uid,
else pathinfo.gid self.target_gid if pathinfo.gid == self.previous_gid
) else pathinfo.gid
)
except Exception:
# Likely to have been caused by external systems
# interacting with this directory tree,
# especially on NFS e.g snapshot dirs.
# Just ignore it and continue on to the next entry
continue
def run(self): def run(self):
LOG.info('Applying nova statedir ownership') LOG.info('Applying nova statedir ownership')
@ -165,5 +180,6 @@ class NovaStatedirOwnershipManager(object):
LOG.info('Nova statedir ownership complete') LOG.info('Nova statedir ownership complete')
if __name__ == '__main__': if __name__ == '__main__':
NovaStatedirOwnershipManager('/var/lib/nova').run() NovaStatedirOwnershipManager('/var/lib/nova').run()

View File

@ -87,7 +87,7 @@ if __name__ == '__main__':
LOG.info('Nova-compute service registered') LOG.info('Nova-compute service registered')
sys.exit(0) sys.exit(0)
LOG.info('Waiting for nova-compute service to register') LOG.info('Waiting for nova-compute service to register')
except Exception as e: except Exception:
LOG.exception( LOG.exception(
'Error while waiting for nova-compute service to register') 'Error while waiting for nova-compute service to register')
time.sleep(timeout) time.sleep(timeout)

View File

@ -54,7 +54,7 @@ if __name__ == '__main__':
password=config.get('placement', 'password'), password=config.get('placement', 'password'),
project_name=config.get('placement', 'project_name'), project_name=config.get('placement', 'project_name'),
project_domain_name=config.get('placement', 'user_domain_name'), project_domain_name=config.get('placement', 'user_domain_name'),
auth_url=config.get('placement', 'auth_url')+'/v3') auth_url=config.get('placement', 'auth_url') + '/v3')
sess = session.Session(auth=auth, verify=False) sess = session.Session(auth=auth, verify=False)
keystone = client.Client(session=sess, interface='internal') keystone = client.Client(session=sess, interface='internal')
@ -75,7 +75,7 @@ if __name__ == '__main__':
LOG.error('Failed to get placement service endpoint!') LOG.error('Failed to get placement service endpoint!')
else: else:
break break
except Exception as e: except Exception:
LOG.exception('Retry - Failed to get placement service endpoint:') LOG.exception('Retry - Failed to get placement service endpoint:')
time.sleep(timeout) time.sleep(timeout)
@ -92,7 +92,7 @@ if __name__ == '__main__':
while iterations > 1: while iterations > 1:
iterations -= 1 iterations -= 1
try: try:
r = requests.get(placement_endpoint_url+'/', verify=False) r = requests.get(placement_endpoint_url + '/', verify=False)
if r.status_code == 200 and response_reg.match(r.text): if r.status_code == 200 and response_reg.match(r.text):
LOG.info('Placement service up! - %s', r.text) LOG.info('Placement service up! - %s', r.text)
sys.exit(0) sys.exit(0)
@ -102,7 +102,7 @@ if __name__ == '__main__':
LOG.info('Placement service not up - %s, %s', LOG.info('Placement service not up - %s, %s',
r.status_code, r.status_code,
r.text) r.text)
except Exception as e: except Exception:
LOG.exception('Error query the placement endpoint:') LOG.exception('Error query the placement endpoint:')
time.sleep(timeout) time.sleep(timeout)

View File

@ -15,14 +15,15 @@
import contextlib import contextlib
import mock import mock
from os import stat as orig_stat
import six import six
import stat import stat
from oslotest import base from oslotest import base
from docker_config_scripts.nova_statedir_ownership import \ from docker_config_scripts.nova_statedir_ownership import \
NovaStatedirOwnershipManager NovaStatedirOwnershipManager # noqa: E402
from docker_config_scripts.nova_statedir_ownership import PathManager from docker_config_scripts.nova_statedir_ownership import PathManager # noqa: E402
# Real chown would require root, so in order to test this we need to fake # Real chown would require root, so in order to test this we need to fake
# all of the methods that interact with the filesystem # all of the methods that interact with the filesystem
@ -43,71 +44,118 @@ class FakeStatInfo(object):
def generate_testtree1(nova_uid, nova_gid): def generate_testtree1(nova_uid, nova_gid):
return { return {
'/var/lib/nova': '/var/lib/nova': {
FakeStatInfo(st_mode=stat.S_IFDIR, 'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_uid=nova_uid, st_uid=nova_uid,
st_gid=nova_gid), st_gid=nova_gid),
'/var/lib/nova/instances': },
FakeStatInfo(st_mode=stat.S_IFDIR, '/var/lib/nova/instances': {
'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_uid=nova_uid, st_uid=nova_uid,
st_gid=nova_gid), st_gid=nova_gid),
'/var/lib/nova/instances/foo': },
FakeStatInfo(st_mode=stat.S_IFDIR, '/var/lib/nova/instances/foo': {
'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_uid=nova_uid, st_uid=nova_uid,
st_gid=nova_gid), st_gid=nova_gid),
'/var/lib/nova/instances/foo/bar': },
FakeStatInfo(st_mode=stat.S_IFREG, '/var/lib/nova/instances/removeddir': {
st_uid=0, 'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_gid=0), st_uid=nova_uid,
'/var/lib/nova/instances/foo/baz': st_gid=nova_gid),
FakeStatInfo(st_mode=stat.S_IFREG, 'removed_when': 'listdir'
st_uid=nova_uid, },
st_gid=nova_gid), '/var/lib/nova/instances/removedfile3': {
'/var/lib/nova/instances/foo/abc': 'removed_when': 'stat'
FakeStatInfo(st_mode=stat.S_IFREG, },
st_uid=0, '/var/lib/nova/instances/foo': {
st_gid=nova_gid), 'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
'/var/lib/nova/instances/foo/def': st_uid=nova_uid,
FakeStatInfo(st_mode=stat.S_IFREG, st_gid=nova_gid),
st_uid=nova_uid, },
st_gid=0), '/var/lib/nova/instances/foo/bar': {
'stat': FakeStatInfo(st_mode=stat.S_IFREG,
st_uid=0,
st_gid=0),
},
'/var/lib/nova/instances/foo/baz': {
'stat': FakeStatInfo(st_mode=stat.S_IFREG,
st_uid=nova_uid,
st_gid=nova_gid),
},
'/var/lib/nova/instances/foo/removeddir': {
'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_uid=nova_uid,
st_gid=nova_gid),
'removed_when': 'listdir'
},
'/var/lib/nova/instances/foo/removeddir2': {
'stat': FakeStatInfo(st_mode=stat.S_IFDIR,
st_uid=0,
st_gid=nova_gid),
'removed_when': 'chown'
},
'/var/lib/nova/instances/foo/abc': {
'stat': FakeStatInfo(st_mode=stat.S_IFREG,
st_uid=0,
st_gid=nova_gid),
},
'/var/lib/nova/instances/foo/def': {
'stat': FakeStatInfo(st_mode=stat.S_IFREG,
st_uid=nova_uid,
st_gid=0),
},
} }
def generate_testtree2(marker_uid, marker_gid, *args, **kwargs): def generate_testtree2(marker_uid, marker_gid, *args, **kwargs):
tree = generate_testtree1(*args, **kwargs) tree = generate_testtree1(*args, **kwargs)
tree.update({ tree.update({
'/var/lib/nova/upgrade_marker': '/var/lib/nova/upgrade_marker': {
FakeStatInfo(st_mode=stat.S_IFREG, 'stat': FakeStatInfo(st_mode=stat.S_IFREG,
st_uid=marker_uid, st_uid=marker_uid,
st_gid=marker_gid) st_gid=marker_gid),
}
}) })
return tree return tree
def check_removed(path, op, testtree):
if op == testtree.get(path, {}).get('removed_when', ''):
raise OSError(2, 'No such file or directory: ' + path)
def generate_fake_stat(testtree): def generate_fake_stat(testtree):
def fake_stat(path): def fake_stat(path):
return testtree.get(path) check_removed(path, 'stat', testtree)
if path.startswith('/var'):
return testtree.get(path, {}).get('stat')
else:
# Tracebacks need to use the real stat
return orig_stat(path)
return fake_stat return fake_stat
def generate_fake_chown(testtree): def generate_fake_chown(testtree):
def fake_chown(path, uid, gid): def fake_chown(path, uid, gid):
check_removed(path, 'chown', testtree)
if uid != -1: if uid != -1:
testtree[path].st_uid = uid testtree[path]['stat'].st_uid = uid
if gid != -1: if gid != -1:
testtree[path].st_gid = gid testtree[path]['stat'].st_gid = gid
return fake_chown return fake_chown
def generate_fake_exists(testtree): def generate_fake_exists(testtree):
def fake_exists(path): def fake_exists(path):
check_removed(path, 'exists', testtree)
return path in testtree return path in testtree
return fake_exists return fake_exists
def generate_fake_listdir(testtree): def generate_fake_listdir(testtree):
def fake_listdir(path): def fake_listdir(path):
check_removed(path, 'listdir', testtree)
path_parts = path.split('/') path_parts = path.split('/')
for entry in testtree: for entry in testtree:
entry_parts = entry.split('/') entry_parts = entry.split('/')
@ -119,6 +167,7 @@ def generate_fake_listdir(testtree):
def generate_fake_unlink(testtree): def generate_fake_unlink(testtree):
def fake_unlink(path): def fake_unlink(path):
check_removed(path, 'unlink', testtree)
del testtree[path] del testtree[path]
return fake_unlink return fake_unlink
@ -152,9 +201,9 @@ def fake_testtree(testtree):
def assert_ids(testtree, path, uid, gid): def assert_ids(testtree, path, uid, gid):
statinfo = testtree[path] statinfo = testtree[path]['stat']
assert (uid, gid) == (statinfo.st_uid, statinfo.st_gid), \ assert (uid, gid) == (statinfo.st_uid, statinfo.st_gid), \
"{}: expected {}:{} actual {}:{}".format( "{}: expected ownership {}:{} actual {}:{}".format(
path, uid, gid, statinfo.st_uid, statinfo.st_gid path, uid, gid, statinfo.st_uid, statinfo.st_gid
) )
@ -192,8 +241,8 @@ class PathManagerCase(base.BaseTestCase):
with fake_testtree(testtree): with fake_testtree(testtree):
pathinfo = PathManager('/var/lib/nova/instances/foo/baz') pathinfo = PathManager('/var/lib/nova/instances/foo/baz')
self.assertTrue(pathinfo.has_owner(current_uid, current_gid)) self.assertTrue(pathinfo.has_owner(current_uid, current_gid))
pathinfo.chown(current_uid+1, current_gid) pathinfo.chown(current_uid + 1, current_gid)
assert_ids(testtree, pathinfo.path, current_uid+1, current_gid) assert_ids(testtree, pathinfo.path, current_uid + 1, current_gid)
def test_chgrp(self): def test_chgrp(self):
testtree = generate_testtree1(current_uid, current_gid) testtree = generate_testtree1(current_uid, current_gid)
@ -201,8 +250,8 @@ class PathManagerCase(base.BaseTestCase):
with fake_testtree(testtree): with fake_testtree(testtree):
pathinfo = PathManager('/var/lib/nova/instances/foo/baz') pathinfo = PathManager('/var/lib/nova/instances/foo/baz')
self.assertTrue(pathinfo.has_owner(current_uid, current_gid)) self.assertTrue(pathinfo.has_owner(current_uid, current_gid))
pathinfo.chown(current_uid, current_gid+1) pathinfo.chown(current_uid, current_gid + 1)
assert_ids(testtree, pathinfo.path, current_uid, current_gid+1) assert_ids(testtree, pathinfo.path, current_uid, current_gid + 1)
def test_chown_chgrp(self): def test_chown_chgrp(self):
testtree = generate_testtree1(current_uid, current_gid) testtree = generate_testtree1(current_uid, current_gid)
@ -210,8 +259,8 @@ class PathManagerCase(base.BaseTestCase):
with fake_testtree(testtree): with fake_testtree(testtree):
pathinfo = PathManager('/var/lib/nova/instances/foo/baz') pathinfo = PathManager('/var/lib/nova/instances/foo/baz')
self.assertTrue(pathinfo.has_owner(current_uid, current_gid)) self.assertTrue(pathinfo.has_owner(current_uid, current_gid))
pathinfo.chown(current_uid+1, current_gid+1) pathinfo.chown(current_uid + 1, current_gid + 1)
assert_ids(testtree, pathinfo.path, current_uid+1, current_gid+1) assert_ids(testtree, pathinfo.path, current_uid + 1, current_gid + 1)
class NovaStatedirOwnershipManagerTestCase(base.BaseTestCase): class NovaStatedirOwnershipManagerTestCase(base.BaseTestCase):
@ -220,7 +269,7 @@ class NovaStatedirOwnershipManagerTestCase(base.BaseTestCase):
with fake_testtree(testtree) as (fake_chown, _, _, _, _): with fake_testtree(testtree) as (fake_chown, _, _, _, _):
NovaStatedirOwnershipManager('/var/lib/nova').run() NovaStatedirOwnershipManager('/var/lib/nova').run()
fake_chown.assert_not_called() fake_chown.assert_called_once_with('/var/lib/nova/instances/foo/removeddir2', 100, -1)
def test_upgrade_marker_no_id_change(self): def test_upgrade_marker_no_id_change(self):
testtree = generate_testtree2(current_uid, testtree = generate_testtree2(current_uid,
@ -230,7 +279,7 @@ class NovaStatedirOwnershipManagerTestCase(base.BaseTestCase):
with fake_testtree(testtree) as (fake_chown, _, _, _, fake_unlink): with fake_testtree(testtree) as (fake_chown, _, _, _, fake_unlink):
NovaStatedirOwnershipManager('/var/lib/nova').run() NovaStatedirOwnershipManager('/var/lib/nova').run()
fake_chown.assert_not_called() fake_chown.assert_called_once_with('/var/lib/nova/instances/foo/removeddir2', 100, -1)
fake_unlink.assert_called_with('/var/lib/nova/upgrade_marker') fake_unlink.assert_called_with('/var/lib/nova/upgrade_marker')
def test_upgrade_marker_id_change(self): def test_upgrade_marker_id_change(self):
@ -247,6 +296,10 @@ class NovaStatedirOwnershipManagerTestCase(base.BaseTestCase):
if k == '/var/lib/nova/upgrade_marker': if k == '/var/lib/nova/upgrade_marker':
# Ignore the marker, it should be deleted # Ignore the marker, it should be deleted
continue continue
if testtree[k].get('removed_when', False):
# Ignore, deleted
continue
v = v['stat']
if v.st_uid == other_uid or v.st_gid == other_gid: if v.st_uid == other_uid or v.st_gid == other_gid:
expected_changes[k] = ( expected_changes[k] = (
current_uid if v.st_uid == other_uid else v.st_uid, current_uid if v.st_uid == other_uid else v.st_uid,

View File

@ -24,11 +24,11 @@ commands =
python ./tools/yaml-validate.py . python ./tools/yaml-validate.py .
bash -c ./tools/roles-data-validation.sh bash -c ./tools/roles-data-validation.sh
bash -c ./tools/check-up-to-date.sh bash -c ./tools/check-up-to-date.sh
flake8 ./docker_config_scripts/ flake8 --ignore E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,W503,W504,E501,E731,W605 ./docker_config_scripts/
[testenv:flake8] [testenv:flake8]
commands = commands =
flake8 ./docker_config_scripts/ flake8 --ignore E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,W503,W504,E501,E731,W605 ./docker_config_scripts/
[testenv:templates] [testenv:templates]
commands = python ./tools/process-templates.py commands = python ./tools/process-templates.py