Ignore unparsable/empty image build ZNode data

As with the parent it seems that we can end up with empty image build
records as well as empty image upload records. Handle the build case as
well so that we can still list image builds properly when in this state.
This logs the problem as well so that you can debug it further.

Change-Id: Ic5521f5e2d2b65db94c2a5794f474913631a7a1b
This commit is contained in:
Clark Boylan 2020-08-14 13:52:34 -07:00 committed by Benjamin Schanzel
parent ee2bc79ae2
commit dfcf770260
4 changed files with 35 additions and 12 deletions

@ -368,7 +368,7 @@ class CleanupWorker(BaseWorker):
# still in progress so that it is checked again later with # still in progress so that it is checked again later with
# its new build state. # its new build state.
b = self._zk.getBuild(image, build.id) b = self._zk.getBuild(image, build.id)
if b.state != zk.BUILDING: if b and b.state != zk.BUILDING:
return True return True
pass pass
except exceptions.ZKLockException: except exceptions.ZKLockException:
@ -780,7 +780,7 @@ class BuildWorker(BaseWorker):
data = self._buildWrapper(diskimage) data = self._buildWrapper(diskimage)
# Remove request on a successful build # Remove request on a successful build
if data.state == zk.READY: if data and data.state == zk.READY:
self._zk.removeBuildRequest(diskimage.name) self._zk.removeBuildRequest(diskimage.name)
except exceptions.ZKLockException: except exceptions.ZKLockException:
@ -1256,7 +1256,7 @@ class UploadWorker(BaseWorker):
# that another thread isn't trying to delete this build just # that another thread isn't trying to delete this build just
# before we upload. # before we upload.
b = self._zk.getBuild(image.name, build.id) b = self._zk.getBuild(image.name, build.id)
if b.state == zk.DELETING: if not b or b.state == zk.DELETING:
return False return False
# New upload number with initial state 'uploading' # New upload number with initial state 'uploading'

@ -190,14 +190,15 @@ def dib_image_list(zk):
paused = zk.getImagePaused(image_name) paused = zk.getImagePaused(image_name)
for build_no in zk.getBuildNumbers(image_name): for build_no in zk.getBuildNumbers(image_name):
build = zk.getBuild(image_name, build_no) build = zk.getBuild(image_name, build_no)
state = paused and 'paused' or build.state if build:
objs.append({'id': '-'.join([image_name, build_no]), state = paused and 'paused' or build.state
'image': image_name, objs.append({'id': '-'.join([image_name, build_no]),
'builder': build.builder, 'image': image_name,
'formats': build.formats, 'builder': build.builder,
'state': state, 'formats': build.formats,
'age': int(build.state_time) 'state': state,
}) 'age': int(build.state_time)
})
return (objs, headers_table) return (objs, headers_table)

@ -326,6 +326,23 @@ class TestZooKeeper(tests.DBTestCase):
self.zk.removeBuildRequest(image) self.zk.removeBuildRequest(image)
self.assertFalse(self.zk.hasBuildRequest(image)) self.assertFalse(self.zk.hasBuildRequest(image))
def test_buildLock_orphan(self):
image = "ubuntu-trusty"
build_number = "0000000003"
path = self.zk._imageBuildNumberLockPath(image, build_number)
# Pretend we still think the image build exists
# (e.g. multiple cleanup workers itertating over builds)
with self.zk.imageBuildNumberLock(image, build_number, blocking=False):
# We now created an empty build number node
pass
self.assertIsNotNone(self.zk.client.exists(path))
# Should not throw an exception because of the empty upload
self.assertIsNone(self.zk.getBuild(image, build_number))
def test_getMostRecentBuilds(self): def test_getMostRecentBuilds(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
v1 = {'state': zk.READY, 'state_time': int(time.time())} v1 = {'state': zk.READY, 'state_time': int(time.time())}

@ -1274,7 +1274,12 @@ class ZooKeeper(object):
except kze.NoNodeError: except kze.NoNodeError:
return None return None
d = ImageBuild.fromDict(self._bytesToDict(data), build_number) try:
d = ImageBuild.fromDict(self._bytesToDict(data), build_number)
except json.decoder.JSONDecodeError:
self.log.exception('Error loading json data from image build %s',
path)
return None
d.stat = stat d.stat = stat
return d return d