zuul/nodepool
Code Issues Proposed changes

Merge "Update references of build "number" to "id""

Browse Source
This commit is contained in:
Zuul 2023-08-14 09:19:46 +00:00 committed by Gerrit Code Review
commit b2c6017d34
6 changed files with 126 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nodepool/builder.py
View File

@ -548,7 +548,7 @@ class CleanupWorker(BaseWorker):
# images. We're about to start deleting files, so # images. We're about to start deleting files, so
# make sure that the dib image state reflects that. # make sure that the dib image state reflects that.
if build.state != zk.DELETING: if build.state != zk.DELETING:
with self._zk.imageBuildNumberLock( with self._zk.imageBuildIdLock(
image, build.id, blocking=False image, build.id, blocking=False
): ):
build.state = zk.DELETING build.state = zk.DELETING

6
nodepool/cmd/nodepoolcmd.py
View File

@ -266,7 +266,7 @@ class NodePoolCmd(NodepoolApp):
uploads = [] uploads = []
for image in provider.diskimages: for image in provider.diskimages:
# Build list of provider images as recorded in ZK # Build list of provider images as recorded in ZK
for bnum in self.zk.getBuildNumbers(image): for bnum in self.zk.getBuildIds(image):
uploads.extend( uploads.extend(
self.zk.getUploads(image, bnum, self.zk.getUploads(image, bnum,
provider.name, provider.name,
@ -325,8 +325,8 @@ class NodePoolCmd(NodepoolApp):
self.list(node_id=node.id) self.list(node_id=node.id)
def dib_image_delete(self): def dib_image_delete(self):
(image, build_num) = self.args.id.rsplit('-', 1) (image, build_id) = self.args.id.rsplit('-', 1)
build = self.zk.getBuild(image, build_num) build = self.zk.getBuild(image, build_id)
if not build: if not build:
print("Build %s not found" % self.args.id) print("Build %s not found" % self.args.id)
return return

4
nodepool/status.py
View File

@ -204,7 +204,7 @@ def dib_image_list(zk):
for image_name in zk.getImageNames(): for image_name in zk.getImageNames():
image_paused[image_name] = \ image_paused[image_name] = \
zk.getImagePaused(image_name) zk.getImagePaused(image_name)
for build_no in zk.getBuildNumbers(image_name): for build_no in zk.getBuildIds(image_name):
build = zk.getBuild(image_name, build_no) build = zk.getBuild(image_name, build_no)
if build: if build:
builds.append(build) builds.append(build)
@ -263,7 +263,7 @@ def image_list(zk):
uploads = zk.getCachedImageUploads() uploads = zk.getCachedImageUploads()
else: else:
for image_name in zk.getImageNames(): for image_name in zk.getImageNames():
for build_no in zk.getBuildNumbers(image_name): for build_no in zk.getBuildIds(image_name):
for provider in zk.getBuildProviders(image_name, build_no): for provider in zk.getBuildProviders(image_name, build_no):
for upload_no in zk.getImageUploadNumbers( for upload_no in zk.getImageUploadNumbers(
image_name, build_no, provider): image_name, build_no, provider):

2
nodepool/tests/__init__.py
View File

@ -673,7 +673,7 @@ class DBTestCase(BaseTestCase):
self.waitForConfig(pool) self.waitForConfig(pool)
actual_uploads = set() actual_uploads = set()
for image_name in self.zk.getImageNames(): for image_name in self.zk.getImageNames():
for build_no in self.zk.getBuildNumbers(image_name): for build_no in self.zk.getBuildIds(image_name):
for provider in self.zk.getBuildProviders( for provider in self.zk.getBuildProviders(
image_name, build_no): image_name, build_no):
for upload_no in self.zk.getImageUploadNumbers( for upload_no in self.zk.getImageUploadNumbers(

54
nodepool/tests/unit/test_zk.py
View File

@ -112,29 +112,29 @@ class TestZooKeeper(tests.DBTestCase):
with self.zk.imageBuildLock(image, blocking=True, timeout=1): with self.zk.imageBuildLock(image, blocking=True, timeout=1):
pass pass
def test_imageBuildNumberLock(self): def test_imageBuildIdLock(self):
path = self.zk._imageBuildNumberLockPath("ubuntu-trusty", "0000") path = self.zk._imageBuildIdLockPath("ubuntu-trusty", "0000")
with self.zk.imageBuildNumberLock( with self.zk.imageBuildIdLock(
"ubuntu-trusty", "0000", blocking=False "ubuntu-trusty", "0000", blocking=False
): ):
self.assertIsNotNone(self.zk.kazoo_client.exists(path)) self.assertIsNotNone(self.zk.kazoo_client.exists(path))
def test_imageBuildNumberLock_exception_nonblocking(self): def test_imageBuildIdLock_exception_nonblocking(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
bnum = "0000000000" bnum = "0000000000"
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
with testtools.ExpectedException( with testtools.ExpectedException(
npe.ZKLockException, "Did not get lock on .*" npe.ZKLockException, "Did not get lock on .*"
): ):
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
pass pass
def test_imageBuildNumberLock_exception_blocking(self): def test_imageBuildIdLock_exception_blocking(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
bnum = "0000000000" bnum = "0000000000"
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
with testtools.ExpectedException(npe.TimeoutException): with testtools.ExpectedException(npe.TimeoutException):
with self.zk.imageBuildNumberLock( with self.zk.imageBuildIdLock(
image, bnum, blocking=True, timeout=1 image, bnum, blocking=True, timeout=1
): ):
pass pass
@ -271,22 +271,22 @@ class TestZooKeeper(tests.DBTestCase):
orig_data.builder_id = 'ABC-123' orig_data.builder_id = 'ABC-123'
orig_data.state = zk.READY orig_data.state = zk.READY
with self.zk.imageBuildLock(image, blocking=True, timeout=1): with self.zk.imageBuildLock(image, blocking=True, timeout=1):
build_num = self.zk.storeBuild(image, orig_data) build_id = self.zk.storeBuild(image, orig_data)
data = self.zk.getBuild(image, build_num) data = self.zk.getBuild(image, build_id)
self.assertEqual(orig_data.builder, data.builder) self.assertEqual(orig_data.builder, data.builder)
self.assertEqual(orig_data.builder_id, data.builder_id) self.assertEqual(orig_data.builder_id, data.builder_id)
self.assertEqual(orig_data.state, data.state) self.assertEqual(orig_data.state, data.state)
self.assertEqual(orig_data.state_time, data.state_time) self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(build_num, data.id) self.assertEqual(build_id, data.id)
self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"]) self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"])
self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), [build_num]) self.assertEqual(self.zk.getBuildIds("ubuntu-trusty"), [build_id])
def test_getImageNames_not_found(self): def test_getImageNames_not_found(self):
self.assertEqual(self.zk.getImageNames(), []) self.assertEqual(self.zk.getImageNames(), [])
def test_getBuildNumbers_not_found(self): def test_getBuildIds_not_found(self):
self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), []) self.assertEqual(self.zk.getBuildIds("ubuntu-trusty"), [])
def test_getBuildProviders_not_found(self): def test_getBuildProviders_not_found(self):
self.assertEqual(self.zk.getBuildProviders( self.assertEqual(self.zk.getBuildProviders(
@ -314,14 +314,14 @@ class TestZooKeeper(tests.DBTestCase):
def test_storeImageUpload_invalid_build(self): def test_storeImageUpload_invalid_build(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
build_number = "0000000001" build_id = uuid.uuid4().hex
provider = "rax" provider = "rax"
orig_data = zk.ImageUpload() orig_data = zk.ImageUpload()
with testtools.ExpectedException( with testtools.ExpectedException(
npe.ZKException, "Cannot find build .*" npe.ZKException, "Cannot find build .*"
): ):
self.zk.storeImageUpload(image, build_number, provider, orig_data) self.zk.storeImageUpload(image, build_id, provider, orig_data)
def test_store_and_get_image_upload(self): def test_store_and_get_image_upload(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
@ -331,10 +331,10 @@ class TestZooKeeper(tests.DBTestCase):
orig_data.state = zk.READY orig_data.state = zk.READY
orig_data.format = "qcow2" orig_data.format = "qcow2"
build_number = self.zk.storeBuild(image, zk.ImageBuild()) build_id = self.zk.storeBuild(image, zk.ImageBuild())
upload_id = self.zk.storeImageUpload(image, build_number, provider, upload_id = self.zk.storeImageUpload(image, build_id, provider,
orig_data) orig_data)
data = self.zk.getImageUpload(image, build_number, provider, upload_id) data = self.zk.getImageUpload(image, build_id, provider, upload_id)
self.assertEqual(upload_id, data.id) self.assertEqual(upload_id, data.id)
self.assertEqual(orig_data.external_id, data.external_id) self.assertEqual(orig_data.external_id, data.external_id)
@ -342,10 +342,10 @@ class TestZooKeeper(tests.DBTestCase):
self.assertEqual(orig_data.state_time, data.state_time) self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(orig_data.format, data.format) self.assertEqual(orig_data.format, data.format)
self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty", self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty",
build_number), build_id),
[provider]) [provider])
self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty", self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty",
build_number, build_id,
provider), provider),
[upload_id]) [upload_id])
@ -364,20 +364,20 @@ class TestZooKeeper(tests.DBTestCase):
def test_buildLock_orphan(self): def test_buildLock_orphan(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
build_number = "0000000003" build_id = uuid.uuid4().hex
path = self.zk._imageBuildNumberLockPath(image, build_number) path = self.zk._imageBuildIdLockPath(image, build_id)
# Pretend we still think the image build exists # Pretend we still think the image build exists
# (e.g. multiple cleanup workers itertating over builds) # (e.g. multiple cleanup workers itertating over builds)
with self.zk.imageBuildNumberLock(image, build_number, blocking=False): with self.zk.imageBuildIdLock(image, build_id, blocking=False):
# We now created an empty build number node # We now created an empty build id node
pass pass
self.assertIsNotNone(self.zk.kazoo_client.exists(path)) self.assertIsNotNone(self.zk.kazoo_client.exists(path))
# Should not throw an exception because of the empty upload # Should not throw an exception because of the empty upload
self.assertIsNone(self.zk.getBuild(image, build_number)) self.assertIsNone(self.zk.getBuild(image, build_id))
def test_getMostRecentBuilds(self): def test_getMostRecentBuilds(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"

184
nodepool/zk/zookeeper.py
View File

@ -1085,17 +1085,17 @@ class ImageCache(NodepoolTreeCache):
def objectFromDict(self, d, key): def objectFromDict(self, d, key):
if len(key) == 4: if len(key) == 4:
image, build_number, provider, upload_number = key image, build_id, provider, upload_number = key
return ImageUpload.fromDict(d, return ImageUpload.fromDict(d,
build_number, build_id,
provider, provider,
image, image,
upload_number) upload_number)
elif len(key) == 2: elif len(key) == 2:
image, build_number = key image, build_id = key
return ImageBuild.fromDict(d, return ImageBuild.fromDict(d,
image, image,
build_number) build_id)
elif len(key) == 1: elif len(key) == 1:
image = key[0] image = key[0]
return Image(image) return Image(image)
@ -1276,8 +1276,8 @@ class ZooKeeper(ZooKeeperBase):
image = parts[1] image = parts[1]
return (image,) return (image,)
def _imageBuildNumberPath(self, image, build_number): def _imageBuildIdPath(self, image, build_id):
return "%s/%s" % (self._imageBuildsPath(image), build_number) return "%s/%s" % (self._imageBuildsPath(image), build_id)
def _parseImageBuildPath(self, path): def _parseImageBuildPath(self, path):
if not path.startswith(self.IMAGE_ROOT): if not path.startswith(self.IMAGE_ROOT):
@ -1295,16 +1295,16 @@ class ZooKeeper(ZooKeeperBase):
def _imageBuildLockPath(self, image): def _imageBuildLockPath(self, image):
return "%s/lock" % self._imageBuildsPath(image) return "%s/lock" % self._imageBuildsPath(image)
def _imageBuildNumberLockPath(self, image, build_number): def _imageBuildIdLockPath(self, image, build_id):
return "%s/lock" % self._imageBuildNumberPath(image, build_number) return "%s/lock" % self._imageBuildIdPath(image, build_id)
def _imageProviderPath(self, image, build_number): def _imageProviderPath(self, image, build_id):
return "%s/%s/providers" % (self._imageBuildsPath(image), return "%s/%s/providers" % (self._imageBuildsPath(image),
build_number) build_id)
def _imageUploadPath(self, image, build_number, provider): def _imageUploadPath(self, image, build_id, provider):
return "%s/%s/providers/%s/images" % (self._imageBuildsPath(image), return "%s/%s/providers/%s/images" % (self._imageBuildsPath(image),
build_number, build_id,
provider) provider)
def _parseImageUploadPath(self, path): def _parseImageUploadPath(self, path):
@ -1322,14 +1322,14 @@ class ZooKeeper(ZooKeeperBase):
upload = parts[7] upload = parts[7]
return image, build, provider, upload return image, build, provider, upload
def _imageUploadLockPath(self, image, build_number, provider): def _imageUploadLockPath(self, image, build_id, provider):
return "%s/lock" % self._imageUploadPath(image, build_number, return "%s/lock" % self._imageUploadPath(image, build_id,
provider) provider)
def _imageUploadNumberLockPath(self, image, build_number, provider, def _imageUploadNumberLockPath(self, image, build_id, provider,
upload_number): upload_number):
return "%s/%s/lock" % ( return "%s/%s/lock" % (
self._imageUploadPath(image, build_number, provider), self._imageUploadPath(image, build_id, provider),
upload_number) upload_number)
def _launcherPath(self, launcher): def _launcherPath(self, launcher):
@ -1406,9 +1406,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageBuildNumberLock(self, image, build_number, def _getImageBuildIdLock(self, image, build_id,
blocking=True, timeout=None): blocking=True, timeout=None):
lock_path = self._imageBuildNumberLockPath(image, build_number) lock_path = self._imageBuildIdLockPath(image, build_id)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
have_lock = lock.acquire(blocking, timeout) have_lock = lock.acquire(blocking, timeout)
@ -1417,8 +1417,8 @@ class ZooKeeper(ZooKeeperBase):
"Timeout trying to acquire lock %s" % lock_path) "Timeout trying to acquire lock %s" % lock_path)
except kze.NoNodeError: except kze.NoNodeError:
have_lock = False have_lock = False
self.log.error("Image build number not found for locking: %s, %s", self.log.error("Image build id not found for locking: %s, %s",
build_number, image) build_id, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@ -1427,9 +1427,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageUploadNumberLock(self, image, build_number, provider, def _getImageUploadNumberLock(self, image, build_id, provider,
upload_number, blocking=True, timeout=None): upload_number, blocking=True, timeout=None):
lock_path = self._imageUploadNumberLockPath(image, build_number, lock_path = self._imageUploadNumberLockPath(image, build_id,
provider, upload_number) provider, upload_number)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
@ -1441,7 +1441,7 @@ class ZooKeeper(ZooKeeperBase):
have_lock = False have_lock = False
self.log.error("Image upload number %s not found for locking: " self.log.error("Image upload number %s not found for locking: "
"%s, %s, %s", "%s, %s, %s",
upload_number, build_number, provider, image) upload_number, build_id, provider, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@ -1450,9 +1450,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageUploadLock(self, image, build_number, provider, def _getImageUploadLock(self, image, build_id, provider,
blocking=True, timeout=None): blocking=True, timeout=None):
lock_path = self._imageUploadLockPath(image, build_number, provider) lock_path = self._imageUploadLockPath(image, build_id, provider)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
have_lock = lock.acquire(blocking, timeout) have_lock = lock.acquire(blocking, timeout)
@ -1462,7 +1462,7 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
have_lock = False have_lock = False
self.log.error("Image upload not found for locking: %s, %s, %s", self.log.error("Image upload not found for locking: %s, %s, %s",
build_number, provider, image) build_id, provider, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@ -1540,17 +1540,17 @@ class ZooKeeper(ZooKeeperBase):
lock.release() lock.release()
@contextmanager @contextmanager
def imageBuildNumberLock(self, image, build_number, def imageBuildIdLock(self, image, build_id,
blocking=True, timeout=None): blocking=True, timeout=None):
''' '''
Context manager to use for locking _specific_ image builds. Context manager to use for locking _specific_ image builds.
Obtains a write lock for the specified image build number. This is Obtains a write lock for the specified image build id. This is
used for locking a build number during the cleanup phase of the used for locking a build id during the cleanup phase of the
builder. builder.
:param str image: Name of the image :param str image: Name of the image
:param str build_number: The image build number to lock. :param str build_id: The image build id to lock.
:param bool blocking: Whether or not to block on trying to :param bool blocking: Whether or not to block on trying to
acquire the lock acquire the lock
:param int timeout: When blocking, how long to wait for the lock :param int timeout: When blocking, how long to wait for the lock
@ -1562,15 +1562,15 @@ class ZooKeeper(ZooKeeperBase):
''' '''
lock = None lock = None
try: try:
lock = self._getImageBuildNumberLock(image, build_number, lock = self._getImageBuildIdLock(image, build_id,
blocking, timeout) blocking, timeout)
yield yield
finally: finally:
if lock: if lock:
lock.release() lock.release()
@contextmanager @contextmanager
def imageUploadLock(self, image, build_number, provider, def imageUploadLock(self, image, build_id, provider,
blocking=True, timeout=None): blocking=True, timeout=None):
''' '''
Context manager to use for locking image uploads. Context manager to use for locking image uploads.
@ -1578,7 +1578,7 @@ class ZooKeeper(ZooKeeperBase):
Obtains a write lock for the specified image upload. Obtains a write lock for the specified image upload.
:param str image: Name of the image. :param str image: Name of the image.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param bool blocking: Whether or not to block on trying to :param bool blocking: Whether or not to block on trying to
acquire the lock acquire the lock
@ -1591,7 +1591,7 @@ class ZooKeeper(ZooKeeperBase):
''' '''
lock = None lock = None
try: try:
lock = self._getImageUploadLock(image, build_number, provider, lock = self._getImageUploadLock(image, build_id, provider,
blocking, timeout) blocking, timeout)
yield yield
finally: finally:
@ -1683,13 +1683,13 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
pass pass
def getBuildNumbers(self, image): def getBuildIds(self, image):
''' '''
Retrieve the builds available for an image. Retrieve the builds available for an image.
:param str image: The image name. :param str image: The image name.
:returns: A list of image build numbers or the empty list. :returns: A list of image build ids or the empty list.
''' '''
path = self._imageBuildsPath(image) path = self._imageBuildsPath(image)
@ -1700,17 +1700,17 @@ class ZooKeeper(ZooKeeperBase):
builds = [x for x in builds if x != 'lock'] builds = [x for x in builds if x != 'lock']
return builds return builds
def getBuildProviders(self, image, build_number): def getBuildProviders(self, image, build_id):
''' '''
Retrieve the providers which have uploads for an image build. Retrieve the providers which have uploads for an image build.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: A list of provider names or the empty list. :returns: A list of provider names or the empty list.
''' '''
path = self._imageProviderPath(image, build_number) path = self._imageProviderPath(image, build_id)
try: try:
providers = self.kazoo_client.get_children(path) providers = self.kazoo_client.get_children(path)
@ -1719,18 +1719,18 @@ class ZooKeeper(ZooKeeperBase):
return sorted(providers) return sorted(providers)
def getImageUploadNumbers(self, image, build_number, provider): def getImageUploadNumbers(self, image, build_id, provider):
''' '''
Retrieve upload numbers for a provider and image build. Retrieve upload numbers for a provider and image build.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:returns: A list of upload numbers or the empty list. :returns: A list of upload numbers or the empty list.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
uploads = self.kazoo_client.get_children(path) uploads = self.kazoo_client.get_children(path)
@ -1740,16 +1740,16 @@ class ZooKeeper(ZooKeeperBase):
uploads = [x for x in uploads if x != 'lock'] uploads = [x for x in uploads if x != 'lock']
return uploads return uploads
def getBuild(self, image, build_number): def getBuild(self, image, build_id):
''' '''
Retrieve the image build data. Retrieve the image build data.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: An ImageBuild object, or None if not found. :returns: An ImageBuild object, or None if not found.
''' '''
path = self._imageBuildsPath(image) + "/%s" % build_number path = self._imageBuildsPath(image) + "/%s" % build_id
try: try:
data, stat = self.kazoo_client.get(path) data, stat = self.kazoo_client.get(path)
@ -1758,7 +1758,7 @@ class ZooKeeper(ZooKeeperBase):
try: try:
d = ImageBuild.fromDict(self._bytesToDict(data), d = ImageBuild.fromDict(self._bytesToDict(data),
image, build_number) image, build_id)
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
self.log.exception('Error loading json data from image build %s', self.log.exception('Error loading json data from image build %s',
path) path)
@ -1832,7 +1832,7 @@ class ZooKeeper(ZooKeeperBase):
builds.sort(key=lambda x: x.state_time, reverse=True) builds.sort(key=lambda x: x.state_time, reverse=True)
return builds[:count] return builds[:count]
def storeBuild(self, image, build_data, build_number=None): def storeBuild(self, image, build_data, build_id=None):
''' '''
Store the image build data. Store the image build data.
@ -1841,40 +1841,40 @@ class ZooKeeper(ZooKeeperBase):
The build data is expected to be represented as a dict. This dict may The build data is expected to be represented as a dict. This dict may
contain any data, as appropriate. contain any data, as appropriate.
If a build number is not supplied, then a new build node/number is If a build id is not supplied, then a new build node/id is
created. The new build number is available in the return value. created. The new build id is available in the return value.
.. important: You should have the image locked before calling this .. important: You should have the image locked before calling this
method. method.
:param str image: The image name for which we have data. :param str image: The image name for which we have data.
:param ImageBuild build_data: The build data. :param ImageBuild build_data: The build data.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: A string for the build number that was updated. :returns: A string for the build id that was updated.
''' '''
# Append trailing / so the sequence node is created as a child node. # Append trailing / so the sequence node is created as a child node.
build_path = self._imageBuildsPath(image) + "/" build_path = self._imageBuildsPath(image) + "/"
if build_number is None: if build_id is None:
build_number = uuid.uuid4().hex build_id = uuid.uuid4().hex
path = build_path + build_number path = build_path + build_id
self.kazoo_client.create( self.kazoo_client.create(
path, path,
value=build_data.serialize(), value=build_data.serialize(),
makepath=True) makepath=True)
else: else:
path = build_path + build_number path = build_path + build_id
self.kazoo_client.set(path, build_data.serialize()) self.kazoo_client.set(path, build_data.serialize())
return build_number return build_id
def getImageUpload(self, image, build_number, provider, upload_number): def getImageUpload(self, image, build_id, provider, upload_number):
''' '''
Retrieve the image upload data. Retrieve the image upload data.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str upload_number: The image upload number. :param str upload_number: The image upload number.
@ -1882,7 +1882,7 @@ class ZooKeeper(ZooKeeperBase):
:raises: ZKException if the image upload path is not found. :raises: ZKException if the image upload path is not found.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
path = path + "/%s" % upload_number path = path + "/%s" % upload_number
try: try:
@ -1892,7 +1892,7 @@ class ZooKeeper(ZooKeeperBase):
try: try:
d = ImageUpload.fromDict(self._bytesToDict(data), d = ImageUpload.fromDict(self._bytesToDict(data),
build_number, build_id,
provider, provider,
image, image,
upload_number) upload_number)
@ -1903,12 +1903,12 @@ class ZooKeeper(ZooKeeperBase):
d.stat = stat d.stat = stat
return d return d
def getUploads(self, image, build_number, provider, states=None): def getUploads(self, image, build_id, provider, states=None):
''' '''
Retrieve all image upload data matching any given states. Retrieve all image upload data matching any given states.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param list states: A list of upload state values to match against. :param list states: A list of upload state values to match against.
A value of None will disable state matching and just return A value of None will disable state matching and just return
@ -1916,7 +1916,7 @@ class ZooKeeper(ZooKeeperBase):
:returns: A list of ImageUpload objects. :returns: A list of ImageUpload objects.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
uploads = self.kazoo_client.get_children(path) uploads = self.kazoo_client.get_children(path)
@ -1927,7 +1927,7 @@ class ZooKeeper(ZooKeeperBase):
for upload in uploads: for upload in uploads:
if upload == 'lock': if upload == 'lock':
continue continue
data = self.getImageUpload(image, build_number, provider, upload) data = self.getImageUpload(image, build_id, provider, upload)
if not data: if not data:
continue continue
if states is None: if states is None:
@ -1937,7 +1937,7 @@ class ZooKeeper(ZooKeeperBase):
return matches return matches
def getMostRecentBuildImageUploads(self, count, image, build_number, def getMostRecentBuildImageUploads(self, count, image, build_id,
provider, state=None): provider, state=None):
''' '''
Retrieve the most recent image upload data with the given state. Retrieve the most recent image upload data with the given state.
@ -1945,7 +1945,7 @@ class ZooKeeper(ZooKeeperBase):
:param int count: A count of the most recent uploads to return. :param int count: A count of the most recent uploads to return.
Use None for all uploads. Use None for all uploads.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str state: The image upload state to match on. Use None to :param str state: The image upload state to match on. Use None to
ignore state. ignore state.
@ -1958,7 +1958,7 @@ class ZooKeeper(ZooKeeperBase):
if state: if state:
states = [state] states = [state]
uploads = self.getUploads(image, build_number, provider, states) uploads = self.getUploads(image, build_id, provider, states)
if not uploads: if not uploads:
return [] return []
@ -1983,8 +1983,8 @@ class ZooKeeper(ZooKeeperBase):
if cached: if cached:
uploads = self.getCachedImageUploads() uploads = self.getCachedImageUploads()
else: else:
for build_number in self.getBuildNumbers(image): for build_id in self.getBuildIds(image):
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
upload_numbers = self.kazoo_client.get_children(path) upload_numbers = self.kazoo_client.get_children(path)
except kze.NoNodeError: except kze.NoNodeError:
@ -1994,7 +1994,7 @@ class ZooKeeper(ZooKeeperBase):
if upload_number == 'lock': # skip the upload lock node if upload_number == 'lock': # skip the upload lock node
continue continue
data = self.getImageUpload( data = self.getImageUpload(
image, build_number, provider, upload_number) image, build_id, provider, upload_number)
if not data or data.state != state: if not data or data.state != state:
continue continue
uploads.append(data) uploads.append(data)
@ -2023,7 +2023,7 @@ class ZooKeeper(ZooKeeperBase):
raise RuntimeError("Caching not enabled") raise RuntimeError("Caching not enabled")
return self._image_cache.getUploads() return self._image_cache.getUploads()
def storeImageUpload(self, image, build_number, provider, image_data, def storeImageUpload(self, image, build_id, provider, image_data,
upload_number=None): upload_number=None):
''' '''
Store the built image's upload data for the given provider. Store the built image's upload data for the given provider.
@ -2038,7 +2038,7 @@ class ZooKeeper(ZooKeeperBase):
return value. return value.
:param str image: The image name for which we have data. :param str image: The image name for which we have data.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param ImageUpload image_data: The image data we want to store. :param ImageUpload image_data: The image data we want to store.
:param str upload_number: The image upload number to update. :param str upload_number: The image upload number to update.
@ -2051,14 +2051,14 @@ class ZooKeeper(ZooKeeperBase):
build_path = self._imageBuildsPath(image) build_path = self._imageBuildsPath(image)
if not self.kazoo_client.exists(build_path): if not self.kazoo_client.exists(build_path):
raise npe.ZKException( raise npe.ZKException(
"Cannot find build %s of image %s" % (build_number, image) "Cannot find build %s of image %s" % (build_id, image)
) )
# Generate a path for the upload. This doesn't have to exist yet # Generate a path for the upload. This doesn't have to exist yet
# since we'll create new provider/upload ID znodes automatically. # since we'll create new provider/upload ID znodes automatically.
# Append trailing / so the sequence node is created as a child node. # Append trailing / so the sequence node is created as a child node.
upload_path = self._imageUploadPath( upload_path = self._imageUploadPath(
image, build_number, provider) + "/" image, build_id, provider) + "/"
if upload_number is None: if upload_number is None:
path = self.kazoo_client.create( path = self.kazoo_client.create(
@ -2087,12 +2087,12 @@ class ZooKeeper(ZooKeeperBase):
return False return False
def _latestImageBuildStat(self, image): def _latestImageBuildStat(self, image):
builds = self.getBuildNumbers(image) builds = self.getBuildIds(image)
if not builds: if not builds:
return return
latest_build, *_ = builds latest_build, *_ = builds
builds_path = self._imageBuildNumberPath(image, latest_build) builds_path = self._imageBuildIdPath(image, latest_build)
return self.kazoo_client.exists(builds_path) return self.kazoo_client.exists(builds_path)
def getBuildRequest(self, image): def getBuildRequest(self, image):
@ -2143,7 +2143,7 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
pass pass
def deleteBuild(self, image, build_number): def deleteBuild(self, image, build_id):
''' '''
Delete an image build from ZooKeeper. Delete an image build from ZooKeeper.
@ -2151,17 +2151,17 @@ class ZooKeeper(ZooKeeperBase):
node can be deleted. node can be deleted.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number to delete. :param str build_id: The image build id to delete.
:returns: True if the build is successfully deleted or did not exist, :returns: True if the build is successfully deleted or did not exist,
False if the provider uploads still exist. False if the provider uploads still exist.
''' '''
path = self._imageBuildsPath(image) path = self._imageBuildsPath(image)
path = path + "/%s" % build_number path = path + "/%s" % build_id
# Verify that no upload znodes exist. # Verify that no upload znodes exist.
for prov in self.getBuildProviders(image, build_number): for prov in self.getBuildProviders(image, build_id):
if self.getImageUploadNumbers(image, build_number, prov): if self.getImageUploadNumbers(image, build_id, prov):
return False return False
try: try:
@ -2172,16 +2172,16 @@ class ZooKeeper(ZooKeeperBase):
return True return True
def deleteUpload(self, image, build_number, provider, upload_number): def deleteUpload(self, image, build_id, provider, upload_number):
''' '''
Delete an image upload from ZooKeeper. Delete an image upload from ZooKeeper.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str upload_number: The image upload number to delete. :param str upload_number: The image upload number to delete.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
path = path + "/%s" % upload_number path = path + "/%s" % upload_number
try: try:
# NOTE: Need to do recursively to remove lock znodes # NOTE: Need to do recursively to remove lock znodes
@ -2834,8 +2834,8 @@ class ZooKeeper(ZooKeeperBase):
provider_builds = {} provider_builds = {}
image_names = self.getImageNames() image_names = self.getImageNames()
for image in image_names: for image in image_names:
build_numbers = self.getBuildNumbers(image) build_ids = self.getBuildIds(image)
for build in build_numbers: for build in build_ids:
providers = self.getBuildProviders(image, build) providers = self.getBuildProviders(image, build)
for p in providers: for p in providers:
if p == provider_name: if p == provider_name:
@ -2855,8 +2855,8 @@ class ZooKeeper(ZooKeeperBase):
provider_uploads = {} provider_uploads = {}
image_names = self.getImageNames() image_names = self.getImageNames()
for image in image_names: for image in image_names:
build_numbers = self.getBuildNumbers(image) build_ids = self.getBuildIds(image)
for build in build_numbers: for build in build_ids:
# If this build is not valid for this provider, move along. # If this build is not valid for this provider, move along.
if provider_name not in self.getBuildProviders(image, build): if provider_name not in self.getBuildProviders(image, build):
continue continue
@ -2928,7 +2928,7 @@ class ZooKeeper(ZooKeeperBase):
if paused: if paused:
paused_path = self._imagePausePath(image_name) paused_path = self._imagePausePath(image_name)
ret[paused_path] = '' ret[paused_path] = ''
for build_no in self.getBuildNumbers(image_name): for build_no in self.getBuildIds(image_name):
build_path = self._imageBuildsPath(image_name) + "/" + build_no build_path = self._imageBuildsPath(image_name) + "/" + build_no
try: try:
build_data, stat = self.kazoo_client.get(build_path) build_data, stat = self.kazoo_client.get(build_path)