zuul/nodepool
Code Issues Proposed changes

Update references of build "number" to "id"

Browse Source 
This follows the previous change and is intended to have little
or no behavior changes (only a few unit tests are updated to use
different placeholder values).  It updates all textual references
of build numbers to build ids to better reflect that they are
UUIDs instead of integers.

Change-Id: I04b5eec732918f5b9b712f8caab2ea4ec90e9a9f
This commit is contained in:
James E. Blair
2023-08-02 10:56:43 -07:00
parent 3815cce7aa
commit 4ef3ebade8
6 changed files with 126 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
nodepool/builder.py
View File

@@ -548,7 +548,7 @@ class CleanupWorker(BaseWorker):
# images. We're about to start deleting files, so # images. We're about to start deleting files, so
# make sure that the dib image state reflects that. # make sure that the dib image state reflects that.
if build.state != zk.DELETING: if build.state != zk.DELETING:
with self._zk.imageBuildNumberLock( with self._zk.imageBuildIdLock(
image, build.id, blocking=False image, build.id, blocking=False
): ):
build.state = zk.DELETING build.state = zk.DELETING

6
nodepool/cmd/nodepoolcmd.py
View File

@@ -266,7 +266,7 @@ class NodePoolCmd(NodepoolApp):
uploads = [] uploads = []
for image in provider.diskimages: for image in provider.diskimages:
# Build list of provider images as recorded in ZK # Build list of provider images as recorded in ZK
for bnum in self.zk.getBuildNumbers(image): for bnum in self.zk.getBuildIds(image):
uploads.extend( uploads.extend(
self.zk.getUploads(image, bnum, self.zk.getUploads(image, bnum,
provider.name, provider.name,
@@ -325,8 +325,8 @@ class NodePoolCmd(NodepoolApp):
self.list(node_id=node.id) self.list(node_id=node.id)
def dib_image_delete(self): def dib_image_delete(self):
(image, build_num) = self.args.id.rsplit('-', 1) (image, build_id) = self.args.id.rsplit('-', 1)
build = self.zk.getBuild(image, build_num) build = self.zk.getBuild(image, build_id)
if not build: if not build:
print("Build %s not found" % self.args.id) print("Build %s not found" % self.args.id)
return return

4
nodepool/status.py
View File

@@ -204,7 +204,7 @@ def dib_image_list(zk):
for image_name in zk.getImageNames(): for image_name in zk.getImageNames():
image_paused[image_name] = \ image_paused[image_name] = \
zk.getImagePaused(image_name) zk.getImagePaused(image_name)
for build_no in zk.getBuildNumbers(image_name): for build_no in zk.getBuildIds(image_name):
build = zk.getBuild(image_name, build_no) build = zk.getBuild(image_name, build_no)
if build: if build:
builds.append(build) builds.append(build)
@@ -263,7 +263,7 @@ def image_list(zk):
uploads = zk.getCachedImageUploads() uploads = zk.getCachedImageUploads()
else: else:
for image_name in zk.getImageNames(): for image_name in zk.getImageNames():
for build_no in zk.getBuildNumbers(image_name): for build_no in zk.getBuildIds(image_name):
for provider in zk.getBuildProviders(image_name, build_no): for provider in zk.getBuildProviders(image_name, build_no):
for upload_no in zk.getImageUploadNumbers( for upload_no in zk.getImageUploadNumbers(
image_name, build_no, provider): image_name, build_no, provider):

2
nodepool/tests/__init__.py
View File

@@ -673,7 +673,7 @@ class DBTestCase(BaseTestCase):
self.waitForConfig(pool) self.waitForConfig(pool)
actual_uploads = set() actual_uploads = set()
for image_name in self.zk.getImageNames(): for image_name in self.zk.getImageNames():
for build_no in self.zk.getBuildNumbers(image_name): for build_no in self.zk.getBuildIds(image_name):
for provider in self.zk.getBuildProviders( for provider in self.zk.getBuildProviders(
image_name, build_no): image_name, build_no):
for upload_no in self.zk.getImageUploadNumbers( for upload_no in self.zk.getImageUploadNumbers(

54
nodepool/tests/unit/test_zk.py
View File

@@ -112,29 +112,29 @@ class TestZooKeeper(tests.DBTestCase):
with self.zk.imageBuildLock(image, blocking=True, timeout=1): with self.zk.imageBuildLock(image, blocking=True, timeout=1):
pass pass
def test_imageBuildNumberLock(self): def test_imageBuildIdLock(self):
path = self.zk._imageBuildNumberLockPath("ubuntu-trusty", "0000") path = self.zk._imageBuildIdLockPath("ubuntu-trusty", "0000")
with self.zk.imageBuildNumberLock( with self.zk.imageBuildIdLock(
"ubuntu-trusty", "0000", blocking=False "ubuntu-trusty", "0000", blocking=False
): ):
self.assertIsNotNone(self.zk.kazoo_client.exists(path)) self.assertIsNotNone(self.zk.kazoo_client.exists(path))
def test_imageBuildNumberLock_exception_nonblocking(self): def test_imageBuildIdLock_exception_nonblocking(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
bnum = "0000000000" bnum = "0000000000"
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
with testtools.ExpectedException( with testtools.ExpectedException(
npe.ZKLockException, "Did not get lock on .*" npe.ZKLockException, "Did not get lock on .*"
): ):
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
pass pass
def test_imageBuildNumberLock_exception_blocking(self): def test_imageBuildIdLock_exception_blocking(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
bnum = "0000000000" bnum = "0000000000"
with self.zk.imageBuildNumberLock(image, bnum, blocking=False): with self.zk.imageBuildIdLock(image, bnum, blocking=False):
with testtools.ExpectedException(npe.TimeoutException): with testtools.ExpectedException(npe.TimeoutException):
with self.zk.imageBuildNumberLock( with self.zk.imageBuildIdLock(
image, bnum, blocking=True, timeout=1 image, bnum, blocking=True, timeout=1
): ):
pass pass
@@ -271,22 +271,22 @@ class TestZooKeeper(tests.DBTestCase):
orig_data.builder_id = 'ABC-123' orig_data.builder_id = 'ABC-123'
orig_data.state = zk.READY orig_data.state = zk.READY
with self.zk.imageBuildLock(image, blocking=True, timeout=1): with self.zk.imageBuildLock(image, blocking=True, timeout=1):
build_num = self.zk.storeBuild(image, orig_data) build_id = self.zk.storeBuild(image, orig_data)
data = self.zk.getBuild(image, build_num) data = self.zk.getBuild(image, build_id)
self.assertEqual(orig_data.builder, data.builder) self.assertEqual(orig_data.builder, data.builder)
self.assertEqual(orig_data.builder_id, data.builder_id) self.assertEqual(orig_data.builder_id, data.builder_id)
self.assertEqual(orig_data.state, data.state) self.assertEqual(orig_data.state, data.state)
self.assertEqual(orig_data.state_time, data.state_time) self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(build_num, data.id) self.assertEqual(build_id, data.id)
self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"]) self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"])
self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), [build_num]) self.assertEqual(self.zk.getBuildIds("ubuntu-trusty"), [build_id])
def test_getImageNames_not_found(self): def test_getImageNames_not_found(self):
self.assertEqual(self.zk.getImageNames(), []) self.assertEqual(self.zk.getImageNames(), [])
def test_getBuildNumbers_not_found(self): def test_getBuildIds_not_found(self):
self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), []) self.assertEqual(self.zk.getBuildIds("ubuntu-trusty"), [])
def test_getBuildProviders_not_found(self): def test_getBuildProviders_not_found(self):
self.assertEqual(self.zk.getBuildProviders( self.assertEqual(self.zk.getBuildProviders(
@@ -314,14 +314,14 @@ class TestZooKeeper(tests.DBTestCase):
def test_storeImageUpload_invalid_build(self): def test_storeImageUpload_invalid_build(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
build_number = "0000000001" build_id = uuid.uuid4().hex
provider = "rax" provider = "rax"
orig_data = zk.ImageUpload() orig_data = zk.ImageUpload()
with testtools.ExpectedException( with testtools.ExpectedException(
npe.ZKException, "Cannot find build .*" npe.ZKException, "Cannot find build .*"
): ):
self.zk.storeImageUpload(image, build_number, provider, orig_data) self.zk.storeImageUpload(image, build_id, provider, orig_data)
def test_store_and_get_image_upload(self): def test_store_and_get_image_upload(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
@@ -331,10 +331,10 @@ class TestZooKeeper(tests.DBTestCase):
orig_data.state = zk.READY orig_data.state = zk.READY
orig_data.format = "qcow2" orig_data.format = "qcow2"
build_number = self.zk.storeBuild(image, zk.ImageBuild()) build_id = self.zk.storeBuild(image, zk.ImageBuild())
upload_id = self.zk.storeImageUpload(image, build_number, provider, upload_id = self.zk.storeImageUpload(image, build_id, provider,
orig_data) orig_data)
data = self.zk.getImageUpload(image, build_number, provider, upload_id) data = self.zk.getImageUpload(image, build_id, provider, upload_id)
self.assertEqual(upload_id, data.id) self.assertEqual(upload_id, data.id)
self.assertEqual(orig_data.external_id, data.external_id) self.assertEqual(orig_data.external_id, data.external_id)
@@ -342,10 +342,10 @@ class TestZooKeeper(tests.DBTestCase):
self.assertEqual(orig_data.state_time, data.state_time) self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(orig_data.format, data.format) self.assertEqual(orig_data.format, data.format)
self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty", self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty",
build_number), build_id),
[provider]) [provider])
self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty", self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty",
build_number, build_id,
provider), provider),
[upload_id]) [upload_id])
@@ -364,20 +364,20 @@ class TestZooKeeper(tests.DBTestCase):
def test_buildLock_orphan(self): def test_buildLock_orphan(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"
build_number = "0000000003" build_id = uuid.uuid4().hex
path = self.zk._imageBuildNumberLockPath(image, build_number) path = self.zk._imageBuildIdLockPath(image, build_id)
# Pretend we still think the image build exists # Pretend we still think the image build exists
# (e.g. multiple cleanup workers itertating over builds) # (e.g. multiple cleanup workers itertating over builds)
with self.zk.imageBuildNumberLock(image, build_number, blocking=False): with self.zk.imageBuildIdLock(image, build_id, blocking=False):
# We now created an empty build number node # We now created an empty build id node
pass pass
self.assertIsNotNone(self.zk.kazoo_client.exists(path)) self.assertIsNotNone(self.zk.kazoo_client.exists(path))
# Should not throw an exception because of the empty upload # Should not throw an exception because of the empty upload
self.assertIsNone(self.zk.getBuild(image, build_number)) self.assertIsNone(self.zk.getBuild(image, build_id))
def test_getMostRecentBuilds(self): def test_getMostRecentBuilds(self):
image = "ubuntu-trusty" image = "ubuntu-trusty"

184
nodepool/zk/zookeeper.py
View File

@@ -1085,17 +1085,17 @@ class ImageCache(NodepoolTreeCache):
def objectFromDict(self, d, key): def objectFromDict(self, d, key):
if len(key) == 4: if len(key) == 4:
image, build_number, provider, upload_number = key image, build_id, provider, upload_number = key
return ImageUpload.fromDict(d, return ImageUpload.fromDict(d,
build_number, build_id,
provider, provider,
image, image,
upload_number) upload_number)
elif len(key) == 2: elif len(key) == 2:
image, build_number = key image, build_id = key
return ImageBuild.fromDict(d, return ImageBuild.fromDict(d,
image, image,
build_number) build_id)
elif len(key) == 1: elif len(key) == 1:
image = key[0] image = key[0]
return Image(image) return Image(image)
@@ -1276,8 +1276,8 @@ class ZooKeeper(ZooKeeperBase):
image = parts[1] image = parts[1]
return (image,) return (image,)
def _imageBuildNumberPath(self, image, build_number): def _imageBuildIdPath(self, image, build_id):
return "%s/%s" % (self._imageBuildsPath(image), build_number) return "%s/%s" % (self._imageBuildsPath(image), build_id)
def _parseImageBuildPath(self, path): def _parseImageBuildPath(self, path):
if not path.startswith(self.IMAGE_ROOT): if not path.startswith(self.IMAGE_ROOT):
@@ -1295,16 +1295,16 @@ class ZooKeeper(ZooKeeperBase):
def _imageBuildLockPath(self, image): def _imageBuildLockPath(self, image):
return "%s/lock" % self._imageBuildsPath(image) return "%s/lock" % self._imageBuildsPath(image)
def _imageBuildNumberLockPath(self, image, build_number): def _imageBuildIdLockPath(self, image, build_id):
return "%s/lock" % self._imageBuildNumberPath(image, build_number) return "%s/lock" % self._imageBuildIdPath(image, build_id)
def _imageProviderPath(self, image, build_number): def _imageProviderPath(self, image, build_id):
return "%s/%s/providers" % (self._imageBuildsPath(image), return "%s/%s/providers" % (self._imageBuildsPath(image),
build_number) build_id)
def _imageUploadPath(self, image, build_number, provider): def _imageUploadPath(self, image, build_id, provider):
return "%s/%s/providers/%s/images" % (self._imageBuildsPath(image), return "%s/%s/providers/%s/images" % (self._imageBuildsPath(image),
build_number, build_id,
provider) provider)
def _parseImageUploadPath(self, path): def _parseImageUploadPath(self, path):
@@ -1322,14 +1322,14 @@ class ZooKeeper(ZooKeeperBase):
upload = parts[7] upload = parts[7]
return image, build, provider, upload return image, build, provider, upload
def _imageUploadLockPath(self, image, build_number, provider): def _imageUploadLockPath(self, image, build_id, provider):
return "%s/lock" % self._imageUploadPath(image, build_number, return "%s/lock" % self._imageUploadPath(image, build_id,
provider) provider)
def _imageUploadNumberLockPath(self, image, build_number, provider, def _imageUploadNumberLockPath(self, image, build_id, provider,
upload_number): upload_number):
return "%s/%s/lock" % ( return "%s/%s/lock" % (
self._imageUploadPath(image, build_number, provider), self._imageUploadPath(image, build_id, provider),
upload_number) upload_number)
def _launcherPath(self, launcher): def _launcherPath(self, launcher):
@@ -1406,9 +1406,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageBuildNumberLock(self, image, build_number, def _getImageBuildIdLock(self, image, build_id,
blocking=True, timeout=None): blocking=True, timeout=None):
lock_path = self._imageBuildNumberLockPath(image, build_number) lock_path = self._imageBuildIdLockPath(image, build_id)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
have_lock = lock.acquire(blocking, timeout) have_lock = lock.acquire(blocking, timeout)
@@ -1417,8 +1417,8 @@ class ZooKeeper(ZooKeeperBase):
"Timeout trying to acquire lock %s" % lock_path) "Timeout trying to acquire lock %s" % lock_path)
except kze.NoNodeError: except kze.NoNodeError:
have_lock = False have_lock = False
self.log.error("Image build number not found for locking: %s, %s", self.log.error("Image build id not found for locking: %s, %s",
build_number, image) build_id, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@@ -1427,9 +1427,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageUploadNumberLock(self, image, build_number, provider, def _getImageUploadNumberLock(self, image, build_id, provider,
upload_number, blocking=True, timeout=None): upload_number, blocking=True, timeout=None):
lock_path = self._imageUploadNumberLockPath(image, build_number, lock_path = self._imageUploadNumberLockPath(image, build_id,
provider, upload_number) provider, upload_number)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
@@ -1441,7 +1441,7 @@ class ZooKeeper(ZooKeeperBase):
have_lock = False have_lock = False
self.log.error("Image upload number %s not found for locking: " self.log.error("Image upload number %s not found for locking: "
"%s, %s, %s", "%s, %s, %s",
upload_number, build_number, provider, image) upload_number, build_id, provider, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@@ -1450,9 +1450,9 @@ class ZooKeeper(ZooKeeperBase):
return lock return lock
def _getImageUploadLock(self, image, build_number, provider, def _getImageUploadLock(self, image, build_id, provider,
blocking=True, timeout=None): blocking=True, timeout=None):
lock_path = self._imageUploadLockPath(image, build_number, provider) lock_path = self._imageUploadLockPath(image, build_id, provider)
try: try:
lock = Lock(self.kazoo_client, lock_path) lock = Lock(self.kazoo_client, lock_path)
have_lock = lock.acquire(blocking, timeout) have_lock = lock.acquire(blocking, timeout)
@@ -1462,7 +1462,7 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
have_lock = False have_lock = False
self.log.error("Image upload not found for locking: %s, %s, %s", self.log.error("Image upload not found for locking: %s, %s, %s",
build_number, provider, image) build_id, provider, image)
# If we aren't blocking, it's possible we didn't get the lock # If we aren't blocking, it's possible we didn't get the lock
# because someone else has it. # because someone else has it.
@@ -1540,17 +1540,17 @@ class ZooKeeper(ZooKeeperBase):
lock.release() lock.release()
@contextmanager @contextmanager
def imageBuildNumberLock(self, image, build_number, def imageBuildIdLock(self, image, build_id,
blocking=True, timeout=None): blocking=True, timeout=None):
''' '''
Context manager to use for locking _specific_ image builds. Context manager to use for locking _specific_ image builds.
Obtains a write lock for the specified image build number. This is Obtains a write lock for the specified image build id. This is
used for locking a build number during the cleanup phase of the used for locking a build id during the cleanup phase of the
builder. builder.
:param str image: Name of the image :param str image: Name of the image
:param str build_number: The image build number to lock. :param str build_id: The image build id to lock.
:param bool blocking: Whether or not to block on trying to :param bool blocking: Whether or not to block on trying to
acquire the lock acquire the lock
:param int timeout: When blocking, how long to wait for the lock :param int timeout: When blocking, how long to wait for the lock
@@ -1562,15 +1562,15 @@ class ZooKeeper(ZooKeeperBase):
''' '''
lock = None lock = None
try: try:
lock = self._getImageBuildNumberLock(image, build_number, lock = self._getImageBuildIdLock(image, build_id,
blocking, timeout) blocking, timeout)
yield yield
finally: finally:
if lock: if lock:
lock.release() lock.release()
@contextmanager @contextmanager
def imageUploadLock(self, image, build_number, provider, def imageUploadLock(self, image, build_id, provider,
blocking=True, timeout=None): blocking=True, timeout=None):
''' '''
Context manager to use for locking image uploads. Context manager to use for locking image uploads.
@@ -1578,7 +1578,7 @@ class ZooKeeper(ZooKeeperBase):
Obtains a write lock for the specified image upload. Obtains a write lock for the specified image upload.
:param str image: Name of the image. :param str image: Name of the image.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param bool blocking: Whether or not to block on trying to :param bool blocking: Whether or not to block on trying to
acquire the lock acquire the lock
@@ -1591,7 +1591,7 @@ class ZooKeeper(ZooKeeperBase):
''' '''
lock = None lock = None
try: try:
lock = self._getImageUploadLock(image, build_number, provider, lock = self._getImageUploadLock(image, build_id, provider,
blocking, timeout) blocking, timeout)
yield yield
finally: finally:
@@ -1683,13 +1683,13 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
pass pass
def getBuildNumbers(self, image): def getBuildIds(self, image):
''' '''
Retrieve the builds available for an image. Retrieve the builds available for an image.
:param str image: The image name. :param str image: The image name.
:returns: A list of image build numbers or the empty list. :returns: A list of image build ids or the empty list.
''' '''
path = self._imageBuildsPath(image) path = self._imageBuildsPath(image)
@@ -1700,17 +1700,17 @@ class ZooKeeper(ZooKeeperBase):
builds = [x for x in builds if x != 'lock'] builds = [x for x in builds if x != 'lock']
return builds return builds
def getBuildProviders(self, image, build_number): def getBuildProviders(self, image, build_id):
''' '''
Retrieve the providers which have uploads for an image build. Retrieve the providers which have uploads for an image build.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: A list of provider names or the empty list. :returns: A list of provider names or the empty list.
''' '''
path = self._imageProviderPath(image, build_number) path = self._imageProviderPath(image, build_id)
try: try:
providers = self.kazoo_client.get_children(path) providers = self.kazoo_client.get_children(path)
@@ -1719,18 +1719,18 @@ class ZooKeeper(ZooKeeperBase):
return sorted(providers) return sorted(providers)
def getImageUploadNumbers(self, image, build_number, provider): def getImageUploadNumbers(self, image, build_id, provider):
''' '''
Retrieve upload numbers for a provider and image build. Retrieve upload numbers for a provider and image build.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:returns: A list of upload numbers or the empty list. :returns: A list of upload numbers or the empty list.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
uploads = self.kazoo_client.get_children(path) uploads = self.kazoo_client.get_children(path)
@@ -1740,16 +1740,16 @@ class ZooKeeper(ZooKeeperBase):
uploads = [x for x in uploads if x != 'lock'] uploads = [x for x in uploads if x != 'lock']
return uploads return uploads
def getBuild(self, image, build_number): def getBuild(self, image, build_id):
''' '''
Retrieve the image build data. Retrieve the image build data.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: An ImageBuild object, or None if not found. :returns: An ImageBuild object, or None if not found.
''' '''
path = self._imageBuildsPath(image) + "/%s" % build_number path = self._imageBuildsPath(image) + "/%s" % build_id
try: try:
data, stat = self.kazoo_client.get(path) data, stat = self.kazoo_client.get(path)
@@ -1758,7 +1758,7 @@ class ZooKeeper(ZooKeeperBase):
try: try:
d = ImageBuild.fromDict(self._bytesToDict(data), d = ImageBuild.fromDict(self._bytesToDict(data),
image, build_number) image, build_id)
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
self.log.exception('Error loading json data from image build %s', self.log.exception('Error loading json data from image build %s',
path) path)
@@ -1832,7 +1832,7 @@ class ZooKeeper(ZooKeeperBase):
builds.sort(key=lambda x: x.state_time, reverse=True) builds.sort(key=lambda x: x.state_time, reverse=True)
return builds[:count] return builds[:count]
def storeBuild(self, image, build_data, build_number=None): def storeBuild(self, image, build_data, build_id=None):
''' '''
Store the image build data. Store the image build data.
@@ -1841,40 +1841,40 @@ class ZooKeeper(ZooKeeperBase):
The build data is expected to be represented as a dict. This dict may The build data is expected to be represented as a dict. This dict may
contain any data, as appropriate. contain any data, as appropriate.
If a build number is not supplied, then a new build node/number is If a build id is not supplied, then a new build node/id is
created. The new build number is available in the return value. created. The new build id is available in the return value.
.. important: You should have the image locked before calling this .. important: You should have the image locked before calling this
method. method.
:param str image: The image name for which we have data. :param str image: The image name for which we have data.
:param ImageBuild build_data: The build data. :param ImageBuild build_data: The build data.
:param str build_number: The image build number. :param str build_id: The image build id.
:returns: A string for the build number that was updated. :returns: A string for the build id that was updated.
''' '''
# Append trailing / so the sequence node is created as a child node. # Append trailing / so the sequence node is created as a child node.
build_path = self._imageBuildsPath(image) + "/" build_path = self._imageBuildsPath(image) + "/"
if build_number is None: if build_id is None:
build_number = uuid.uuid4().hex build_id = uuid.uuid4().hex
path = build_path + build_number path = build_path + build_id
self.kazoo_client.create( self.kazoo_client.create(
path, path,
value=build_data.serialize(), value=build_data.serialize(),
makepath=True) makepath=True)
else: else:
path = build_path + build_number path = build_path + build_id
self.kazoo_client.set(path, build_data.serialize()) self.kazoo_client.set(path, build_data.serialize())
return build_number return build_id
def getImageUpload(self, image, build_number, provider, upload_number): def getImageUpload(self, image, build_id, provider, upload_number):
''' '''
Retrieve the image upload data. Retrieve the image upload data.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str upload_number: The image upload number. :param str upload_number: The image upload number.
@@ -1882,7 +1882,7 @@ class ZooKeeper(ZooKeeperBase):
:raises: ZKException if the image upload path is not found. :raises: ZKException if the image upload path is not found.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
path = path + "/%s" % upload_number path = path + "/%s" % upload_number
try: try:
@@ -1892,7 +1892,7 @@ class ZooKeeper(ZooKeeperBase):
try: try:
d = ImageUpload.fromDict(self._bytesToDict(data), d = ImageUpload.fromDict(self._bytesToDict(data),
build_number, build_id,
provider, provider,
image, image,
upload_number) upload_number)
@@ -1903,12 +1903,12 @@ class ZooKeeper(ZooKeeperBase):
d.stat = stat d.stat = stat
return d return d
def getUploads(self, image, build_number, provider, states=None): def getUploads(self, image, build_id, provider, states=None):
''' '''
Retrieve all image upload data matching any given states. Retrieve all image upload data matching any given states.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param list states: A list of upload state values to match against. :param list states: A list of upload state values to match against.
A value of None will disable state matching and just return A value of None will disable state matching and just return
@@ -1916,7 +1916,7 @@ class ZooKeeper(ZooKeeperBase):
:returns: A list of ImageUpload objects. :returns: A list of ImageUpload objects.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
uploads = self.kazoo_client.get_children(path) uploads = self.kazoo_client.get_children(path)
@@ -1927,7 +1927,7 @@ class ZooKeeper(ZooKeeperBase):
for upload in uploads: for upload in uploads:
if upload == 'lock': if upload == 'lock':
continue continue
data = self.getImageUpload(image, build_number, provider, upload) data = self.getImageUpload(image, build_id, provider, upload)
if not data: if not data:
continue continue
if states is None: if states is None:
@@ -1937,7 +1937,7 @@ class ZooKeeper(ZooKeeperBase):
return matches return matches
def getMostRecentBuildImageUploads(self, count, image, build_number, def getMostRecentBuildImageUploads(self, count, image, build_id,
provider, state=None): provider, state=None):
''' '''
Retrieve the most recent image upload data with the given state. Retrieve the most recent image upload data with the given state.
@@ -1945,7 +1945,7 @@ class ZooKeeper(ZooKeeperBase):
:param int count: A count of the most recent uploads to return. :param int count: A count of the most recent uploads to return.
Use None for all uploads. Use None for all uploads.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str state: The image upload state to match on. Use None to :param str state: The image upload state to match on. Use None to
ignore state. ignore state.
@@ -1958,7 +1958,7 @@ class ZooKeeper(ZooKeeperBase):
if state: if state:
states = [state] states = [state]
uploads = self.getUploads(image, build_number, provider, states) uploads = self.getUploads(image, build_id, provider, states)
if not uploads: if not uploads:
return [] return []
@@ -1983,8 +1983,8 @@ class ZooKeeper(ZooKeeperBase):
if cached: if cached:
uploads = self.getCachedImageUploads() uploads = self.getCachedImageUploads()
else: else:
for build_number in self.getBuildNumbers(image): for build_id in self.getBuildIds(image):
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
try: try:
upload_numbers = self.kazoo_client.get_children(path) upload_numbers = self.kazoo_client.get_children(path)
except kze.NoNodeError: except kze.NoNodeError:
@@ -1994,7 +1994,7 @@ class ZooKeeper(ZooKeeperBase):
if upload_number == 'lock': # skip the upload lock node if upload_number == 'lock': # skip the upload lock node
continue continue
data = self.getImageUpload( data = self.getImageUpload(
image, build_number, provider, upload_number) image, build_id, provider, upload_number)
if not data or data.state != state: if not data or data.state != state:
continue continue
uploads.append(data) uploads.append(data)
@@ -2023,7 +2023,7 @@ class ZooKeeper(ZooKeeperBase):
raise RuntimeError("Caching not enabled") raise RuntimeError("Caching not enabled")
return self._image_cache.getUploads() return self._image_cache.getUploads()
def storeImageUpload(self, image, build_number, provider, image_data, def storeImageUpload(self, image, build_id, provider, image_data,
upload_number=None): upload_number=None):
''' '''
Store the built image's upload data for the given provider. Store the built image's upload data for the given provider.
@@ -2038,7 +2038,7 @@ class ZooKeeper(ZooKeeperBase):
return value. return value.
:param str image: The image name for which we have data. :param str image: The image name for which we have data.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param ImageUpload image_data: The image data we want to store. :param ImageUpload image_data: The image data we want to store.
:param str upload_number: The image upload number to update. :param str upload_number: The image upload number to update.
@@ -2051,14 +2051,14 @@ class ZooKeeper(ZooKeeperBase):
build_path = self._imageBuildsPath(image) build_path = self._imageBuildsPath(image)
if not self.kazoo_client.exists(build_path): if not self.kazoo_client.exists(build_path):
raise npe.ZKException( raise npe.ZKException(
"Cannot find build %s of image %s" % (build_number, image) "Cannot find build %s of image %s" % (build_id, image)
) )
# Generate a path for the upload. This doesn't have to exist yet # Generate a path for the upload. This doesn't have to exist yet
# since we'll create new provider/upload ID znodes automatically. # since we'll create new provider/upload ID znodes automatically.
# Append trailing / so the sequence node is created as a child node. # Append trailing / so the sequence node is created as a child node.
upload_path = self._imageUploadPath( upload_path = self._imageUploadPath(
image, build_number, provider) + "/" image, build_id, provider) + "/"
if upload_number is None: if upload_number is None:
path = self.kazoo_client.create( path = self.kazoo_client.create(
@@ -2087,12 +2087,12 @@ class ZooKeeper(ZooKeeperBase):
return False return False
def _latestImageBuildStat(self, image): def _latestImageBuildStat(self, image):
builds = self.getBuildNumbers(image) builds = self.getBuildIds(image)
if not builds: if not builds:
return return
latest_build, *_ = builds latest_build, *_ = builds
builds_path = self._imageBuildNumberPath(image, latest_build) builds_path = self._imageBuildIdPath(image, latest_build)
return self.kazoo_client.exists(builds_path) return self.kazoo_client.exists(builds_path)
def getBuildRequest(self, image): def getBuildRequest(self, image):
@@ -2143,7 +2143,7 @@ class ZooKeeper(ZooKeeperBase):
except kze.NoNodeError: except kze.NoNodeError:
pass pass
def deleteBuild(self, image, build_number): def deleteBuild(self, image, build_id):
''' '''
Delete an image build from ZooKeeper. Delete an image build from ZooKeeper.
@@ -2151,17 +2151,17 @@ class ZooKeeper(ZooKeeperBase):
node can be deleted. node can be deleted.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number to delete. :param str build_id: The image build id to delete.
:returns: True if the build is successfully deleted or did not exist, :returns: True if the build is successfully deleted or did not exist,
False if the provider uploads still exist. False if the provider uploads still exist.
''' '''
path = self._imageBuildsPath(image) path = self._imageBuildsPath(image)
path = path + "/%s" % build_number path = path + "/%s" % build_id
# Verify that no upload znodes exist. # Verify that no upload znodes exist.
for prov in self.getBuildProviders(image, build_number): for prov in self.getBuildProviders(image, build_id):
if self.getImageUploadNumbers(image, build_number, prov): if self.getImageUploadNumbers(image, build_id, prov):
return False return False
try: try:
@@ -2172,16 +2172,16 @@ class ZooKeeper(ZooKeeperBase):
return True return True
def deleteUpload(self, image, build_number, provider, upload_number): def deleteUpload(self, image, build_id, provider, upload_number):
''' '''
Delete an image upload from ZooKeeper. Delete an image upload from ZooKeeper.
:param str image: The image name. :param str image: The image name.
:param str build_number: The image build number. :param str build_id: The image build id.
:param str provider: The provider name owning the image. :param str provider: The provider name owning the image.
:param str upload_number: The image upload number to delete. :param str upload_number: The image upload number to delete.
''' '''
path = self._imageUploadPath(image, build_number, provider) path = self._imageUploadPath(image, build_id, provider)
path = path + "/%s" % upload_number path = path + "/%s" % upload_number
try: try:
# NOTE: Need to do recursively to remove lock znodes # NOTE: Need to do recursively to remove lock znodes
@@ -2834,8 +2834,8 @@ class ZooKeeper(ZooKeeperBase):
provider_builds = {} provider_builds = {}
image_names = self.getImageNames() image_names = self.getImageNames()
for image in image_names: for image in image_names:
build_numbers = self.getBuildNumbers(image) build_ids = self.getBuildIds(image)
for build in build_numbers: for build in build_ids:
providers = self.getBuildProviders(image, build) providers = self.getBuildProviders(image, build)
for p in providers: for p in providers:
if p == provider_name: if p == provider_name:
@@ -2855,8 +2855,8 @@ class ZooKeeper(ZooKeeperBase):
provider_uploads = {} provider_uploads = {}
image_names = self.getImageNames() image_names = self.getImageNames()
for image in image_names: for image in image_names:
build_numbers = self.getBuildNumbers(image) build_ids = self.getBuildIds(image)
for build in build_numbers: for build in build_ids:
# If this build is not valid for this provider, move along. # If this build is not valid for this provider, move along.
if provider_name not in self.getBuildProviders(image, build): if provider_name not in self.getBuildProviders(image, build):
continue continue
@@ -2928,7 +2928,7 @@ class ZooKeeper(ZooKeeperBase):
if paused: if paused:
paused_path = self._imagePausePath(image_name) paused_path = self._imagePausePath(image_name)
ret[paused_path] = '' ret[paused_path] = ''
for build_no in self.getBuildNumbers(image_name): for build_no in self.getBuildIds(image_name):
build_path = self._imageBuildsPath(image_name) + "/" + build_no build_path = self._imageBuildsPath(image_name) + "/" + build_no
try: try:
build_data, stat = self.kazoo_client.get(build_path) build_data, stat = self.kazoo_client.get(build_path)