zuul
/
nodepool
Code Issues Proposed changes

Do pep8 housekeeping according to zuul rules 

The pep8 rules used in nodepool are somewhat broken. In preparation to
use the pep8 ruleset from zuul we need to fix the findings upfront.

Change-Id: I9fb2a80db7671c590cdb8effbd1a1102aaa3aff8
This commit is contained in:
Tobias Henkel 2017-11-26 08:29:40 +01:00 committed by Tristan Cacqueray
parent be4b8f1416
commit 7d79770840
18 changed files with 114 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
nodepool/builder.py
View File

@ -32,9 +32,11 @@ from nodepool import zk
MINS = 60
HOURS = 60 * MINS
IMAGE_TIMEOUT = 6 * HOURS # How long to wait for an image save
SUSPEND_WAIT_TIME = 30 # How long to wait between checks for
# ZooKeeper connectivity if it disappears.
# How long to wait for an image save
IMAGE_TIMEOUT = 6 * HOURS
# How long to wait between checks for ZooKeeper connectivity if it disappears.
SUSPEND_WAIT_TIME = 30
# HP Cloud requires qemu compat with 0.10. That version works elsewhere,
# so just hardcode it for all qcow2 building
@ -151,7 +153,8 @@ class CleanupWorker(BaseWorker):
interval, zk):
super(CleanupWorker, self).__init__(builder_id, config_path,
secure_path, interval, zk)
self.log = logging.getLogger("nodepool.builder.CleanupWorker.%s" % name)
self.log = logging.getLogger(
"nodepool.builder.CleanupWorker.%s" % name)
self.name = 'CleanupWorker.%s' % name
def _buildUploadRecencyTable(self):
@ -395,7 +398,8 @@ class CleanupWorker(BaseWorker):
self.log.info("Removing failed upload record: %s" % upload)
self._zk.deleteUpload(image, build_id, provider, upload.id)
elif upload.state == zk.DELETING:
self.log.info("Removing deleted upload and record: %s" % upload)
self.log.info(
"Removing deleted upload and record: %s" % upload)
self._deleteUpload(upload)
elif upload.state == zk.FAILED:
self.log.info("Removing failed upload and record: %s" % upload)
@ -576,6 +580,7 @@ class BuildWorker(BaseWorker):
or (now - builds[0].state_time) >= diskimage.rebuild_age
or not set(builds[0].formats).issuperset(diskimage.image_types)
):
try:
with self._zk.imageBuildLock(diskimage.name, blocking=False):
# To avoid locking each image repeatedly, we have an
@ -584,7 +589,8 @@ class BuildWorker(BaseWorker):
# lock acquisition. If it's not the same build as
# identified in the first check above, assume another
# BuildWorker created the build for us and continue.
builds2 = self._zk.getMostRecentBuilds(1, diskimage.name, zk.READY)
builds2 = self._zk.getMostRecentBuilds(
1, diskimage.name, zk.READY)
if builds2 and builds[0].id != builds2[0].id:
return
@ -746,7 +752,8 @@ class BuildWorker(BaseWorker):
self._zk.resetLostFlag()
build_data.state = zk.FAILED
elif p.returncode:
self.log.info("DIB failed creating %s (%s)" % (diskimage.name, p.returncode))
self.log.info(
"DIB failed creating %s (%s)" % (diskimage.name, p.returncode))
build_data.state = zk.FAILED
else:
self.log.info("DIB image %s is built" % diskimage.name)
@ -756,7 +763,8 @@ class BuildWorker(BaseWorker):
if self._statsd:
# record stats on the size of each image we create
for ext in img_types.split(','):
key = 'nodepool.dib_image_build.%s.%s.size' % (diskimage.name, ext)
key = 'nodepool.dib_image_build.%s.%s.size' % (
diskimage.name, ext)
# A bit tricky because these image files may be sparse
# files; we only want the true size of the file for
# purposes of watching if we've added too much stuff
@ -1101,9 +1109,9 @@ class NodePoolBuilder(object):
# startup process has completed.
self._start_lock = threading.Lock()
#=======================================================================
# ======================================================================
# Private methods
#=======================================================================
# ======================================================================
def _getBuilderID(self, id_file):
if not os.path.exists(id_file):
@ -1126,9 +1134,9 @@ class NodePoolBuilder(object):
raise RuntimeError('No images-dir specified in config.')
return config
#=======================================================================
# ======================================================================
# Public methods
#=======================================================================
# ======================================================================
def start(self):
'''

4
nodepool/driver/__init__.py
View File

@ -106,9 +106,9 @@ class NodeRequestHandler(object):
return 0
return self.launch_manager.alive_thread_count
#----------------------------------------------------------------
# ---------------------------------------------------------------
# Public methods
#----------------------------------------------------------------
# ---------------------------------------------------------------
def unlockNodeSet(self, clear_allocation=False):
'''

4
nodepool/driver/openstack/config.py
View File

@ -200,7 +200,6 @@ class OpenStackProviderConfig(ProviderConfig):
top_label = config.labels[pl.name]
top_label.pools.append(pp)
def get_schema(self):
provider_diskimage = {
'name': str,
@ -238,7 +237,8 @@ class OpenStackProviderConfig(ProviderConfig):
label_diskimage = v.Schema({v.Required('diskimage'): str}, extra=True)
label_cloud_image = v.Schema({v.Required('cloud-image'): str}, extra=True)
label_cloud_image = v.Schema({v.Required('cloud-image'): str},
extra=True)
pool_label = v.All(pool_label_main,
v.Any(label_min_ram, label_flavor_name),

7
nodepool/driver/openstack/handler.py
View File

@ -61,8 +61,8 @@ class NodeLauncher(threading.Thread, stats.StatsReporter):
self._pool = self._label.pool
self._provider_config = self._pool.provider
if self._label.diskimage:
self._diskimage = \
self._provider_config.diskimages[self._label.diskimage.name]
self._diskimage = self._provider_config.diskimages[
self._label.diskimage.name]
else:
self._diskimage = None
@ -542,8 +542,7 @@ class OpenStackNodeRequestHandler(NodeRequestHandler):
elif not self._imagesAvailable():
declined_reasons.append('images are not available')
elif (self.pool.max_servers == 0 or
not self._hasProviderQuota(self.request.node_types)
):
not self._hasProviderQuota(self.request.node_types)):
declined_reasons.append('it would exceed quota')
# TODO(tobiash): Maybe also calculate the quota prediction here and
# backoff for some seconds if the used quota would be exceeded?

2
nodepool/exceptions.py
View File

@ -64,8 +64,10 @@ class ServerDeleteException(TimeoutException):
class ImageCreateException(TimeoutException):
statsd_key = 'error.imagetimeout'
class ZKException(Exception):
pass
class ZKLockException(ZKException):
pass

26
nodepool/launcher.py
View File

@ -35,10 +35,14 @@ from nodepool.driver.openstack.handler import OpenStackNodeRequestHandler
MINS = 60
HOURS = 60 * MINS
WATERMARK_SLEEP = 10 # Interval between checking if new servers needed
LOCK_CLEANUP = 8 * HOURS # When to delete node request lock znodes
SUSPEND_WAIT_TIME = 30 # How long to wait between checks for ZooKeeper
# connectivity if it disappears.
# Interval between checking if new servers needed
WATERMARK_SLEEP = 10
# When to delete node request lock znodes
LOCK_CLEANUP = 8 * HOURS
# How long to wait between checks for ZooKeeper connectivity if it disappears.
SUSPEND_WAIT_TIME = 30
class NodeDeleter(threading.Thread, stats.StatsReporter):
@ -137,9 +141,9 @@ class PoolWorker(threading.Thread):
os.getpid(),
self.name)
#----------------------------------------------------------------
# ---------------------------------------------------------------
# Private methods
#----------------------------------------------------------------
# ---------------------------------------------------------------
def _get_node_request_handler(self, provider, request):
if provider.driver.name == 'fake':
@ -177,8 +181,7 @@ class PoolWorker(threading.Thread):
# Short-circuit for limited request handling
if (provider.max_concurrency > 0 and
active_threads >= provider.max_concurrency
):
active_threads >= provider.max_concurrency):
self.log.debug("Request handling limited: %s active threads ",
"with max concurrency of %s",
active_threads, provider.max_concurrency)
@ -238,9 +241,9 @@ class PoolWorker(threading.Thread):
active_reqs = [r.request.id for r in self.request_handlers]
self.log.debug("Active requests: %s", active_reqs)
#----------------------------------------------------------------
# ---------------------------------------------------------------
# Public methods
#----------------------------------------------------------------
# ---------------------------------------------------------------
def activeThreads(self):
'''
@ -604,8 +607,7 @@ class DeletedNodeWorker(BaseCleanupWorker):
# If a ready node has been allocated to a request, but that
# request is now missing, deallocate it.
if (node.state == zk.READY and node.allocated_to
and not zk_conn.getNodeRequest(node.allocated_to)
):
and not zk_conn.getNodeRequest(node.allocated_to)):
try:
zk_conn.lockNode(node, blocking=False)
except exceptions.ZKLockException:

5
nodepool/nodeutils.py
View File

@ -29,9 +29,8 @@ from nodepool import exceptions
log = logging.getLogger("nodepool.utils")
ITERATE_INTERVAL = 2 # How long to sleep while waiting for something
# in a loop
# How long to sleep while waiting for something in a loop
ITERATE_INTERVAL = 2
def iterate_timeout(max_seconds, exc, purpose):

2
nodepool/stats.py
View File

@ -24,6 +24,7 @@ from nodepool import zk
log = logging.getLogger("nodepool.stats")
def get_client():
"""Return a statsd client object setup from environment variables; or
None if they are not set
@ -87,7 +88,6 @@ class StatsReporter(object):
self._statsd.timing(key, dt)
self._statsd.incr(key)
def updateNodeStats(self, zk_conn, provider):
'''
Refresh statistics for all known nodes.

2
nodepool/status.py
View File

@ -154,6 +154,7 @@ def dib_image_list_json(zk):
})
return json.dumps(objs)
def image_list(zk):
t = PrettyTable(["Build ID", "Upload ID", "Provider", "Image",
"Provider Image Name", "Provider Image ID", "State",
@ -173,6 +174,7 @@ def image_list(zk):
age(upload.state_time)])
return str(t)
def request_list(zk):
t = PrettyTable(["Request ID", "State", "Requestor", "Node Types", "Nodes",
"Declined By"])

1
nodepool/task_manager.py
View File

@ -26,6 +26,7 @@ import requests.exceptions
from nodepool import stats
class ManagerStoppedException(Exception):
pass

4
nodepool/tests/test_builder.py
View File

@ -86,6 +86,7 @@ class TestNodepoolBuilderDibImage(tests.BaseTestCase):
image = builder.DibImageFile('myid1234')
self.assertRaises(exceptions.BuilderError, image.to_path, '/imagedir/')
class TestNodePoolBuilder(tests.DBTestCase):
def test_start_stop(self):
@ -155,7 +156,8 @@ class TestNodePoolBuilder(tests.DBTestCase):
image = self.zk.getMostRecentImageUpload('fake-provider', 'fake-image')
self.replace_config(configfile, 'node_two_provider_remove.yaml')
self.waitForImageDeletion('fake-provider2', 'fake-image')
image2 = self.zk.getMostRecentImageUpload('fake-provider', 'fake-image')
image2 = self.zk.getMostRecentImageUpload('fake-provider',
'fake-image')
self.assertEqual(image, image2)
def test_image_addition(self):

3
nodepool/tests/test_commands.py
View File

@ -52,7 +52,8 @@ class TestNodepoolCMD(tests.DBTestCase):
self.assertEquals(rows_with_val, count)
def assert_alien_images_listed(self, configfile, image_cnt, image_id):
self.assert_listed(configfile, ['alien-image-list'], 2, image_id, image_cnt)
self.assert_listed(configfile, ['alien-image-list'], 2, image_id,
image_cnt)
def assert_alien_images_empty(self, configfile):
self.assert_alien_images_listed(configfile, 0, 0)

1
nodepool/tests/test_launcher.py
View File

@ -258,7 +258,6 @@ class TestLauncher(tests.DBTestCase):
max_instances=math.inf,
max_ram=2 * 8192)
@mock.patch('nodepool.driver.fake.provider.get_fake_quota')
def test_over_quota(self, mock_quota,
config='node_quota_cloud.yaml'):

12
nodepool/tests/test_zk.py
View File

@ -104,7 +104,8 @@ class TestZooKeeper(tests.DBTestCase):
with testtools.ExpectedException(
npe.ZKLockException, "Did not get lock on .*"
):
with self.zk.imageUploadLock(image, bnum, prov, blocking=False):
with self.zk.imageUploadLock(image, bnum, prov,
blocking=False):
pass
def test_imageUploadLock_exception_blocking(self):
@ -254,7 +255,8 @@ class TestZooKeeper(tests.DBTestCase):
self.zk.storeImageUpload(image, bnum, provider, up3)
# up2 should be the most recent 'ready' upload
data = self.zk.getMostRecentBuildImageUploads(1, image, bnum, provider, zk.READY)
data = self.zk.getMostRecentBuildImageUploads(
1, image, bnum, provider, zk.READY)
self.assertNotEqual([], data)
self.assertEqual(1, len(data))
self.assertEqual(data[0].id, up2_id)
@ -278,7 +280,8 @@ class TestZooKeeper(tests.DBTestCase):
up3_id = self.zk.storeImageUpload(image, bnum, provider, up3)
# up3 should be the most recent upload, regardless of state
data = self.zk.getMostRecentBuildImageUploads(1, image, bnum, provider, None)
data = self.zk.getMostRecentBuildImageUploads(
1, image, bnum, provider, None)
self.assertNotEqual([], data)
self.assertEqual(1, len(data))
self.assertEqual(data[0].id, up3_id)
@ -875,4 +878,5 @@ class TestZKModel(tests.BaseTestCase):
self.assertEqual(n.connection_port, 22, "Default port not 22")
n.connection_port = 22022
d = n.toDict()
self.assertEqual(d["connection_port"], 22022, "Custom ssh port not set")
self.assertEqual(d["connection_port"], 22022,
"Custom ssh port not set")

29
nodepool/zk.py
View File

@ -583,9 +583,9 @@ class ZooKeeper(object):
self.client = None
self._became_lost = False
#========================================================================
# =======================================================================
# Private Methods
#========================================================================
# =======================================================================
def _imagePath(self, image):
return "%s/%s" % (self.IMAGE_ROOT, image)
@ -709,10 +709,9 @@ class ZooKeeper(object):
else:
self.log.debug("ZooKeeper connection: CONNECTED")
#========================================================================
# =======================================================================
# Public Methods and Properties
#========================================================================
# =======================================================================
@property
def connected(self):
@ -1065,9 +1064,11 @@ class ZooKeeper(object):
except kze.NoNodeError:
return None
d = ImageUpload.fromDict(
self._bytesToDict(data), build_number, provider, image, upload_number
)
d = ImageUpload.fromDict(self._bytesToDict(data),
build_number,
provider,
image,
upload_number)
d.stat = stat
return d
@ -1158,7 +1159,8 @@ class ZooKeeper(object):
for upload in uploads:
if upload == 'lock': # skip the upload lock node
continue
data = self.getImageUpload(image, build_number, provider, upload)
data = self.getImageUpload(
image, build_number, provider, upload)
if not data or data.state != state:
continue
elif (recent_data is None or
@ -1201,7 +1203,8 @@ class ZooKeeper(object):
# Generate a path for the upload. This doesn't have to exist yet
# since we'll create new provider/upload ID znodes automatically.
# Append trailing / so the sequence node is created as a child node.
upload_path = self._imageUploadPath(image, build_number, provider) + "/"
upload_path = self._imageUploadPath(
image, build_number, provider) + "/"
if upload_number is None:
path = self.client.create(
@ -1486,7 +1489,8 @@ class ZooKeeper(object):
:raises: ZKLockException if the request is not currently locked.
'''
if request.lock is None:
raise npe.ZKLockException("Request %s does not hold a lock" % request)
raise npe.ZKLockException(
"Request %s does not hold a lock" % request)
request.lock.release()
request.lock = None
@ -1630,8 +1634,7 @@ class ZooKeeper(object):
ret = {}
for node in self.nodeIterator():
if (node.state == READY and
not node.allocated_to and node.type in labels
):
not node.allocated_to and node.type in labels):
if node.type not in ret:
ret[node.type] = []
ret[node.type].append(node)