zuul
/
zuul
Code Issues Proposed changes

Move shadow layout to item 

It turns out layouts can be large[citation needed].  The layout used
for a prior buildset is only of historic interest.  The shadow layout
is effectively used only to freeze a job graph, and once a buildset
has been superceded, that is no longer necessary.  Drop the reference
at this point to allow the memory to be reclaimed sooner.

Change-Id: I2bf651ab99f1b1bfbe0e2442cf26c40475022fe5
This commit is contained in:
James E. Blair 2017-10-02 15:04:56 -07:00
parent 88985e2d30
commit 29a24fd6cc
5 changed files with 24 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/unit/test_model.py
View File

@ -262,7 +262,7 @@ class TestJob(BaseTestCase):
# Test master # Test master
change.branch = 'master' change.branch = 'master'
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
@ -291,7 +291,7 @@ class TestJob(BaseTestCase):
# Test diablo # Test diablo
change.branch = 'stable/diablo' change.branch = 'stable/diablo'
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
@ -321,7 +321,7 @@ class TestJob(BaseTestCase):
# Test essex # Test essex
change.branch = 'stable/essex' change.branch = 'stable/essex'
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
@ -554,7 +554,7 @@ class TestJob(BaseTestCase):
change = model.Change(self.project) change = model.Change(self.project)
change.branch = 'master' change.branch = 'master'
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
@ -568,7 +568,7 @@ class TestJob(BaseTestCase):
change.branch = 'stable/diablo' change.branch = 'stable/diablo'
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
@ -625,7 +625,7 @@ class TestJob(BaseTestCase):
change.branch = 'master' change.branch = 'master'
change.files = ['/COMMIT_MSG', 'ignored-file'] change.files = ['/COMMIT_MSG', 'ignored-file']
item = queue.enqueueChange(change) item = queue.enqueueChange(change)
item.current_build_set.layout = layout item.layout = layout
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change)) self.assertFalse(python27.changeMatches(change))
@ -700,7 +700,7 @@ class TestJob(BaseTestCase):
# Test master # Test master
change.branch = 'master' change.branch = 'master'
item = self.queue.enqueueChange(change) item = self.queue.enqueueChange(change)
item.current_build_set.layout = self.layout item.layout = self.layout
with testtools.ExpectedException( with testtools.ExpectedException(
Exception, Exception,
"Project project2 is not allowed to run job job"): "Project project2 is not allowed to run job job"):
@ -736,7 +736,7 @@ class TestJob(BaseTestCase):
# Test master # Test master
change.branch = 'master' change.branch = 'master'
item = self.queue.enqueueChange(change) item = self.queue.enqueueChange(change)
item.current_build_set.layout = self.layout item.layout = self.layout
with testtools.ExpectedException( with testtools.ExpectedException(
Exception, Exception,
"Pre-review pipeline gate does not allow post-review job"): "Pre-review pipeline gate does not allow post-review job"):

2
tests/unit/test_scheduler.py
View File

@ -5589,7 +5589,7 @@ class TestSemaphoreInRepo(ZuulTestCase):
queue = queue_candidate queue = queue_candidate
break break
queue_item = queue.queue[0] queue_item = queue.queue[0]
item_dynamic_layout = queue_item.current_build_set.layout item_dynamic_layout = queue_item.layout
dynamic_test_semaphore = \ dynamic_test_semaphore = \
item_dynamic_layout.semaphores.get('test-semaphore') item_dynamic_layout.semaphores.get('test-semaphore')
self.assertEqual(dynamic_test_semaphore.max, 1) self.assertEqual(dynamic_test_semaphore.max, 1)

2
zuul/executor/client.py
View File

@ -238,7 +238,7 @@ class ExecutorClient(object):
required_projects = set() required_projects = set()
def make_project_dict(project, override_branch=None): def make_project_dict(project, override_branch=None):
project_config = item.current_build_set.layout.project_configs.get( project_config = item.layout.project_configs.get(
project.canonical_name, None) project.canonical_name, None)
if project_config: if project_config:
project_default_branch = project_config.default_branch project_default_branch = project_config.default_branch

18
zuul/manager/__init__.py
View File

@ -236,11 +236,11 @@ class PipelineManager(object):
# in-repo files stored in the buildset. # in-repo files stored in the buildset.
# 3) None in the case that a fetch of the files from # 3) None in the case that a fetch of the files from
# the merger is still pending. # the merger is still pending.
item.current_build_set.layout = self.getLayout(item) item.layout = self.getLayout(item)
# Rebuild the frozen job tree from the new layout, if # Rebuild the frozen job tree from the new layout, if
# we have one. If not, it will be built later. # we have one. If not, it will be built later.
if item.current_build_set.layout: if item.layout:
item.freezeJobGraph() item.freezeJobGraph()
# Re-set build results in case any new jobs have been # Re-set build results in case any new jobs have been
@ -373,7 +373,7 @@ class PipelineManager(object):
def executeJobs(self, item): def executeJobs(self, item):
# TODO(jeblair): This should return a value indicating a job # TODO(jeblair): This should return a value indicating a job
# was executed. Appears to be a longstanding bug. # was executed. Appears to be a longstanding bug.
if not item.current_build_set.layout: if not item.layout:
return False return False
jobs = item.findJobsToRun( jobs = item.findJobsToRun(
@ -465,7 +465,7 @@ class PipelineManager(object):
def getLayout(self, item): def getLayout(self, item):
if not item.change.updatesConfig(): if not item.change.updatesConfig():
if item.item_ahead: if item.item_ahead:
return item.item_ahead.current_build_set.layout return item.item_ahead.layout
else: else:
return item.queue.pipeline.layout return item.queue.pipeline.layout
# This item updates the config, ask the merger for the result. # This item updates the config, ask the merger for the result.
@ -516,10 +516,9 @@ class PipelineManager(object):
def prepareJobs(self, item): def prepareJobs(self, item):
# This only runs once the item is in the pipeline's action window # This only runs once the item is in the pipeline's action window
# Returns True if the item is ready, false otherwise # Returns True if the item is ready, false otherwise
build_set = item.current_build_set if not item.layout:
if not build_set.layout: item.layout = self.getLayout(item)
build_set.layout = self.getLayout(item) if not item.layout:
if not build_set.layout:
return False return False
if not item.job_graph: if not item.job_graph:
@ -745,8 +744,7 @@ class PipelineManager(object):
# pipeline, use the dynamic layout if available, otherwise, # pipeline, use the dynamic layout if available, otherwise,
# fall back to the current static layout as a best # fall back to the current static layout as a best
# approximation. # approximation.
layout = (item.current_build_set.layout or layout = (item.layout or self.pipeline.layout)
self.pipeline.layout)
project_in_pipeline = True project_in_pipeline = True
if not layout.getProjectPipelineConfig(item.change.project, if not layout.getProjectPipelineConfig(item.change.project,

13
zuul/model.py
View File

@ -1271,7 +1271,6 @@ class BuildSet(object):
self.node_requests = {} # job -> reqs self.node_requests = {} # job -> reqs
self.files = RepoFiles() self.files = RepoFiles()
self.repo_state = {} self.repo_state = {}
self.layout = None
self.tries = {} self.tries = {}
@property @property
@ -1366,7 +1365,7 @@ class BuildSet(object):
item = self.item item = self.item
layout = None layout = None
while item: while item:
layout = item.current_build_set.layout layout = item.layout
if layout: if layout:
break break
item = item.item_ahead item = item.item_ahead
@ -1410,7 +1409,7 @@ class QueueItem(object):
self.quiet = False self.quiet = False
self.active = False # Whether an item is within an active window self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all self.live = True # Whether an item is intended to be processed at all
# TODO(jeblair): move job_graph to buildset self.layout = None
self.job_graph = None self.job_graph = None
def __repr__(self): def __repr__(self):
@ -1428,6 +1427,7 @@ class QueueItem(object):
old.next_build_set = self.current_build_set old.next_build_set = self.current_build_set
self.current_build_set.previous_build_set = old self.current_build_set.previous_build_set = old
self.build_sets.append(self.current_build_set) self.build_sets.append(self.current_build_set)
self.layout = None
self.job_graph = None self.job_graph = None
def addBuild(self, build): def addBuild(self, build):
@ -1443,8 +1443,7 @@ class QueueItem(object):
def freezeJobGraph(self): def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and """Find or create actual matching jobs for this item's change and
store the resulting job tree.""" store the resulting job tree."""
layout = self.current_build_set.layout job_graph = self.layout.createJobGraph(self)
job_graph = layout.createJobGraph(self)
for job in job_graph.getJobs(): for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully # Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not. # accessible. This will raise an exception if not.
@ -2527,14 +2526,14 @@ class SemaphoreHandler(object):
@staticmethod @staticmethod
def _max_count(item, semaphore_name): def _max_count(item, semaphore_name):
if not item.current_build_set.layout: if not item.layout:
# This should not occur as the layout of the item must already be # This should not occur as the layout of the item must already be
# built when acquiring or releasing a semaphore for a job. # built when acquiring or releasing a semaphore for a job.
raise Exception("Item {} has no layout".format(item)) raise Exception("Item {} has no layout".format(item))
# find the right semaphore # find the right semaphore
default_semaphore = Semaphore(semaphore_name, 1) default_semaphore = Semaphore(semaphore_name, 1)
semaphores = item.current_build_set.layout.semaphores semaphores = item.layout.semaphores
return semaphores.get(semaphore_name, default_semaphore).max return semaphores.get(semaphore_name, default_semaphore).max