Move shadow layout to item

It turns out layouts can be large[citation needed].  The layout used
for a prior buildset is only of historic interest.  The shadow layout
is effectively used only to freeze a job graph, and once a buildset
has been superceded, that is no longer necessary.  Drop the reference
at this point to allow the memory to be reclaimed sooner.

Change-Id: I2bf651ab99f1b1bfbe0e2442cf26c40475022fe5
This commit is contained in:
James E. Blair 2017-10-02 15:04:56 -07:00
parent 88985e2d30
commit 29a24fd6cc
5 changed files with 24 additions and 27 deletions

View File

@ -262,7 +262,7 @@ class TestJob(BaseTestCase):
# Test master
change.branch = 'master'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@ -291,7 +291,7 @@ class TestJob(BaseTestCase):
# Test diablo
change.branch = 'stable/diablo'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@ -321,7 +321,7 @@ class TestJob(BaseTestCase):
# Test essex
change.branch = 'stable/essex'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@ -554,7 +554,7 @@ class TestJob(BaseTestCase):
change = model.Change(self.project)
change.branch = 'master'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@ -568,7 +568,7 @@ class TestJob(BaseTestCase):
change.branch = 'stable/diablo'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@ -625,7 +625,7 @@ class TestJob(BaseTestCase):
change.branch = 'master'
change.files = ['/COMMIT_MSG', 'ignored-file']
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
item.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change))
@ -700,7 +700,7 @@ class TestJob(BaseTestCase):
# Test master
change.branch = 'master'
item = self.queue.enqueueChange(change)
item.current_build_set.layout = self.layout
item.layout = self.layout
with testtools.ExpectedException(
Exception,
"Project project2 is not allowed to run job job"):
@ -736,7 +736,7 @@ class TestJob(BaseTestCase):
# Test master
change.branch = 'master'
item = self.queue.enqueueChange(change)
item.current_build_set.layout = self.layout
item.layout = self.layout
with testtools.ExpectedException(
Exception,
"Pre-review pipeline gate does not allow post-review job"):

View File

@ -5589,7 +5589,7 @@ class TestSemaphoreInRepo(ZuulTestCase):
queue = queue_candidate
break
queue_item = queue.queue[0]
item_dynamic_layout = queue_item.current_build_set.layout
item_dynamic_layout = queue_item.layout
dynamic_test_semaphore = \
item_dynamic_layout.semaphores.get('test-semaphore')
self.assertEqual(dynamic_test_semaphore.max, 1)

View File

@ -238,7 +238,7 @@ class ExecutorClient(object):
required_projects = set()
def make_project_dict(project, override_branch=None):
project_config = item.current_build_set.layout.project_configs.get(
project_config = item.layout.project_configs.get(
project.canonical_name, None)
if project_config:
project_default_branch = project_config.default_branch

View File

@ -236,11 +236,11 @@ class PipelineManager(object):
# in-repo files stored in the buildset.
# 3) None in the case that a fetch of the files from
# the merger is still pending.
item.current_build_set.layout = self.getLayout(item)
item.layout = self.getLayout(item)
# Rebuild the frozen job tree from the new layout, if
# we have one. If not, it will be built later.
if item.current_build_set.layout:
if item.layout:
item.freezeJobGraph()
# Re-set build results in case any new jobs have been
@ -373,7 +373,7 @@ class PipelineManager(object):
def executeJobs(self, item):
# TODO(jeblair): This should return a value indicating a job
# was executed. Appears to be a longstanding bug.
if not item.current_build_set.layout:
if not item.layout:
return False
jobs = item.findJobsToRun(
@ -465,7 +465,7 @@ class PipelineManager(object):
def getLayout(self, item):
if not item.change.updatesConfig():
if item.item_ahead:
return item.item_ahead.current_build_set.layout
return item.item_ahead.layout
else:
return item.queue.pipeline.layout
# This item updates the config, ask the merger for the result.
@ -516,10 +516,9 @@ class PipelineManager(object):
def prepareJobs(self, item):
# This only runs once the item is in the pipeline's action window
# Returns True if the item is ready, false otherwise
build_set = item.current_build_set
if not build_set.layout:
build_set.layout = self.getLayout(item)
if not build_set.layout:
if not item.layout:
item.layout = self.getLayout(item)
if not item.layout:
return False
if not item.job_graph:
@ -745,8 +744,7 @@ class PipelineManager(object):
# pipeline, use the dynamic layout if available, otherwise,
# fall back to the current static layout as a best
# approximation.
layout = (item.current_build_set.layout or
self.pipeline.layout)
layout = (item.layout or self.pipeline.layout)
project_in_pipeline = True
if not layout.getProjectPipelineConfig(item.change.project,

View File

@ -1271,7 +1271,6 @@ class BuildSet(object):
self.node_requests = {} # job -> reqs
self.files = RepoFiles()
self.repo_state = {}
self.layout = None
self.tries = {}
@property
@ -1366,7 +1365,7 @@ class BuildSet(object):
item = self.item
layout = None
while item:
layout = item.current_build_set.layout
layout = item.layout
if layout:
break
item = item.item_ahead
@ -1410,7 +1409,7 @@ class QueueItem(object):
self.quiet = False
self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all
# TODO(jeblair): move job_graph to buildset
self.layout = None
self.job_graph = None
def __repr__(self):
@ -1428,6 +1427,7 @@ class QueueItem(object):
old.next_build_set = self.current_build_set
self.current_build_set.previous_build_set = old
self.build_sets.append(self.current_build_set)
self.layout = None
self.job_graph = None
def addBuild(self, build):
@ -1443,8 +1443,7 @@ class QueueItem(object):
def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
layout = self.current_build_set.layout
job_graph = layout.createJobGraph(self)
job_graph = self.layout.createJobGraph(self)
for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not.
@ -2527,14 +2526,14 @@ class SemaphoreHandler(object):
@staticmethod
def _max_count(item, semaphore_name):
if not item.current_build_set.layout:
if not item.layout:
# This should not occur as the layout of the item must already be
# built when acquiring or releasing a semaphore for a job.
raise Exception("Item {} has no layout".format(item))
# find the right semaphore
default_semaphore = Semaphore(semaphore_name, 1)
semaphores = item.current_build_set.layout.semaphores
semaphores = item.layout.semaphores
return semaphores.get(semaphore_name, default_semaphore).max