Remove layout attribute from queue items

We want to restrict layout access to job freezing so that fewer
actions (like creating a build request after a job is frozen) require
access to it.

To prevent components other than the pipeline manager from accessing the
layout, we will remove the layout as an attribute from the queue item
and store it in an internal cache of the manager.

Queue items will reference their (dynamic) layout via the layout's UUID.
An item's layout UUID will change if any of it's input to the layout
creation changed. Those inputs are the tenant layout and list of items
ahead. This means that during a re-enqueue and in case of a gate reset
we will set the layout UUID of an item back to None.

This also prepares us for the scale-out scheduler where we have to
re-calculate a layout on a scheduler if it is not available in the
cache, as we are not storing the layout in Zookeeper due to it's size.

Since the project metadata is needed by the executor client, which
should not access to layout anymore, the metadata is now available via
the job graph.

Change-Id: I93c7a3932fbf9ad8915d1d3d1fff9682778b28f8

tests/unit/test_model.py

@@ -51,6 +51,7 @@ class TestJob(BaseTestCase):
         self.tenant = model.Tenant('tenant')
         self.tenant.default_ansible_version = AnsibleManager().default_version
         self.layout = model.Layout(self.tenant)
+        self.tenant.layout = self.layout
         self.project = model.Project('project', self.source)
         self.context = model.SourceContext(self.project, 'master',
                                            'test', True)
@@ -204,13 +205,12 @@ class TestJob(BaseTestCase):
         change = model.Change(self.project)
         change.branch = 'master'
         item = queue.enqueueChange(change, None)
-        item.layout = self.layout
 
         self.assertTrue(base.changeMatchesBranch(change))
         self.assertTrue(python27.changeMatchesBranch(change))
         self.assertFalse(python27diablo.changeMatchesBranch(change))
 
-        item.freezeJobGraph()
+        item.freezeJobGraph(self.layout)
         self.assertEqual(len(item.getJobs()), 1)
         job = item.getJobs()[0]
         self.assertEqual(job.name, 'python27')
@@ -218,13 +218,12 @@ class TestJob(BaseTestCase):
 
         change.branch = 'stable/diablo'
         item = queue.enqueueChange(change, None)
-        item.layout = self.layout
 
         self.assertTrue(base.changeMatchesBranch(change))
         self.assertTrue(python27.changeMatchesBranch(change))
         self.assertTrue(python27diablo.changeMatchesBranch(change))
 
-        item.freezeJobGraph()
+        item.freezeJobGraph(self.layout)
         self.assertEqual(len(item.getJobs()), 1)
         job = item.getJobs()[0]
         self.assertEqual(job.name, 'python27')
@@ -267,12 +266,11 @@ class TestJob(BaseTestCase):
         change.branch = 'master'
         change.files = ['/COMMIT_MSG', 'ignored-file']
         item = queue.enqueueChange(change, None)
-        item.layout = self.layout
 
         self.assertTrue(base.changeMatchesFiles(change))
         self.assertFalse(python27.changeMatchesFiles(change))
 
-        item.freezeJobGraph()
+        item.freezeJobGraph(self.layout)
         self.assertEqual([], item.getJobs())
 
     def test_job_source_project(self):
@@ -340,7 +338,7 @@ class TestJob(BaseTestCase):
         with testtools.ExpectedException(
                 Exception,
                 "Pre-review pipeline gate does not allow post-review job"):
-            item.freezeJobGraph()
+            item.freezeJobGraph(self.layout)
 
 
 class TestJobTimeData(BaseTestCase):