Revert "Bootstrap-bridge as top-level job"

This reverts commit d616ec9d9a.

We are hitting ERROR! 'zuul_return' is not a valid attribute for a Play
in the bootstrap-bridge job.

Change-Id: Iebb49ae9c01ea62e8877860fdb0bf1e3d4080607

playbooks/zuul/run-production-bootstrap-bridge.yaml

@@ -3,9 +3,3 @@
     - add-bastion-host
 
 - import_playbook: ../bootstrap-bridge.yaml
-
-- name: Wait for child jobs
-  zuul_return:
-    data:
-      zuul:
-        pause: true

zuul.d/infra-prod.yaml

@@ -2,59 +2,14 @@
 # in projects.yaml because it's easier to keep an overall view of
 # what's happening in there.
 
-# Make sure only one run happens at a time.  The deploy pipeline
+# Make sure only one run of a system-config playbook happens at a time
-# should keep things in order, but this is to stop perodic jobs
-# jumping in.
 - semaphore:
-    name: infra-prod-deployment
+    name: infra-prod-playbook
     max: 1
 
-# This semaphore limits the total number of production playbook
-# jobs that can run on bridge at one time. We want things to run in
-# parallel but we have a lot of jobs (particularly in the periodic
-# pipeline) that we don't want to run all at once.
-- semaphore:
-    name: infra-prod-playbook-limit
-    # TODO(clarkb) this semaphore allows us to stage the rollout of
-    # parallel infra-prod job exceution in two steps. First we reorganize
-    # everything but roughly keep the same behaviors as before (max: 1).
-    # When we are happy with that we can bump this to 2 or higher and see
-    # things run in parallel.
-    max: 1
-
-- job:
-    name: infra-prod-bootstrap-bridge
-    parent: opendev-infra-prod-setup-src
-    semaphores: infra-prod-deployment
-    description: |
-        Configure the bastion host (bridge)
-
-        This job does minimal configuration on the bastion host
-        (bridge.openstack.org) to allow it to run system-config
-        playbooks against our production hosts.  It sets up Ansible
-        and root keys on the host.  It also synchronizes the
-        system-config repo from the executor to the bastion.
-
-        Note that this is separate to infra-prod-service-bridge;
-        bridge in it's role as the bastion host actaully runs that
-        against itself; it includes things not strictly needed to make
-        the host able to deploy system-config.
-
-        This job is the parent of all deployment jobs, and will pause
-        until they finish.  This prevents conflicts between deployment
-        jobs from changes and periodic runs (which use HEAD of
-        master).
-    run: playbooks/zuul/run-production-bootstrap-bridge.yaml
-    # Do not set file matchers on this job. We must always run this job
-    # before any other infra-prod jobs to ensure system-config is up to
-    # date on bridge before we run our playbooks.
-    nodeset:
-      nodes: []
-
 - job:
     name: infra-prod-playbook
-    parent: opendev-infra-prod-setup-keys
+    parent: opendev-infra-prod-base
-    semaphores: infra-prod-playbook-limit
     description: |
       Run specified playbook against productions hosts.
 
@@ -64,6 +19,7 @@
       /home/zuul/src/opendev.org/opendev/system-config/playbooks
       on the bastion host.
     abstract: true
+    semaphores: infra-prod-playbook
     run: playbooks/zuul/run-production-playbook.yaml
     post-run: playbooks/zuul/run-production-playbook-post.yaml
     required-projects:
@@ -74,12 +30,41 @@
       infra_prod_playbook_encrypt_log: true
     nodeset:
       nodes: []
-    dependencies:
+
-      - name: infra-prod-bootstrap-bridge
+- job:
-        # This is a hard dependency because we require the bootstrap job to
+    name: infra-prod-bootstrap-bridge
-        # have run before we start any playbook jobs, otherwise our buildset
+    parent: opendev-infra-prod-setup-src
-        # would not hold the bridge semaphore and we may not have the correct
+    description: |
-        # system-config state on bridge.
+        Configure the bastion host (bridge)
+
+        This job does minimal configuration on the bastion host
+        (bridge.openstack.org) to allow it to run system-config
+        playbooks against our production hosts.  It sets up Ansible
+        and root keys on the host. It also synchronizes the system-config
+        repo from the executor to the bastion. This is necessary to
+        emit an up to date known_hosts file when adding new hosts to
+        the inventory.
+
+        Note that this is separate to infra-prod-service-bridge;
+        bridge in it's role as the bastion host actaully runs that
+        against itself; it includes things not strictly needed to make
+        the host able to deploy system-config.
+    # While we don't run the infra-prod-playbook in this job we do run
+    # system-config git repo updates. Until we're ready to stop running
+    # system-config updates in every job we use this semaphore to ensure
+    # exclusivity.
+    semaphores: infra-prod-playbook
+    run: playbooks/zuul/run-production-bootstrap-bridge.yaml
+    files:
+      - playbooks/bootstrap-bridge.yaml
+      - playbooks/zuul/run-production-bootstrap-bridge.yaml
+      - playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml
+      - playbooks/roles/install-ansible/
+      - playbooks/roles/root-keys/
+      - inventory/base/hosts.yaml
+      - inventory/service/group_vars/bastion.yaml
+    nodeset:
+      nodes: []
 
 - job:
     name: infra-prod-base

zuul.d/project.yaml

@@ -340,10 +340,7 @@
         # NOTE: infra-prod-* jobs have a hierarchy below that ensure
         # they can run in parallel.  We are deliberately keeping their
         # dependencies here rather than job definitions to help keep
-        # these relationships clear. The one exception to this is the
+        # these relationships clear.
-        # base infra-prod-playbook job depends on infra-prod-bootstrap-bridge.
-        # We make this exception because it is vital that bootstrap-bridge
-        # run before everything else always.
 
         # This installs the ansible on bridge that all the infra-prod
         # jobs will run with.  Note the jobs use this ansible to then
@@ -351,7 +348,10 @@
         - infra-prod-bootstrap-bridge
 
         # From now on, all jobs should depend on base
-        - infra-prod-base
+        - infra-prod-base: &infra-prod-base
+            dependencies:
+              - name: infra-prod-bootstrap-bridge
+                soft: true
 
         # Legacy puppet hosts
         - infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
@@ -635,7 +635,7 @@
         # Nightly runs of ansible things for catchup
         # Keep in order from above
         - infra-prod-bootstrap-bridge
-        - infra-prod-base
+        - infra-prod-base: *infra-prod-base
         - infra-prod-remote-puppet-else: *infra-prod-remote-puppet-else
         - infra-prod-letsencrypt: *infra-prod-letsencrypt
         - infra-prod-service-bridge: *infra-prod-service-bridge