Split the base playbook into services

This is a first step toward making smaller playbooks which can be
run by Zuul in CD.

Zuul should be able to handle missing projects now, so remove it
from the puppet_git playbook and into puppet.

Make the base playbook be merely the base roles.

Make service playbooks for each service.

Remove the run-docker job because it's covered by service jobs.

Stop testing that puppet is installed in testinfra. It's accidentally
working due to the selection of non-puppeted hosts only being on
bionic nodes and not installing puppet on bionic. Instead, we can now
rely on actually *running* puppet when it's important, such as in the
eavesdrop job. Also remove the installation of puppet on the nodes in
the base job, since it's only useful to test that a synthetic test
of installing puppet on nodes we don't use works.

Don't run remote_puppet_git on gitea for now - it's too slow. A
followup patch will rework gitea project creation to not take hours.

Change-Id: Ibb78341c2c6be28005cea73542e829d8f7cfab08

.zuul.yaml

@@ -406,7 +406,9 @@
       - .zuul.yaml
       - playbooks/group_vars/eavesdrop.yaml
       - testinfra/test_eavesdrop.py
-
+    vars:
+      run_playbooks:
+        - playbooks/remote_puppet_else.yaml
 
 - job:
     name: system-config-run-letsencrypt
@@ -423,6 +425,10 @@
           label: ubuntu-bionic
         - name: letsencrypt02.opendev.org
           label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-nameserver.yaml
+        - playbooks/service-letsencrypt.yaml
     host-vars:
       letsencrypt01.opendev.org:
         host_copy_output:
@@ -448,6 +454,10 @@
           label: ubuntu-xenial
         - name: nb01.openstack.org
           label: ubuntu-xenial
+    vars:
+      run_playbooks:
+        - playbooks/service-nodepool.yaml
+        - playbooks/remote_puppet_else.yaml
     files:
       - .zuul.yaml
       - playbooks/group_vars/nodepool.yaml
@@ -457,22 +467,6 @@
       - playbooks/templates/clouds/
       - testinfra/test_nodepool.py
 
-- job:
-    name: system-config-run-docker
-    parent: system-config-run
-    description: |
-      Test docker installation and setup
-    nodeset:
-      nodes:
-        - name: bridge.openstack.org
-          label: ubuntu-bionic
-        - name: bionic-docker
-          label: ubuntu-bionic
-    files:
-      - .zuul.yaml
-      - playbooks/roles/install-docker
-      - testinfra/test_docker.py
-
 - job:
     name: system-config-run-dns
     parent: system-config-run
@@ -489,6 +483,9 @@
           label: ubuntu-bionic
         - name: ns1.opendev.org
           label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-nameserver.yaml
     host-vars:
       adns1.opendev.org:
         host_copy_output:
@@ -518,6 +515,9 @@
           label: ubuntu-bionic
         - name: insecure-ci-registry01.opendev.org
           label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-registry.yaml
     host-vars:
       insecure-ci-registry01.opendev.org:
         host_copy_output:
@@ -546,7 +546,9 @@
         - name: gitea01.opendev.org
           label: ubuntu-bionic
     vars:
-      run_base_test_playbook: playbooks/zuul/test-gitea.yaml
+      run_playbooks:
+        - playbooks/service-gitea-lb.yaml
+      run_test_playbook: playbooks/test-gitea.yaml
     host-vars:
       gitea01.opendev.org:
         host_copy_output:
@@ -583,6 +585,9 @@
           label: ubuntu-bionic
         - name: zp01.opendev.org
           label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-zuul.yaml
     files:
       - .zuul.yaml
       - playbooks/roles/zuul-preview/
@@ -634,7 +639,6 @@
         - system-config-run-dns
         - system-config-run-eavesdrop
         - system-config-run-nodepool
-        - system-config-run-docker
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:
@@ -659,7 +663,6 @@
         - system-config-run-dns
         - system-config-run-eavesdrop
         - system-config-run-nodepool
-        - system-config-run-docker
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:

playbooks/base.yaml

@@ -22,89 +22,3 @@
   roles:
     - snmpd
     - iptables
-
-- hosts: bridge.openstack.org:!disabled
-  name: "Base: configure cloud credentials on bridge"
-  roles:
-    - install-kubectl
-    - configure-kubectl
-  tasks:
-    - include_role:
-        name: configure-openstacksdk
-      vars:
-        openstacksdk_config_file: '{{ openstacksdk_config_dir }}/all-clouds.yaml'
-        openstacksdk_config_template: clouds/bridge_all_clouds.yaml.j2
-    - include_role:
-        name: configure-openstacksdk
-      vars:
-        openstacksdk_config_template: clouds/bridge_clouds.yaml.j2
-
-- hosts: nodepool-launcher:nodepool-builder:!disabled
-  name: "Base: configure OpenStackSDK on nodepool"
-  strategy: free
-  roles:
-    - minimal-nodepool
-    - configure-openstacksdk
-    - configure-kubectl
-
-- hosts: "puppet:!disabled"
-  name: "Base: install and configure puppet on puppet hosts"
-  roles:
-    - puppet-install
-    - disable-puppet-agent
-
-- hosts: adns:!disabled
-  name: "Base: configure adns server"
-  roles:
-    - master-nameserver
-
-- hosts: "ns1.opendev.org:ns2.opendev.org:!disabled"
-  name: "Base: configure authoritative nameservers"
-  roles:
-    - nameserver
-
-- hosts: "docker:!disabled"
-  name: "Base: install and configure docker on docker hosts"
-  roles:
-    - install-docker
-
-- hosts: "registry:!disabled"
-  name: "Base: configure registry"
-  roles:
-    - install-docker
-    - registry
-
-- hosts: "gitea:!disabled"
-  name: "Base: configure gitea"
-  roles:
-    - install-docker
-    - gitea
-
-- hosts: "gitea-lb:!disabled"
-  name: "Base: configure gitea load balancer"
-  roles:
-    - install-docker
-    - haproxy
-
-- hosts: "zuul-preview:!disabled"
-  name: "Base: configure zuul-preview"
-  roles:
-    - install-docker
-    - zuul-preview
-
-# This next section needs to happen in order.  letsencrypt hosts
-# export their TXT authentication records which is installed onto
-# adns1, and then the hosts verify to issue/renew keys
-- hosts: "letsencrypt:!disabled"
-  name: "Base: deploy and renew certificates"
-  roles:
-    - letsencrypt-acme-sh-install
-    - letsencrypt-request-certs
-- hosts: "adns:!disabled"
-  name: "Install txt records"
-  roles:
-    - letsencrypt-install-txt-record
-- hosts: "letsencrypt:!disabled"
-  name: "Create certs"
-  roles:
-    - letsencrypt-create-certs

playbooks/bridge.yaml

@@ -1,5 +1,5 @@
-- hosts: bridge.openstack.org
+- hosts: bridge.openstack.org:!disabled
-  name: "Bridge: configure the bastion host"
+  name: "Bridge: boostrap the bastion host"
   become: true
   roles:
     - pip3
@@ -21,6 +21,3 @@
       install_ansible_ara_name: '{{ bridge_ara_name | default("ara") }}'
       install_ansible_ara_version: '{{ bridge_ara_version | default("0.16.1") }}'
     - root-keys
-    - ansible-cron
-    - cloud-launcher-cron
-    - edit-secrets-script

playbooks/remote_puppet_afs.yaml

@@ -1,3 +1,9 @@
+- hosts: "afs:afsdb:!disabled"
+  name: "Base: install and configure puppet on puppet hosts"
+  roles:
+    - puppet-install
+    - disable-puppet-agent
+
 - hosts: "afs:afsdb:!disabled"
   name: "AFS: run puppet on the AFS servers"
   strategy: free

playbooks/remote_puppet_else.yaml

@@ -1,5 +1,7 @@
-- hosts: 'puppet:!review:!zuul-scheduler:!afs:!afsdb:!puppetmaster*:!disabled'
+- hosts: 'puppet:!review:!afs:!afsdb:!puppetmaster*:!disabled'
   name: "Puppet-else: run puppet on all other servers"
   strategy: free
   roles:
+    - puppet-install
+    - disable-puppet-agent
     - puppet

playbooks/remote_puppet_git.yaml

@@ -1,3 +1,15 @@
+- hosts: "gitea:!disabled"
+  name: "Base: configure gitea"
+  roles:
+    - install-docker
+    - gitea
+
+- hosts: "review:!disabled"
+  name: "Base: install and configure puppet on puppet hosts"
+  roles:
+    - puppet-install
+    - disable-puppet-agent
+
 - hosts: "localhost:!disabled"
   name: "Puppet-git: Collect the project-config ref"
   strategy: free
@@ -27,13 +39,3 @@
       facts:
         project_config_ref: "{{ hostvars.localhost.gitinfo.after }}"
       puppet_timeout: 60m
-
-- hosts: "zuul-scheduler:!disabled"
-  name: "Puppet-git: Run puppet on the Zuul scheduler"
-  strategy: free
-  gather_facts: true
-  roles:
-    - role: puppet
-      facts:
-        project_config_ref: "{{ hostvars.localhost.gitinfo.after }}"
-      puppet_timeout: 60m

playbooks/roles/gitea/defaults/main.yaml

@@ -0,0 +1 @@
+gitea_no_log: true

playbooks/roles/gitea/tasks/main.yaml

@@ -59,7 +59,7 @@
   block:
     - name: Create root user
       command: "docker exec -t giteadocker_gitea-web_1 gitea admin create-user --name root --password {{ gitea_root_password }} --email {{ gitea_root_email }} --admin"
-      no_log: true
+      no_log: "{{ gitea_no_log }}"
 - name: Check if gerrit user exists
   uri:
     url: "https://localhost:3000/api/v1/users/gerrit"

playbooks/service-bridge.yaml

@@ -0,0 +1,18 @@
+- hosts: bridge.openstack.org:!disabled
+  name: "Bridge: configure the bastion host"
+  roles:
+    - ansible-cron
+    - cloud-launcher-cron
+    - edit-secrets-script
+    - install-kubectl
+    - configure-kubectl
+  tasks:
+    - include_role:
+        name: configure-openstacksdk
+      vars:
+        openstacksdk_config_file: '{{ openstacksdk_config_dir }}/all-clouds.yaml'
+        openstacksdk_config_template: clouds/bridge_all_clouds.yaml.j2
+    - include_role:
+        name: configure-openstacksdk
+      vars:
+        openstacksdk_config_template: clouds/bridge_clouds.yaml.j2

playbooks/service-gitea-lb.yaml

@@ -0,0 +1,5 @@
+- hosts: "gitea-lb:!disabled"
+  name: "Base: configure gitea load balancer"
+  roles:
+    - install-docker
+    - haproxy

playbooks/service-letsencrypt.yaml

@@ -0,0 +1,16 @@
+# This needs to happen in order.  letsencrypt hosts export their TXT
+# authentication records which is installed onto adns1, and then the
+# hosts verify to issue/renew keys
+- hosts: "letsencrypt:!disabled"
+  name: "Base: deploy and renew certificates"
+  roles:
+    - letsencrypt-acme-sh-install
+    - letsencrypt-request-certs
+- hosts: "adns:!disabled"
+  name: "Install txt records"
+  roles:
+    - letsencrypt-install-txt-record
+- hosts: "letsencrypt:!disabled"
+  name: "Create certs"
+  roles:
+    - letsencrypt-create-certs

playbooks/service-nameserver.yaml

@@ -0,0 +1,10 @@
+- hosts: adns:!disabled
+  name: "Base: configure adns server"
+  roles:
+    - master-nameserver
+
+- hosts: "ns1.opendev.org:ns2.opendev.org:!disabled"
+  name: "Base: configure authoritative nameservers"
+  roles:
+    - nameserver
+

playbooks/service-nodepool.yaml

@@ -0,0 +1,7 @@
+- hosts: nodepool-launcher:nodepool-builder:!disabled
+  name: "Base: configure OpenStackSDK on nodepool"
+  strategy: free
+  roles:
+    - minimal-nodepool
+    - configure-openstacksdk
+    - configure-kubectl

playbooks/service-registry.yaml

@@ -0,0 +1,5 @@
+- hosts: "registry:!disabled"
+  name: "Base: configure registry"
+  roles:
+    - install-docker
+    - registry

playbooks/service-zuul.yaml

@@ -0,0 +1,5 @@
+- hosts: "zuul-preview:!disabled"
+  name: "Base: configure zuul-preview"
+  roles:
+    - install-docker
+    - zuul-preview

playbooks/test-gitea.yaml

@@ -14,6 +14,14 @@
         dest: "/opt/project-config/gerrit/projects.yaml"
         content: "{{ projects }}"
 
+# TODO(mordred) This should be part of the service, once we refactor
+# the project creation and are running remote_puppet_git.
+- hosts: "gitea"
+  name: "Install and configure gitea"
+  roles:
+    - install-docker
+    - gitea
+
 - hosts: "gitea"
   name: "Create repos on gitea servers"
   strategy: free

playbooks/zuul/run-base.yaml

@@ -18,6 +18,17 @@
         repo: /home/zuul/src/opendev.org/opendev/system-config
         dest: /opt/system-config
         force: yes
+    # TODO: the next two tasks are update-system-config.yaml and
+    # should be removed or refactored out of here to a shared
+    # location.
+    - name: Clone puppet modules to /etc/puppet/modules
+      command: ./install_modules.sh
+      args:
+        chdir: /opt/system-config
+    - name: Install ansible roles to /etc/ansible/roles
+      command: ansible-galaxy install --roles-path /etc/ansible/roles --force -r roles.yaml
+      args:
+        chdir: /opt/system-config
     - name: Add groups config for test nodes
       template:
         src: "templates/gate-groups.yaml.j2"
@@ -33,7 +44,7 @@
         path: /etc/ansible/ansible.cfg
         section: defaults
         option: inventory_plugins
-        value: /home/zuul/src/opendev.org/opendev/system-config/playbooks/roles/install-ansible/files/inventory_plugins
+        value: /opt/system-config/playbooks/roles/install-ansible/files/inventory_plugins
     - name: Update ansible.cfg to configure inventory plugins
       ini_file:
         path: /etc/ansible/ansible.cfg
@@ -74,9 +85,15 @@
       command: ansible localhost -m debug -a 'var=groups'
     - name: Run base.yaml
       command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml
+    - name: Run bridge service playbook
+      command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml
+    - name: Run playbook
+      when: run_playbooks is defined
+      loop: "{{ run_playbooks }}"
+      command: "ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ item }}"
     - name: Run test playbook
-      when: run_base_test_playbook is defined
+      when: run_test_playbook is defined
-      shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_base_test_playbook }}"
+      shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_test_playbook }}"
     - name: Run testinfra to validate configuration
       include_role:
         name: tox

playbooks/zuul/templates/gate-groups.yaml.j2

@@ -2,12 +2,6 @@
 # for gate jobs are put in the right groups for testing
 plugin: yamlgroup
 groups:
-  puppet:
-    - trusty
-    - xenial
-    - centos7
-    # note: bionic currently isn't puppeted
-
   docker:
     - bionic-docker
 

playbooks/zuul/templates/group_vars/gitea.yaml.j2

@@ -5,6 +5,7 @@ gitea_root_db_password: TlG1lNXKLfruXN0j
 gitea_db_username: gitea
 gitea_db_password: 5bfuOBKtltff0XZX
 gitea_root_password: BUbBcpToMwR05ZCB
+gitea_no_log: false
 gitea_gerrit_password: yVpMWIUIvT7f6NwA
 gitea_tls_cert: |
   -----BEGIN CERTIFICATE-----

run_all.sh

@@ -92,6 +92,31 @@ start_timer
 timeout -k 2m 120m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/base.yaml
 send_timer base
 
+# Service playbooks
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea-lb.yaml
+send_timer gitea-lb
+
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-letsencrypt.yaml
+send_timer letsencrypt
+
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nameserver.yaml
+send_timer nameserver
+
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
+send_timer nodepool
+
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
+send_timer registry
+
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-zuul.yaml
+send_timer zuul
+
 # Run the git/gerrit/zuul sequence, since it's important that they all work together
 start_timer
 timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_git.yaml

testinfra/test_base.py

@@ -40,24 +40,6 @@ def test_exim_is_installed(host):
     assert cmd.rc == 0
 
 
-def test_puppet(host):
-    # We only install puppet on trusty, xenial and centos 7
-    if (host.system_info.codename in ['trusty', 'xenial'] or
-            host.system_info.distribution in ['centos']):
-        # Package name differs depending on puppet release version
-        # just check one version of puppet is installed.
-        puppet = host.package("puppet")
-        puppet_agent = host.package("puppet-agent")
-        assert puppet.is_installed or puppet_agent.is_installed
-        service = host.service("puppet")
-        assert not service.is_running
-        assert not service.is_enabled
-    else:
-        puppet = host.package("puppet")
-        puppet_agent = host.package("puppet-agent")
-        assert not puppet.is_installed and not puppet_agent.is_installed
-
-
 def test_iptables(host):
     rules = host.iptables.rules()
     rules = [x.strip() for x in rules]

testinfra/test_docker.py

@@ -1,21 +0,0 @@
-# Copyright 2018 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-testinfra_hosts = ['bionic-docker']
-
-
-def test_docker_service(host):
-    docker = host.service('docker')
-    assert docker.is_running