Use docker_custom_config variable

In the Train cycle, Kolla Ansible added support for
docker_custom_config, and writes out configuration to
/etc/docker/daemon.json. This will conflict with Kayobe's configuration
of that file, and changes made by kayobe will be reversed when
kolla-ansible bootstrap-servers is run.

This change uses the new variable to pass daemon.json configuration
through to kolla ansible. Because the ordering has changed, we also need
to separate out the devicemapper setup and run this prior to starting
docker.

Change-Id: Idc3fa9fefd8242ef9db76d4d773885e3594b453a
Depends-On: https://review.opendev.org/691001
Story: 2006764
Task: 37277

ansible/docker-devicemapper.yml

@@ -0,0 +1,11 @@
+---
+- name: Ensure docker devicemapper storage is configured
+  hosts: docker
+  tags:
+    - docker
+    - docker-devicemapper
+  tasks:
+    - name: Ensure docker devicemapper storage is configured
+      include_role:
+        name: docker-devicemapper
+      when: docker_storage_driver == 'devicemapper'

ansible/docker.yml

@@ -7,4 +7,3 @@
     - docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
   roles:
     - role: docker
-      docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"

ansible/kolla-ansible.yml

@@ -303,3 +303,4 @@
         # While kayobe has its own support for installing an NTP daemon, the
         # kolla-ansible baremetal role does a one-time sync which is useful.
         kolla_enable_host_ntp: "{{ ntp_service_enabled }}"
+        docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"

ansible/roles/docker-devicemapper/defaults/main.yml

@@ -0,0 +1,25 @@
+---
+# Name of the docker storage driver.
+docker_storage_driver: devicemapper
+
+# Name of the docker storage LVM volume group.
+docker_storage_volume_group:
+
+# Name of the docker storage data LVM volume.
+docker_storage_volume_thinpool:
+
+# Size of the docker storage data LVM volume (see lvol module size argument).
+docker_storage_volume_thinpool_size:
+
+# Name of the docker storage metadata LVM volume.
+docker_storage_volume_thinpool_meta:
+
+# Size of the docker storage metadata LVM volume (see lvol module size
+# argument).
+docker_storage_volume_thinpool_meta_size:
+
+# Threshold at which to extend thin-provisioned docker storage volumes.
+docker_storage_thinpool_autoextend_threshold: 80
+
+# Percentage by which to extend thin-provisioned docker storage volumes.
+docker_storage_thinpool_autoextend_percent: 20

ansible/roles/docker-devicemapper/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Ensure the docker storage volume is converted to a thinpool
+  command: >
+    lvconvert -y --zero n -c 512K
+    --thinpool {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }}
+    --poolmetadata {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool_meta }}
+  become: True
+
+- name: Ensure the docker storage metadata profile is applied
+  command: >
+    lvchange --metadataprofile docker-thinpool
+    {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }}
+  become: True

ansible/roles/docker-devicemapper/tasks/main.yml

@@ -1,21 +1,33 @@
 ---
-- name: Ensure the docker daemon is stopped
-  service:
-    name: docker
-    state: stopped
-  become: True
-  notify: restart docker service
+- name: Query docker daemon information
+  command: "docker info"
+  register: docker_info
+  changed_when: False
+  failed_when: False
 
-- name: Ensure loopback storage state is absent
-  file:
-    path: "{{ item }}"
-    state: absent
-  with_items:
-    - "/var/lib/docker/devicemapper"
-    - "/var/lib/docker/images"
-    - "/var/lib/docker/containers"
+- name: Fail when non-devicemapper containers or images exist
+  fail:
+    msg: >
+      Not configuring docker storage in {{ docker_storage_driver }} mode as
+      non-devicemapper containers or images exist.
+  when:
+    - docker_info.rc == 0
+    - "'Data loop file' in docker_info.stdout or 'devicemapper' not in docker_info.stdout"
+    - "'Images: 0' not in docker_info.stdout or 'Containers: 0' not in docker_info.stdout"
+
+- name: Ensure the docker storage metadata profile exists
+  template:
+    src: docker-thinpool.profile.j2
+    dest: /etc/lvm/profile/docker-thinpool.profile
   become: True
 
+- name: Query LVM thinpool volume
+  command: "lvs {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }}"
+  register: lvs_result
+  changed_when: false
+  failed_when: false
+  become: true
+
 - block:
     - name: Ensure the docker storage data and metadata volumes exist
       lvol:
@@ -38,15 +50,9 @@
         --poolmetadata {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool_meta }}
       become: True
 
-    - name: Ensure the docker storage metadata profile exists
-      template:
-        src: docker-thinpool.profile.j2
-        dest: /etc/lvm/profile/docker-thinpool.profile
-      become: True
-
     - name: Ensure the docker storage metadata profile is applied
       command: >
         lvchange --metadataprofile docker-thinpool
         {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }}
       become: True
-  when: docker_storage_driver == 'devicemapper'
+  when: lvs_result.rc != 0

ansible/roles/docker/defaults/main.yml

@@ -1,44 +1,10 @@
 ---
-# Name of the docker storage driver.
-docker_storage_driver: devicemapper
-
-# Name of the docker storage LVM volume group.
-docker_storage_volume_group:
-
-# Name of the docker storage data LVM volume.
-docker_storage_volume_thinpool:
-
-# Size of the docker storage data LVM volume (see lvol module size argument).
-docker_storage_volume_thinpool_size:
-
-# Name of the docker storage metadata LVM volume.
-docker_storage_volume_thinpool_meta:
-
-# Size of the docker storage metadata LVM volume (see lvol module size
-# argument).
-docker_storage_volume_thinpool_meta_size:
-
-# Threshold at which to extend thin-provisioned docker storage volumes.
-docker_storage_thinpool_autoextend_threshold: 80
-
-# Percentage by which to extend thin-provisioned docker storage volumes.
-docker_storage_thinpool_autoextend_percent: 20
-
 # URL of docker registry
 docker_registry:
 
 # CA of docker registry
 docker_registry_ca:
 
-# List of Docker registry mirrors.
-docker_registry_mirrors: []
-
-# MTU to pass through to containers not using net=host
-docker_daemon_mtu: 1500
-
-# Enable live-restore on docker daemon
-docker_daemon_live_restore: false
-
 # Upper constraints file which is passed to pip when installing packages
 # into a venv.
 docker_upper_constraints_file:

ansible/roles/docker/handlers/main.yml

@@ -1,10 +1,4 @@
 ---
-- name: restart docker service
-  service:
-    name: docker
-    state: restarted
-  become: True
-
 - name: reload docker service
   service:
     name: docker

ansible/roles/docker/tasks/config.yml

@@ -1,22 +0,0 @@
----
-- name: Ensure the docker daemon configuration file exists
-  template:
-    src: daemon.json.j2
-    dest: /etc/docker/daemon.json
-  become: True
-  notify: restart docker service
-
-- name: Ensure the path for CA file for private registry exists
-  file:
-    path: "/etc/docker/certs.d/{{ docker_registry }}"
-    state: directory
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-
-- name: Ensure the CA file for private registry exists
-  copy:
-    src: "{{ docker_registry_ca }}"
-    dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt"
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-  notify: reload docker service

ansible/roles/docker/tasks/main.yml

@@ -57,24 +57,17 @@
     state: started
   become: True
 
-- name: Query docker daemon information
-  command: "docker info"
-  register: docker_info
-  changed_when: False
-  until: docker_info is success
-  retries: 3
-  delay: 5
+- name: Ensure the path for CA file for private registry exists
+  file:
+    path: "/etc/docker/certs.d/{{ docker_registry }}"
+    state: directory
+  become: True
+  when: docker_registry is not none and docker_registry_ca is not none
 
-- name: Fail when loopback-mode containers or images exist
-  fail:
-    msg: >
-      Not configuring docker storage in {{ docker_storage_driver }} mode as
-      loopback-backed containers or images exist.
-  when:
-    - "'Data loop file' in docker_info.stdout or docker_storage_driver not in docker_info.stdout"
-    - "'Images: 0' not in docker_info.stdout or 'Containers: 0' not in docker_info.stdout"
-
-- include_tasks: storage.yml
-  when: "'Data loop file' in docker_info.stdout or docker_storage_driver not in docker_info.stdout"
-
-- include_tasks: config.yml
+- name: Ensure the CA file for private registry exists
+  copy:
+    src: "{{ docker_registry_ca }}"
+    dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt"
+  become: True
+  when: docker_registry is not none and docker_registry_ca is not none
+  notify: reload docker service

ansible/roles/kolla-ansible/defaults/main.yml

@@ -312,3 +312,30 @@ kolla_selinux_state:
 
 # Whether to enable the NTP daemon.
 kolla_enable_host_ntp:
+
+###############################################################################
+# Docker configuration.
+
+# Name of the docker storage driver.
+docker_storage_driver: devicemapper
+
+# Name of the docker storage LVM volume group.
+docker_storage_volume_group:
+
+# Name of the docker storage data LVM volume.
+docker_storage_volume_thinpool:
+
+# URL of docker registry
+docker_registry:
+
+# CA of docker registry
+docker_registry_ca:
+
+# List of Docker registry mirrors.
+docker_registry_mirrors: []
+
+# MTU to pass through to containers not using net=host
+docker_daemon_mtu: 1500
+
+# Enable live-restore on docker daemon
+docker_daemon_live_restore: false

ansible/roles/kolla-ansible/tasks/config.yml

@@ -45,6 +45,8 @@
     src: "globals.yml.j2"
     dest: "{{ kolla_config_path }}/globals.yml"
     mode: 0640
+  vars:
+    kolla_docker_custom_config: "{{ lookup('template', 'daemon.json.j2') }}"
 
 - name: Ensure the Kolla seed inventory file exists
   copy:

ansible/roles/kolla-ansible/templates/daemon.json.j2

@@ -6,7 +6,6 @@
 {%- endfor %}
   ],
 {%- endif %}
-  "storage-driver": "{{ docker_storage_driver }}",
 {% if docker_daemon_mtu %}
   "mtu": {{ docker_daemon_mtu }},
 {% endif %}

ansible/roles/kolla-ansible/templates/globals.yml.j2

@@ -68,6 +68,8 @@ docker_namespace: "{{ kolla_docker_namespace }}"
 docker_registry_username: "{{ kolla_docker_registry_username }}"
 docker_registry_password: "{{ kolla_docker_registry_password }}"
 {% endif %}
+docker_storage_driver: "{{ docker_storage_driver }}"
+docker_custom_config: {{ kolla_docker_custom_config | to_nice_json | indent(2) }}
 
 ###################
 # Messaging options

kayobe/cli/commands.py

@@ -551,7 +551,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         playbooks += _build_playbook_list(
             "users", "yum", "dev-tools", "disable-selinux", "network",
             "sysctl", "ip-routing", "snat", "disable-glean", "ntp", "mdadm",
-            "lvm")
+            "lvm", "docker-devicemapper")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
 
         self.generate_kolla_ansible_config(parsed_args, service_config=False)
@@ -951,7 +951,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         playbooks += _build_playbook_list(
             "users", "yum", "dev-tools", "disable-selinux", "network",
             "sysctl", "disable-glean", "disable-cloud-init", "ntp", "mdadm",
-            "lvm")
+            "lvm", "docker-devicemapper")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
         self.generate_kolla_ansible_config(parsed_args, service_config=False)

kayobe/tests/unit/cli/test_commands.py

@@ -513,6 +513,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "ntp.yml"),
                     utils.get_data_files_path("ansible", "mdadm.yml"),
                     utils.get_data_files_path("ansible", "lvm.yml"),
+                    utils.get_data_files_path("ansible",
+                                              "docker-devicemapper.yml"),
                 ],
                 limit="seed",
             ),
@@ -1138,6 +1140,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "ntp.yml"),
                     utils.get_data_files_path("ansible", "mdadm.yml"),
                     utils.get_data_files_path("ansible", "lvm.yml"),
+                    utils.get_data_files_path("ansible",
+                                              "docker-devicemapper.yml"),
                 ],
                 limit="overcloud",
             ),

releasenotes/notes/docker-custom-config-5103260d5ddb7223.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    Uses the new Kolla Ansible variable ``docker_custom_config`` to populate
+    Docker's ``daemon.json`` configuration file.