Ubuntu: add support for Apt repository configuration

This change adds support for configuring Apt repositories on Ubuntu
hosts during host configuration.

Repositories are configured in a single file
(/etc/apt/sources.list.d/kayobe.sources), using the modern deb822
format [1]. This format is more flexible and readable than the original
single-line format, particularly if multiple options are used.

Using a single file allows us to more easily keep the set of
repositories in sync, since Ansible doesn't make it easy to clean things
up.

Support is added for marking repositories as signed by a particular GPG
key. This approach is now preferred over the deprecated [2] apt-key
tool, which resulted in a set of globally trusted keys.

It is also possible to disable the repositories in
/etc/apt/sources.list via apt_disable_sources_list. This allows for
replacing the standard repositories with a local mirror.

CI tests and documentation are provided.

[1] https://manpages.ubuntu.com/manpages/focal/en/man5/sources.list.5.html
[2] https://manpages.ubuntu.com/manpages/groovy/man8/apt-key.8.html

Story: 2009655
Task: 43818

Change-Id: I3f821937b0930a0ac9341178de7ae5123d82b957

ansible/group_vars/all/apt

@@ -10,3 +10,30 @@ apt_proxy_http:
 
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 apt_proxy_https: "{{ apt_proxy_http }}"
+
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+apt_keys: []
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+apt_repositories: []
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+apt_disable_sources_list: false

ansible/roles/apt/defaults/main.yml

@@ -10,3 +10,33 @@ apt_proxy_http:
 
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 apt_proxy_https: "{{ apt_proxy_http }}"
+
+# Directory containing GPG keyrings for apt repos.
+apt_keys_path: "/usr/local/share/keyrings"
+
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+apt_keys: []
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+apt_repositories: []
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+apt_disable_sources_list: false

ansible/roles/apt/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Update apt cache
+  package:
+    update_cache: true
+  become: true

ansible/roles/apt/tasks/keys.yml

@@ -0,0 +1,19 @@
+---
+- name: Ensure keys directory exists
+  file:
+    path: "{{ apt_keys_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+  become: true
+
+- name: Ensure keys exist
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ apt_keys_path ~ '/' ~ item.filename | basename }}"
+    owner: root
+    group: root
+    mode: 0644
+  loop: "{{ apt_keys }}"
+  become: true

ansible/roles/apt/tasks/main.yml

@@ -1,17 +1,6 @@
 ---
-- name: Configure apt proxy
-  template:
-    src: "01proxy.j2"
-    dest: /etc/apt/apt.conf.d/01proxy
-    owner: root
-    group: root
-    mode: 0664
-  become: true
-  when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0
+- import_tasks: proxy.yml
 
-- name: Remove old apt proxy config
-  file:
-    path: /etc/apt/apt.conf.d/01proxy
-    state: absent
-  become: true
-  when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0
+- import_tasks: keys.yml
+
+- import_tasks: repos.yml

ansible/roles/apt/tasks/proxy.yml

@@ -0,0 +1,17 @@
+---
+- name: Configure apt proxy
+  template:
+    src: "01proxy.j2"
+    dest: /etc/apt/apt.conf.d/01proxy
+    owner: root
+    group: root
+    mode: 0664
+  become: true
+  when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0
+
+- name: Remove old apt proxy config
+  file:
+    path: /etc/apt/apt.conf.d/01proxy
+    state: absent
+  become: true
+  when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0

ansible/roles/apt/tasks/repos.yml

@@ -0,0 +1,22 @@
+---
+# NOTE(mgoddard): Use the modern deb822 repository format rather than the old
+# format used by the apt_repository module.
+- name: Configure apt repositories
+  template:
+    src: "kayobe.sources.j2"
+    dest: "/etc/apt/sources.list.d/kayobe.sources"
+    owner: root
+    group: root
+    mode: 0644
+  become: true
+  notify:
+    - Update apt cache
+
+- name: Disable repositories in /etc/apt/sources.list
+  replace:
+    backup: true
+    path: /etc/apt/sources.list
+    regexp: '^(deb.*)'
+    replace: '# \1'
+  when: apt_disable_sources_list | bool
+  become: true

ansible/roles/apt/templates/kayobe.sources.j2

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+{% for repo in apt_repositories %}
+Types: {{ repo.types | default('deb') }}
+URIs: {{ repo.url }}
+Suites: {{ repo.suites | default(ansible_facts.distribution_release) }}
+Components: {{ repo.components | default('main') }}
+{% if repo.signed_by is defined %}
+Signed-by: {{ apt_keys_path }}/{{ repo.signed_by }}
+{% endif %}
+{% if repo.architecture is defined %}
+Architecture: {{ repo.architecture }}
+{% endif %}
+
+{% endfor %}

doc/source/configuration/reference/hosts.rst

@@ -316,8 +316,7 @@ oversight or testing.
 Apt
 ===
 
-On Ubuntu, Apt is used to manage packages and package repositories. Currently
-Kayobe does not provide support for configuring custom Apt repositories.
+On Ubuntu, Apt is used to manage packages and package repositories.
 
 Apt cache
 ---------
@@ -325,10 +324,100 @@ Apt cache
 The Apt cache timeout may be configured via ``apt_cache_valid_time`` (in
 seconds) in ``etc/kayobe/apt.yml``, and defaults to 3600.
 
+Apt proxy
+---------
+
 Apt can be configured to use a proxy via ``apt_proxy_http`` and
 ``apt_proxy_https`` in ``etc/kayobe/apt.yml``. These should be set to the full
 URL of the relevant proxy (e.g. ``http://squid.example.com:3128``).
 
+Apt repositories
+----------------
+
+Kayobe supports configuration of custom Apt repositories via the
+``apt_repositories`` variable in ``etc/kayobe/apt.yml`` since the Yoga release.
+The format is a list, with each item mapping to a dict/map with the following
+items:
+
+* ``types``: whitespace-separated list of repository types, e.g. ``deb`` or
+  ``deb-src`` (optional, default is ``deb``)
+* ``url``: URL of the repository
+* ``suites``: whitespace-separated list of suites, e.g. ``focal`` (optional,
+  default is ``ansible_facts.distribution_release``)
+* ``components``: whitespace-separated list of components, e.g. ``main``
+  (optional, default is ``main``)
+* ``signed_by``: whitespace-separated list of names of GPG keyring files in
+  ``apt_keys_path`` (optional, default is unset)
+* ``architecture``: whitespace-separated list of architectures that will be used
+  (optional, default is unset)
+
+The default of ``apt_repositories`` is an empty list.
+
+For example, the following configuration defines a single Apt repository:
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_repositories:
+     - types: deb
+       url: https://example.com/repo
+       suites: focal
+       components: all
+
+In the following example, the Ubuntu Focal 20.04 repositories are consumed from
+a local package mirror. The ``apt_disable_sources_list`` variable is set to
+``true``, which disables all repositories in ``/etc/apt/sources.list``,
+including the default Ubuntu ones.
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_repositories:
+     - url: http://mirror.example.com/ubuntu/
+       suites: focal focal-updates
+       components: main restricted universe multiverse
+     - url: http://mirror.example.com/ubuntu/
+       suites: focal-security
+       components: main restricted universe multiverse
+
+   apt_disable_sources_list: true
+
+Apt keys
+--------
+
+Some repositories may be signed by a key that is not one of Apt's trusted keys.
+Kayobe avoids the use of the deprecated ``apt-key`` utility, and instead allows
+keys to be downloaded to a directory. This enables repositories to use the
+``SignedBy`` option to state that they are signed by a specific key. This
+approach is more secure than using globally trusted keys.
+
+Keys to be downloaded are defined by the ``apt_keys`` variable. The format is a
+list, with each item mapping to a dict/map with the following items:
+
+* ``url``: URL of key
+* ``filename``: Name of a file in which to store the downloaded key in
+  ``apt_keys_path``. The extension should be ``.asc`` for ASCII-armoured keys,
+  or ``.gpg`` otherwise.
+
+The default value of ``apt_keys`` is an empty list.
+
+In the following example, a key is downloaded, and a repository is configured
+that is signed by the key.
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_keys:
+     - url: https://example.com/GPG-key
+       filename: example-key.asc
+
+   apt_repositories:
+     - types: deb
+       url: https://example.com/repo
+       suites: focal
+       components: all
+       signed_by: example-key.asc
+
 SELinux
 =======
 *tags:*

etc/kayobe/apt.yml

@@ -11,6 +11,33 @@
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 #apt_proxy_https:
 
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+#apt_keys:
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+#apt_repositories:
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+#apt_disable_sources_list:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2

@@ -114,6 +114,25 @@ docker_storage_driver: devicemapper
 # Set Honolulu time.
 timezone: Pacific/Honolulu
 
+{% if ansible_os_family == "Debian" %}
+apt_keys:
+  - url: https://packages.treasuredata.com/GPG-KEY-td-agent
+    filename: td-agent.asc
+apt_repositories:
+  # Ubuntu focal repositories.
+  - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/"
+    suites: focal focal-updates
+    components: main restricted universe multiverse
+  - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/"
+    suites: focal-security
+    components: main restricted universe multiverse
+  # Treasuredata repository.
+  - url:  http://packages.treasuredata.com/4/ubuntu/focal/
+    components: contrib
+    signed_by: td-agent.asc
+apt_disable_sources_list: true
+{% endif %}
+
 {% if ansible_os_family in ['RedHat', 'Rocky'] %}
 # Use a local DNF mirror.
 dnf_use_local_mirror: true

playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py

@@ -16,6 +16,11 @@ def _is_firewalld_supported():
     return info in ['centos', 'rocky']
 
 
+def _is_apt():
+    info = distro.linux_distribution()
+    return info[0].startswith('Ubuntu')
+
+
 def _is_dnf():
     info = distro.id()
     return info in ['centos', 'rocky']
@@ -187,6 +192,13 @@ def test_ntp_clock_synchronized(host):
     assert "synchronized: yes" in status_output
 
 
+@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu")
+def test_apt_custom_package_repository_is_available(host):
+    with host.sudo():
+        host.check_output("apt -y install td-agent")
+    assert host.package("td-agent").is_installed
+
+
 @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel",
                                   "epel-modular"])
 @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8")

releasenotes/notes/apt-repositories-850efef70ba34946.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for configuring Apt repositories on Ubuntu hosts. See `story
+    2009655 <https://storyboard.openstack.org/#!/story/2009655>`__ for details.