Merge "Ubuntu: add support for Apt repository configuration"

ansible/group_vars/all/apt

@@ -10,3 +10,30 @@ apt_proxy_http:
 
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 apt_proxy_https: "{{ apt_proxy_http }}"
+
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+apt_keys: []
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+apt_repositories: []
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+apt_disable_sources_list: false

ansible/roles/apt/defaults/main.yml

@@ -10,3 +10,33 @@ apt_proxy_http:
 
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 apt_proxy_https: "{{ apt_proxy_http }}"
+
+# Directory containing GPG keyrings for apt repos.
+apt_keys_path: "/usr/local/share/keyrings"
+
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+apt_keys: []
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+apt_repositories: []
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+apt_disable_sources_list: false

ansible/roles/apt/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Update apt cache
+  package:
+    update_cache: true
+  become: true

ansible/roles/apt/tasks/keys.yml

@@ -0,0 +1,19 @@
+---
+- name: Ensure keys directory exists
+  file:
+    path: "{{ apt_keys_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+  become: true
+
+- name: Ensure keys exist
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ apt_keys_path ~ '/' ~ item.filename | basename }}"
+    owner: root
+    group: root
+    mode: 0644
+  loop: "{{ apt_keys }}"
+  become: true

ansible/roles/apt/tasks/main.yml

@@ -1,17 +1,6 @@
 ---
-- name: Configure apt proxy
-  template:
-    src: "01proxy.j2"
-    dest: /etc/apt/apt.conf.d/01proxy
-    owner: root
-    group: root
-    mode: 0664
-  become: true
-  when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0
+- import_tasks: proxy.yml
 
-- name: Remove old apt proxy config
-  file:
-    path: /etc/apt/apt.conf.d/01proxy
-    state: absent
-  become: true
-  when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0
+- import_tasks: keys.yml
+
+- import_tasks: repos.yml

ansible/roles/apt/tasks/proxy.yml

@@ -0,0 +1,17 @@
+---
+- name: Configure apt proxy
+  template:
+    src: "01proxy.j2"
+    dest: /etc/apt/apt.conf.d/01proxy
+    owner: root
+    group: root
+    mode: 0664
+  become: true
+  when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0
+
+- name: Remove old apt proxy config
+  file:
+    path: /etc/apt/apt.conf.d/01proxy
+    state: absent
+  become: true
+  when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0

ansible/roles/apt/tasks/repos.yml

@@ -0,0 +1,22 @@
+---
+# NOTE(mgoddard): Use the modern deb822 repository format rather than the old
+# format used by the apt_repository module.
+- name: Configure apt repositories
+  template:
+    src: "kayobe.sources.j2"
+    dest: "/etc/apt/sources.list.d/kayobe.sources"
+    owner: root
+    group: root
+    mode: 0644
+  become: true
+  notify:
+    - Update apt cache
+
+- name: Disable repositories in /etc/apt/sources.list
+  replace:
+    backup: true
+    path: /etc/apt/sources.list
+    regexp: '^(deb.*)'
+    replace: '# \1'
+  when: apt_disable_sources_list | bool
+  become: true

ansible/roles/apt/templates/kayobe.sources.j2

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+{% for repo in apt_repositories %}
+Types: {{ repo.types | default('deb') }}
+URIs: {{ repo.url }}
+Suites: {{ repo.suites | default(ansible_facts.distribution_release) }}
+Components: {{ repo.components | default('main') }}
+{% if repo.signed_by is defined %}
+Signed-by: {{ apt_keys_path }}/{{ repo.signed_by }}
+{% endif %}
+{% if repo.architecture is defined %}
+Architecture: {{ repo.architecture }}
+{% endif %}
+
+{% endfor %}

doc/source/configuration/reference/hosts.rst

@@ -316,8 +316,7 @@ oversight or testing.
 Apt
 ===
 
-On Ubuntu, Apt is used to manage packages and package repositories. Currently
-Kayobe does not provide support for configuring custom Apt repositories.
+On Ubuntu, Apt is used to manage packages and package repositories.
 
 Apt cache
 ---------
@@ -325,10 +324,100 @@ Apt cache
 The Apt cache timeout may be configured via ``apt_cache_valid_time`` (in
 seconds) in ``etc/kayobe/apt.yml``, and defaults to 3600.
 
+Apt proxy
+---------
+
 Apt can be configured to use a proxy via ``apt_proxy_http`` and
 ``apt_proxy_https`` in ``etc/kayobe/apt.yml``. These should be set to the full
 URL of the relevant proxy (e.g. ``http://squid.example.com:3128``).
 
+Apt repositories
+----------------
+
+Kayobe supports configuration of custom Apt repositories via the
+``apt_repositories`` variable in ``etc/kayobe/apt.yml`` since the Yoga release.
+The format is a list, with each item mapping to a dict/map with the following
+items:
+
+* ``types``: whitespace-separated list of repository types, e.g. ``deb`` or
+  ``deb-src`` (optional, default is ``deb``)
+* ``url``: URL of the repository
+* ``suites``: whitespace-separated list of suites, e.g. ``focal`` (optional,
+  default is ``ansible_facts.distribution_release``)
+* ``components``: whitespace-separated list of components, e.g. ``main``
+  (optional, default is ``main``)
+* ``signed_by``: whitespace-separated list of names of GPG keyring files in
+  ``apt_keys_path`` (optional, default is unset)
+* ``architecture``: whitespace-separated list of architectures that will be used
+  (optional, default is unset)
+
+The default of ``apt_repositories`` is an empty list.
+
+For example, the following configuration defines a single Apt repository:
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_repositories:
+     - types: deb
+       url: https://example.com/repo
+       suites: focal
+       components: all
+
+In the following example, the Ubuntu Focal 20.04 repositories are consumed from
+a local package mirror. The ``apt_disable_sources_list`` variable is set to
+``true``, which disables all repositories in ``/etc/apt/sources.list``,
+including the default Ubuntu ones.
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_repositories:
+     - url: http://mirror.example.com/ubuntu/
+       suites: focal focal-updates
+       components: main restricted universe multiverse
+     - url: http://mirror.example.com/ubuntu/
+       suites: focal-security
+       components: main restricted universe multiverse
+
+   apt_disable_sources_list: true
+
+Apt keys
+--------
+
+Some repositories may be signed by a key that is not one of Apt's trusted keys.
+Kayobe avoids the use of the deprecated ``apt-key`` utility, and instead allows
+keys to be downloaded to a directory. This enables repositories to use the
+``SignedBy`` option to state that they are signed by a specific key. This
+approach is more secure than using globally trusted keys.
+
+Keys to be downloaded are defined by the ``apt_keys`` variable. The format is a
+list, with each item mapping to a dict/map with the following items:
+
+* ``url``: URL of key
+* ``filename``: Name of a file in which to store the downloaded key in
+  ``apt_keys_path``. The extension should be ``.asc`` for ASCII-armoured keys,
+  or ``.gpg`` otherwise.
+
+The default value of ``apt_keys`` is an empty list.
+
+In the following example, a key is downloaded, and a repository is configured
+that is signed by the key.
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_keys:
+     - url: https://example.com/GPG-key
+       filename: example-key.asc
+
+   apt_repositories:
+     - types: deb
+       url: https://example.com/repo
+       suites: focal
+       components: all
+       signed_by: example-key.asc
+
 SELinux
 =======
 *tags:*

etc/kayobe/apt.yml

@@ -11,6 +11,33 @@
 # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
 #apt_proxy_https:
 
+# List of apt keys. Each item is a dict containing the following keys:
+# * url: URL of key
+# * filename: Name of a file in which to store the downloaded key. The
+#   extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise.
+# Default is an empty list.
+#apt_keys:
+
+# A list of Apt repositories. Each item is a dict with the following keys:
+# * types: whitespace-separated list of repository types, e.g. deb or deb-src
+#   (optional, default is 'deb')
+# * url: URL of the repository
+# * suites: whitespace-separated list of suites, e.g. focal (optional, default
+#   is ansible_facts.distribution_release)
+# * components: whitespace-separated list of components, e.g. main (optional,
+#   default is 'main')
+# * signed_by: whitespace-separated list of names of GPG keyring files in
+#   apt_keys_path (optional, default is unset)
+# * architecture: whitespace-separated list of architectures that will be used
+#   (optional, default is unset)
+# Default is an empty list.
+#apt_repositories:
+
+# Whether to disable repositories in /etc/apt/sources.list. This may be used
+# when replacing the distribution repositories via apt_repositories.
+# Default is false.
+#apt_disable_sources_list:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2

@@ -114,6 +114,25 @@ docker_storage_driver: devicemapper
 # Set Honolulu time.
 timezone: Pacific/Honolulu
 
+{% if ansible_os_family == "Debian" %}
+apt_keys:
+  - url: https://packages.treasuredata.com/GPG-KEY-td-agent
+    filename: td-agent.asc
+apt_repositories:
+  # Ubuntu focal repositories.
+  - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/"
+    suites: focal focal-updates
+    components: main restricted universe multiverse
+  - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/"
+    suites: focal-security
+    components: main restricted universe multiverse
+  # Treasuredata repository.
+  - url:  http://packages.treasuredata.com/4/ubuntu/focal/
+    components: contrib
+    signed_by: td-agent.asc
+apt_disable_sources_list: true
+{% endif %}
+
 {% if ansible_os_family in ['RedHat', 'Rocky'] %}
 # Use a local DNF mirror.
 dnf_use_local_mirror: true

playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py

@@ -16,6 +16,11 @@ def _is_firewalld_supported():
     return info in ['centos', 'rocky']
 
 
+def _is_apt():
+    info = distro.linux_distribution()
+    return info[0].startswith('Ubuntu')
+
+
 def _is_dnf():
     info = distro.id()
     return info in ['centos', 'rocky']
@@ -187,6 +192,13 @@ def test_ntp_clock_synchronized(host):
     assert "synchronized: yes" in status_output
 
 
+@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu")
+def test_apt_custom_package_repository_is_available(host):
+    with host.sudo():
+        host.check_output("apt -y install td-agent")
+    assert host.package("td-agent").is_installed
+
+
 @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel",
                                   "epel-modular"])
 @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8")

releasenotes/notes/apt-repositories-850efef70ba34946.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for configuring Apt repositories on Ubuntu hosts. See `story
+    2009655 <https://storyboard.openstack.org/#!/story/2009655>`__ for details.