From c603be253641aab2199330c1afb2c83051be3e3a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 2 Nov 2021 10:40:56 +0000 Subject: [PATCH] Ubuntu: add support for Apt repository configuration This change adds support for configuring Apt repositories on Ubuntu hosts during host configuration. Repositories are configured in a single file (/etc/apt/sources.list.d/kayobe.sources), using the modern deb822 format [1]. This format is more flexible and readable than the original single-line format, particularly if multiple options are used. Using a single file allows us to more easily keep the set of repositories in sync, since Ansible doesn't make it easy to clean things up. Support is added for marking repositories as signed by a particular GPG key. This approach is now preferred over the deprecated [2] apt-key tool, which resulted in a set of globally trusted keys. It is also possible to disable the repositories in /etc/apt/sources.list via apt_disable_sources_list. This allows for replacing the standard repositories with a local mirror. CI tests and documentation are provided. [1] https://manpages.ubuntu.com/manpages/focal/en/man5/sources.list.5.html [2] https://manpages.ubuntu.com/manpages/groovy/man8/apt-key.8.html Story: 2009655 Task: 43818 Change-Id: I3f821937b0930a0ac9341178de7ae5123d82b957 --- ansible/group_vars/all/apt | 27 ++++++ ansible/roles/apt/defaults/main.yml | 30 ++++++ ansible/roles/apt/handlers/main.yml | 5 + ansible/roles/apt/tasks/keys.yml | 19 ++++ ansible/roles/apt/tasks/main.yml | 19 +--- ansible/roles/apt/tasks/proxy.yml | 17 ++++ ansible/roles/apt/tasks/repos.yml | 22 +++++ ansible/roles/apt/templates/kayobe.sources.j2 | 15 +++ doc/source/configuration/reference/hosts.rst | 93 ++++++++++++++++++- etc/kayobe/apt.yml | 27 ++++++ .../overrides.yml.j2 | 19 ++++ .../tests/test_overcloud_host_configure.py | 12 +++ .../apt-repositories-850efef70ba34946.yaml | 5 + 13 files changed, 293 insertions(+), 17 deletions(-) create mode 100644 ansible/roles/apt/handlers/main.yml create mode 100644 ansible/roles/apt/tasks/keys.yml create mode 100644 ansible/roles/apt/tasks/proxy.yml create mode 100644 ansible/roles/apt/tasks/repos.yml create mode 100644 ansible/roles/apt/templates/kayobe.sources.j2 create mode 100644 releasenotes/notes/apt-repositories-850efef70ba34946.yaml diff --git a/ansible/group_vars/all/apt b/ansible/group_vars/all/apt index fad722dcd..ded0cdf0a 100644 --- a/ansible/group_vars/all/apt +++ b/ansible/group_vars/all/apt @@ -10,3 +10,30 @@ apt_proxy_http: # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. apt_proxy_https: "{{ apt_proxy_http }}" + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +apt_keys: [] + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +apt_repositories: [] + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +apt_disable_sources_list: false diff --git a/ansible/roles/apt/defaults/main.yml b/ansible/roles/apt/defaults/main.yml index fad722dcd..43ce85b36 100644 --- a/ansible/roles/apt/defaults/main.yml +++ b/ansible/roles/apt/defaults/main.yml @@ -10,3 +10,33 @@ apt_proxy_http: # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. apt_proxy_https: "{{ apt_proxy_http }}" + +# Directory containing GPG keyrings for apt repos. +apt_keys_path: "/usr/local/share/keyrings" + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +apt_keys: [] + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +apt_repositories: [] + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +apt_disable_sources_list: false diff --git a/ansible/roles/apt/handlers/main.yml b/ansible/roles/apt/handlers/main.yml new file mode 100644 index 000000000..2d39add43 --- /dev/null +++ b/ansible/roles/apt/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Update apt cache + package: + update_cache: true + become: true diff --git a/ansible/roles/apt/tasks/keys.yml b/ansible/roles/apt/tasks/keys.yml new file mode 100644 index 000000000..4c1cda1e0 --- /dev/null +++ b/ansible/roles/apt/tasks/keys.yml @@ -0,0 +1,19 @@ +--- +- name: Ensure keys directory exists + file: + path: "{{ apt_keys_path }}" + owner: root + group: root + mode: 0755 + state: directory + become: true + +- name: Ensure keys exist + get_url: + url: "{{ item.url }}" + dest: "{{ apt_keys_path ~ '/' ~ item.filename | basename }}" + owner: root + group: root + mode: 0644 + loop: "{{ apt_keys }}" + become: true diff --git a/ansible/roles/apt/tasks/main.yml b/ansible/roles/apt/tasks/main.yml index 16205b6be..4bbd0b665 100644 --- a/ansible/roles/apt/tasks/main.yml +++ b/ansible/roles/apt/tasks/main.yml @@ -1,17 +1,6 @@ --- -- name: Configure apt proxy - template: - src: "01proxy.j2" - dest: /etc/apt/apt.conf.d/01proxy - owner: root - group: root - mode: 0664 - become: true - when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0 +- import_tasks: proxy.yml -- name: Remove old apt proxy config - file: - path: /etc/apt/apt.conf.d/01proxy - state: absent - become: true - when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0 +- import_tasks: keys.yml + +- import_tasks: repos.yml diff --git a/ansible/roles/apt/tasks/proxy.yml b/ansible/roles/apt/tasks/proxy.yml new file mode 100644 index 000000000..16205b6be --- /dev/null +++ b/ansible/roles/apt/tasks/proxy.yml @@ -0,0 +1,17 @@ +--- +- name: Configure apt proxy + template: + src: "01proxy.j2" + dest: /etc/apt/apt.conf.d/01proxy + owner: root + group: root + mode: 0664 + become: true + when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0 + +- name: Remove old apt proxy config + file: + path: /etc/apt/apt.conf.d/01proxy + state: absent + become: true + when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0 diff --git a/ansible/roles/apt/tasks/repos.yml b/ansible/roles/apt/tasks/repos.yml new file mode 100644 index 000000000..ef401bb5e --- /dev/null +++ b/ansible/roles/apt/tasks/repos.yml @@ -0,0 +1,22 @@ +--- +# NOTE(mgoddard): Use the modern deb822 repository format rather than the old +# format used by the apt_repository module. +- name: Configure apt repositories + template: + src: "kayobe.sources.j2" + dest: "/etc/apt/sources.list.d/kayobe.sources" + owner: root + group: root + mode: 0644 + become: true + notify: + - Update apt cache + +- name: Disable repositories in /etc/apt/sources.list + replace: + backup: true + path: /etc/apt/sources.list + regexp: '^(deb.*)' + replace: '# \1' + when: apt_disable_sources_list | bool + become: true diff --git a/ansible/roles/apt/templates/kayobe.sources.j2 b/ansible/roles/apt/templates/kayobe.sources.j2 new file mode 100644 index 000000000..91f6bf6b4 --- /dev/null +++ b/ansible/roles/apt/templates/kayobe.sources.j2 @@ -0,0 +1,15 @@ +# {{ ansible_managed }} + +{% for repo in apt_repositories %} +Types: {{ repo.types | default('deb') }} +URIs: {{ repo.url }} +Suites: {{ repo.suites | default(ansible_facts.distribution_release) }} +Components: {{ repo.components | default('main') }} +{% if repo.signed_by is defined %} +Signed-by: {{ apt_keys_path }}/{{ repo.signed_by }} +{% endif %} +{% if repo.architecture is defined %} +Architecture: {{ repo.architecture }} +{% endif %} + +{% endfor %} diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 68ee55c1f..ed50bfd79 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -316,8 +316,7 @@ oversight or testing. Apt === -On Ubuntu, Apt is used to manage packages and package repositories. Currently -Kayobe does not provide support for configuring custom Apt repositories. +On Ubuntu, Apt is used to manage packages and package repositories. Apt cache --------- @@ -325,10 +324,100 @@ Apt cache The Apt cache timeout may be configured via ``apt_cache_valid_time`` (in seconds) in ``etc/kayobe/apt.yml``, and defaults to 3600. +Apt proxy +--------- + Apt can be configured to use a proxy via ``apt_proxy_http`` and ``apt_proxy_https`` in ``etc/kayobe/apt.yml``. These should be set to the full URL of the relevant proxy (e.g. ``http://squid.example.com:3128``). +Apt repositories +---------------- + +Kayobe supports configuration of custom Apt repositories via the +``apt_repositories`` variable in ``etc/kayobe/apt.yml`` since the Yoga release. +The format is a list, with each item mapping to a dict/map with the following +items: + +* ``types``: whitespace-separated list of repository types, e.g. ``deb`` or + ``deb-src`` (optional, default is ``deb``) +* ``url``: URL of the repository +* ``suites``: whitespace-separated list of suites, e.g. ``focal`` (optional, + default is ``ansible_facts.distribution_release``) +* ``components``: whitespace-separated list of components, e.g. ``main`` + (optional, default is ``main``) +* ``signed_by``: whitespace-separated list of names of GPG keyring files in + ``apt_keys_path`` (optional, default is unset) +* ``architecture``: whitespace-separated list of architectures that will be used + (optional, default is unset) + +The default of ``apt_repositories`` is an empty list. + +For example, the following configuration defines a single Apt repository: + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_repositories: + - types: deb + url: https://example.com/repo + suites: focal + components: all + +In the following example, the Ubuntu Focal 20.04 repositories are consumed from +a local package mirror. The ``apt_disable_sources_list`` variable is set to +``true``, which disables all repositories in ``/etc/apt/sources.list``, +including the default Ubuntu ones. + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_repositories: + - url: http://mirror.example.com/ubuntu/ + suites: focal focal-updates + components: main restricted universe multiverse + - url: http://mirror.example.com/ubuntu/ + suites: focal-security + components: main restricted universe multiverse + + apt_disable_sources_list: true + +Apt keys +-------- + +Some repositories may be signed by a key that is not one of Apt's trusted keys. +Kayobe avoids the use of the deprecated ``apt-key`` utility, and instead allows +keys to be downloaded to a directory. This enables repositories to use the +``SignedBy`` option to state that they are signed by a specific key. This +approach is more secure than using globally trusted keys. + +Keys to be downloaded are defined by the ``apt_keys`` variable. The format is a +list, with each item mapping to a dict/map with the following items: + +* ``url``: URL of key +* ``filename``: Name of a file in which to store the downloaded key in + ``apt_keys_path``. The extension should be ``.asc`` for ASCII-armoured keys, + or ``.gpg`` otherwise. + +The default value of ``apt_keys`` is an empty list. + +In the following example, a key is downloaded, and a repository is configured +that is signed by the key. + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_keys: + - url: https://example.com/GPG-key + filename: example-key.asc + + apt_repositories: + - types: deb + url: https://example.com/repo + suites: focal + components: all + signed_by: example-key.asc + SELinux ======= *tags:* diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 5f278e322..c4314a0aa 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -11,6 +11,33 @@ # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. #apt_proxy_https: +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +#apt_keys: + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +#apt_repositories: + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +#apt_disable_sources_list: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 index a6de4e74f..6591d5e68 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 @@ -114,6 +114,25 @@ docker_storage_driver: devicemapper # Set Honolulu time. timezone: Pacific/Honolulu +{% if ansible_os_family == "Debian" %} +apt_keys: + - url: https://packages.treasuredata.com/GPG-KEY-td-agent + filename: td-agent.asc +apt_repositories: + # Ubuntu focal repositories. + - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/" + suites: focal focal-updates + components: main restricted universe multiverse + - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/" + suites: focal-security + components: main restricted universe multiverse + # Treasuredata repository. + - url: http://packages.treasuredata.com/4/ubuntu/focal/ + components: contrib + signed_by: td-agent.asc +apt_disable_sources_list: true +{% endif %} + {% if ansible_os_family in ['RedHat', 'Rocky'] %} # Use a local DNF mirror. dnf_use_local_mirror: true diff --git a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py index 8b335c11b..a15429be8 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py +++ b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py @@ -16,6 +16,11 @@ def _is_firewalld_supported(): return info in ['centos', 'rocky'] +def _is_apt(): + info = distro.linux_distribution() + return info[0].startswith('Ubuntu') + + def _is_dnf(): info = distro.id() return info in ['centos', 'rocky'] @@ -187,6 +192,13 @@ def test_ntp_clock_synchronized(host): assert "synchronized: yes" in status_output +@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu") +def test_apt_custom_package_repository_is_available(host): + with host.sudo(): + host.check_output("apt -y install td-agent") + assert host.package("td-agent").is_installed + + @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel", "epel-modular"]) @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8") diff --git a/releasenotes/notes/apt-repositories-850efef70ba34946.yaml b/releasenotes/notes/apt-repositories-850efef70ba34946.yaml new file mode 100644 index 000000000..1e82c8a1e --- /dev/null +++ b/releasenotes/notes/apt-repositories-850efef70ba34946.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for configuring Apt repositories on Ubuntu hosts. See `story + 2009655 `__ for details.