Debian/Ubuntu: handle APT keys in proper way

apt-key is deprecated and in Debian 'bullseye' it complain when we use
it the way we did.

The right way is to store key files in separate directory and then, for
each repo source file, point to proper key. And this is what this patch
does.

Both distros use the same keys now.

More info: https://michael-prokop.at/blog/2021/02/16/how-to-properly-use-3rd-party-debian-repository-signing-keys-with-apt/

Change-Id: Ie19bf46236907d79cb42a5e94cd1067e952b8cbb
This commit is contained in:
Marcin Juszkiewicz 2021-04-06 13:14:49 +02:00
parent ee84d3b8b9
commit ffaf102fec
5 changed files with 239 additions and 90 deletions

View File

@ -252,13 +252,11 @@ ENV DEBIAN_FRONTEND noninteractive
COPY dpkg_reducing_disk_footprint /etc/dpkg/dpkg.cfg.d/dpkg_reducing_disk_footprint
{% block base_ubuntu_package_pre %}
# Need apt-transport-https and ca-certificates before replacing sources.list or
# apt-get update will not work if any repositories are accessed via HTTPS
# curl and ca-certificates to fetch remote keys via http
# gnupg to fetch keys directly from keyserver
{% set base_ubuntu_package_pre_packages = [
'apt-transport-https',
'ca-certificates',
'curl',
'dirmngr',
'gnupg'
] %}
{{ macros.install_packages(base_ubuntu_package_pre_packages | customizable("base_ubuntu_package_pre_packages")) }}
@ -294,7 +292,6 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
{% set base_apt_packages = [
'apt-utils',
'curl',
'dumb-init',
'gawk',
'iproute2',
@ -311,55 +308,29 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
'tgt'
] %}
{% if base_distro == 'ubuntu' %}
{# 391A9AA2147192839E9DB0315EDB1B62EC4926EA -- Canonical Cloud Archive Signing Key <ftpmaster@canonical.com> #}
{# 46095ACC8548582C1A2699A9D27D666CD88E42B4 -- Elasticsearch (Elasticsearch Signing Key) <dev_ops@elasticsearch.org> #}
{# 58118E89F3A912897C070ADBF76221572C52609D -- Docker Release Tool (releasedocker) <docker@docker.com> #}
{# 4D8EB5FDA37AB55F41A135203BF88A0C6A770882 -- Apache Qpid PPA Signing Key <dev@qpid.apache.org> #}
{# 901F9177AB97ACBE -- Treasure Data, Inc (Treasure Agent Official Signing key) <support@treasure-data.com> #}
{# A20F259AEB9C94BB -- Sensuapp (Freight) <support@hw-ops.com> #}
{# F1656F24C74CD1D8 -- MariaDB Signing Key <signing-key@mariadb.org> #}
{# F77F1EDA57EBB1CC -- Launchpad RabbitMQ Erlang PPA key #}
{# F6609E60DC62814E -- PackageCloud RabbitMQ repository key #}
{% set base_apt_keys = [
'391A9AA2147192839E9DB0315EDB1B62EC4926EA',
'46095ACC8548582C1A2699A9D27D666CD88E42B4',
'49B07274951063870A8B7EAE7B8AA1A344C05248',
'58118E89F3A912897C070ADBF76221572C52609D',
'4D8EB5FDA37AB55F41A135203BF88A0C6A770882',
'901F9177AB97ACBE',
'A20F259AEB9C94BB',
'F1656F24C74CD1D8',
'F77F1EDA57EBB1CC',
'F6609E60DC62814E',
] %}
{% set remote_apt_keys = [
'https://packages.grafana.com/gpg.key',
'https://repos.influxdata.com/influxdb.key',
] %}
{% elif base_distro == 'debian' %}
{% set base_apt_keys = [
'46095ACC8548582C1A2699A9D27D666CD88E42B4',
'F77F1EDA57EBB1CC',
'F6609E60DC62814E',
] %}
{% set remote_apt_keys = [
'https://download.docker.com/linux/debian/gpg',
'https://packages.grafana.com/gpg.key',
'https://packages.treasuredata.com/GPG-KEY-td-agent',
'https://repos.influxdata.com/influxdb.key',
] %}
{% endif %}
{% set base_apt_keys = [
{'name': 'erlang', 'keyid': 'F77F1EDA57EBB1CC'},
{'name': 'rabbitmq', 'keyid': 'F6609E60DC62814E'},
] %}
{# NOTE(hrw): type field defaults to 'asc' which is used for single keys #}
{% set remote_apt_keys = [
{'name': 'elasticsearch', 'url': 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'},
{'name': 'grafana', 'url': 'https://packages.grafana.com/gpg.key'},
{'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdb.key'},
{'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'},
{'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'},
] %}
{% block base_ubuntu_package_installation %}
{%- block base_ubuntu_package_key_installation %}
{% for key in base_apt_keys | customizable('apt_keys') %}
{%- if loop.first %}RUN {% else %} && {% endif %}apt-key adv --no-tty --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key }}
{%- if loop.first %}RUN mkdir -p /etc/kolla/apt-keys/{% endif %} && gpg --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 {{ key.keyid }} && gpg --export {{ key.keyid }} >/etc/kolla/apt-keys/{{ key.name }}.gpg
{%- if not loop.last %} \
{% endif -%}
{% endfor %}
{% for key in remote_apt_keys | customizable('remote_apt_keys') %}
{%- if loop.first %} RUN {% else %} && {% endif %}curl {{ key }} | apt-key add -
{%- if loop.first %} RUN mkdir -p /etc/kolla/apt-keys/ {% endif %} && curl {{ key.url }} -o /etc/kolla/apt-keys/{{ key.name }}.{{ key.type | default('asc') }}
{%- if not loop.last %} \
{% endif -%}
{% endfor %}

View File

@ -115,8 +115,16 @@ def handle_repos(context, reponames, mode):
commands += ' %s %s' % (rpm_switch, repo_list[repo])
elif base_package_type == 'deb':
if mode == 'enable':
commands += 'echo "%s" ' % repo_list[repo]
commands += '>/etc/apt/sources.list.d/%s.list && ' % repo
commands += f"""echo 'Uris: {repo_list[repo]['url']}' \
>/etc/apt/sources.list.d/{repo}.sources \
&& echo 'Components: {repo_list[repo]['component']}' \
>>/etc/apt/sources.list.d/{repo}.sources \
&& echo 'Types: deb' >>/etc/apt/sources.list.d/{repo}.sources \
&& echo 'Suites: {repo_list[repo]['suite']}' \
>>/etc/apt/sources.list.d/{repo}.sources \
&& echo 'Signed-By: /etc/kolla/apt-keys/{repo_list[repo]['gpg_key']}' \
>>/etc/apt/sources.list.d/{repo}.sources \
&& """
except KeyError:
# NOTE(hrw): we ignore missing repositories for a given
# distro/arch

View File

@ -37,45 +37,177 @@ centos-aarch64:
rabbitmq: "rabbitmq_rabbitmq-server"
td-agent: "treasuredata"
# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
debian:
elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
grafana: "deb https://packages.grafana.com/oss/deb stable main"
influxdb: "deb https://repos.influxdata.com/debian buster stable"
logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "bionic"
component: "main"
gpg_key: "erlang.gpg"
grafana:
url: "https://packages.grafana.com/oss/deb"
suite: "stable"
component: "main"
gpg_key: "grafana.asc"
influxdb:
url: "https://repos.influxdata.com/debian"
suite: "buster"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
rabbitmq:
url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
suite: "buster"
component: "main"
gpg_key: "rabbitmq.gpg"
td-agent:
url: "http://packages.treasuredata.com/4/debian/buster"
suite: "buster"
component: "contrib"
gpg_key: "treasuredata.asc"
debian-aarch64:
elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main"
grafana: "deb https://packages.grafana.com/oss/deb stable main"
influxdb: "deb https://repos.influxdata.com/debian buster stable"
logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main"
td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
# NOTE(mnasiadka): Erlang repo - Debian Buster/Bullseye needs to use bionic as per RabbitMQ docs
url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "bionic"
component: "main"
gpg_key: "erlang.gpg"
grafana:
url: "https://packages.grafana.com/oss/deb"
suite: "stable"
component: "main"
gpg_key: "grafana.asc"
influxdb:
url: "https://repos.influxdata.com/debian"
suite: "buster"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
rabbitmq:
url: "https://packagecloud.io/rabbitmq/rabbitmq-server/debian/"
suite: "buster"
component: "main"
gpg_key: "rabbitmq.gpg"
td-agent:
url: "http://packages.treasuredata.com/4/debian/buster"
suite: "buster"
component: "contrib"
gpg_key: "treasuredata.asc"
ubuntu:
elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
grafana: "deb https://packages.grafana.com/oss/deb stable main"
influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ focal main"
rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "focal"
component: "main"
gpg_key: "erlang.gpg"
grafana:
url: "https://packages.grafana.com/oss/deb"
suite: "stable"
component: "main"
gpg_key: "grafana.asc"
influxdb:
url: "https://repos.influxdata.com/ubuntu"
suite: "focal"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
kibana:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
suite: "focal"
component: "main"
gpg_key: "mariadb.gpg"
rabbitmq:
url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
suite: "focal"
component: "main"
gpg_key: "rabbitmq.gpg"
td-agent:
url: "http://packages.treasuredata.com/4/ubuntu/focal/"
suite: "focal"
component: "contrib"
gpg_key: "treasuredata.asc"
ubuntu-aarch64:
elasticsearch: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main"
grafana: "deb https://packages.grafana.com/oss/deb stable main"
influxdb: "deb https://repos.influxdata.com/ubuntu focal stable"
logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main"
mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu focal main"
rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main"
td-agent: "deb http://packages.treasuredata.com/4/ubuntu/focal/ focal contrib"
elasticsearch:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
erlang:
url: "http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu"
suite: "focal"
component: "main"
gpg_key: "erlang.gpg"
grafana:
url: "https://packages.grafana.com/oss/deb"
suite: "stable"
component: "main"
gpg_key: "grafana.asc"
influxdb:
url: "https://repos.influxdata.com/ubuntu"
suite: "focal"
component: "stable"
gpg_key: "influxdb.asc"
logstash:
url: "https://artifacts.elastic.co/packages/oss-7.x/apt"
suite: "stable"
component: "main"
gpg_key: "elasticsearch.asc"
mariadb:
url: "http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu"
suite: "focal"
component: "main"
gpg_key: "mariadb.gpg"
rabbitmq:
url: "https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/"
suite: "focal"
component: "main"
gpg_key: "rabbitmq.gpg"
td-agent:
url: "http://packages.treasuredata.com/4/ubuntu/focal/"
suite: "focal"
component: "contrib"
gpg_key: "treasuredata.asc"

View File

@ -66,8 +66,16 @@ class MethodsTest(base.TestCase):
}
result = methods.handle_repos(template_vars, ['grafana'], 'enable')
expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list'
expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Components: main' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Types: deb' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Suites: stable' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources"
self.assertEqual(expectCmd, result)
def test_enable_repos_debian_missing_repo(self):
@ -91,11 +99,29 @@ class MethodsTest(base.TestCase):
result = methods.handle_repos(template_vars, ['grafana', 'kibana'],
'enable')
expectCmd = 'RUN echo "deb https://packages.grafana.com/oss/deb '
expectCmd += 'stable main" >/etc/apt/sources.list.d/grafana.list && '
expectCmd += 'echo "deb '
expectCmd += 'https://artifacts.elastic.co/packages/oss-7.x/apt '
expectCmd += 'stable main" >/etc/apt/sources.list.d/kibana.list'
expectCmd = "RUN echo 'Uris: https://packages.grafana.com/oss/deb' "
expectCmd += ">/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Components: main' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Types: deb' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Suites: stable' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/grafana.asc' "
expectCmd += ">>/etc/apt/sources.list.d/grafana.sources && "
expectCmd += "echo 'Uris: "
expectCmd += "https://artifacts.elastic.co/packages/oss-7.x/apt' "
expectCmd += ">/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Components: main' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Types: deb' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Suites: stable' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources && "
expectCmd += "echo 'Signed-By: /etc/kolla/apt-keys/elasticsearch.asc' "
expectCmd += ">>/etc/apt/sources.list.d/kibana.sources"
self.assertEqual(expectCmd, result)
def test_disable_repos_centos(self):

View File

@ -0,0 +1,12 @@
---
upgrade:
- |
Format of APT keys has changed from simple list into dictionary. For
``base_apt_keys`` we now use name and key ids and for ``remote_apt_keys``
names and URLs.
This allows to instruct APT to use those keys only for their repositories
instead of trusting them for all possible packages.
If you override ``base_apt_keys`` or ``remote_apt_keys`` then please adapt
to the new format.