From 3b378b0a10570b91549a23055863d14e869768b5 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 10 Jan 2020 16:40:50 +0100 Subject: [PATCH] [CI] Trust only infra mirrors and fix APT mirrors Infra mirrors get their indices rebuilt to avoid broken indices (due to partial update). Unfortunately, this wipes out the cryptographic signatures. Our approach so far was disabling apt security features globally. However, this is not a valid choice for external repos. It hid an issue we introduced with new RabbitMQ repos missing proper keys installed in the image. This caused permanent failures outside of our CI. Our process should be as close as possible to users' experience. This patch makes CI trust only the mirrors that have their indices rebuilt (so infra mirrors). For Train, fixed missing override parts as well. And it includes also: - I8209770ca1353fd89aacef0859ca3a373645c6b2 - Icb745f628c1c50d9f20df022b08ddc46fccc270c - Id3b55ac470a977be7ee7decdef3b49e8d89a130f Change-Id: Ic5abc4b87fd76f87aba383abf43e95ba70629fcb Co-Authored-By: Marcin Juszkiewicz (cherry picked from commit c364c8be85f4653bf7af1c7ac0378823553e7441) (cherry picked from commit 4a482380408d107c59119b379add34f4b8e87ad7) --- docker/base/Dockerfile.j2 | 2 + docker/base/sources.list.ubuntu | 6 +-- ...ntu-uses-mirrors-now-0858d579944eea48.yaml | 5 +++ tests/templates/template_overrides.j2 | 42 ++++++++++--------- 4 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 releasenotes/notes/ubuntu-uses-mirrors-now-0858d579944eea48.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index f89b392a2d..a683edcd70 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -448,6 +448,8 @@ COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list COPY sources.list /etc/apt/sources.list.d/kolla-custom.list {% endblock %} +{% block base_debian_after_sources_list %}{% endblock %} + {% block base_ubuntu_package_apt_preferences %} COPY apt_preferences.{{ base_distro }} /etc/apt/preferences COPY apt_preferences /etc/apt/preferences.d/kolla-custom diff --git a/docker/base/sources.list.ubuntu b/docker/base/sources.list.ubuntu index df69e38d2b..6f17853498 100644 --- a/docker/base/sources.list.ubuntu +++ b/docker/base/sources.list.ubuntu @@ -1,9 +1,9 @@ # For non-x86 architectures we use sources.list.ubuntu. # Default repos -deb http://archive.ubuntu.com/ubuntu/ bionic main universe -deb http://archive.ubuntu.com/ubuntu/ bionic-updates main universe -deb http://archive.ubuntu.com/ubuntu/ bionic-security main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt bionic main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main universe +deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main universe # Backports have a lower priority and must be explicitly installed to be used deb http://archive.ubuntu.com/ubuntu/ bionic-backports main universe diff --git a/releasenotes/notes/ubuntu-uses-mirrors-now-0858d579944eea48.yaml b/releasenotes/notes/ubuntu-uses-mirrors-now-0858d579944eea48.yaml new file mode 100644 index 0000000000..aa5d418ea6 --- /dev/null +++ b/releasenotes/notes/ubuntu-uses-mirrors-now-0858d579944eea48.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Ubuntu based images use APT mirrors now. May affect builds done behind http + proxies. diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index 43c9c0e54e..679cfb1982 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -13,15 +13,6 @@ RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \ && ln -s /etc/npmrc /usr/etc/npmrc {% raw %} -{% if base_distro == 'ubuntu' %} -{% endraw %} - -# NOTE(hrw): Debian 'buster' and Ubuntu 18.04 refuse to use unsigned repos -RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated \ - && echo 'Acquire::AllowInsecureRepositories "true";' > /etc/apt/apt.conf.d/99allow-insecure-repos - -{% raw %} -{% endif %} {% endblock %} {% block base_centos_repo_overrides_post_copy %} @@ -72,15 +63,22 @@ RUN sed -i \ {% raw %} {% endblock %} -{% block base_footer %} -{% if base_distro == "ubuntu" %} +{% block base_debian_after_sources_list %} +{% if base_distro == "debian" %} {% endraw %} -## TODO(yoctozepto): replace base repo earlier as well (like RHEL-based do) -RUN sed -i -e "s|http://archive.ubuntu.com|http://{{ nodepool_mirror_host }}|" \ - -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \ - /etc/apt/sources.list \ - && apt-get update +RUN sed -i -e "s|http://deb.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \ + -e "s|http://security.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \ + /etc/apt/sources.list + +{% raw %} +{% elif base_distro == "ubuntu" %} +{% endraw %} + +RUN sed -i -e "s|mirror://mirrors.ubuntu.com/mirrors.txt|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu/|" \ + -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \ + -e "s|http://ports.ubuntu.com|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-ports|" \ + /etc/apt/sources.list {% raw %} {% endif %} @@ -128,13 +126,17 @@ RUN sed -i \ {% raw %} {% endif %} {% elif base_package_type == 'deb' %} +{% if base_distro == "debian" %} {% endraw %} -RUN rm -f /etc/apt/apt.conf.d/99allow-unauthenticated /etc/apt/apt.conf.d/99allow-insecure-repos +RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://deb.debian.org|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://security.debian.org|" \ + /etc/apt/sources.list {% raw %} -{% if base_distro == "ubuntu" %} +{% elif base_distro == "ubuntu" %} {% endraw %} -RUN sed -i -e "s|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \ - -e "s|http://{{ nodepool_mirror_host }}|http://archive.ubuntu.com|" \ +RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu/|mirror://mirrors.ubuntu.com/mirrors.txt|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \ + -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-ports|http://ports.ubuntu.com|" \ /etc/apt/sources.list {% raw %} {% endif %}