Merge "[CI] Trust only infra mirrors and fix APT mirrors" into stable/train

docker/base/Dockerfile.j2

@@ -448,6 +448,8 @@ COPY sources.list.{{ base_distro }}.{{ base_arch }} /etc/apt/sources.list
 COPY sources.list /etc/apt/sources.list.d/kolla-custom.list
 {% endblock %}
 
+{% block base_debian_after_sources_list %}{% endblock %}
+
 {% block base_ubuntu_package_apt_preferences %}
 COPY apt_preferences.{{ base_distro }} /etc/apt/preferences
 COPY apt_preferences /etc/apt/preferences.d/kolla-custom

docker/base/sources.list.ubuntu

@@ -1,9 +1,9 @@
 # For non-x86 architectures we use sources.list.ubuntu.<arch>
 
 # Default repos
-deb http://archive.ubuntu.com/ubuntu/ bionic main universe
+deb mirror://mirrors.ubuntu.com/mirrors.txt bionic main universe
-deb http://archive.ubuntu.com/ubuntu/ bionic-updates main universe
+deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main universe
-deb http://archive.ubuntu.com/ubuntu/ bionic-security main universe
+deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main universe
 
 # Backports have a lower priority and must be explicitly installed to be used
 deb http://archive.ubuntu.com/ubuntu/ bionic-backports main universe

releasenotes/notes/ubuntu-uses-mirrors-now-0858d579944eea48.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    Ubuntu based images use APT mirrors now. May affect builds done behind http
+    proxies.

tests/templates/template_overrides.j2

@@ -13,15 +13,6 @@ RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \
     && ln -s /etc/npmrc /usr/etc/npmrc
 
 {% raw %}
-{% if base_distro == 'ubuntu' %}
-{% endraw %}
-
-# NOTE(hrw): Debian 'buster' and Ubuntu 18.04 refuse to use unsigned repos
-RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated \
- && echo 'Acquire::AllowInsecureRepositories "true";' > /etc/apt/apt.conf.d/99allow-insecure-repos
-
-{% raw %}
-{% endif %}
 {% endblock %}
 
 {% block base_centos_repo_overrides_post_copy %}
@@ -72,15 +63,22 @@ RUN sed -i \
 {% raw %}
 {% endblock %}
 
-{% block base_footer %}
+{% block base_debian_after_sources_list %}
-{% if base_distro == "ubuntu" %}
+{% if base_distro == "debian" %}
 {% endraw %}
 
-## TODO(yoctozepto): replace base repo earlier as well (like RHEL-based do)
+RUN sed -i -e "s|http://deb.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \
-RUN sed -i -e "s|http://archive.ubuntu.com|http://{{ nodepool_mirror_host }}|" \
+        -e "s|http://security.debian.org|[trusted=yes] http://{{ nodepool_mirror_host }}|" \
-        -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \
+        /etc/apt/sources.list
-        /etc/apt/sources.list \
+
-    && apt-get update
+{% raw %}
+{% elif base_distro == "ubuntu" %}
+{% endraw %}
+
+RUN sed -i -e "s|mirror://mirrors.ubuntu.com/mirrors.txt|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu/|" \
+        -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \
+        -e "s|http://ports.ubuntu.com|[trusted=yes] http://{{ nodepool_mirror_host }}/ubuntu-ports|" \
+        /etc/apt/sources.list
 
 {% raw %}
 {% endif %}
@@ -128,13 +126,17 @@ RUN sed -i \
 {% raw %}
 {% endif %}
 {% elif base_package_type == 'deb' %}
+{% if base_distro == "debian" %}
 {% endraw %}
-RUN rm -f /etc/apt/apt.conf.d/99allow-unauthenticated /etc/apt/apt.conf.d/99allow-insecure-repos
+RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://deb.debian.org|" \
+        -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}|http://security.debian.org|" \
+        /etc/apt/sources.list
 {% raw %}
-{% if base_distro == "ubuntu" %}
+{% elif base_distro == "ubuntu" %}
 {% endraw %}
-RUN sed -i -e "s|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \
+RUN sed -i -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu/|mirror://mirrors.ubuntu.com/mirrors.txt|" \
-        -e "s|http://{{ nodepool_mirror_host }}|http://archive.ubuntu.com|" \
+        -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|http://ubuntu-cloud.archive.canonical.com/ubuntu|" \
+        -e "s|\[trusted=yes\] http://{{ nodepool_mirror_host }}/ubuntu-ports|http://ports.ubuntu.com|" \
         /etc/apt/sources.list
 {% raw %}
 {% endif %}