DNM testing custom pypi mirrors (696337)

Testing https://review.opendev.org/#/c/696337/ via base-test

Change-Id: Iebb98174f5f031c6cb7f6d27b3dabc88af35cc76
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
This commit is contained in:
Bogdan Dobrelya 2019-12-02 12:40:25 +01:00
parent 77bc616dc5
commit 4baa3452e4
39 changed files with 486 additions and 0 deletions

View File

@ -0,0 +1,25 @@
An ansible role to configure services to use mirrors.
**Role Variables**
.. zuul:rolevar:: mirror_fqdn
:default: {{ zuul_site_mirror_fqdn }}
The base host for mirror servers.
.. zuul:rolevar:: pypi_fqdn
:default: {{ mirror_fqdn }}
The base host for PyPi mirror server.
.. zuul:rolevar:: pypi_mirror
URL to override the generated pypi mirror url based on
:zuul:rolevar:`configure-mirrors.pypi_fqdn`.
.. zuul:rolevar:: set_apt_mirrors_trusted
:default: False
Set to True in order to tag APT mirrors as trusted, needed
when accessing unsigned mirrors with newer releases like
Ubuntu Bionic.

View File

@ -0,0 +1,5 @@
mirror_fqdn: "{{ zuul_site_mirror_fqdn|default(omit) }}"
pypi_fqdn: "{{ mirror_fqdn }}"
pypi_mirror: "http://{{ pypi_fqdn }}/pypi/simple"
set_apt_mirrors_trusted: False
wheel_mirror: "http://{{ mirror_fqdn }}/wheel/{{ ansible_distribution | lower }}-{{ ansible_distribution_version }}-{{ ansible_architecture | lower }}"

View File

@ -0,0 +1,29 @@
# The apt module can not be used for this since it installs python-apt
# which can not work until this command fixes the cache.
- name: Update apt cache
become: yes
command: apt-get update
# ANSIBLE0006: ignore, since we cannot use the apt module
tags:
- skip_ansible_lint
- name: Update yum/dnf cache
become: yes
command: "{{ item }}"
args:
warn: false
with_items:
- "{{ ansible_pkg_mgr }} clean all"
- "{{ ansible_pkg_mgr }} makecache -v"
# verbose is needed in order to make it possible to debug potential failures
- name: Update zypper cache
become: yes
command: "{{ item }}"
with_items:
- zypper clean
- zypper refresh
- name: Update Gentoo cache
become: yes
command: emerge-webrsync

View File

@ -0,0 +1,9 @@
- name: gather needed facts
when: ansible_pkg_mgr is not defined
setup:
gather_subset: pkg_mgr
- name: Set up infrastructure mirrors
include: mirror.yaml
when: mirror_fqdn is defined
static: no

View File

@ -0,0 +1,31 @@
- name: Remove .pydistutils.cfg configuration in homedir
file:
path: ~/.pydistutils.cfg
state: absent
- name: Include OS-specific variables
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml"
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- "default.yaml"
- name: Install /etc/pip.conf configuration
become: yes
template:
dest: /etc/pip.conf
group: root
mode: 0644
owner: root
src: etc/pip.conf.j2
- name: Setup distribution specific packaging mirrors
include: "{{ item }}"
static: no
with_first_found:
- "mirror/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "mirror/{{ ansible_distribution }}.yaml"
- "mirror/{{ ansible_os_family }}.yaml"
- "mirror/default.yaml"

View File

@ -0,0 +1,25 @@
- name: Install CentOS repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "centos7/{{ item }}.j2"
with_items:
- etc/yum.repos.d/CentOS-Base.repo
- etc/yum.repos.d/epel.repo
notify:
- Update yum/dnf cache
# http://dnf.readthedocs.io/en/latest/conf_ref.html#options-for-both-main-and-repo
# deltarpm is useful when the bottleneck is the network throughput.
# It also requires additional drpm packages to be hosted by the mirrors which
# is not done by default.
- name: Disable deltrarpm
become: yes
ini_file:
path: /etc/yum.conf
section: main
option: deltarpm
value: 0

View File

@ -0,0 +1,28 @@
- name: Install CentOS 8 repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "centos8/{{ item }}.j2"
with_items:
- etc/yum.repos.d/CentOS-AppStream.repo
- etc/yum.repos.d/CentOS-Base.repo
- etc/yum.repos.d/CentOS-Extras.repo
- etc/yum.repos.d/CentOS-PowerTools.repo
- etc/yum.repos.d/epel.repo
notify:
- Update yum/dnf cache
# http://dnf.readthedocs.io/en/latest/conf_ref.html#options-for-both-main-and-repo
# deltarpm is useful when the bottleneck is the network throughput.
# It also requires additional drpm packages to be hosted by the mirrors which
# is not done by default.
- name: Disable deltrarpm
become: yes
ini_file:
path: /etc/dnf.conf
section: main
option: deltarpm
value: 0

View File

@ -0,0 +1,16 @@
- name: Install Debian repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "apt/{{ item }}.j2"
with_items:
- etc/apt/sources.list.d/default.list
- etc/apt/sources.list.d/updates.list
- etc/apt/sources.list.d/backports.list
- etc/apt/sources.list.d/security.list
- etc/apt/apt.conf.d/99unauthenticated
notify:
- Update apt cache

View File

@ -0,0 +1,25 @@
- name: Install Fedora repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "fedora/{{ item }}.j2"
with_items:
- etc/yum.repos.d/fedora.repo
- etc/yum.repos.d/fedora-updates.repo
notify:
- Update yum/dnf cache
# http://dnf.readthedocs.io/en/latest/conf_ref.html#options-for-both-main-and-repo
# deltarpm is useful when the bottleneck is the network throughput.
# It also requires additional drpm packages to be hosted by the mirrors which
# is not done by default.
- name: Disable deltrarpm
become: yes
ini_file:
path: /etc/dnf/dnf.conf
section: main
option: deltarpm
value: "false"

View File

@ -0,0 +1,4 @@
- name: Update Gentoo repository
command: echo "Syncs the Gentoo repository"
notify:
- Update Gentoo cache

View File

@ -0,0 +1,34 @@
- name: set zypper base package repository (openSUSE Leap)
set_fact:
opensuse_repo_baseurl: "{{ package_mirror }}/distribution/leap/$releasever/repo/oss/"
when: not ansible_distribution | search("Tumbleweed")
- name: set zypper base package repository (openSUSE Tumbleweed)
set_fact:
opensuse_repo_baseurl: "{{ package_mirror }}/tumbleweed/repo/oss/"
when: ansible_distribution | search("Tumbleweed")
- name: Install Suse repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "suse/{{ item }}.j2"
with_items:
- etc/zypp/repos.d/repo-oss.repo
notify:
- Update zypper cache
- name: Install openSUSE Leap Update repository
become: yes
template:
dest: /etc/zypp/repos.d/repo-update.repo
group: root
mode: 0644
owner: root
src: suse/etc/zypp/repos.d/repo-update.repo.j2
notify:
- Update zypper cache
when: not ansible_distribution | search("Tumbleweed")

View File

@ -0,0 +1,13 @@
- name: Install Ubuntu repository files
become: yes
template:
dest: "/{{ item }}"
group: root
mode: 0644
owner: root
src: "apt/{{ item }}.j2"
with_items:
- etc/apt/sources.list
- etc/apt/apt.conf.d/99unauthenticated
notify:
- Update apt cache

View File

@ -0,0 +1,6 @@
- name: Warn about unsupported distribution
debug:
msg: >
WARNING: {{ ansible_distribution }} mirrors are not supported either
by this role yet. The execution of the job will continue without setting
up cached mirrors.

View File

@ -0,0 +1,2 @@
# {{ ansible_managed }}
APT::Get::AllowUnauthenticated "true";

View File

@ -0,0 +1,3 @@
# {{ ansible_managed }}
deb {{ package_mirror }} {{ ansible_distribution_release }}-backports main
deb-src {{ package_mirror }} {{ ansible_distribution_release }}-backports main

View File

@ -0,0 +1,3 @@
# {{ ansible_managed }}
deb {{ package_mirror }} {{ ansible_distribution_release }} main
deb-src {{ package_mirror }} {{ ansible_distribution_release }} main

View File

@ -0,0 +1,3 @@
# {{ ansible_managed }}
deb {{ security_mirror }} {{ ansible_distribution_release }} main
deb-src {{ security_mirror }} {{ ansible_distribution_release }} main

View File

@ -0,0 +1,3 @@
# {{ ansible_managed }}
deb {{ package_mirror }} {{ ansible_distribution_release }}-updates main
deb-src {{ package_mirror }} {{ ansible_distribution_release }}-updates main

View File

@ -0,0 +1,5 @@
# {{ ansible_managed }}
deb {% if set_apt_mirrors_trusted %}[ trusted=yes ] {% endif %}{{ package_mirror }} {{ ansible_distribution_release }} main universe
deb {% if set_apt_mirrors_trusted %}[ trusted=yes ] {% endif %}{{ package_mirror }} {{ ansible_distribution_release }}-updates main universe
deb {% if set_apt_mirrors_trusted %}[ trusted=yes ] {% endif %}{{ package_mirror }} {{ ansible_distribution_release }}-backports main universe
deb {% if set_apt_mirrors_trusted %}[ trusted=yes ] {% endif %}{{ package_mirror }} {{ ansible_distribution_release }}-security main universe

View File

@ -0,0 +1,20 @@
# {{ ansible_managed }}
[base]
name=CentOS-$releasever - Base
baseurl={{ package_mirror }}/$releasever/os/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
#released updates
[updates]
name=CentOS-$releasever - Updates
baseurl={{ package_mirror }}/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
baseurl={{ package_mirror }}/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

View File

@ -0,0 +1,24 @@
# {{ ansible_managed }}
[epel]
name=Extra Packages for Enterprise Linux 7 - $basearch
baseurl={{ epel_mirror }}/7/$basearch
failovermethod=priority
enabled=0
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
[epel-debuginfo]
name=Extra Packages for Enterprise Linux 7 - $basearch - Debug
baseurl={{ epel_mirror }}/7/$basearch/debug
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=1
[epel-source]
name=Extra Packages for Enterprise Linux 7 - $basearch - Source
baseurl={{ epel_mirror }}/7/SRPMS
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=1

View File

@ -0,0 +1,7 @@
# {{ ansible_managed }}
[AppStream]
name=CentOS-$releasever - AppStream
baseurl={{ package_mirror }}/$releasever/AppStream/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

View File

@ -0,0 +1,7 @@
# {{ ansible_managed }}
[BaseOS]
name=CentOS-$releasever - Base
baseurl={{ package_mirror }}/$releasever/BaseOS/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

View File

@ -0,0 +1,7 @@
# {{ ansible_managed }}
[extras]
name=CentOS-$releasever - Extras
baseurl={{ package_mirror }}/$releasever/extras/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

View File

@ -0,0 +1,7 @@
# {{ ansible_managed }}
[PowerTools]
name=CentOS-$releasever - PowerTools
baseurl={{ package_mirror }}/$releasever/PowerTools/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

View File

@ -0,0 +1,21 @@
# {{ ansible_managed }}
[epel]
name=Extra Packages for Enterprise Linux 8 - $basearch
baseurl={{ epel_mirror }}/8/Everything/$basearch
enabled=0
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8
[epel-debuginfo]
name=Extra Packages for Enterprise Linux 8 - $basearch - Debug
baseurl={{ epel_mirror }}/8/Everything/$basearch/debug
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8
gpgcheck=1
[epel-source]
name=Extra Packages for Enterprise Linux 8 - $basearch - Source
baseurl={{ epel_mirror }}/8/Everything/SRPMS
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8
gpgcheck=1

View File

@ -0,0 +1,6 @@
# {{ ansible_managed }}
[global]
timeout = 60
index-url = {{ pypi_mirror }}
trusted-host = {{ pypi_fqdn }}
extra-index-url = {{ wheel_mirror }}

View File

@ -0,0 +1,48 @@
# {{ ansible_managed }}
[updates]
name=Fedora $releasever - $basearch - Updates
failovermethod=priority
{% if ansible_distribution_version | version_compare('28', '<') %}
baseurl={{ package_mirror }}/updates/$releasever/$basearch/
{% else %}
baseurl={{ package_mirror }}/updates/$releasever/Everything/$basearch/
{% endif %}
enabled=1
repo_gpgcheck=0
type=rpm
gpgcheck=1
metadata_expire=6h
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False
[updates-debuginfo]
name=Fedora $releasever - $basearch - Updates - Debug
failovermethod=priority
{% if ansible_distribution_version | version_compare('28', '<') %}
baseurl={{ package_mirror }}/updates/$releasever/$basearch/debug/
{% else %}
baseurl={{ package_mirror }}/updates/$releasever/Everything/$basearch/debug/tree/
{% endif %}
enabled=0
repo_gpgcheck=0
type=rpm
gpgcheck=1
metadata_expire=6h
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False
[updates-source]
name=Fedora $releasever - Updates Source
failovermethod=priority
{% if ansible_distribution_version | version_compare('28', '<') %}
baseurl={{ package_mirror }}/updates/$releasever/SRPMS/
{% else %}
baseurl={{ package_mirror }}/updates/$releasever/Everything/source/tree/
{% endif %}
enabled=0
repo_gpgcheck=0
type=rpm
gpgcheck=1
metadata_expire=6h
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False

View File

@ -0,0 +1,36 @@
# {{ ansible_managed }}
[fedora]
name=Fedora $releasever - $basearch
failovermethod=priority
baseurl={{ package_mirror }}/releases/$releasever/Everything/$basearch/os/
enabled=1
metadata_expire=7d
repo_gpgcheck=0
type=rpm
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False
[fedora-debuginfo]
name=Fedora $releasever - $basearch - Debug
failovermethod=priority
baseurl={{ package_mirror }}/releases/$releasever/Everything/$basearch/debug/tree/
enabled=0
metadata_expire=7d
repo_gpgcheck=0
type=rpm
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False
[fedora-source]
name=Fedora $releasever - Source
failovermethod=priority
baseurl={{ package_mirror }}/releases/$releasever/Everything/source/tree/
enabled=0
metadata_expire=7d
repo_gpgcheck=0
type=rpm
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False

View File

@ -0,0 +1,8 @@
# {{ ansible_managed }}
[repo-oss]
name=repo-oss
enabled=1
autorefresh=0
baseurl={{ opensuse_repo_baseurl }}
type=yast2
keeppackages=0

View File

@ -0,0 +1,8 @@
# {{ ansible_managed }}
[repo-update]
name=repo-update
enabled=1
autorefresh=0
baseurl={{ package_mirror }}/update/leap/$releasever/oss/
type=rpm-md
keeppackages=0

View File

@ -0,0 +1,2 @@
package_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}"
epel_mirror: "http://{{ mirror_fqdn }}/epel"

View File

@ -0,0 +1,2 @@
package_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}"
security_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}-security"

View File

@ -0,0 +1,3 @@
mirror_fqdn: ftp.up.pt
pypi_fqdn: mirror01.ord.rax.openstack.org
package_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}"

View File

@ -0,0 +1,7 @@
package_mirror: "http://{{ mirror_fqdn }}/opensuse"
wheels_slug: "{%- if ansible_distribution == 'openSUSE Tumbleweed' -%}
opensuse-tumbleweed-{{ ansible_architecture | lower }}
{%- else -%}
{{ ansible_distribution | lower }}-{{ ansible_distribution_version }}-{{ ansible_architecture | lower }}
{%- endif -%}"
wheel_mirror: "http://{{ mirror_fqdn }}/wheel/{{ wheels_slug }}"

View File

@ -0,0 +1 @@
package_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}-ports"

View File

@ -0,0 +1 @@
package_mirror: "http://{{ mirror_fqdn }}/{{ ansible_distribution | lower }}"

View File

@ -0,0 +1 @@
package_mirror:

View File

@ -21,6 +21,7 @@
- job:
name: multinode
parent: base-test
abstract: true
description: |
Do additional setup needed for multi-node jobs such as setting up