Clean-up gate code to use mirror_info and pin ceph-ansible SHA

1. The OpenStack infrastructure provides us with a list of mirrors
   which are already templated so this will simply the process of
   using those mirrors instead of the previous implementation which
   tries to derive the right URL's.
2. Instead of rewriting the host's sources file for Ubuntu, we leave
   it alone.
3. We place the overrides into a new override file to simplify the
   implementation and no longer require a bunch of conditionals in
   the user_variables template. Some more clean-up can be done, but
   it will follow on later.
4. The ceph-ansible project has created a new library which is held
   in the root folder of the repo. We need to ensure that the library
   path is included, but that results in other breakages. To help
   get patches moving through the system again we pin the SHA to just
   prior the upstream changes.
5. The MariaDB mirror setting has been removed due to the default
   MariaDB setup being done via a specific pin. The apt repo setup
   for MariaDB is no longer being used.

Co-Authored-By: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Change-Id: Ib908e5503ecc93255d554d027ec62539c6316099

ansible-role-requirements.yml

@@ -185,7 +185,7 @@
 - name: ceph-ansible
   scm: git
   src: https://github.com/ceph/ceph-ansible
-  version: master
+  version: 78c1f1938f9365809cc5e7d09cb77566c7045e3b
 - name: opendaylight
   scm: git
   src: https://github.com/opendaylight/integration-packaging-ansible-opendaylight

tests/roles/bootstrap-host/defaults/main.yml

@@ -217,21 +217,6 @@ bootstrap_host_octavia: "{{ (bootstrap_host_scenario in ['octavia', 'translation
 
 ### Optional Settings ###
 
-# Set the apt repository URL's configured for the host and containers.
-# By default the configuration will be derived from the host.
-#bootstrap_host_ubuntu_repo: http://archive.ubuntu.com/ubuntu/
-#bootstrap_host_ubuntu_security_repo: http://archive.ubuntu.com/ubuntu/
-
-# Set the distribution suffixes that will be included in the apt repository configuration
-bootstrap_host_apt_distribution_suffix_list:
-  - updates
-  - backports
-
-# Set the components that will be included in the apt repository configuration
-bootstrap_host_apt_components:
-  - main
-  - universe
-
 # Specify the public IP address for the host.
 # By default the address will be set to the ipv4 address of the
 # host's network interface that has the default route on it.

tests/roles/bootstrap-host/tasks/install_packages.yml

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 - include: "install_packages_{{ ansible_os_family | lower }}.yml"
+  when:
+    - ansible_pkg_mgr != 'apt'
 
 - name: Remove known problem packages
   package:
@@ -26,9 +28,7 @@
   package:
     name: "{{ packages_install }}"
     state: present
-    # NOTE(hwoarang) Ubuntu has updated its cache before so no need to do it
-    # again
-    update_cache: "{{ (ansible_pkg_mgr == 'zypper') | ternary('yes', omit) }}"
+    update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
   tags:
     - install-packages
 

tests/roles/bootstrap-host/tasks/install_packages_debian.yml

@@ -1,77 +0,0 @@
----
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Ensure that apt supports HTTPS package sources
-  apt:
-    name: apt-transport-https
-    state: present
-  tags:
-    - apt-install-prerequisites
-
-- name: Determine the existing Ubuntu repo configuration
-  command: 'grep -oP "^deb \K(\[?.*\]?.*ubuntu\S*\/?)(?= {{ ansible_distribution_release }} main)" /etc/apt/sources.list'
-  register: ubuntu_repo
-  when:
-    - bootstrap_host_ubuntu_repo is not defined
-  changed_when: false
-  tags:
-    - find-apt-repo
-
-- name: Determine the existing Ubuntu Security repo configuration
-  command: 'grep -oP "^deb \K(\[?.*\]?.*ubuntu\S*\/?)(?= {{ ansible_distribution_release }}-security main)" /etc/apt/sources.list'
-  register: ubuntu_security_repo
-  when:
-    - bootstrap_host_ubuntu_security_repo is not defined
-  changed_when: false
-  failed_when: false
-  tags:
-    - find-apt-security-repo
-
-- name: Set apt repo facts based on discovered information
-  set_fact:
-    bootstrap_host_ubuntu_repo: "{{ ubuntu_repo.stdout_lines[0] }}"
-  when:
-    - bootstrap_host_ubuntu_repo is not defined
-    - ubuntu_repo is defined
-
-- name: Set apt security repo facts based on discovered information
-  set_fact:
-    bootstrap_host_ubuntu_security_repo: "{{ ubuntu_security_repo.stdout_lines[0] }}"
-  when:
-    - bootstrap_host_ubuntu_security_repo is not defined
-    - ubuntu_security_repo is defined
-    - ubuntu_security_repo.stdout_lines | length > 0
-
-- name: Configure apt's sources.list
-  template:
-    src: apt-sources.list.j2
-    dest: /etc/apt/sources.list
-    backup: yes
-  when:
-    - bootstrap_host_ubuntu_repo is defined
-  register: apt_sources_configure
-
-- name: Update apt-cache
-  apt:
-    update_cache: yes
-  when:
-    - apt_sources_configure is defined
-    - apt_sources_configure | changed
-  register: _update
-  retries: 5
-  delay: 2
-  until: _update | succeeded
-  tags:
-    - apt-cache-update

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -22,7 +22,6 @@
   tags:
     - create-directories
 
-
 - name: Deploy user conf.d configuration
   config_template:
     src: "{{ item.path | default(bootstrap_host_aio_config_path ~ '/conf.d') }}/{{ item.name }}"
@@ -63,26 +62,43 @@
     path: /etc/nodepool
   register: nodepool_dir
 
-- name: Set the UCA repository URL in OpenStack-CI
-  set_fact:
-    uca_apt_repo_url: "{{ bootstrap_host_ubuntu_repo | netorigin }}/ubuntu-cloud-archive"
-  when:
-    - nodepool_dir.stat.exists
-    - bootstrap_host_ubuntu_repo is defined
-
-- name: Set the MariaDB repository URL in OpenStack-CI
-  set_fact:
-    galera_repo_url: "{{ bootstrap_host_ubuntu_repo | netorigin }}/ubuntu-mariadb/10.1"
-  when:
-    - nodepool_dir.stat.exists
-    - bootstrap_host_ubuntu_repo is defined
-
-- name: Set the LXC image repository URL in OpenStack-CI
-  set_fact:
-    lxc_image_cache_server: "{{ bootstrap_host_ubuntu_repo | netloc_no_port }}:8080/images.linuxcontainers"
+- name: Discover the OpenStack-Infra mirrors
+  shell: |
+    source /etc/ci/mirror_info.sh
+    NODEPOOL_OVERRIDES="/etc/openstack_deploy/user_openstackci.yml"
+    echo "uca_apt_repo_url: '${NODEPOOL_UCA_MIRROR}'" >> ${NODEPOOL_OVERRIDES}
+    echo "openstack_hosts_centos_mirror_url: '${NODEPOOL_CENTOS_MIRROR}'" >> ${NODEPOOL_OVERRIDES}
+  args:
+    executable: /bin/bash
   when:
     - nodepool_dir.stat.exists | bool
-    - bootstrap_host_ubuntu_repo is defined
+  tags:
+    - skip_ansible_lint
+
+- name: Discover the OpenStack-Infra pypi/wheel mirror
+  shell: |
+    source /etc/ci/mirror_info.sh
+    echo "${NODEPOOL_PYPI_MIRROR}"
+    echo "${NODEPOOL_WHEEL_MIRROR}"
+  args:
+    executable: /bin/bash
+  register: _pypi_wheel_mirror
+  when:
+    - nodepool_dir.stat.exists | bool
+  tags:
+    - skip_ansible_lint
+
+- name: Discover the OpenStack-Infra LXC reverse proxy
+  shell: |
+    source /etc/ci/mirror_info.sh
+    echo ${NODEPOOL_LXC_IMAGE_PROXY}
+  register: _lxc_mirror
+  args:
+    executable: /bin/bash
+  when:
+    - nodepool_dir.stat.exists | bool
+  tags:
+    - skip_ansible_lint
 
 - name: Set the package cache timeout to 60 mins in OpenStack-CI
   set_fact:
@@ -107,7 +123,6 @@
     - ansible_selinux.status is defined
     - ansible_selinux.status == "enabled"
 
-
 # This is a very dirty hack due to images.linuxcontainers.org
 # constantly failing to resolve in openstack-infra.
 - name: Implement hard-coded hosts entries for consistently failing name
@@ -219,15 +234,3 @@
     - path: /etc/openstack_deploy/host_vars/localhost.yml
       line: 'tintin: "milou"'
   when: "(lookup('env','ACTION') | default(false,true)) == 'varstest'"
-
-- name: Discover the pypi mirror URL when in nodepool
-  shell: |
-    source /etc/ci/mirror_info.sh
-    echo "${NODEPOOL_PYPI_MIRROR}"
-  args:
-    executable: /bin/bash
-  register: _pypi_mirror
-  when:
-    - nodepool_dir.stat.exists | bool
-  tags:
-    - skip_ansible_lint

tests/roles/bootstrap-host/templates/apt-sources.list.j2

@@ -1,16 +0,0 @@
-# {{ ansible_managed }}
-
-# Base repository
-deb {{ bootstrap_host_ubuntu_repo }} {{ ansible_distribution_release }} {{ bootstrap_host_apt_components | join(" ") }}
-
-{% if bootstrap_host_apt_distribution_suffix_list | length > 0 %}
-# Additional distribution repositories
-{%   for suffix in bootstrap_host_apt_distribution_suffix_list %}
-deb {{ bootstrap_host_ubuntu_repo }} {{ ansible_distribution_release }}-{{ suffix }} {{ bootstrap_host_apt_components | join(" ") }}
-{%   endfor %}
-{% endif %}
-
-{% if bootstrap_host_ubuntu_security_repo is defined %}
-# Security repository
-deb {{ bootstrap_host_ubuntu_security_repo }} {{ ansible_distribution_release }}-security {{ bootstrap_host_apt_components | join(" ") }}
-{% endif %}

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -150,21 +150,10 @@ repo_build_pip_extra_indexes:
 {{ repo_build_pip_extra_indexes | to_nice_yaml }}
 {% endif %}
 
-{% if uca_apt_repo_url is defined %}
-## Ubuntu Cloud Archive mirror to use
-uca_apt_repo_url: {{ uca_apt_repo_url }}
-{% endif %}
-
-{% if galera_repo_url is defined %}
-## MariaDB mirror to use
-galera_repo_url: {{ galera_repo_url }}
-galera_client_apt_repo_url: {{ galera_repo_url }}
-{% endif %}
-
-{% if lxc_image_cache_server is defined %}
+{% if _lxc_mirror is defined and _lxc_mirror.stdout_lines is defined %}
 ## images.linuxcontainers.org reverse proxy
 lxc_image_cache_server_mirrors:
-  - "http://{{ lxc_image_cache_server }}"
+  - "http://{{ _lxc_mirror.stdout_lines[0] }}"
 {% endif %}
 
 {% if cache_timeout is defined %}
@@ -201,6 +190,8 @@ nova_service_negate:
   - "nova-agent.service"
   - "nova-resetnetwork.service"
 
-{% if _pypi_mirror is defined and _pypi_mirror.stdout is defined %}
-repo_nginx_pypi_upstream: "{{ _pypi_mirror.stdout | netloc }}"
+{% if _pypi_wheel_mirror is defined and _pypi_wheel_mirror.stdout_lines is defined %}
+repo_nginx_pypi_upstream: "{{ _pypi_wheel_mirror.stdout_lines[0] | netloc }}"
+repo_build_pip_extra_indexes:
+ - "{{ _pypi_wheel_mirror.stdout_lines[1] }}"
 {% endif %}

tests/roles/bootstrap-host/vars/ubuntu.yml

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 packages_install:
+  - apt-transport-https
   - bridge-utils
   - build-essential
   - curl