Remove apt-cacher-ng

The repo container's package cache causes quite a bit of confusion
given that it's a 'hidden' feature which catches deployers off-guard
when they already have their own cache configured. This is really
the kind of service which people should manage outside of OSA. It
also makes no sense if the deployer is using their own local mirror
which is a fairly common practise. Adding to that, it seems that it
is broken in bionic, causing massive delays in package installs.
Finally, it also adds to quite a bit of complexity due to the fact
that it's in a container - so in the playbooks prior to the container's
existence we have to detect whether it's there and add/remove the config
accordingly.

Let's just remove it and let deployers managing their own caching
infrastructure if they want it.

Change-Id: I829b9cfa16fbd1f9f4d33b5943f1e46623e1b157
This commit is contained in:
Jesse Pretorius 2018-10-08 11:13:12 +01:00
parent 94b4eb098a
commit 6663637374
14 changed files with 77 additions and 207 deletions

View File

@ -45,27 +45,10 @@ repo_server_port: 8181
# Pypi Server port
repo_pypiserver_port: 8280
# Toggle the implementation of the Package Cache service
repo_pkg_cache_enabled: true
# Set the listening port for the Package Cache service
repo_pkg_cache_port: 3142
# Set the listening address for the Package Cache service
repo_pkg_cache_bind: "0.0.0.0"
# Set the git file paths
repo_git_cache_dirname: openstackgit
repo_git_cache_dir: "{{ repo_service_home_folder }}/repo/{{ repo_git_cache_dirname }}"
# Set the Package Cache Service files path
repo_pkg_cache_dirname: pkg-cache
repo_pkg_cache_dir: "{{ repo_service_home_folder }}/repo/{{ repo_pkg_cache_dirname }}"
# Set the Package Cache Service owner and group
repo_pkg_cache_owner: apt-cacher-ng
repo_pkg_cache_group: apt-cacher-ng
# Set the log directory
repo_service_log_dir: /var/log/apt-cacher-ng
@ -94,4 +77,3 @@ repo_pypiserver_start_options: >-
# config override var for systemd init file
repo_pypiserver_init_overrides: {}

View File

@ -35,17 +35,6 @@
retries: 5
delay: 2
- name: reload acng
service:
name: "apt-cacher-ng"
enabled: yes
state: restarted
daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
register: _restart
until: _restart is success
retries: 5
delay: 2
- name: reload lsyncd
service:
name: "lsyncd"
@ -90,3 +79,36 @@
retries: 5
delay: 2
# TODO(odyssey4me):
# Remove these tasks in T. They are only present for the
# Q->R upgrade or for R->S upgrades for environments which
# were installed prior to R's release.
- name: Remove apt-cacher-ng package
package:
name: apt-cacher-ng
state: absent
purge: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}"
listen:
- cleanup apt-cacher-ng service
- name: Remove apt-cacher-ng data folders
file:
path: "{{ repo_service_home_folder }}/repo/{{ repo_pkg_cache_dirname | default('pkg-cache') }}"
state: absent
listen:
- cleanup apt-cacher-ng service
- name: Remove apt-cacher-ng user
user:
name: "{{ repo_pkg_cache_owner | default('apt-cacher-ng') }}"
state: absent
listen:
- cleanup apt-cacher-ng service
- name: Remove apt-cacher-ng group
group:
name: "{{ repo_pkg_cache_group | default('apt-cacher-ng') }}"
state: absent
listen:
- cleanup apt-cacher-ng service

View File

@ -0,0 +1,22 @@
---
deprecations:
- |
The package cache on the repo server has been removed. If caching of
packages is desired, it should be setup outside of OpenStack-Ansible
and the variable ``lxc_container_cache_files`` (for LXC containers)
or ``nspawn_container_cache_files_from_host`` (for nspawn containers)
can be used to copy the appropriate host configuration from the host
into the containers on creation. Alternatively, environment variables
can be set to use the cache in the host /etc/environment file prior
to container creation, or the ``deployment_environment_variables``
can have the right variables set to use it. The following variables
have been removed.
* ``repo_pkg_cache_enabled``
* ``repo_pkg_cache_port``
* ``repo_pkg_cache_bind``
* ``repo_pkg_cache_dirname``
* ``repo_pkg_cache_dir``
* ``repo_pkg_cache_owner``
* ``repo_pkg_cache_group``

View File

@ -36,10 +36,6 @@
tags:
- repo_server-config
- include: repo_cacher.yml
static: no
when: repo_pkg_cache_enabled | bool
- include: repo_key_populate.yml
tags:
- repo_server-config

View File

@ -1,92 +0,0 @@
---
# Copyright 2016, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install repo caching server packages
package:
name: "{{ repo_pkg_cache_server_distro_packages }}"
state: "{{ repo_server_package_state }}"
register: install_packages
until: install_packages is success
retries: 5
delay: 5
when:
- repo_pkg_cache_enabled | bool
tags:
- repo_server-install
- name: Create cache directory
file:
path: "{{ repo_pkg_cache_dir }}"
state: "directory"
owner: "{{ repo_pkg_cache_owner }}"
group: "{{ repo_service_group_name }}"
mode: "02775"
tags:
- repo_server-install
- name: Create log directory
file:
path: "{{ repo_service_log_dir }}"
state: "directory"
owner: "{{ repo_pkg_cache_owner }}"
group: "{{ repo_pkg_cache_group }}"
mode: "02775"
tags:
- repo_server-install
- name: Stat the cache path
stat:
path: /var/cache/apt-cacher-ng
register: acs
tags:
- repo_server-install
- name: Remove cacher directory if its a directory
file:
path: "/var/cache/apt-cacher-ng"
state: "absent"
when:
- acs.stat.isdir is defined and acs.stat.isdir
tags:
- repo_server-install
- name: Link cacher to the repo path
file:
src: "{{ repo_service_home_folder }}/repo/pkg-cache"
dest: "/var/cache/apt-cacher-ng"
state: "link"
tags:
- repo_server-install
- name: Create yum merged mirror list
shell: |
curl https://www.centos.org/download/full-mirrorlist.csv | sed 's/^.*"http:/http:/' | sed 's/".*$//' | grep ^http >/etc/apt-cacher-ng/centos_mirrors
echo "http://mirror.centos.org/centos/" >>/etc/apt-cacher-ng/centos_mirrors
args:
warn: no
when:
- ansible_pkg_mgr == 'yum'
tags:
- repo_server-config
- skip_ansible_lint
- name: Drop acng.conf
template:
src: "acng.conf.j2"
dest: "/etc/apt-cacher-ng/acng.conf"
notify:
- reload acng
tags:
- repo_server-config

View File

@ -32,7 +32,7 @@
gpgcheck: yes
enabled: yes
state: present
includepkgs: 'apt-cacher-ng lsyncd'
includepkgs: 'lsyncd'
when:
- ansible_pkg_mgr in ['yum', 'dnf']
register: install_epel_repo

View File

@ -22,8 +22,27 @@
- { src: "lsyncd.defaults.j2", dest: "{{ repo_lsyncd_defaults_file }}" }
notify:
- reload lsyncd
- cleanup apt-cacher-ng service
when:
- groups['repo_all']|length > 1
- groups['repo_all'] | length > 1
tags:
- repo-lsyncd
- repo-config
# TODO(odyssey4me):
# Remove this task in T. It is only present for the
# Q->R upgrade or for R->S upgrades for environments which
# were installed prior to R's release.
# It is implemented as a handler to prevent the service
# being removed before lsync has restarted with the new
# config. This task in particular cater for when lsync
# is not used because there is only one repo container.
- name: Remove apt-cacher-ng service
command: "true"
when:
- groups['repo_all'] | length == 1
notify:
- cleanup apt-cacher-ng service
tags:
- skip_ansible_lint

View File

@ -1,31 +0,0 @@
# {{ ansible_managed }}
CacheDir: {{ repo_pkg_cache_dir }}
LogDir: /var/log/apt-cacher-ng
Port: {{ repo_pkg_cache_port }}
BindAddress: {{ repo_pkg_cache_bind }}
Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives
Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu # Ubuntu Archives
Remap-debvol: file:debvol_mirror*.gz /debian-volatile ; file:backends_debvol # Debian Volatile Archives
Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here
Remap-sfnet: file:sfnet_mirrors # ; file:backends_sfnet # incomplete, please create this file or specify preferred mirrors here
Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
Remap-fedora: file:fedora_mirrors # Fedora Linux
Remap-slrep: file:sl_mirrors # Scientific Linux
Remap-centos: file:centos_mirrors /centos #centos
ReportPage: acng-report.html
PidFile: /var/run/apt-cacher-ng
ExTreshold: 4
LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng
PassThroughPattern: .*
{% if proxy_env_url is defined %}
Proxy: {{ proxy_env_url }}
{% endif %}
VfilePatternEx: ^/\?release=[0-9]+&arch=
# NOTE(mhayden): Caching the CentOS mirror list causes yum to throw
# 503 errors intermittently since the remote file is dynamic. Also,
# yum has issues with retrieving the mariadb.org repodata bz2 and
# that causes more intermittent 503 errors. This DontCache line
# tells apt-cacher-ng to allow requests for these to pass through
# without being cached.
DontCache: (mirrorlist\.centos\.org)|(mariadb\.org.*\.bz2$)

View File

@ -593,7 +593,7 @@ sync {
rsync,
source = "{{ repo_service_home_folder }}/repo",
target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_service_home_folder }}/repo",
exclude = {"{{ repo_pkg_cache_dirname }}", "{{ repo_git_cache_dir }}"},
exclude = {"{{ repo_git_cache_dir }}"},
rsync = {
compress = true,
acls = true,
@ -614,20 +614,5 @@ sync {
postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-git"
}
}
{% if repo_pkg_cache_enabled | bool %}
sync {
rsync,
source = "{{ repo_pkg_cache_dir }}",
target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_pkg_cache_dir }}",
delete = false,
rsync = {
compress = true,
acls = true,
rsh = "/usr/bin/ssh -l {{ repo_service_user_name }} -i {{ repo_service_home_folder }}/.ssh/id_rsa -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=5",
precmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh pre-pkg",
postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-pkg"
}
}
{% endif %}
{% endif %}
{% endfor %}

View File

@ -1,15 +1,6 @@
#!/bin/bash
# This script is called by lsyncd to perform 'pre' and 'post' rsync tasks.
#
PKG_CACHE={{ repo_pkg_cache_dir }}
function chg_owner {
NEW_OWNER=$1
if [ -d $PKG_CACHE ]; then
chown -R $NEW_OWNER $PKG_CACHE
fi
}
CMD=$1
case $CMD in
pre-www)
@ -26,12 +17,6 @@ pre-git)
post-git)
systemctl start git.socket
;;
pre-pkg)
chg_owner {{ repo_service_user_name }}
;;
post-pkg)
chg_owner {{ repo_pkg_cache_owner }}
;;
*)
echo "Unknown command."
esac

View File

@ -58,12 +58,3 @@
dest: /tmp/repo_server
version: master
accept_hostkey: yes
- name: Check apt-cacher-ng is running
command: "pgrep apt-cacher-ng"
register: pgrep_apt_cacher_ng
until: pgrep_apt_cacher_ng is success
retries: 5
delay: 2
tags:
- skip_ansible_lint

View File

@ -27,9 +27,6 @@ repo_server_distro_packages:
- openssh-server
- rsync
repo_pkg_cache_server_distro_packages:
- apt-cacher-ng
repo_lsyncd_config_file: /etc/lsyncd/lsyncd.conf.lua
repo_lsyncd_defaults_file: /etc/default/lsyncd

View File

@ -29,9 +29,6 @@ repo_server_distro_packages:
- openssh-server
- rsync
repo_pkg_cache_server_distro_packages:
- apt-cacher-ng
repo_lsyncd_config_file: /etc/lsyncd.conf
repo_lsyncd_defaults_file: /etc/sysconfig/lsyncd

View File

@ -28,9 +28,6 @@ repo_server_distro_packages:
- openssh
- rsync
repo_pkg_cache_server_distro_packages:
- apt-cacher-ng
repo_lsyncd_config_file: /etc/lsyncd/lsyncd.conf
repo_lsyncd_defaults_file: /etc/sysconfig/lsyncd