Remove apt-cacher-ng

The repo container's package cache causes quite a bit of confusion
given that it's a 'hidden' feature which catches deployers off-guard
when they already have their own cache configured. This is really
the kind of service which people should manage outside of OSA. It
also makes no sense if the deployer is using their own local mirror
which is a fairly common practise. Adding to that, it seems that it
is broken in bionic, causing massive delays in package installs.
Finally, it also adds to quite a bit of complexity due to the fact
that it's in a container - so in the playbooks prior to the container's
existence we have to detect whether it's there and add/remove the config
accordingly.

Let's just remove it and let deployers managing their own caching
infrastructure if they want it.

Change-Id: I829b9cfa16fbd1f9f4d33b5943f1e46623e1b157

defaults/main.yml

@@ -45,27 +45,10 @@ repo_server_port: 8181
 # Pypi Server port
 repo_pypiserver_port: 8280
 
-# Toggle the implementation of the Package Cache service
-repo_pkg_cache_enabled: true
-
-# Set the listening port for the Package Cache service
-repo_pkg_cache_port: 3142
-
-# Set the listening address for the Package Cache service
-repo_pkg_cache_bind: "0.0.0.0"
-
 # Set the git file paths
 repo_git_cache_dirname: openstackgit
 repo_git_cache_dir: "{{ repo_service_home_folder }}/repo/{{ repo_git_cache_dirname }}"
 
-# Set the Package Cache Service files path
-repo_pkg_cache_dirname: pkg-cache
-repo_pkg_cache_dir: "{{ repo_service_home_folder }}/repo/{{ repo_pkg_cache_dirname }}"
-
-# Set the Package Cache Service owner and group
-repo_pkg_cache_owner: apt-cacher-ng
-repo_pkg_cache_group: apt-cacher-ng
-
 # Set the log directory
 repo_service_log_dir: /var/log/apt-cacher-ng
 
@@ -94,4 +77,3 @@ repo_pypiserver_start_options: >-
 
 # config override var for systemd init file
 repo_pypiserver_init_overrides: {}
-

handlers/main.yml

@@ -35,17 +35,6 @@
   retries: 5
   delay: 2
 
-- name: reload acng
-  service:
-    name: "apt-cacher-ng"
-    enabled: yes
-    state: restarted
-    daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
-  register: _restart
-  until: _restart is success
-  retries: 5
-  delay: 2
-
 - name: reload lsyncd
   service:
     name: "lsyncd"
@@ -90,3 +79,36 @@
   retries: 5
   delay: 2
 
+# TODO(odyssey4me):
+# Remove these tasks in T. They are only present for the
+# Q->R upgrade or for R->S upgrades for environments which
+# were installed prior to R's release.
+- name: Remove apt-cacher-ng package
+  package:
+    name: apt-cacher-ng
+    state: absent
+    purge: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}"
+  listen:
+    - cleanup apt-cacher-ng service
+
+- name: Remove apt-cacher-ng data folders
+  file:
+    path: "{{ repo_service_home_folder }}/repo/{{ repo_pkg_cache_dirname | default('pkg-cache') }}"
+    state: absent
+  listen:
+    - cleanup apt-cacher-ng service
+
+- name: Remove apt-cacher-ng user
+  user:
+    name: "{{ repo_pkg_cache_owner | default('apt-cacher-ng') }}"
+    state: absent
+  listen:
+    - cleanup apt-cacher-ng service
+
+- name: Remove apt-cacher-ng group
+  group:
+    name: "{{ repo_pkg_cache_group | default('apt-cacher-ng') }}"
+    state: absent
+  listen:
+    - cleanup apt-cacher-ng service
+

releasenotes/notes/remove-pkg-cache-afba3577138dc0a0.yaml

@@ -0,0 +1,22 @@
+---
+deprecations:
+  - |
+    The package cache on the repo server has been removed. If caching of
+    packages is desired, it should be setup outside of OpenStack-Ansible
+    and the variable ``lxc_container_cache_files`` (for LXC containers)
+    or ``nspawn_container_cache_files_from_host`` (for nspawn containers)
+    can be used to copy the appropriate host configuration from the host
+    into the containers on creation. Alternatively, environment variables
+    can be set to use the cache in the host /etc/environment file prior
+    to container creation, or the ``deployment_environment_variables``
+    can have the right variables set to use it. The following variables
+    have been removed.
+
+    * ``repo_pkg_cache_enabled``
+    * ``repo_pkg_cache_port``
+    * ``repo_pkg_cache_bind``
+    * ``repo_pkg_cache_dirname``
+    * ``repo_pkg_cache_dir``
+    * ``repo_pkg_cache_owner``
+    * ``repo_pkg_cache_group``
+

tasks/main.yml

@@ -36,10 +36,6 @@
   tags:
     - repo_server-config
 
-- include: repo_cacher.yml
-  static: no
-  when: repo_pkg_cache_enabled | bool
-
 - include: repo_key_populate.yml
   tags:
     - repo_server-config

tasks/repo_cacher.yml

@@ -1,92 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Install repo caching server packages
-  package:
-    name: "{{ repo_pkg_cache_server_distro_packages }}"
-    state: "{{ repo_server_package_state }}"
-  register: install_packages
-  until: install_packages is success
-  retries: 5
-  delay: 5
-  when:
-    - repo_pkg_cache_enabled | bool
-  tags:
-    - repo_server-install
-
-- name: Create cache directory
-  file:
-    path: "{{ repo_pkg_cache_dir }}"
-    state: "directory"
-    owner: "{{ repo_pkg_cache_owner }}"
-    group: "{{ repo_service_group_name }}"
-    mode: "02775"
-  tags:
-    - repo_server-install
-
-- name: Create log directory
-  file:
-    path: "{{ repo_service_log_dir }}"
-    state: "directory"
-    owner: "{{ repo_pkg_cache_owner }}"
-    group: "{{ repo_pkg_cache_group }}"
-    mode: "02775"
-  tags:
-    - repo_server-install
-
-- name: Stat the cache path
-  stat:
-    path: /var/cache/apt-cacher-ng
-  register: acs
-  tags:
-    - repo_server-install
-
-- name: Remove cacher directory if its a directory
-  file:
-    path: "/var/cache/apt-cacher-ng"
-    state: "absent"
-  when:
-    - acs.stat.isdir is defined and acs.stat.isdir
-  tags:
-    - repo_server-install
-
-- name: Link cacher to the repo path
-  file:
-    src: "{{ repo_service_home_folder }}/repo/pkg-cache"
-    dest: "/var/cache/apt-cacher-ng"
-    state: "link"
-  tags:
-    - repo_server-install
-
-- name: Create yum merged mirror list
-  shell: |
-    curl https://www.centos.org/download/full-mirrorlist.csv | sed 's/^.*"http:/http:/' | sed 's/".*$//' | grep ^http >/etc/apt-cacher-ng/centos_mirrors
-    echo "http://mirror.centos.org/centos/" >>/etc/apt-cacher-ng/centos_mirrors
-  args:
-    warn: no
-  when:
-    - ansible_pkg_mgr == 'yum'
-  tags:
-    - repo_server-config
-    - skip_ansible_lint
-
-- name: Drop acng.conf
-  template:
-    src: "acng.conf.j2"
-    dest: "/etc/apt-cacher-ng/acng.conf"
-  notify:
-    - reload acng
-  tags:
-    - repo_server-config

tasks/repo_install.yml

@@ -32,7 +32,7 @@
     gpgcheck: yes
     enabled: yes
     state: present
-    includepkgs: 'apt-cacher-ng lsyncd'
+    includepkgs: 'lsyncd'
   when:
     - ansible_pkg_mgr in ['yum', 'dnf']
   register: install_epel_repo

tasks/repo_sync_manager.yml

@@ -22,8 +22,27 @@
     - { src: "lsyncd.defaults.j2", dest: "{{ repo_lsyncd_defaults_file }}" }
   notify:
     - reload lsyncd
+    - cleanup apt-cacher-ng service
   when:
-    - groups['repo_all']|length > 1
+    - groups['repo_all'] | length > 1
   tags:
     - repo-lsyncd
     - repo-config
+
+# TODO(odyssey4me):
+# Remove this task in T. It is only present for the
+# Q->R upgrade or for R->S upgrades for environments which
+# were installed prior to R's release.
+# It is implemented as a handler to prevent the service
+# being removed before lsync has restarted with the new
+# config. This task in particular cater for when lsync
+# is not used because there is only one repo container.
+- name: Remove apt-cacher-ng service
+  command: "true"
+  when:
+    - groups['repo_all'] | length == 1
+  notify:
+    - cleanup apt-cacher-ng service
+  tags:
+    - skip_ansible_lint
+

templates/acng.conf.j2

@@ -1,31 +0,0 @@
-# {{ ansible_managed }}
-
-CacheDir: {{ repo_pkg_cache_dir }}
-LogDir: /var/log/apt-cacher-ng
-Port: {{ repo_pkg_cache_port }}
-BindAddress: {{ repo_pkg_cache_bind }}
-Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives
-Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu # Ubuntu Archives
-Remap-debvol: file:debvol_mirror*.gz /debian-volatile ; file:backends_debvol # Debian Volatile Archives
-Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here
-Remap-sfnet:  file:sfnet_mirrors # ; file:backends_sfnet # incomplete, please create this file or specify preferred mirrors here
-Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
-Remap-fedora: file:fedora_mirrors # Fedora Linux
-Remap-slrep:  file:sl_mirrors # Scientific Linux
-Remap-centos: file:centos_mirrors /centos #centos
-ReportPage: acng-report.html
-PidFile: /var/run/apt-cacher-ng
-ExTreshold: 4
-LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng
-PassThroughPattern: .*
-{% if proxy_env_url is defined %}
-Proxy: {{ proxy_env_url }}
-{% endif %}
-VfilePatternEx: ^/\?release=[0-9]+&arch=
-# NOTE(mhayden): Caching the CentOS mirror list causes yum to throw
-# 503 errors intermittently since the remote file is dynamic. Also,
-# yum has issues with retrieving the mariadb.org repodata bz2 and
-# that causes more intermittent 503 errors. This DontCache line
-# tells apt-cacher-ng to allow requests for these to pass through
-# without being cached.
-DontCache: (mirrorlist\.centos\.org)|(mariadb\.org.*\.bz2$)

templates/lsyncd.lua.j2

@@ -593,7 +593,7 @@ sync {
     rsync,
     source = "{{ repo_service_home_folder }}/repo",
     target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_service_home_folder }}/repo",
-    exclude = {"{{ repo_pkg_cache_dirname }}", "{{ repo_git_cache_dir }}"},
+    exclude = {"{{ repo_git_cache_dir }}"},
     rsync = {
         compress = true,
         acls = true,
@@ -614,20 +614,5 @@ sync {
         postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-git"
     }
 }
-{% if repo_pkg_cache_enabled | bool %}
-sync {
-    rsync,
-    source = "{{ repo_pkg_cache_dir }}",
-    target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_pkg_cache_dir }}",
-    delete = false,
-    rsync = {
-        compress = true,
-        acls = true,
-        rsh = "/usr/bin/ssh -l {{ repo_service_user_name }} -i {{ repo_service_home_folder }}/.ssh/id_rsa -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=5",
-        precmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh pre-pkg",
-        postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-pkg"
-    }
-}
-{% endif %}
 {% endif %}
 {% endfor %}

templates/repo_prepost_cmd.sh.j2

@@ -1,15 +1,6 @@
 #!/bin/bash
 #  This script is called by lsyncd to perform 'pre' and 'post' rsync tasks.
 #
-PKG_CACHE={{ repo_pkg_cache_dir }}
-
-function chg_owner {
-  NEW_OWNER=$1
-  if [ -d $PKG_CACHE ]; then
-    chown -R $NEW_OWNER $PKG_CACHE
-  fi
-}
-
 CMD=$1
 case $CMD in
 pre-www)
@@ -26,12 +17,6 @@ pre-git)
 post-git)
   systemctl start git.socket
   ;;
-pre-pkg)
-  chg_owner {{ repo_service_user_name }}
-  ;;
-post-pkg)
-  chg_owner {{ repo_pkg_cache_owner }}
-  ;;
 *)
   echo "Unknown command."
 esac

tests/test-repo-server-functional.yml

@@ -58,12 +58,3 @@
         dest: /tmp/repo_server
         version: master
         accept_hostkey: yes
-
-    - name: Check apt-cacher-ng is running
-      command: "pgrep apt-cacher-ng"
-      register: pgrep_apt_cacher_ng
-      until: pgrep_apt_cacher_ng is success
-      retries: 5
-      delay: 2
-      tags:
-        - skip_ansible_lint

vars/debian.yml

@@ -27,9 +27,6 @@ repo_server_distro_packages:
   - openssh-server
   - rsync
 
-repo_pkg_cache_server_distro_packages:
-  - apt-cacher-ng
-
 repo_lsyncd_config_file: /etc/lsyncd/lsyncd.conf.lua
 repo_lsyncd_defaults_file: /etc/default/lsyncd
 

vars/redhat.yml

@@ -29,9 +29,6 @@ repo_server_distro_packages:
   - openssh-server
   - rsync
 
-repo_pkg_cache_server_distro_packages:
-  - apt-cacher-ng
-
 repo_lsyncd_config_file: /etc/lsyncd.conf
 repo_lsyncd_defaults_file: /etc/sysconfig/lsyncd
 

vars/suse.yml

@@ -28,9 +28,6 @@ repo_server_distro_packages:
   - openssh
   - rsync
 
-repo_pkg_cache_server_distro_packages:
-  - apt-cacher-ng
-
 repo_lsyncd_config_file: /etc/lsyncd/lsyncd.conf
 repo_lsyncd_defaults_file: /etc/sysconfig/lsyncd