From b85282c046e93b2bed9b1b02a051e664a09abca3 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 27 Jun 2019 14:20:02 +1000 Subject: [PATCH] Move rsync mirror updates to new opendev.org mirror-update host This move was prompted by wishing to expose the mirror update logs for the rsync updates so that debugging problems does not require a root user (note: not actually done in this change; will be a follow-on). Rather than start hacking at puppet, the rsync mirror scripts make a nice delination point for starting an Ansible-first/Bionic update. Most magic is included in the scripts, so there is not much more to do than copy them. The host uses the existing kerberos and openafs roles and copies the key material into place (to be added before merge). Note the scripts are removed from the extant puppet so we don't have two updates happening simultaneously. This will also require a manual clean to remove the cron jobs as a once-off when merging. The other part of mirror-update is the reprepro based scripts for the various debuntu repositories. They are left as future work for now. Testing is added to ensure dependencies and scripts are all in place. Change-Id: I525ac18b55f0e11b0a541b51fa97ee5d6512bf70 --- .zuul.yaml | 23 ++++ inventory/groups.yaml | 3 + .../manifests/mirror_update.pp | 129 ------------------ playbooks/roles/mirror-update/README.rst | 15 ++ .../roles/mirror-update/files}/cache-stats.sh | 0 .../mirror-update/files/centos-mirror-update | 0 .../mirror-update/files/epel-mirror-update | 0 .../mirror-update/files/fedora-mirror-update | 0 .../files/opensuse-mirror-update | 0 .../files/yum-puppetlabs-mirror-update | 0 playbooks/roles/mirror-update/tasks/main.yaml | 27 ++++ .../roles/mirror-update/tasks/rsync.yaml | 56 ++++++++ playbooks/service-mirror-update.yaml | 11 ++ playbooks/zuul/run-base.yaml | 1 + .../mirror-update01.opendev.org.yaml.j2 | 12 ++ testinfra/test_mirror-update.py | 50 +++++++ 16 files changed, 198 insertions(+), 129 deletions(-) create mode 100644 playbooks/roles/mirror-update/README.rst rename {modules/openstack_project/files/mirror => playbooks/roles/mirror-update/files}/cache-stats.sh (100%) mode change 100644 => 100755 rename modules/openstack_project/files/mirror/centos-mirror-update.sh => playbooks/roles/mirror-update/files/centos-mirror-update (100%) rename modules/openstack_project/files/mirror/epel-mirror-update.sh => playbooks/roles/mirror-update/files/epel-mirror-update (100%) rename modules/openstack_project/files/mirror/fedora-mirror-update.sh => playbooks/roles/mirror-update/files/fedora-mirror-update (100%) mode change 100644 => 100755 rename modules/openstack_project/files/mirror/opensuse-mirror-update.sh => playbooks/roles/mirror-update/files/opensuse-mirror-update (100%) mode change 100644 => 100755 rename modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh => playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update (100%) create mode 100644 playbooks/roles/mirror-update/tasks/main.yaml create mode 100644 playbooks/roles/mirror-update/tasks/rsync.yaml create mode 100644 playbooks/service-mirror-update.yaml create mode 100644 playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 create mode 100644 testinfra/test_mirror-update.py diff --git a/.zuul.yaml b/.zuul.yaml index 388d297a15..493c941b04 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -571,6 +571,27 @@ host_copy_output: '/var/log/apache2/': logs +- job: + name: system-config-run-mirror-update + parent: system-config-run + description: | + Run the playbook for a mirror update node + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: mirror-update01.opendev.org + label: ubuntu-bionic + vars: + run_playbooks: + - playbooks/service-mirror-update.yaml + files: + - .zuul.yaml + - roles/ + - playbooks/roles/mirror-update/ + - playbooks/service-mirror-update.yaml + - testinfra/test_mirror-update.py + - job: name: system-config-run-docker-registry parent: system-config-run @@ -710,6 +731,7 @@ - system-config-run-lists - system-config-run-nodepool - system-config-run-mirror + - system-config-run-mirror-update - system-config-run-docker-registry - system-config-run-gitea: dependencies: @@ -737,6 +759,7 @@ - system-config-run-lists - system-config-run-nodepool - system-config-run-mirror + - system-config-run-mirror-update - system-config-run-docker-registry - system-config-run-gitea: dependencies: diff --git a/inventory/groups.yaml b/inventory/groups.yaml index a79c772b8c..62e82cf6c9 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -5,6 +5,7 @@ groups: afs-client: - review-dev[0-9]*.open*.org - mirror[0-9]*.open*.org + - mirror-update[0-9]*.opendev.org - files[0-9]*.open*.org - ze[0-9]*.open*.org - afsdb*.open*.org @@ -67,6 +68,8 @@ groups: - mirror[0-9]*.openstack.org mirror_opendev: - mirror[0-9]*.opendev.org + mirror-update: + - mirror-update[0-9]*.opendev.org nodepool: - nb[0-9]*.open*.org - nl[0-9]*.open*.org diff --git a/modules/openstack_project/manifests/mirror_update.pp b/modules/openstack_project/manifests/mirror_update.pp index 435723b4a9..b1adeaa064 100644 --- a/modules/openstack_project/manifests/mirror_update.pp +++ b/modules/openstack_project/manifests/mirror_update.pp @@ -373,135 +373,6 @@ class openstack_project::mirror_update ( ] } - ### RDO mirror ### - file { '/etc/rdo.keytab': - ensure => absent, - } - - file { '/usr/local/bin/rdo-mirror-update': - ensure => absent, - } - - cron { 'rdo mirror': - ensure => absent, - } - - ### EPEL mirror ### - file { '/etc/epel.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $epel_keytab, - } - - file { '/usr/local/bin/epel-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/epel-mirror-update.sh', - } - - cron { 'epel mirror': - user => 'root', - minute => fqdn_rand(45, 'epel-mirror'), - hour => '*/2', - command => 'flock -n /var/run/epel-mirror.lock epel-mirror-update mirror.epel >>/var/log/epel-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/epel-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/epel.keytab'], - ] - } - - ### Puppetlabs / CentOS mirror ### - file { '/etc/yum-puppetlabs.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $yum_puppetlabs_keytab, - } - - file { '/usr/local/bin/yum-puppetlabs-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/yum-puppetlabs-mirror-update.sh', - } - - cron { 'yum-puppetlabs mirror': - user => 'root', - minute => fqdn_rand(45, 'yum-puppetlabs'), - hour => '*/2', - command => 'flock -n /var/run/yum-puppetlabs-mirror.lock yum-puppetlabs-mirror-update mirror.yum-puppetlabs >>/var/log/yum-puppetlabs-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/yum-puppetlabs-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/yum-puppetlabs.keytab'], - ] - } - - ### Fedora mirror ### - file { '/etc/fedora.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $fedora_keytab, - } - - file { '/usr/local/bin/fedora-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/fedora-mirror-update.sh', - } - - cron { 'fedora mirror': - user => 'root', - minute => fqdn_rand(45, 'fedora-mirror'), - hour => '*/2', - command => 'flock -n /var/run/fedora-mirror.lock fedora-mirror-update mirror.fedora >>/var/log/fedora-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/fedora-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/fedora.keytab'], - ] - } - - ### openSUSE mirror ### - file { '/etc/opensuse.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $opensuse_keytab, - } - - file { '/usr/local/bin/opensuse-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/opensuse-mirror-update.sh', - } - - cron { 'opensuse mirror': - user => 'root', - minute => fqdn_rand(45, 'opensuse-mirror'), - hour => '*/2', - command => 'flock -n /var/run/opensuse-mirror.lock opensuse-mirror-update mirror.opensuse >>/var/log/opensuse-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/opensuse-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/opensuse.keytab'], - ] - } - ### Ubuntu Cloud Archive Mirror ### ::openstack_project::reprepro { 'ubuntu-cloud-archive-reprepro-mirror': confdir => '/etc/reprepro/ubuntu-cloud-archive', diff --git a/playbooks/roles/mirror-update/README.rst b/playbooks/roles/mirror-update/README.rst new file mode 100644 index 0000000000..abe7e333e0 --- /dev/null +++ b/playbooks/roles/mirror-update/README.rst @@ -0,0 +1,15 @@ +mirror-update + +This role sets up the ``mirror-update`` host, which does the periodic +sync of upstream mirrors to the AFS volumes. + +It is not intended to be a particularly generic or flexible role, as +there is usually only one instance of the mirror-update host (to avoid +conflicting updates). + +At this stage, it handles the mirrors that are updated by ``rsync`` +only. It is expected that it will grow to cover mirroring other +volumes that are currently done by the legacy ``openstack.org`` host +and managed by puppet. + +**Role Variables** diff --git a/modules/openstack_project/files/mirror/cache-stats.sh b/playbooks/roles/mirror-update/files/cache-stats.sh old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/cache-stats.sh rename to playbooks/roles/mirror-update/files/cache-stats.sh diff --git a/modules/openstack_project/files/mirror/centos-mirror-update.sh b/playbooks/roles/mirror-update/files/centos-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/centos-mirror-update.sh rename to playbooks/roles/mirror-update/files/centos-mirror-update diff --git a/modules/openstack_project/files/mirror/epel-mirror-update.sh b/playbooks/roles/mirror-update/files/epel-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/epel-mirror-update.sh rename to playbooks/roles/mirror-update/files/epel-mirror-update diff --git a/modules/openstack_project/files/mirror/fedora-mirror-update.sh b/playbooks/roles/mirror-update/files/fedora-mirror-update old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/fedora-mirror-update.sh rename to playbooks/roles/mirror-update/files/fedora-mirror-update diff --git a/modules/openstack_project/files/mirror/opensuse-mirror-update.sh b/playbooks/roles/mirror-update/files/opensuse-mirror-update old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/opensuse-mirror-update.sh rename to playbooks/roles/mirror-update/files/opensuse-mirror-update diff --git a/modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh b/playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh rename to playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update diff --git a/playbooks/roles/mirror-update/tasks/main.yaml b/playbooks/roles/mirror-update/tasks/main.yaml new file mode 100644 index 0000000000..7e124d453f --- /dev/null +++ b/playbooks/roles/mirror-update/tasks/main.yaml @@ -0,0 +1,27 @@ +# NOTE(ianw) : this does not feel like a a great way to write out +# binary data. But you can't do what you'd logically think at first +# with like +# +# copy: +# content: {{ string | b64decode }} +# +# because jinja treats the content as utf-8, and ends up mangling +# "real" binary data like a keytab. See issues like: +# https://github.com/ansible/ansible/issues/20150 +- name: Install afsadmin keytab + shell: 'echo {{ mirror_update_keytab_afsadmin }} | base64 -d > /etc/afsadmin.keytab' + args: + creates: /etc/afsadmin.keytab +#no_log: True + +- name: Ensure permissions on afsadmin keytab + file: + path: '/etc/afsadmin.keytab' + owner: root + group: root + mode: '0400' + +- name: Setup rsync mirror scripts + include_tasks: rsync.yaml + +# TODO: reprepro and other mirror components diff --git a/playbooks/roles/mirror-update/tasks/rsync.yaml b/playbooks/roles/mirror-update/tasks/rsync.yaml new file mode 100644 index 0000000000..d53ecce229 --- /dev/null +++ b/playbooks/roles/mirror-update/tasks/rsync.yaml @@ -0,0 +1,56 @@ +# Mirror scripts that use rsync + +- name: Create rsync log output directory + file: + path: /var/log/rsync-mirrors + state: directory + owner: root + group: root + mode: '0755' + +- name: Set update script names + set_fact: + rsync_update_scripts: + - centos + - epel + - fedora + - opensuse + - yum-puppetlabs + +- name: Copy keytab files in place + shell: 'echo {{ lookup("vars", "mirror_update_keytab_" + item) }} | base64 -d > /etc/{{ item }}.keytab' + args: + creates: '/etc/{{ item }}.keytab' + loop: '{{ rsync_update_scripts }}' +# no_log: True + +- name: Ensure keytab permissions + file: + path: '/etc/{{ item }}.keytab' + owner: root + group: root + mode: '0400' + loop: '{{ rsync_update_scripts }}' + +- name: Copy rsync mirror scripts in place + copy: + src: '{{ item }}-mirror-update' + dest: '/usr/local/bin/{{ item }}-mirror-update' + mode: '0755' + loop: '{{ rsync_update_scripts }}' + +- name: Install update cron jobs + cron: + name: '{{ item }} mirror sync' + state: present + job: 'flock -n /var/run/{{ item }}-mirror.lock {{ item }}-mirror-update mirror.{{ item }} >> /var/log/rsync-mirrors/{{ item }}.log 2>&1' + hour: '*/2' + minute: '{{ 45 | random(seed=inventory_hostname) }}' + loop: '{{ rsync_update_scripts }}' + +- name: Install logrotate rules + include_role: + name: logrotate + vars: + logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log' + loop: '{{ rsync_update_scripts }}' \ No newline at end of file diff --git a/playbooks/service-mirror-update.yaml b/playbooks/service-mirror-update.yaml new file mode 100644 index 0000000000..24865ca433 --- /dev/null +++ b/playbooks/service-mirror-update.yaml @@ -0,0 +1,11 @@ +- hosts: "mirror-update:!disabled" + name: "Configure mirror-update" + roles: + - role: kerberos-client + kerberos_realm: 'OPENSTACK.ORG' + kerberos_admin_server: 'kdc.openstack.org' + kerberos_kdcs: + - kdc03.openstack.org + - kdc04.openstack.org + - role: openafs-client + - role: mirror-update diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 10a918324b..e412df1c1a 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -83,6 +83,7 @@ - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml - host_vars/mirror01.openafs.provider.opendev.org.yaml + - host_vars/mirror-update01.opendev.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' - name: Run base.yaml diff --git a/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..8b5bfd472f --- /dev/null +++ b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 @@ -0,0 +1,12 @@ +mirror_update_keytab_afsadmin: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_centos: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_epel: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_fedora: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_opensuse: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_yum-puppetlabs: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB diff --git a/testinfra/test_mirror-update.py b/testinfra/test_mirror-update.py new file mode 100644 index 0000000000..9eb762dfa0 --- /dev/null +++ b/testinfra/test_mirror-update.py @@ -0,0 +1,50 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +testinfra_hosts = ['mirror-update01.opendev.org'] + +# Manually calculated from the "secret" value in the test host vars +KEYTAB_SHA256 = '8f4e9384338ffa41b927ed3c15463512384cb7268693a7c60c1e1254f690b7d0' + +def test_tools(host): + f = host.file('/usr/bin/k5start') + assert f.exists + f = host.file('/usr/bin/rsync') + assert f.exists + f = host.file('/usr/bin/vos') + assert f.exists + +def test_rsync_scripts(host): + for script in ['centos', + 'epel', + 'fedora', + 'opensuse', + 'yum-puppetlabs']: + f = host.file('/usr/local/bin/%s-mirror-update' % script) + assert f.exists + +def test_keytabs(host): + for keytab in ['/etc/afsadmin.keytab', + '/etc/centos.keytab', + '/etc/epel.keytab', + '/etc/fedora.keytab', + '/etc/opensuse.keytab', + '/etc/yum-puppetlabs.keytab']: + + f = host.file(keytab) + assert f.exists + assert f.sha256sum == KEYTAB_SHA256 + assert f.mode == 0o400 +