diff --git a/.zuul.yaml b/.zuul.yaml index 388d297a15..493c941b04 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -571,6 +571,27 @@ host_copy_output: '/var/log/apache2/': logs +- job: + name: system-config-run-mirror-update + parent: system-config-run + description: | + Run the playbook for a mirror update node + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: mirror-update01.opendev.org + label: ubuntu-bionic + vars: + run_playbooks: + - playbooks/service-mirror-update.yaml + files: + - .zuul.yaml + - roles/ + - playbooks/roles/mirror-update/ + - playbooks/service-mirror-update.yaml + - testinfra/test_mirror-update.py + - job: name: system-config-run-docker-registry parent: system-config-run @@ -710,6 +731,7 @@ - system-config-run-lists - system-config-run-nodepool - system-config-run-mirror + - system-config-run-mirror-update - system-config-run-docker-registry - system-config-run-gitea: dependencies: @@ -737,6 +759,7 @@ - system-config-run-lists - system-config-run-nodepool - system-config-run-mirror + - system-config-run-mirror-update - system-config-run-docker-registry - system-config-run-gitea: dependencies: diff --git a/inventory/groups.yaml b/inventory/groups.yaml index a79c772b8c..62e82cf6c9 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -5,6 +5,7 @@ groups: afs-client: - review-dev[0-9]*.open*.org - mirror[0-9]*.open*.org + - mirror-update[0-9]*.opendev.org - files[0-9]*.open*.org - ze[0-9]*.open*.org - afsdb*.open*.org @@ -67,6 +68,8 @@ groups: - mirror[0-9]*.openstack.org mirror_opendev: - mirror[0-9]*.opendev.org + mirror-update: + - mirror-update[0-9]*.opendev.org nodepool: - nb[0-9]*.open*.org - nl[0-9]*.open*.org diff --git a/modules/openstack_project/manifests/mirror_update.pp b/modules/openstack_project/manifests/mirror_update.pp index 435723b4a9..b1adeaa064 100644 --- a/modules/openstack_project/manifests/mirror_update.pp +++ b/modules/openstack_project/manifests/mirror_update.pp @@ -373,135 +373,6 @@ class openstack_project::mirror_update ( ] } - ### RDO mirror ### - file { '/etc/rdo.keytab': - ensure => absent, - } - - file { '/usr/local/bin/rdo-mirror-update': - ensure => absent, - } - - cron { 'rdo mirror': - ensure => absent, - } - - ### EPEL mirror ### - file { '/etc/epel.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $epel_keytab, - } - - file { '/usr/local/bin/epel-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/epel-mirror-update.sh', - } - - cron { 'epel mirror': - user => 'root', - minute => fqdn_rand(45, 'epel-mirror'), - hour => '*/2', - command => 'flock -n /var/run/epel-mirror.lock epel-mirror-update mirror.epel >>/var/log/epel-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/epel-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/epel.keytab'], - ] - } - - ### Puppetlabs / CentOS mirror ### - file { '/etc/yum-puppetlabs.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $yum_puppetlabs_keytab, - } - - file { '/usr/local/bin/yum-puppetlabs-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/yum-puppetlabs-mirror-update.sh', - } - - cron { 'yum-puppetlabs mirror': - user => 'root', - minute => fqdn_rand(45, 'yum-puppetlabs'), - hour => '*/2', - command => 'flock -n /var/run/yum-puppetlabs-mirror.lock yum-puppetlabs-mirror-update mirror.yum-puppetlabs >>/var/log/yum-puppetlabs-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/yum-puppetlabs-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/yum-puppetlabs.keytab'], - ] - } - - ### Fedora mirror ### - file { '/etc/fedora.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $fedora_keytab, - } - - file { '/usr/local/bin/fedora-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/fedora-mirror-update.sh', - } - - cron { 'fedora mirror': - user => 'root', - minute => fqdn_rand(45, 'fedora-mirror'), - hour => '*/2', - command => 'flock -n /var/run/fedora-mirror.lock fedora-mirror-update mirror.fedora >>/var/log/fedora-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/fedora-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/fedora.keytab'], - ] - } - - ### openSUSE mirror ### - file { '/etc/opensuse.keytab': - owner => 'root', - group => 'root', - mode => '0400', - content => $opensuse_keytab, - } - - file { '/usr/local/bin/opensuse-mirror-update': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/mirror/opensuse-mirror-update.sh', - } - - cron { 'opensuse mirror': - user => 'root', - minute => fqdn_rand(45, 'opensuse-mirror'), - hour => '*/2', - command => 'flock -n /var/run/opensuse-mirror.lock opensuse-mirror-update mirror.opensuse >>/var/log/opensuse-mirror.log 2>&1', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/usr/local/bin/opensuse-mirror-update'], - File['/etc/afsadmin.keytab'], - File['/etc/opensuse.keytab'], - ] - } - ### Ubuntu Cloud Archive Mirror ### ::openstack_project::reprepro { 'ubuntu-cloud-archive-reprepro-mirror': confdir => '/etc/reprepro/ubuntu-cloud-archive', diff --git a/playbooks/roles/mirror-update/README.rst b/playbooks/roles/mirror-update/README.rst new file mode 100644 index 0000000000..abe7e333e0 --- /dev/null +++ b/playbooks/roles/mirror-update/README.rst @@ -0,0 +1,15 @@ +mirror-update + +This role sets up the ``mirror-update`` host, which does the periodic +sync of upstream mirrors to the AFS volumes. + +It is not intended to be a particularly generic or flexible role, as +there is usually only one instance of the mirror-update host (to avoid +conflicting updates). + +At this stage, it handles the mirrors that are updated by ``rsync`` +only. It is expected that it will grow to cover mirroring other +volumes that are currently done by the legacy ``openstack.org`` host +and managed by puppet. + +**Role Variables** diff --git a/modules/openstack_project/files/mirror/cache-stats.sh b/playbooks/roles/mirror-update/files/cache-stats.sh old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/cache-stats.sh rename to playbooks/roles/mirror-update/files/cache-stats.sh diff --git a/modules/openstack_project/files/mirror/centos-mirror-update.sh b/playbooks/roles/mirror-update/files/centos-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/centos-mirror-update.sh rename to playbooks/roles/mirror-update/files/centos-mirror-update diff --git a/modules/openstack_project/files/mirror/epel-mirror-update.sh b/playbooks/roles/mirror-update/files/epel-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/epel-mirror-update.sh rename to playbooks/roles/mirror-update/files/epel-mirror-update diff --git a/modules/openstack_project/files/mirror/fedora-mirror-update.sh b/playbooks/roles/mirror-update/files/fedora-mirror-update old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/fedora-mirror-update.sh rename to playbooks/roles/mirror-update/files/fedora-mirror-update diff --git a/modules/openstack_project/files/mirror/opensuse-mirror-update.sh b/playbooks/roles/mirror-update/files/opensuse-mirror-update old mode 100644 new mode 100755 similarity index 100% rename from modules/openstack_project/files/mirror/opensuse-mirror-update.sh rename to playbooks/roles/mirror-update/files/opensuse-mirror-update diff --git a/modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh b/playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update similarity index 100% rename from modules/openstack_project/files/mirror/yum-puppetlabs-mirror-update.sh rename to playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update diff --git a/playbooks/roles/mirror-update/tasks/main.yaml b/playbooks/roles/mirror-update/tasks/main.yaml new file mode 100644 index 0000000000..7e124d453f --- /dev/null +++ b/playbooks/roles/mirror-update/tasks/main.yaml @@ -0,0 +1,27 @@ +# NOTE(ianw) : this does not feel like a a great way to write out +# binary data. But you can't do what you'd logically think at first +# with like +# +# copy: +# content: {{ string | b64decode }} +# +# because jinja treats the content as utf-8, and ends up mangling +# "real" binary data like a keytab. See issues like: +# https://github.com/ansible/ansible/issues/20150 +- name: Install afsadmin keytab + shell: 'echo {{ mirror_update_keytab_afsadmin }} | base64 -d > /etc/afsadmin.keytab' + args: + creates: /etc/afsadmin.keytab +#no_log: True + +- name: Ensure permissions on afsadmin keytab + file: + path: '/etc/afsadmin.keytab' + owner: root + group: root + mode: '0400' + +- name: Setup rsync mirror scripts + include_tasks: rsync.yaml + +# TODO: reprepro and other mirror components diff --git a/playbooks/roles/mirror-update/tasks/rsync.yaml b/playbooks/roles/mirror-update/tasks/rsync.yaml new file mode 100644 index 0000000000..d53ecce229 --- /dev/null +++ b/playbooks/roles/mirror-update/tasks/rsync.yaml @@ -0,0 +1,56 @@ +# Mirror scripts that use rsync + +- name: Create rsync log output directory + file: + path: /var/log/rsync-mirrors + state: directory + owner: root + group: root + mode: '0755' + +- name: Set update script names + set_fact: + rsync_update_scripts: + - centos + - epel + - fedora + - opensuse + - yum-puppetlabs + +- name: Copy keytab files in place + shell: 'echo {{ lookup("vars", "mirror_update_keytab_" + item) }} | base64 -d > /etc/{{ item }}.keytab' + args: + creates: '/etc/{{ item }}.keytab' + loop: '{{ rsync_update_scripts }}' +# no_log: True + +- name: Ensure keytab permissions + file: + path: '/etc/{{ item }}.keytab' + owner: root + group: root + mode: '0400' + loop: '{{ rsync_update_scripts }}' + +- name: Copy rsync mirror scripts in place + copy: + src: '{{ item }}-mirror-update' + dest: '/usr/local/bin/{{ item }}-mirror-update' + mode: '0755' + loop: '{{ rsync_update_scripts }}' + +- name: Install update cron jobs + cron: + name: '{{ item }} mirror sync' + state: present + job: 'flock -n /var/run/{{ item }}-mirror.lock {{ item }}-mirror-update mirror.{{ item }} >> /var/log/rsync-mirrors/{{ item }}.log 2>&1' + hour: '*/2' + minute: '{{ 45 | random(seed=inventory_hostname) }}' + loop: '{{ rsync_update_scripts }}' + +- name: Install logrotate rules + include_role: + name: logrotate + vars: + logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log' + loop: '{{ rsync_update_scripts }}' \ No newline at end of file diff --git a/playbooks/service-mirror-update.yaml b/playbooks/service-mirror-update.yaml new file mode 100644 index 0000000000..24865ca433 --- /dev/null +++ b/playbooks/service-mirror-update.yaml @@ -0,0 +1,11 @@ +- hosts: "mirror-update:!disabled" + name: "Configure mirror-update" + roles: + - role: kerberos-client + kerberos_realm: 'OPENSTACK.ORG' + kerberos_admin_server: 'kdc.openstack.org' + kerberos_kdcs: + - kdc03.openstack.org + - kdc04.openstack.org + - role: openafs-client + - role: mirror-update diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 10a918324b..e412df1c1a 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -83,6 +83,7 @@ - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml - host_vars/mirror01.openafs.provider.opendev.org.yaml + - host_vars/mirror-update01.opendev.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' - name: Run base.yaml diff --git a/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..8b5bfd472f --- /dev/null +++ b/playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2 @@ -0,0 +1,12 @@ +mirror_update_keytab_afsadmin: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_centos: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_epel: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_fedora: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_opensuse: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB +mirror_update_keytab_yum-puppetlabs: |- + AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB diff --git a/testinfra/test_mirror-update.py b/testinfra/test_mirror-update.py new file mode 100644 index 0000000000..9eb762dfa0 --- /dev/null +++ b/testinfra/test_mirror-update.py @@ -0,0 +1,50 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +testinfra_hosts = ['mirror-update01.opendev.org'] + +# Manually calculated from the "secret" value in the test host vars +KEYTAB_SHA256 = '8f4e9384338ffa41b927ed3c15463512384cb7268693a7c60c1e1254f690b7d0' + +def test_tools(host): + f = host.file('/usr/bin/k5start') + assert f.exists + f = host.file('/usr/bin/rsync') + assert f.exists + f = host.file('/usr/bin/vos') + assert f.exists + +def test_rsync_scripts(host): + for script in ['centos', + 'epel', + 'fedora', + 'opensuse', + 'yum-puppetlabs']: + f = host.file('/usr/local/bin/%s-mirror-update' % script) + assert f.exists + +def test_keytabs(host): + for keytab in ['/etc/afsadmin.keytab', + '/etc/centos.keytab', + '/etc/epel.keytab', + '/etc/fedora.keytab', + '/etc/opensuse.keytab', + '/etc/yum-puppetlabs.keytab']: + + f = host.file(keytab) + assert f.exists + assert f.sha256sum == KEYTAB_SHA256 + assert f.mode == 0o400 +