Move rsync mirror updates to new opendev.org mirror-update host

This move was prompted by wishing to expose the mirror update logs for
the rsync updates so that debugging problems does not require a root
user (note: not actually done in this change; will be a follow-on).

Rather than start hacking at puppet, the rsync mirror scripts make a
nice delination point for starting an Ansible-first/Bionic update.

Most magic is included in the scripts, so there is not much more to do
than copy them.  The host uses the existing kerberos and openafs roles
and copies the key material into place (to be added before merge).

Note the scripts are removed from the extant puppet so we don't have
two updates happening simultaneously.  This will also require a manual
clean to remove the cron jobs as a once-off when merging.

The other part of mirror-update is the reprepro based scripts for the
various debuntu repositories.  They are left as future work for now.

Testing is added to ensure dependencies and scripts are all in place.

Change-Id: I525ac18b55f0e11b0a541b51fa97ee5d6512bf70

.zuul.yaml

@@ -571,6 +571,27 @@
         host_copy_output:
           '/var/log/apache2/': logs
 
+- job:
+    name: system-config-run-mirror-update
+    parent: system-config-run
+    description: |
+      Run the playbook for a mirror update node
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: mirror-update01.opendev.org
+          label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-mirror-update.yaml
+    files:
+      - .zuul.yaml
+      - roles/
+      - playbooks/roles/mirror-update/
+      - playbooks/service-mirror-update.yaml
+      - testinfra/test_mirror-update.py
+
 - job:
     name: system-config-run-docker-registry
     parent: system-config-run
@@ -710,6 +731,7 @@
         - system-config-run-lists
         - system-config-run-nodepool
         - system-config-run-mirror
+        - system-config-run-mirror-update
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:
@@ -737,6 +759,7 @@
         - system-config-run-lists
         - system-config-run-nodepool
         - system-config-run-mirror
+        - system-config-run-mirror-update
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:

inventory/groups.yaml

@@ -5,6 +5,7 @@ groups:
   afs-client:
     - review-dev[0-9]*.open*.org
     - mirror[0-9]*.open*.org
+    - mirror-update[0-9]*.opendev.org
     - files[0-9]*.open*.org
     - ze[0-9]*.open*.org
     - afsdb*.open*.org
@@ -67,6 +68,8 @@ groups:
     - mirror[0-9]*.openstack.org
   mirror_opendev:
     - mirror[0-9]*.opendev.org
+  mirror-update:
+    - mirror-update[0-9]*.opendev.org
   nodepool:
     - nb[0-9]*.open*.org
     - nl[0-9]*.open*.org

modules/openstack_project/manifests/mirror_update.pp

@@ -373,135 +373,6 @@ class openstack_project::mirror_update (
     ]
   }
 
-  ### RDO mirror ###
-  file { '/etc/rdo.keytab':
-    ensure => absent,
-  }
-
-  file { '/usr/local/bin/rdo-mirror-update':
-    ensure => absent,
-  }
-
-  cron { 'rdo mirror':
-    ensure => absent,
-  }
-
-  ### EPEL mirror ###
-  file { '/etc/epel.keytab':
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0400',
-    content => $epel_keytab,
-  }
-
-  file { '/usr/local/bin/epel-mirror-update':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/mirror/epel-mirror-update.sh',
-  }
-
-  cron { 'epel mirror':
-    user        => 'root',
-    minute      => fqdn_rand(45, 'epel-mirror'),
-    hour        => '*/2',
-    command     => 'flock -n /var/run/epel-mirror.lock epel-mirror-update mirror.epel >>/var/log/epel-mirror.log 2>&1',
-    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
-    require     => [
-       File['/usr/local/bin/epel-mirror-update'],
-       File['/etc/afsadmin.keytab'],
-       File['/etc/epel.keytab'],
-    ]
-  }
-
-  ### Puppetlabs / CentOS mirror ###
-  file { '/etc/yum-puppetlabs.keytab':
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0400',
-    content => $yum_puppetlabs_keytab,
-  }
-
-  file { '/usr/local/bin/yum-puppetlabs-mirror-update':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/mirror/yum-puppetlabs-mirror-update.sh',
-  }
-
-  cron { 'yum-puppetlabs mirror':
-    user        => 'root',
-    minute      => fqdn_rand(45, 'yum-puppetlabs'),
-    hour        => '*/2',
-    command     => 'flock -n /var/run/yum-puppetlabs-mirror.lock yum-puppetlabs-mirror-update mirror.yum-puppetlabs >>/var/log/yum-puppetlabs-mirror.log 2>&1',
-    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
-    require     => [
-       File['/usr/local/bin/yum-puppetlabs-mirror-update'],
-       File['/etc/afsadmin.keytab'],
-       File['/etc/yum-puppetlabs.keytab'],
-    ]
-  }
-
-  ### Fedora mirror ###
-  file { '/etc/fedora.keytab':
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0400',
-    content => $fedora_keytab,
-  }
-
-  file { '/usr/local/bin/fedora-mirror-update':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/mirror/fedora-mirror-update.sh',
-  }
-
-  cron { 'fedora mirror':
-    user        => 'root',
-    minute      => fqdn_rand(45, 'fedora-mirror'),
-    hour        => '*/2',
-    command     => 'flock -n /var/run/fedora-mirror.lock fedora-mirror-update mirror.fedora >>/var/log/fedora-mirror.log 2>&1',
-    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
-    require     => [
-       File['/usr/local/bin/fedora-mirror-update'],
-       File['/etc/afsadmin.keytab'],
-       File['/etc/fedora.keytab'],
-    ]
-  }
-
-  ### openSUSE mirror ###
-  file { '/etc/opensuse.keytab':
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0400',
-    content => $opensuse_keytab,
-  }
-
-  file { '/usr/local/bin/opensuse-mirror-update':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/mirror/opensuse-mirror-update.sh',
-  }
-
-  cron { 'opensuse mirror':
-    user        => 'root',
-    minute      => fqdn_rand(45, 'opensuse-mirror'),
-    hour        => '*/2',
-    command     => 'flock -n /var/run/opensuse-mirror.lock opensuse-mirror-update mirror.opensuse >>/var/log/opensuse-mirror.log 2>&1',
-    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
-    require     => [
-       File['/usr/local/bin/opensuse-mirror-update'],
-       File['/etc/afsadmin.keytab'],
-       File['/etc/opensuse.keytab'],
-    ]
-  }
-
   ### Ubuntu Cloud Archive Mirror ###
   ::openstack_project::reprepro { 'ubuntu-cloud-archive-reprepro-mirror':
     confdir       => '/etc/reprepro/ubuntu-cloud-archive',

playbooks/roles/mirror-update/README.rst

@@ -0,0 +1,15 @@
+mirror-update
+
+This role sets up the ``mirror-update`` host, which does the periodic
+sync of upstream mirrors to the AFS volumes.
+
+It is not intended to be a particularly generic or flexible role, as
+there is usually only one instance of the mirror-update host (to avoid
+conflicting updates).
+
+At this stage, it handles the mirrors that are updated by ``rsync``
+only.  It is expected that it will grow to cover mirroring other
+volumes that are currently done by the legacy ``openstack.org`` host
+and managed by puppet.
+
+**Role Variables**

playbooks/roles/mirror-update/tasks/main.yaml

@@ -0,0 +1,27 @@
+# NOTE(ianw) : this does not feel like a a great way to write out
+# binary data.  But you can't do what you'd logically think at first
+# with like
+#
+#  copy:
+#   content: {{ string | b64decode }}
+#
+# because jinja treats the content as utf-8, and ends up mangling
+# "real" binary data like a keytab.  See issues like:
+#  https://github.com/ansible/ansible/issues/20150
+- name: Install afsadmin keytab
+  shell: 'echo {{ mirror_update_keytab_afsadmin }} | base64 -d > /etc/afsadmin.keytab'
+  args:
+    creates: /etc/afsadmin.keytab
+#no_log: True
+
+- name: Ensure permissions on afsadmin keytab
+  file:
+    path: '/etc/afsadmin.keytab'
+    owner: root
+    group: root
+    mode: '0400'
+
+- name: Setup rsync mirror scripts
+  include_tasks: rsync.yaml
+
+# TODO: reprepro and other mirror components

playbooks/roles/mirror-update/tasks/rsync.yaml

@@ -0,0 +1,56 @@
+# Mirror scripts that use rsync
+
+- name: Create rsync log output directory
+  file:
+    path: /var/log/rsync-mirrors
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Set update script names
+  set_fact:
+    rsync_update_scripts:
+      - centos
+      - epel
+      - fedora
+      - opensuse
+      - yum-puppetlabs
+
+- name: Copy keytab files in place
+  shell: 'echo {{ lookup("vars", "mirror_update_keytab_" + item) }} | base64 -d > /etc/{{ item }}.keytab'
+  args:
+    creates: '/etc/{{ item }}.keytab'
+  loop: '{{ rsync_update_scripts }}'
+#  no_log: True
+
+- name: Ensure keytab permissions
+  file:
+    path: '/etc/{{ item }}.keytab'
+    owner: root
+    group: root
+    mode: '0400'
+  loop: '{{ rsync_update_scripts }}'
+
+- name: Copy rsync mirror scripts in place
+  copy:
+    src: '{{ item }}-mirror-update'
+    dest: '/usr/local/bin/{{ item }}-mirror-update'
+    mode: '0755'
+  loop: '{{ rsync_update_scripts }}'
+
+- name: Install update cron jobs
+  cron:
+    name: '{{ item }} mirror sync'
+    state: present
+    job: 'flock -n /var/run/{{ item }}-mirror.lock {{ item }}-mirror-update mirror.{{ item }} >> /var/log/rsync-mirrors/{{ item }}.log 2>&1'
+    hour: '*/2'
+    minute: '{{ 45 | random(seed=inventory_hostname) }}'
+  loop: '{{ rsync_update_scripts }}'
+
+- name: Install logrotate rules
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log'
+  loop: '{{ rsync_update_scripts }}'

playbooks/service-mirror-update.yaml

@@ -0,0 +1,11 @@
+- hosts: "mirror-update:!disabled"
+  name: "Configure mirror-update"
+  roles:
+    - role: kerberos-client
+      kerberos_realm: 'OPENSTACK.ORG'
+      kerberos_admin_server: 'kdc.openstack.org'
+      kerberos_kdcs:
+        - kdc03.openstack.org
+        - kdc04.openstack.org
+    - role: openafs-client
+    - role: mirror-update

playbooks/zuul/run-base.yaml

@@ -83,6 +83,7 @@
         - host_vars/letsencrypt01.opendev.org.yaml
         - host_vars/letsencrypt02.opendev.org.yaml
         - host_vars/mirror01.openafs.provider.opendev.org.yaml
+        - host_vars/mirror-update01.opendev.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
     - name: Run base.yaml

playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2

@@ -0,0 +1,12 @@
+mirror_update_keytab_afsadmin: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_centos: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_epel: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_fedora: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_opensuse: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
+mirror_update_keytab_yum-puppetlabs: |-
+  AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB

testinfra/test_mirror-update.py

@@ -0,0 +1,50 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['mirror-update01.opendev.org']
+
+# Manually calculated from the "secret" value in the test host vars
+KEYTAB_SHA256 = '8f4e9384338ffa41b927ed3c15463512384cb7268693a7c60c1e1254f690b7d0'
+
+def test_tools(host):
+    f = host.file('/usr/bin/k5start')
+    assert f.exists
+    f = host.file('/usr/bin/rsync')
+    assert f.exists
+    f = host.file('/usr/bin/vos')
+    assert f.exists
+
+def test_rsync_scripts(host):
+    for script in ['centos',
+                   'epel',
+                   'fedora',
+                   'opensuse',
+                   'yum-puppetlabs']:
+        f = host.file('/usr/local/bin/%s-mirror-update' % script)
+        assert f.exists
+
+def test_keytabs(host):
+    for keytab in ['/etc/afsadmin.keytab',
+                   '/etc/centos.keytab',
+                   '/etc/epel.keytab',
+                   '/etc/fedora.keytab',
+                   '/etc/opensuse.keytab',
+                   '/etc/yum-puppetlabs.keytab']:
+
+        f = host.file(keytab)
+        assert f.exists
+        assert f.sha256sum == KEYTAB_SHA256
+        assert f.mode == 0o400
+