Merge "Move rsync mirror updates to new opendev.org mirror-update host"
This commit is contained in:
commit
5d0d5725ec
23
.zuul.yaml
23
.zuul.yaml
@ -571,6 +571,27 @@
|
||||
host_copy_output:
|
||||
'/var/log/apache2/': logs
|
||||
|
||||
- job:
|
||||
name: system-config-run-mirror-update
|
||||
parent: system-config-run
|
||||
description: |
|
||||
Run the playbook for a mirror update node
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: bridge.openstack.org
|
||||
label: ubuntu-bionic
|
||||
- name: mirror-update01.opendev.org
|
||||
label: ubuntu-bionic
|
||||
vars:
|
||||
run_playbooks:
|
||||
- playbooks/service-mirror-update.yaml
|
||||
files:
|
||||
- .zuul.yaml
|
||||
- roles/
|
||||
- playbooks/roles/mirror-update/
|
||||
- playbooks/service-mirror-update.yaml
|
||||
- testinfra/test_mirror-update.py
|
||||
|
||||
- job:
|
||||
name: system-config-run-docker-registry
|
||||
parent: system-config-run
|
||||
@ -710,6 +731,7 @@
|
||||
- system-config-run-lists
|
||||
- system-config-run-nodepool
|
||||
- system-config-run-mirror
|
||||
- system-config-run-mirror-update
|
||||
- system-config-run-docker-registry
|
||||
- system-config-run-gitea:
|
||||
dependencies:
|
||||
@ -737,6 +759,7 @@
|
||||
- system-config-run-lists
|
||||
- system-config-run-nodepool
|
||||
- system-config-run-mirror
|
||||
- system-config-run-mirror-update
|
||||
- system-config-run-docker-registry
|
||||
- system-config-run-gitea:
|
||||
dependencies:
|
||||
|
@ -5,6 +5,7 @@ groups:
|
||||
afs-client:
|
||||
- review-dev[0-9]*.open*.org
|
||||
- mirror[0-9]*.open*.org
|
||||
- mirror-update[0-9]*.opendev.org
|
||||
- files[0-9]*.open*.org
|
||||
- ze[0-9]*.open*.org
|
||||
- afsdb*.open*.org
|
||||
@ -67,6 +68,8 @@ groups:
|
||||
- mirror[0-9]*.openstack.org
|
||||
mirror_opendev:
|
||||
- mirror[0-9]*.opendev.org
|
||||
mirror-update:
|
||||
- mirror-update[0-9]*.opendev.org
|
||||
nodepool:
|
||||
- nb[0-9]*.open*.org
|
||||
- nl[0-9]*.open*.org
|
||||
|
@ -373,135 +373,6 @@ class openstack_project::mirror_update (
|
||||
]
|
||||
}
|
||||
|
||||
### RDO mirror ###
|
||||
file { '/etc/rdo.keytab':
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/rdo-mirror-update':
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
cron { 'rdo mirror':
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
### EPEL mirror ###
|
||||
file { '/etc/epel.keytab':
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0400',
|
||||
content => $epel_keytab,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/epel-mirror-update':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/mirror/epel-mirror-update.sh',
|
||||
}
|
||||
|
||||
cron { 'epel mirror':
|
||||
user => 'root',
|
||||
minute => fqdn_rand(45, 'epel-mirror'),
|
||||
hour => '*/2',
|
||||
command => 'flock -n /var/run/epel-mirror.lock epel-mirror-update mirror.epel >>/var/log/epel-mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/usr/local/bin/epel-mirror-update'],
|
||||
File['/etc/afsadmin.keytab'],
|
||||
File['/etc/epel.keytab'],
|
||||
]
|
||||
}
|
||||
|
||||
### Puppetlabs / CentOS mirror ###
|
||||
file { '/etc/yum-puppetlabs.keytab':
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0400',
|
||||
content => $yum_puppetlabs_keytab,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/yum-puppetlabs-mirror-update':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/mirror/yum-puppetlabs-mirror-update.sh',
|
||||
}
|
||||
|
||||
cron { 'yum-puppetlabs mirror':
|
||||
user => 'root',
|
||||
minute => fqdn_rand(45, 'yum-puppetlabs'),
|
||||
hour => '*/2',
|
||||
command => 'flock -n /var/run/yum-puppetlabs-mirror.lock yum-puppetlabs-mirror-update mirror.yum-puppetlabs >>/var/log/yum-puppetlabs-mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/usr/local/bin/yum-puppetlabs-mirror-update'],
|
||||
File['/etc/afsadmin.keytab'],
|
||||
File['/etc/yum-puppetlabs.keytab'],
|
||||
]
|
||||
}
|
||||
|
||||
### Fedora mirror ###
|
||||
file { '/etc/fedora.keytab':
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0400',
|
||||
content => $fedora_keytab,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/fedora-mirror-update':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/mirror/fedora-mirror-update.sh',
|
||||
}
|
||||
|
||||
cron { 'fedora mirror':
|
||||
user => 'root',
|
||||
minute => fqdn_rand(45, 'fedora-mirror'),
|
||||
hour => '*/2',
|
||||
command => 'flock -n /var/run/fedora-mirror.lock fedora-mirror-update mirror.fedora >>/var/log/fedora-mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/usr/local/bin/fedora-mirror-update'],
|
||||
File['/etc/afsadmin.keytab'],
|
||||
File['/etc/fedora.keytab'],
|
||||
]
|
||||
}
|
||||
|
||||
### openSUSE mirror ###
|
||||
file { '/etc/opensuse.keytab':
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0400',
|
||||
content => $opensuse_keytab,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/opensuse-mirror-update':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/mirror/opensuse-mirror-update.sh',
|
||||
}
|
||||
|
||||
cron { 'opensuse mirror':
|
||||
user => 'root',
|
||||
minute => fqdn_rand(45, 'opensuse-mirror'),
|
||||
hour => '*/2',
|
||||
command => 'flock -n /var/run/opensuse-mirror.lock opensuse-mirror-update mirror.opensuse >>/var/log/opensuse-mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/usr/local/bin/opensuse-mirror-update'],
|
||||
File['/etc/afsadmin.keytab'],
|
||||
File['/etc/opensuse.keytab'],
|
||||
]
|
||||
}
|
||||
|
||||
### Ubuntu Cloud Archive Mirror ###
|
||||
::openstack_project::reprepro { 'ubuntu-cloud-archive-reprepro-mirror':
|
||||
confdir => '/etc/reprepro/ubuntu-cloud-archive',
|
||||
|
15
playbooks/roles/mirror-update/README.rst
Normal file
15
playbooks/roles/mirror-update/README.rst
Normal file
@ -0,0 +1,15 @@
|
||||
mirror-update
|
||||
|
||||
This role sets up the ``mirror-update`` host, which does the periodic
|
||||
sync of upstream mirrors to the AFS volumes.
|
||||
|
||||
It is not intended to be a particularly generic or flexible role, as
|
||||
there is usually only one instance of the mirror-update host (to avoid
|
||||
conflicting updates).
|
||||
|
||||
At this stage, it handles the mirrors that are updated by ``rsync``
|
||||
only. It is expected that it will grow to cover mirroring other
|
||||
volumes that are currently done by the legacy ``openstack.org`` host
|
||||
and managed by puppet.
|
||||
|
||||
**Role Variables**
|
0
modules/openstack_project/files/mirror/cache-stats.sh → playbooks/roles/mirror-update/files/cache-stats.sh
Normal file → Executable file
0
modules/openstack_project/files/mirror/cache-stats.sh → playbooks/roles/mirror-update/files/cache-stats.sh
Normal file → Executable file
0
modules/openstack_project/files/mirror/fedora-mirror-update.sh → playbooks/roles/mirror-update/files/fedora-mirror-update
Normal file → Executable file
0
modules/openstack_project/files/mirror/fedora-mirror-update.sh → playbooks/roles/mirror-update/files/fedora-mirror-update
Normal file → Executable file
0
modules/openstack_project/files/mirror/opensuse-mirror-update.sh → playbooks/roles/mirror-update/files/opensuse-mirror-update
Normal file → Executable file
0
modules/openstack_project/files/mirror/opensuse-mirror-update.sh → playbooks/roles/mirror-update/files/opensuse-mirror-update
Normal file → Executable file
27
playbooks/roles/mirror-update/tasks/main.yaml
Normal file
27
playbooks/roles/mirror-update/tasks/main.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
# NOTE(ianw) : this does not feel like a a great way to write out
|
||||
# binary data. But you can't do what you'd logically think at first
|
||||
# with like
|
||||
#
|
||||
# copy:
|
||||
# content: {{ string | b64decode }}
|
||||
#
|
||||
# because jinja treats the content as utf-8, and ends up mangling
|
||||
# "real" binary data like a keytab. See issues like:
|
||||
# https://github.com/ansible/ansible/issues/20150
|
||||
- name: Install afsadmin keytab
|
||||
shell: 'echo {{ mirror_update_keytab_afsadmin }} | base64 -d > /etc/afsadmin.keytab'
|
||||
args:
|
||||
creates: /etc/afsadmin.keytab
|
||||
#no_log: True
|
||||
|
||||
- name: Ensure permissions on afsadmin keytab
|
||||
file:
|
||||
path: '/etc/afsadmin.keytab'
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0400'
|
||||
|
||||
- name: Setup rsync mirror scripts
|
||||
include_tasks: rsync.yaml
|
||||
|
||||
# TODO: reprepro and other mirror components
|
56
playbooks/roles/mirror-update/tasks/rsync.yaml
Normal file
56
playbooks/roles/mirror-update/tasks/rsync.yaml
Normal file
@ -0,0 +1,56 @@
|
||||
# Mirror scripts that use rsync
|
||||
|
||||
- name: Create rsync log output directory
|
||||
file:
|
||||
path: /var/log/rsync-mirrors
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Set update script names
|
||||
set_fact:
|
||||
rsync_update_scripts:
|
||||
- centos
|
||||
- epel
|
||||
- fedora
|
||||
- opensuse
|
||||
- yum-puppetlabs
|
||||
|
||||
- name: Copy keytab files in place
|
||||
shell: 'echo {{ lookup("vars", "mirror_update_keytab_" + item) }} | base64 -d > /etc/{{ item }}.keytab'
|
||||
args:
|
||||
creates: '/etc/{{ item }}.keytab'
|
||||
loop: '{{ rsync_update_scripts }}'
|
||||
# no_log: True
|
||||
|
||||
- name: Ensure keytab permissions
|
||||
file:
|
||||
path: '/etc/{{ item }}.keytab'
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0400'
|
||||
loop: '{{ rsync_update_scripts }}'
|
||||
|
||||
- name: Copy rsync mirror scripts in place
|
||||
copy:
|
||||
src: '{{ item }}-mirror-update'
|
||||
dest: '/usr/local/bin/{{ item }}-mirror-update'
|
||||
mode: '0755'
|
||||
loop: '{{ rsync_update_scripts }}'
|
||||
|
||||
- name: Install update cron jobs
|
||||
cron:
|
||||
name: '{{ item }} mirror sync'
|
||||
state: present
|
||||
job: 'flock -n /var/run/{{ item }}-mirror.lock {{ item }}-mirror-update mirror.{{ item }} >> /var/log/rsync-mirrors/{{ item }}.log 2>&1'
|
||||
hour: '*/2'
|
||||
minute: '{{ 45 | random(seed=inventory_hostname) }}'
|
||||
loop: '{{ rsync_update_scripts }}'
|
||||
|
||||
- name: Install logrotate rules
|
||||
include_role:
|
||||
name: logrotate
|
||||
vars:
|
||||
logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log'
|
||||
loop: '{{ rsync_update_scripts }}'
|
11
playbooks/service-mirror-update.yaml
Normal file
11
playbooks/service-mirror-update.yaml
Normal file
@ -0,0 +1,11 @@
|
||||
- hosts: "mirror-update:!disabled"
|
||||
name: "Configure mirror-update"
|
||||
roles:
|
||||
- role: kerberos-client
|
||||
kerberos_realm: 'OPENSTACK.ORG'
|
||||
kerberos_admin_server: 'kdc.openstack.org'
|
||||
kerberos_kdcs:
|
||||
- kdc03.openstack.org
|
||||
- kdc04.openstack.org
|
||||
- role: openafs-client
|
||||
- role: mirror-update
|
@ -83,6 +83,7 @@
|
||||
- host_vars/letsencrypt01.opendev.org.yaml
|
||||
- host_vars/letsencrypt02.opendev.org.yaml
|
||||
- host_vars/mirror01.openafs.provider.opendev.org.yaml
|
||||
- host_vars/mirror-update01.opendev.org.yaml
|
||||
- name: Display group membership
|
||||
command: ansible localhost -m debug -a 'var=groups'
|
||||
- name: Run base.yaml
|
||||
|
@ -0,0 +1,12 @@
|
||||
mirror_update_keytab_afsadmin: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
||||
mirror_update_keytab_centos: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
||||
mirror_update_keytab_epel: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
||||
mirror_update_keytab_fedora: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
||||
mirror_update_keytab_opensuse: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
||||
mirror_update_keytab_yum-puppetlabs: |-
|
||||
AQIDBAUGBwgJEBESExQVFm9wZW5kZXYub3JnIHNhbXBsZSBrZXl0YWIWFRQTEhEQCQgHBgUEAwIB
|
50
testinfra/test_mirror-update.py
Normal file
50
testinfra/test_mirror-update.py
Normal file
@ -0,0 +1,50 @@
|
||||
# Copyright 2019 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
testinfra_hosts = ['mirror-update01.opendev.org']
|
||||
|
||||
# Manually calculated from the "secret" value in the test host vars
|
||||
KEYTAB_SHA256 = '8f4e9384338ffa41b927ed3c15463512384cb7268693a7c60c1e1254f690b7d0'
|
||||
|
||||
def test_tools(host):
|
||||
f = host.file('/usr/bin/k5start')
|
||||
assert f.exists
|
||||
f = host.file('/usr/bin/rsync')
|
||||
assert f.exists
|
||||
f = host.file('/usr/bin/vos')
|
||||
assert f.exists
|
||||
|
||||
def test_rsync_scripts(host):
|
||||
for script in ['centos',
|
||||
'epel',
|
||||
'fedora',
|
||||
'opensuse',
|
||||
'yum-puppetlabs']:
|
||||
f = host.file('/usr/local/bin/%s-mirror-update' % script)
|
||||
assert f.exists
|
||||
|
||||
def test_keytabs(host):
|
||||
for keytab in ['/etc/afsadmin.keytab',
|
||||
'/etc/centos.keytab',
|
||||
'/etc/epel.keytab',
|
||||
'/etc/fedora.keytab',
|
||||
'/etc/opensuse.keytab',
|
||||
'/etc/yum-puppetlabs.keytab']:
|
||||
|
||||
f = host.file(keytab)
|
||||
assert f.exists
|
||||
assert f.sha256sum == KEYTAB_SHA256
|
||||
assert f.mode == 0o400
|
||||
|
Loading…
Reference in New Issue
Block a user