opendev/system-config
Code Issues Proposed changes

Merge "backups: remove all bup"

Browse Source
This commit is contained in:
Zuul 2021-02-21 22:41:41 +00:00 committed by Gerrit Code Review
commit 1b2435c349
23 changed files with 2 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
inventory/base/hosts.yaml
View File

@ -49,13 +49,6 @@ all:
region_name: DFW
public_ipv4: 104.239.149.165
public_ipv6: 2001:4800:7819:105:be76:4eff:fe01:e6ff
backup01.ca-ymq-1.vexxhost.opendev.org:
ansible_host: 199.204.45.119
location:
cloud: openstackci-vexxhost
region_name: ca-ymq-1
public_v4: 199.204.45.119
public_v6: 2604:e100:1:0:f816:3eff:feab:d678
backup02.ca-ymq-1.vexxhost.opendev.org:
ansible_host: 199.204.45.196
location:
@ -70,13 +63,6 @@ all:
region_name: ORD
public_v4: 23.253.160.180
public_v6: 2001:4801:7825:103:be76:4eff:fe10:1b1
backup01.ord.rax.ci.openstack.org:
ansible_host: 23.253.20.173
location:
cloud: openstackci-rax
region_name: ORD
public_v4: 23.253.20.173
public_v6: 2001:4801:7824:101:be76:4eff:fe10:20cf
bridge.openstack.org:
ansible_host: 23.253.234.219
location:

23
inventory/service/groups.yaml
View File

@ -19,27 +19,6 @@ groups:
afs-admin:
- mirror-update[0-9]*.openstack.org
ask: ask*.open*.org
# NOTE: By default we keep the backup-server group empty as an
# emergency escape hatch if a problem were to propage through
# production servers. However, this also means if you add a server to
# the "backup" group to be backed up, you should uncomment the
# "backup-server" group for an Ansible pulse so the users & keys are
# setup on the server(s). You can submit a follow-on change to revert
# this at the same time.
backup:
- gitea01.opendev.org
- review[0-9]*.openstack.org
- review-dev[0-9]*.open*.org
- zuul[0-9]*.open*.org
# All these servers are "special-cased" in specifically
# as they are puppet and should be replaced "soon"
- ethercalc02.openstack.org
- ask01.openstack.org
- lists.openstack.org
- storyboard01.opendev.org
- translate01.openstack.org
backup-server:
- backup01.ca-ymq-1.vexxhost.opendev.org
borg-backup:
- etherpad[0-9]*.opendev.org
- gitea01.opendev.org
@ -66,7 +45,6 @@ groups:
control-plane-clouds:
- bridge.openstack.org
disabled:
- backup01.ord.rax.ci.openstack.org
- corvustest
- idp.openstackid.org
- lists-dev01.openstack.org
@ -146,7 +124,6 @@ groups:
- pbx[0-9]*.opendev.org
puppet:
- ask*.open*.org
- backup[0-9]*.openstack.org
- cacti[0-9]*.open*.org
- corvustest
- eavesdrop[0-9]*.open*.org

8
manifests/site.pp
View File

@ -355,14 +355,6 @@ node /^pbx\d*\.open.*\.org$/ {
}
}
# Node-OS: xenial
# A backup machine. Don't run cron or puppet agent on it.
node /^backup\d+\..*\.ci\.open.*\.org$/ {
$group = "ci-backup"
class { 'openstack_project::server': }
include openstack_project::backup_server
}
# Node-OS: xenial
node /^openstackid\d*(\.openstack)?\.org$/ {
$group = "openstackid"

7
modules/openstack_project/manifests/backup_server.pp
View File

@ -1,7 +0,0 @@
# == Class: openstack_project::backup_server
#
class openstack_project::backup_server {
package { 'bup':
ensure => present,
}
}

10
modules/openstack_project/manifests/ethercalc.pp
View File

@ -21,14 +21,4 @@ class openstack_project::ethercalc (
include ethercalc::redis
# Redis creates a snapshot at /var/lib/redis/dump.rdb periodically
# (at worst every 15 minutes if at least one change is made to redis)
# which can be used to recover the Redis DB. Bup will automagically
# pick this file up during its normal operation so no other DB dumping
# is required like with mysql.
include bup
bup::site { 'ord.rax':
backup_user => "bup-$::hostname",
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}

6
modules/openstack_project/manifests/lists.pp
View File

@ -42,12 +42,6 @@ class openstack_project::lists(
user::virtual::disable { 'oubiwann': }
user::virtual::disable { 'rockstar': }
include bup
bup::site { 'ord.rax':
backup_user => 'bup-lists',
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
# Begin user servicable parts
mailman::site { 'openstack':

5
modules/openstack_project/manifests/storyboard.pp
View File

@ -86,9 +86,4 @@ class openstack_project::storyboard(
source => $superusers,
}
include bup
bup::site { 'ord.rax':
backup_user => 'bup-storyboard',
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}

8
modules/openstack_project/manifests/wiki.pp
View File

@ -75,14 +75,6 @@ class openstack_project::wiki (
require => File['/srv/mediawiki'],
}
if $bup_user != undef {
include bup
bup::site { 'ord.rax':
backup_user => $bup_user,
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}
class { '::elasticsearch':
es_template_config => {
'bootstrap.mlockall' => true,

15
playbooks/roles/backup-server/README.rst
View File

@ -1,15 +0,0 @@
Setup backup server
This role configures backup server(s) in the ``backup-server`` group
to accept backups from remote hosts.
Note that the ``backup`` role must have run on each host in the
``backup`` group before this role. That role will create a
``bup_user`` tuple in the hostvars for for each host consisting of the
required username and public key.
Each required user gets a separate home directory in ``/opt/backups``.
Their ``authorized_keys`` file is configured with the public key to
allow the remote host to log in and only run ``bup``.
**Role Variables**

1
playbooks/roles/backup-server/defaults/main.yaml
View File

@ -1 +0,0 @@
bup_users: []

21
playbooks/roles/backup-server/tasks/main.yaml
View File

@ -1,21 +0,0 @@
- name: Create backup directory
file:
state: directory
path: /opt/backups
- name: Install bup
package:
name:
- bup
state: present
- name: Build all bup users from backup hosts
set_fact:
bup_users: '{{ bup_users }} + [ {{ hostvars[item]["bup_user"] }} ]'
with_inventory_hostnames: 'backup:!disabled'
- name: Create bup users
include_tasks: user.yaml
loop: '{{ bup_users }}'
loop_control:
loop_var: bup_user

32
playbooks/roles/backup-server/tasks/user.yaml
View File

@ -1,32 +0,0 @@
# note bup_user is the parent loop variable name; this works on each
# element from the bup_users global.
- name: Set variables
set_fact:
user_name: '{{ bup_user[0] }}'
user_key: '{{ bup_user[1] }}'
- name: Create bup user
user:
name: '{{ user_name }}'
comment: 'Backup user'
shell: /bin/bash
home: '/opt/backups/{{ user_name }}'
create_home: yes
register: homedir
- name: Create bup user authorized key
authorized_key:
user: '{{ user_name }}'
state: present
key: '{{ user_key }}'
key_options: 'command="BUP_DEBUG=0 BUP_FORCE_TTY=3 bup server",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty'
# ansible-lint wants this in a handler, it should be done here and
# now; this isn't like a service restart where multiple things might
# call it.
- name: Initalise bup
shell: |
BUP_DIR=/opt/backups/{{ user_name }}/.bup bup init
become: yes
become_user: '{{ user_name }}'
when: homedir.changed

23
playbooks/roles/backup/README.rst
View File

@ -1,23 +0,0 @@
Configure a host to be backed up
This role setups a host to use ``bup`` for backup to any hosts in the
``backup-server`` group.
A separate ssh key will be generated for root to connect to the backup
server(s) and the host key for the backup servers will be accepted to
the host.
The ``bup`` tool is installed and a cron job is setup to run the
backup periodically.
Note the ``backup-server`` role must run after this to create the user
correctly on the backup server. This role sets a tuple ``bup_user``
with the username and public key; the ``backup-server`` role uses this
variable for each host in the ``backup`` group to initalise users.
**Role Variables**
.. zuul:rolevar:: bup_username
The username to connect to the backup server. If this is left
undefined, it will be automatically set to ``bup-$(hostname)``

25
playbooks/roles/backup/files/bup-excludes
View File

@ -1,25 +0,0 @@
/proc/*
/sys/*
/dev/*
/tmp/*
/floppy/*
/cdrom/*
/var/spool/squid/*
/var/spool/exim/*
/media/*
/mnt/*
/var/agentx/*
/run/*
/root/backup-restore-*
/root/.bup
/etc/puppet/modules/*
/etc/puppet/hieradata/*
/var/cache/*
/var/lib/docker/*
/var/lib/puppet/reports/*
/var/lib/postgresql/*
/var/lib/lxcfs/*
/var/lib/zuul/backup/*
/var/lib/zuul/times/*
/opt/system-config/*
/afs/*

57
playbooks/roles/backup/tasks/main.yaml
View File

@ -1,57 +0,0 @@
- name: Generate bup username for this host
set_fact:
bup_username: 'bup-{{ inventory_hostname.split(".", 1)[0] }}'
when: bup_username is not defined
- debug:
var: bup_username
- name: Install bup
package:
name:
- bup
state: absent
- name: Remove old keypair
file:
path: /root/.ssh/id_backup_ed25519
state: absent
- name: Remove old keypair
file:
path: /root/.ssh/id_backup_ed25519.pub
state: absent
- name: Remove old config directory
file:
path: /root/.bup
state: absent
- name: Remove ssh config
blockinfile:
path: /root/.ssh/config
state: absent
create: false
block: |
Host {{ item }}
HostName {{ item }}
IdentityFile /root/.ssh/id_backup_ed25519
User {{ bup_username }}
mode: 0600
with_inventory_hostnames: backup-server
ignore_errors: True
- name: Remove /etc/bup-excludes
file:
path: /etc/bup-excludes
state: absent
- name: Remove backup cronjob
cron:
name: "Run bup backup"
job: "tar -X /etc/bup-excludes -cPF - / | bup split -r {{ bup_username }}@{{ item }}: -n root -q"
user: root
hour: '5'
minute: '{{ 59|random(seed=item) }}'
state: absent
with_inventory_hostnames: backup-server

6
playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
View File

@ -38,15 +38,13 @@ results:
- mirror
review01.openstack.org:
- backup
- borg-backup
- gerrit
- letsencrypt
- review
backup01.ord.rax.ci.openstack.org:
- disabled
- puppet
backup01.ord.rax.opendev.org:
- borg-backup-server
ze01.openstack.org:
- afs-client

8
playbooks/service-backup.yaml
View File

@ -1,8 +0,0 @@
# NOTE(ianw) : we are removing bup for borg. This just needs to run
# once to remove bup parts from the backup clients, then we will
# remove it completely.
- hosts: "backup:!disabled"
name: "Base: Generate backup users and keys"
roles:
- iptables
- backup

2
playbooks/zuul/run-base.yaml
View File

@ -83,8 +83,6 @@
- host_vars/mirror01.openafs.provider.opendev.org.yaml
- host_vars/mirror02.openafs.provider.opendev.org.yaml
- host_vars/mirror-update01.opendev.org.yaml
- host_vars/backup-test01.opendev.org.yaml
- host_vars/backup-test02.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'

1
playbooks/zuul/templates/host_vars/backup-test01.opendev.org.yaml.j2
View File

@ -1 +0,0 @@
bup_username: bup-backup01

2
playbooks/zuul/templates/host_vars/backup-test02.opendev.org.yaml.j2
View File

@ -1,2 +0,0 @@
# Intentionally left blank to test autogeneration of name
#bup_username: bup-backup-test02

13
zuul.d/infra-prod.yaml
View File

@ -275,19 +275,6 @@
- playbooks/roles/static/
- playbooks/roles/zuul-user/
- job:
name: infra-prod-service-backup
parent: infra-prod-service-base
description: Run service-backup.yaml playbook.
vars:
playbook_name: service-backup.yaml
files:
- inventory/
- playbooks/service-backup.yaml
- playbooks/roles/backup/
- playbooks/roles/backup-server/
- playbooks/roles/iptables/
- job:
name: infra-prod-service-borg-backup
parent: infra-prod-service-base

3
zuul.d/project.yaml
View File

@ -13,7 +13,6 @@
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-backup
- system-config-run-borg-backup
- system-config-run-dns
- system-config-run-eavesdrop:
@ -281,7 +280,6 @@
- infra-prod-service-mirror-update
- infra-prod-service-mirror
- infra-prod-service-static
- infra-prod-service-backup
- infra-prod-service-borg-backup
- infra-prod-service-registry
- infra-prod-service-refstack
@ -326,7 +324,6 @@
- infra-prod-service-mirror
- infra-prod-service-static
- infra-prod-service-borg-backup
- infra-prod-service-backup
- infra-prod-service-zookeeper
- infra-prod-service-review
- infra-prod-service-review-dev

24
zuul.d/system-config-run.yaml
View File

@ -305,30 +305,6 @@
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-backup
parent: system-config-run
description: |
Run the playbook for backup configuration
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: backup01.region.provider.opendev.org
label: ubuntu-bionic
- name: backup-test01.opendev.org
label: ubuntu-bionic
- name: backup-test02.opendev.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-backup.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/backup
- playbooks/zuul/templates/host_vars/backup
- testinfra/test_backups.py
- job:
name: system-config-run-borg-backup
parent: system-config-run