opendev/system-config
Code Issues Proposed changes

backups: remove all bup

Browse Source 
All hosts are now running thier backups via borg to servers in
vexxhost and rax.ord.

For reference, the servers being backed up at this time are:

 borg-ask01
 borg-ethercalc02
 borg-etherpad01
 borg-gitea01
 borg-lists
 borg-review-dev01
 borg-review01
 borg-storyboard01
 borg-translate01
 borg-wiki-update-test
 borg-zuul01

This removes the old bup backup hosts, the no-longer used ansible
roles for the bup backup server and client roles, and any remaining
bup related configuration.

For simplicity, we will remove any remaining bup cron jobs on the
above servers manually after this merges.

Change-Id: I32554ca857a81ae8a250ce082421a7ede460ea3c
This commit is contained in:
Ian Wienand 2020-12-11 09:19:28 +11:00
parent ebdd2144bf
commit 39ffc685d6
23 changed files with 2 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
inventory/base/hosts.yaml
View File

@ -49,13 +49,6 @@ all:
region_name: DFW
public_ipv4: 104.239.149.165
public_ipv6: 2001:4800:7819:105:be76:4eff:fe01:e6ff
backup01.ca-ymq-1.vexxhost.opendev.org:
ansible_host: 199.204.45.119
location:
cloud: openstackci-vexxhost
region_name: ca-ymq-1
public_v4: 199.204.45.119
public_v6: 2604:e100:1:0:f816:3eff:feab:d678
backup02.ca-ymq-1.vexxhost.opendev.org:
ansible_host: 199.204.45.196
location:
@ -70,13 +63,6 @@ all:
region_name: ORD
public_v4: 23.253.160.180
public_v6: 2001:4801:7825:103:be76:4eff:fe10:1b1
backup01.ord.rax.ci.openstack.org:
ansible_host: 23.253.20.173
location:
cloud: openstackci-rax
region_name: ORD
public_v4: 23.253.20.173
public_v6: 2001:4801:7824:101:be76:4eff:fe10:20cf
bridge.openstack.org:
ansible_host: 23.253.234.219
location:

23
inventory/service/groups.yaml
View File

@ -19,27 +19,6 @@ groups:
afs-admin:
- mirror-update[0-9]*.openstack.org
ask: ask*.open*.org
# NOTE: By default we keep the backup-server group empty as an
# emergency escape hatch if a problem were to propage through
# production servers. However, this also means if you add a server to
# the "backup" group to be backed up, you should uncomment the
# "backup-server" group for an Ansible pulse so the users & keys are
# setup on the server(s). You can submit a follow-on change to revert
# this at the same time.
backup:
- gitea01.opendev.org
- review[0-9]*.openstack.org
- review-dev[0-9]*.open*.org
- zuul[0-9]*.open*.org
# All these servers are "special-cased" in specifically
# as they are puppet and should be replaced "soon"
- ethercalc02.openstack.org
- ask01.openstack.org
- lists.openstack.org
- storyboard01.opendev.org
- translate01.openstack.org
backup-server:
- backup01.ca-ymq-1.vexxhost.opendev.org
borg-backup:
- etherpad[0-9]*.opendev.org
- gitea01.opendev.org
@ -66,7 +45,6 @@ groups:
control-plane-clouds:
- bridge.openstack.org
disabled:
- backup01.ord.rax.ci.openstack.org
- corvustest
- idp.openstackid.org
- lists-dev01.openstack.org
@ -146,7 +124,6 @@ groups:
- pbx[0-9]*.opendev.org
puppet:
- ask*.open*.org
- backup[0-9]*.openstack.org
- cacti[0-9]*.open*.org
- corvustest
- eavesdrop[0-9]*.open*.org

8
manifests/site.pp
View File

@ -355,14 +355,6 @@ node /^pbx\d*\.open.*\.org$/ {
}
}
# Node-OS: xenial
# A backup machine. Don't run cron or puppet agent on it.
node /^backup\d+\..*\.ci\.open.*\.org$/ {
$group = "ci-backup"
class { 'openstack_project::server': }
include openstack_project::backup_server
}
# Node-OS: xenial
node /^openstackid\d*(\.openstack)?\.org$/ {
$group = "openstackid"

7
modules/openstack_project/manifests/backup_server.pp
View File

@ -1,7 +0,0 @@
# == Class: openstack_project::backup_server
#
class openstack_project::backup_server {
package { 'bup':
ensure => present,
}
}

10
modules/openstack_project/manifests/ethercalc.pp
View File

@ -21,14 +21,4 @@ class openstack_project::ethercalc (
include ethercalc::redis
# Redis creates a snapshot at /var/lib/redis/dump.rdb periodically
# (at worst every 15 minutes if at least one change is made to redis)
# which can be used to recover the Redis DB. Bup will automagically
# pick this file up during its normal operation so no other DB dumping
# is required like with mysql.
include bup
bup::site { 'ord.rax':
backup_user => "bup-$::hostname",
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}

6
modules/openstack_project/manifests/lists.pp
View File

@ -42,12 +42,6 @@ class openstack_project::lists(
user::virtual::disable { 'oubiwann': }
user::virtual::disable { 'rockstar': }
include bup
bup::site { 'ord.rax':
backup_user => 'bup-lists',
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
# Begin user servicable parts
mailman::site { 'openstack':

5
modules/openstack_project/manifests/storyboard.pp
View File

@ -86,9 +86,4 @@ class openstack_project::storyboard(
source => $superusers,
}
include bup
bup::site { 'ord.rax':
backup_user => 'bup-storyboard',
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}

8
modules/openstack_project/manifests/wiki.pp
View File

@ -75,14 +75,6 @@ class openstack_project::wiki (
require => File['/srv/mediawiki'],
}
if $bup_user != undef {
include bup
bup::site { 'ord.rax':
backup_user => $bup_user,
backup_server => 'backup01.ord.rax.ci.openstack.org',
}
}
class { '::elasticsearch':
es_template_config => {
'bootstrap.mlockall' => true,

15
playbooks/roles/backup-server/README.rst
View File

@ -1,15 +0,0 @@
Setup backup server
This role configures backup server(s) in the ``backup-server`` group
to accept backups from remote hosts.
Note that the ``backup`` role must have run on each host in the
``backup`` group before this role. That role will create a
``bup_user`` tuple in the hostvars for for each host consisting of the
required username and public key.
Each required user gets a separate home directory in ``/opt/backups``.
Their ``authorized_keys`` file is configured with the public key to
allow the remote host to log in and only run ``bup``.
**Role Variables**

1
playbooks/roles/backup-server/defaults/main.yaml
View File

@ -1 +0,0 @@
bup_users: []

21
playbooks/roles/backup-server/tasks/main.yaml
View File

@ -1,21 +0,0 @@
- name: Create backup directory
file:
state: directory
path: /opt/backups
- name: Install bup
package:
name:
- bup
state: present
- name: Build all bup users from backup hosts
set_fact:
bup_users: '{{ bup_users }} + [ {{ hostvars[item]["bup_user"] }} ]'
with_inventory_hostnames: 'backup:!disabled'
- name: Create bup users
include_tasks: user.yaml
loop: '{{ bup_users }}'
loop_control:
loop_var: bup_user

32
playbooks/roles/backup-server/tasks/user.yaml
View File

@ -1,32 +0,0 @@
# note bup_user is the parent loop variable name; this works on each
# element from the bup_users global.
- name: Set variables
set_fact:
user_name: '{{ bup_user[0] }}'
user_key: '{{ bup_user[1] }}'
- name: Create bup user
user:
name: '{{ user_name }}'
comment: 'Backup user'
shell: /bin/bash
home: '/opt/backups/{{ user_name }}'
create_home: yes
register: homedir
- name: Create bup user authorized key
authorized_key:
user: '{{ user_name }}'
state: present
key: '{{ user_key }}'
key_options: 'command="BUP_DEBUG=0 BUP_FORCE_TTY=3 bup server",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty'
# ansible-lint wants this in a handler, it should be done here and
# now; this isn't like a service restart where multiple things might
# call it.
- name: Initalise bup
shell: |
BUP_DIR=/opt/backups/{{ user_name }}/.bup bup init
become: yes
become_user: '{{ user_name }}'
when: homedir.changed

23
playbooks/roles/backup/README.rst
View File

@ -1,23 +0,0 @@
Configure a host to be backed up
This role setups a host to use ``bup`` for backup to any hosts in the
``backup-server`` group.
A separate ssh key will be generated for root to connect to the backup
server(s) and the host key for the backup servers will be accepted to
the host.
The ``bup`` tool is installed and a cron job is setup to run the
backup periodically.
Note the ``backup-server`` role must run after this to create the user
correctly on the backup server. This role sets a tuple ``bup_user``
with the username and public key; the ``backup-server`` role uses this
variable for each host in the ``backup`` group to initalise users.
**Role Variables**
.. zuul:rolevar:: bup_username
The username to connect to the backup server. If this is left
undefined, it will be automatically set to ``bup-$(hostname)``

25
playbooks/roles/backup/files/bup-excludes
View File

@ -1,25 +0,0 @@
/proc/*
/sys/*
/dev/*
/tmp/*
/floppy/*
/cdrom/*
/var/spool/squid/*
/var/spool/exim/*
/media/*
/mnt/*
/var/agentx/*
/run/*
/root/backup-restore-*
/root/.bup
/etc/puppet/modules/*
/etc/puppet/hieradata/*
/var/cache/*
/var/lib/docker/*
/var/lib/puppet/reports/*
/var/lib/postgresql/*
/var/lib/lxcfs/*
/var/lib/zuul/backup/*
/var/lib/zuul/times/*
/opt/system-config/*
/afs/*

57
playbooks/roles/backup/tasks/main.yaml
View File

@ -1,57 +0,0 @@
- name: Generate bup username for this host
set_fact:
bup_username: 'bup-{{ inventory_hostname.split(".", 1)[0] }}'
when: bup_username is not defined
- debug:
var: bup_username
- name: Install bup
package:
name:
- bup
state: absent
- name: Remove old keypair
file:
path: /root/.ssh/id_backup_ed25519
state: absent
- name: Remove old keypair
file:
path: /root/.ssh/id_backup_ed25519.pub
state: absent
- name: Remove old config directory
file:
path: /root/.bup
state: absent
- name: Remove ssh config
blockinfile:
path: /root/.ssh/config
state: absent
create: false
block: |
Host {{ item }}
HostName {{ item }}
IdentityFile /root/.ssh/id_backup_ed25519
User {{ bup_username }}
mode: 0600
with_inventory_hostnames: backup-server
ignore_errors: True
- name: Remove /etc/bup-excludes
file:
path: /etc/bup-excludes
state: absent
- name: Remove backup cronjob
cron:
name: "Run bup backup"
job: "tar -X /etc/bup-excludes -cPF - / | bup split -r {{ bup_username }}@{{ item }}: -n root -q"
user: root
hour: '5'
minute: '{{ 59|random(seed=item) }}'
state: absent
with_inventory_hostnames: backup-server

6
playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
View File

@ -38,15 +38,13 @@ results:
- mirror
review01.openstack.org:
- backup
- borg-backup
- gerrit
- letsencrypt
- review
backup01.ord.rax.ci.openstack.org:
- disabled
- puppet
backup01.ord.rax.opendev.org:
- borg-backup-server
ze01.openstack.org:
- afs-client

8
playbooks/service-backup.yaml
View File

@ -1,8 +0,0 @@
# NOTE(ianw) : we are removing bup for borg. This just needs to run
# once to remove bup parts from the backup clients, then we will
# remove it completely.
- hosts: "backup:!disabled"
name: "Base: Generate backup users and keys"
roles:
- iptables
- backup

2
playbooks/zuul/run-base.yaml
View File

@ -83,8 +83,6 @@
- host_vars/mirror01.openafs.provider.opendev.org.yaml
- host_vars/mirror02.openafs.provider.opendev.org.yaml
- host_vars/mirror-update01.opendev.org.yaml
- host_vars/backup-test01.opendev.org.yaml
- host_vars/backup-test02.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'

1
playbooks/zuul/templates/host_vars/backup-test01.opendev.org.yaml.j2
View File

@ -1 +0,0 @@
bup_username: bup-backup01

2
playbooks/zuul/templates/host_vars/backup-test02.opendev.org.yaml.j2
View File

@ -1,2 +0,0 @@
# Intentionally left blank to test autogeneration of name
#bup_username: bup-backup-test02

13
zuul.d/infra-prod.yaml
View File

@ -275,19 +275,6 @@
- playbooks/roles/static/
- playbooks/roles/zuul-user/
- job:
name: infra-prod-service-backup
parent: infra-prod-service-base
description: Run service-backup.yaml playbook.
vars:
playbook_name: service-backup.yaml
files:
- inventory/
- playbooks/service-backup.yaml
- playbooks/roles/backup/
- playbooks/roles/backup-server/
- playbooks/roles/iptables/
- job:
name: infra-prod-service-borg-backup
parent: infra-prod-service-base

3
zuul.d/project.yaml
View File

@ -13,7 +13,6 @@
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-backup
- system-config-run-borg-backup
- system-config-run-dns
- system-config-run-eavesdrop:
@ -271,7 +270,6 @@
- infra-prod-service-mirror-update
- infra-prod-service-mirror
- infra-prod-service-static
- infra-prod-service-backup
- infra-prod-service-borg-backup
- infra-prod-service-registry
- infra-prod-service-refstack
@ -316,7 +314,6 @@
- infra-prod-service-mirror
- infra-prod-service-static
- infra-prod-service-borg-backup
- infra-prod-service-backup
- infra-prod-service-zookeeper
- infra-prod-service-review
- infra-prod-service-review-dev

24
zuul.d/system-config-run.yaml
View File

@ -305,30 +305,6 @@
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-backup
parent: system-config-run
description: |
Run the playbook for backup configuration
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: backup01.region.provider.opendev.org
label: ubuntu-bionic
- name: backup-test01.opendev.org
label: ubuntu-bionic
- name: backup-test02.opendev.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-backup.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/backup
- playbooks/zuul/templates/host_vars/backup
- testinfra/test_backups.py
- job:
name: system-config-run-borg-backup
parent: system-config-run