Merge "backups: remove all bup"
This commit is contained in:
Zuul
Gerrit Code Review
commit
1b2435c349
@ -49,13 +49,6 @@ all:
|
|||||||
region_name: DFW
|
region_name: DFW
|
||||||
public_ipv4: 104.239.149.165
|
public_ipv4: 104.239.149.165
|
||||||
public_ipv6: 2001:4800:7819:105:be76:4eff:fe01:e6ff
|
public_ipv6: 2001:4800:7819:105:be76:4eff:fe01:e6ff
|
||||||
backup01.ca-ymq-1.vexxhost.opendev.org:
|
|
||||||
ansible_host: 199.204.45.119
|
|
||||||
location:
|
|
||||||
cloud: openstackci-vexxhost
|
|
||||||
region_name: ca-ymq-1
|
|
||||||
public_v4: 199.204.45.119
|
|
||||||
public_v6: 2604:e100:1:0:f816:3eff:feab:d678
|
|
||||||
backup02.ca-ymq-1.vexxhost.opendev.org:
|
backup02.ca-ymq-1.vexxhost.opendev.org:
|
||||||
ansible_host: 199.204.45.196
|
ansible_host: 199.204.45.196
|
||||||
location:
|
location:
|
||||||
@ -70,13 +63,6 @@ all:
|
|||||||
region_name: ORD
|
region_name: ORD
|
||||||
public_v4: 23.253.160.180
|
public_v4: 23.253.160.180
|
||||||
public_v6: 2001:4801:7825:103:be76:4eff:fe10:1b1
|
public_v6: 2001:4801:7825:103:be76:4eff:fe10:1b1
|
||||||
backup01.ord.rax.ci.openstack.org:
|
|
||||||
ansible_host: 23.253.20.173
|
|
||||||
location:
|
|
||||||
cloud: openstackci-rax
|
|
||||||
region_name: ORD
|
|
||||||
public_v4: 23.253.20.173
|
|
||||||
public_v6: 2001:4801:7824:101:be76:4eff:fe10:20cf
|
|
||||||
bridge.openstack.org:
|
bridge.openstack.org:
|
||||||
ansible_host: 23.253.234.219
|
ansible_host: 23.253.234.219
|
||||||
location:
|
location:
|
||||||
|
|||||||
@ -19,27 +19,6 @@ groups:
|
|||||||
afs-admin:
|
afs-admin:
|
||||||
- mirror-update[0-9]*.openstack.org
|
- mirror-update[0-9]*.openstack.org
|
||||||
ask: ask*.open*.org
|
ask: ask*.open*.org
|
||||||
# NOTE: By default we keep the backup-server group empty as an
|
|
||||||
# emergency escape hatch if a problem were to propage through
|
|
||||||
# production servers. However, this also means if you add a server to
|
|
||||||
# the "backup" group to be backed up, you should uncomment the
|
|
||||||
# "backup-server" group for an Ansible pulse so the users & keys are
|
|
||||||
# setup on the server(s). You can submit a follow-on change to revert
|
|
||||||
# this at the same time.
|
|
||||||
backup:
|
|
||||||
- gitea01.opendev.org
|
|
||||||
- review[0-9]*.openstack.org
|
|
||||||
- review-dev[0-9]*.open*.org
|
|
||||||
- zuul[0-9]*.open*.org
|
|
||||||
# All these servers are "special-cased" in specifically
|
|
||||||
# as they are puppet and should be replaced "soon"
|
|
||||||
- ethercalc02.openstack.org
|
|
||||||
- ask01.openstack.org
|
|
||||||
- lists.openstack.org
|
|
||||||
- storyboard01.opendev.org
|
|
||||||
- translate01.openstack.org
|
|
||||||
backup-server:
|
|
||||||
- backup01.ca-ymq-1.vexxhost.opendev.org
|
|
||||||
borg-backup:
|
borg-backup:
|
||||||
- etherpad[0-9]*.opendev.org
|
- etherpad[0-9]*.opendev.org
|
||||||
- gitea01.opendev.org
|
- gitea01.opendev.org
|
||||||
@ -66,7 +45,6 @@ groups:
|
|||||||
control-plane-clouds:
|
control-plane-clouds:
|
||||||
- bridge.openstack.org
|
- bridge.openstack.org
|
||||||
disabled:
|
disabled:
|
||||||
- backup01.ord.rax.ci.openstack.org
|
|
||||||
- corvustest
|
- corvustest
|
||||||
- idp.openstackid.org
|
- idp.openstackid.org
|
||||||
- lists-dev01.openstack.org
|
- lists-dev01.openstack.org
|
||||||
@ -146,7 +124,6 @@ groups:
|
|||||||
- pbx[0-9]*.opendev.org
|
- pbx[0-9]*.opendev.org
|
||||||
puppet:
|
puppet:
|
||||||
- ask*.open*.org
|
- ask*.open*.org
|
||||||
- backup[0-9]*.openstack.org
|
|
||||||
- cacti[0-9]*.open*.org
|
- cacti[0-9]*.open*.org
|
||||||
- corvustest
|
- corvustest
|
||||||
- eavesdrop[0-9]*.open*.org
|
- eavesdrop[0-9]*.open*.org
|
||||||
|
|||||||
@ -355,14 +355,6 @@ node /^pbx\d*\.open.*\.org$/ {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# Node-OS: xenial
|
|
||||||
# A backup machine. Don't run cron or puppet agent on it.
|
|
||||||
node /^backup\d+\..*\.ci\.open.*\.org$/ {
|
|
||||||
$group = "ci-backup"
|
|
||||||
class { 'openstack_project::server': }
|
|
||||||
include openstack_project::backup_server
|
|
||||||
}
|
|
||||||
|
|
||||||
# Node-OS: xenial
|
# Node-OS: xenial
|
||||||
node /^openstackid\d*(\.openstack)?\.org$/ {
|
node /^openstackid\d*(\.openstack)?\.org$/ {
|
||||||
$group = "openstackid"
|
$group = "openstackid"
|
||||||
|
|||||||
@ -1,7 +0,0 @@
|
|||||||
# == Class: openstack_project::backup_server
|
|
||||||
#
|
|
||||||
class openstack_project::backup_server {
|
|
||||||
package { 'bup':
|
|
||||||
ensure => present,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -21,14 +21,4 @@ class openstack_project::ethercalc (
|
|||||||
|
|
||||||
include ethercalc::redis
|
include ethercalc::redis
|
||||||
|
|
||||||
# Redis creates a snapshot at /var/lib/redis/dump.rdb periodically
|
|
||||||
# (at worst every 15 minutes if at least one change is made to redis)
|
|
||||||
# which can be used to recover the Redis DB. Bup will automagically
|
|
||||||
# pick this file up during its normal operation so no other DB dumping
|
|
||||||
# is required like with mysql.
|
|
||||||
include bup
|
|
||||||
bup::site { 'ord.rax':
|
|
||||||
backup_user => "bup-$::hostname",
|
|
||||||
backup_server => 'backup01.ord.rax.ci.openstack.org',
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -42,12 +42,6 @@ class openstack_project::lists(
|
|||||||
user::virtual::disable { 'oubiwann': }
|
user::virtual::disable { 'oubiwann': }
|
||||||
user::virtual::disable { 'rockstar': }
|
user::virtual::disable { 'rockstar': }
|
||||||
|
|
||||||
include bup
|
|
||||||
bup::site { 'ord.rax':
|
|
||||||
backup_user => 'bup-lists',
|
|
||||||
backup_server => 'backup01.ord.rax.ci.openstack.org',
|
|
||||||
}
|
|
||||||
|
|
||||||
# Begin user servicable parts
|
# Begin user servicable parts
|
||||||
|
|
||||||
mailman::site { 'openstack':
|
mailman::site { 'openstack':
|
||||||
|
|||||||
@ -86,9 +86,4 @@ class openstack_project::storyboard(
|
|||||||
source => $superusers,
|
source => $superusers,
|
||||||
}
|
}
|
||||||
|
|
||||||
include bup
|
|
||||||
bup::site { 'ord.rax':
|
|
||||||
backup_user => 'bup-storyboard',
|
|
||||||
backup_server => 'backup01.ord.rax.ci.openstack.org',
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -75,14 +75,6 @@ class openstack_project::wiki (
|
|||||||
require => File['/srv/mediawiki'],
|
require => File['/srv/mediawiki'],
|
||||||
}
|
}
|
||||||
|
|
||||||
if $bup_user != undef {
|
|
||||||
include bup
|
|
||||||
bup::site { 'ord.rax':
|
|
||||||
backup_user => $bup_user,
|
|
||||||
backup_server => 'backup01.ord.rax.ci.openstack.org',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class { '::elasticsearch':
|
class { '::elasticsearch':
|
||||||
es_template_config => {
|
es_template_config => {
|
||||||
'bootstrap.mlockall' => true,
|
'bootstrap.mlockall' => true,
|
||||||
|
|||||||
@ -1,15 +0,0 @@
|
|||||||
Setup backup server
|
|
||||||
|
|
||||||
This role configures backup server(s) in the ``backup-server`` group
|
|
||||||
to accept backups from remote hosts.
|
|
||||||
|
|
||||||
Note that the ``backup`` role must have run on each host in the
|
|
||||||
``backup`` group before this role. That role will create a
|
|
||||||
``bup_user`` tuple in the hostvars for for each host consisting of the
|
|
||||||
required username and public key.
|
|
||||||
|
|
||||||
Each required user gets a separate home directory in ``/opt/backups``.
|
|
||||||
Their ``authorized_keys`` file is configured with the public key to
|
|
||||||
allow the remote host to log in and only run ``bup``.
|
|
||||||
|
|
||||||
**Role Variables**
|
|
||||||
@ -1 +0,0 @@
|
|||||||
bup_users: []
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
- name: Create backup directory
|
|
||||||
file:
|
|
||||||
state: directory
|
|
||||||
path: /opt/backups
|
|
||||||
|
|
||||||
- name: Install bup
|
|
||||||
package:
|
|
||||||
name:
|
|
||||||
- bup
|
|
||||||
state: present
|
|
||||||
|
|
||||||
- name: Build all bup users from backup hosts
|
|
||||||
set_fact:
|
|
||||||
bup_users: '{{ bup_users }} + [ {{ hostvars[item]["bup_user"] }} ]'
|
|
||||||
with_inventory_hostnames: 'backup:!disabled'
|
|
||||||
|
|
||||||
- name: Create bup users
|
|
||||||
include_tasks: user.yaml
|
|
||||||
loop: '{{ bup_users }}'
|
|
||||||
loop_control:
|
|
||||||
loop_var: bup_user
|
|
||||||
@ -1,32 +0,0 @@
|
|||||||
# note bup_user is the parent loop variable name; this works on each
|
|
||||||
# element from the bup_users global.
|
|
||||||
- name: Set variables
|
|
||||||
set_fact:
|
|
||||||
user_name: '{{ bup_user[0] }}'
|
|
||||||
user_key: '{{ bup_user[1] }}'
|
|
||||||
|
|
||||||
- name: Create bup user
|
|
||||||
user:
|
|
||||||
name: '{{ user_name }}'
|
|
||||||
comment: 'Backup user'
|
|
||||||
shell: /bin/bash
|
|
||||||
home: '/opt/backups/{{ user_name }}'
|
|
||||||
create_home: yes
|
|
||||||
register: homedir
|
|
||||||
|
|
||||||
- name: Create bup user authorized key
|
|
||||||
authorized_key:
|
|
||||||
user: '{{ user_name }}'
|
|
||||||
state: present
|
|
||||||
key: '{{ user_key }}'
|
|
||||||
key_options: 'command="BUP_DEBUG=0 BUP_FORCE_TTY=3 bup server",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty'
|
|
||||||
|
|
||||||
# ansible-lint wants this in a handler, it should be done here and
|
|
||||||
# now; this isn't like a service restart where multiple things might
|
|
||||||
# call it.
|
|
||||||
- name: Initalise bup
|
|
||||||
shell: |
|
|
||||||
BUP_DIR=/opt/backups/{{ user_name }}/.bup bup init
|
|
||||||
become: yes
|
|
||||||
become_user: '{{ user_name }}'
|
|
||||||
when: homedir.changed
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
Configure a host to be backed up
|
|
||||||
|
|
||||||
This role setups a host to use ``bup`` for backup to any hosts in the
|
|
||||||
``backup-server`` group.
|
|
||||||
|
|
||||||
A separate ssh key will be generated for root to connect to the backup
|
|
||||||
server(s) and the host key for the backup servers will be accepted to
|
|
||||||
the host.
|
|
||||||
|
|
||||||
The ``bup`` tool is installed and a cron job is setup to run the
|
|
||||||
backup periodically.
|
|
||||||
|
|
||||||
Note the ``backup-server`` role must run after this to create the user
|
|
||||||
correctly on the backup server. This role sets a tuple ``bup_user``
|
|
||||||
with the username and public key; the ``backup-server`` role uses this
|
|
||||||
variable for each host in the ``backup`` group to initalise users.
|
|
||||||
|
|
||||||
**Role Variables**
|
|
||||||
|
|
||||||
.. zuul:rolevar:: bup_username
|
|
||||||
|
|
||||||
The username to connect to the backup server. If this is left
|
|
||||||
undefined, it will be automatically set to ``bup-$(hostname)``
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
/proc/*
|
|
||||||
/sys/*
|
|
||||||
/dev/*
|
|
||||||
/tmp/*
|
|
||||||
/floppy/*
|
|
||||||
/cdrom/*
|
|
||||||
/var/spool/squid/*
|
|
||||||
/var/spool/exim/*
|
|
||||||
/media/*
|
|
||||||
/mnt/*
|
|
||||||
/var/agentx/*
|
|
||||||
/run/*
|
|
||||||
/root/backup-restore-*
|
|
||||||
/root/.bup
|
|
||||||
/etc/puppet/modules/*
|
|
||||||
/etc/puppet/hieradata/*
|
|
||||||
/var/cache/*
|
|
||||||
/var/lib/docker/*
|
|
||||||
/var/lib/puppet/reports/*
|
|
||||||
/var/lib/postgresql/*
|
|
||||||
/var/lib/lxcfs/*
|
|
||||||
/var/lib/zuul/backup/*
|
|
||||||
/var/lib/zuul/times/*
|
|
||||||
/opt/system-config/*
|
|
||||||
/afs/*
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
- name: Generate bup username for this host
|
|
||||||
set_fact:
|
|
||||||
bup_username: 'bup-{{ inventory_hostname.split(".", 1)[0] }}'
|
|
||||||
when: bup_username is not defined
|
|
||||||
|
|
||||||
- debug:
|
|
||||||
var: bup_username
|
|
||||||
|
|
||||||
- name: Install bup
|
|
||||||
package:
|
|
||||||
name:
|
|
||||||
- bup
|
|
||||||
state: absent
|
|
||||||
|
|
||||||
- name: Remove old keypair
|
|
||||||
file:
|
|
||||||
path: /root/.ssh/id_backup_ed25519
|
|
||||||
state: absent
|
|
||||||
|
|
||||||
- name: Remove old keypair
|
|
||||||
file:
|
|
||||||
path: /root/.ssh/id_backup_ed25519.pub
|
|
||||||
state: absent
|
|
||||||
|
|
||||||
- name: Remove old config directory
|
|
||||||
file:
|
|
||||||
path: /root/.bup
|
|
||||||
state: absent
|
|
||||||
|
|
||||||
- name: Remove ssh config
|
|
||||||
blockinfile:
|
|
||||||
path: /root/.ssh/config
|
|
||||||
state: absent
|
|
||||||
create: false
|
|
||||||
block: |
|
|
||||||
Host {{ item }}
|
|
||||||
HostName {{ item }}
|
|
||||||
IdentityFile /root/.ssh/id_backup_ed25519
|
|
||||||
User {{ bup_username }}
|
|
||||||
mode: 0600
|
|
||||||
with_inventory_hostnames: backup-server
|
|
||||||
ignore_errors: True
|
|
||||||
|
|
||||||
- name: Remove /etc/bup-excludes
|
|
||||||
file:
|
|
||||||
path: /etc/bup-excludes
|
|
||||||
state: absent
|
|
||||||
|
|
||||||
- name: Remove backup cronjob
|
|
||||||
cron:
|
|
||||||
name: "Run bup backup"
|
|
||||||
job: "tar -X /etc/bup-excludes -cPF - / | bup split -r {{ bup_username }}@{{ item }}: -n root -q"
|
|
||||||
user: root
|
|
||||||
hour: '5'
|
|
||||||
minute: '{{ 59|random(seed=item) }}'
|
|
||||||
state: absent
|
|
||||||
with_inventory_hostnames: backup-server
|
|
||||||
@ -38,15 +38,13 @@ results:
|
|||||||
- mirror
|
- mirror
|
||||||
|
|
||||||
review01.openstack.org:
|
review01.openstack.org:
|
||||||
- backup
|
|
||||||
- borg-backup
|
- borg-backup
|
||||||
- gerrit
|
- gerrit
|
||||||
- letsencrypt
|
- letsencrypt
|
||||||
- review
|
- review
|
||||||
|
|
||||||
backup01.ord.rax.ci.openstack.org:
|
backup01.ord.rax.opendev.org:
|
||||||
- disabled
|
- borg-backup-server
|
||||||
- puppet
|
|
||||||
|
|
||||||
ze01.openstack.org:
|
ze01.openstack.org:
|
||||||
- afs-client
|
- afs-client
|
||||||
|
|||||||
@ -1,8 +0,0 @@
|
|||||||
# NOTE(ianw) : we are removing bup for borg. This just needs to run
|
|
||||||
# once to remove bup parts from the backup clients, then we will
|
|
||||||
# remove it completely.
|
|
||||||
- hosts: "backup:!disabled"
|
|
||||||
name: "Base: Generate backup users and keys"
|
|
||||||
roles:
|
|
||||||
- iptables
|
|
||||||
- backup
|
|
||||||
@ -83,8 +83,6 @@
|
|||||||
- host_vars/mirror01.openafs.provider.opendev.org.yaml
|
- host_vars/mirror01.openafs.provider.opendev.org.yaml
|
||||||
- host_vars/mirror02.openafs.provider.opendev.org.yaml
|
- host_vars/mirror02.openafs.provider.opendev.org.yaml
|
||||||
- host_vars/mirror-update01.opendev.org.yaml
|
- host_vars/mirror-update01.opendev.org.yaml
|
||||||
- host_vars/backup-test01.opendev.org.yaml
|
|
||||||
- host_vars/backup-test02.opendev.org.yaml
|
|
||||||
- host_vars/refstack01.openstack.org.yaml
|
- host_vars/refstack01.openstack.org.yaml
|
||||||
- name: Display group membership
|
- name: Display group membership
|
||||||
command: ansible localhost -m debug -a 'var=groups'
|
command: ansible localhost -m debug -a 'var=groups'
|
||||||
|
|||||||
@ -1 +0,0 @@
|
|||||||
bup_username: bup-backup01
|
|
||||||
@ -1,2 +0,0 @@
|
|||||||
# Intentionally left blank to test autogeneration of name
|
|
||||||
#bup_username: bup-backup-test02
|
|
||||||
@ -275,19 +275,6 @@
|
|||||||
- playbooks/roles/static/
|
- playbooks/roles/static/
|
||||||
- playbooks/roles/zuul-user/
|
- playbooks/roles/zuul-user/
|
||||||
|
|
||||||
- job:
|
|
||||||
name: infra-prod-service-backup
|
|
||||||
parent: infra-prod-service-base
|
|
||||||
description: Run service-backup.yaml playbook.
|
|
||||||
vars:
|
|
||||||
playbook_name: service-backup.yaml
|
|
||||||
files:
|
|
||||||
- inventory/
|
|
||||||
- playbooks/service-backup.yaml
|
|
||||||
- playbooks/roles/backup/
|
|
||||||
- playbooks/roles/backup-server/
|
|
||||||
- playbooks/roles/iptables/
|
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: infra-prod-service-borg-backup
|
name: infra-prod-service-borg-backup
|
||||||
parent: infra-prod-service-base
|
parent: infra-prod-service-base
|
||||||
|
|||||||
@ -13,7 +13,6 @@
|
|||||||
- system-config-run-base
|
- system-config-run-base
|
||||||
- system-config-run-base-ansible-devel:
|
- system-config-run-base-ansible-devel:
|
||||||
voting: false
|
voting: false
|
||||||
- system-config-run-backup
|
|
||||||
- system-config-run-borg-backup
|
- system-config-run-borg-backup
|
||||||
- system-config-run-dns
|
- system-config-run-dns
|
||||||
- system-config-run-eavesdrop:
|
- system-config-run-eavesdrop:
|
||||||
@ -281,7 +280,6 @@
|
|||||||
- infra-prod-service-mirror-update
|
- infra-prod-service-mirror-update
|
||||||
- infra-prod-service-mirror
|
- infra-prod-service-mirror
|
||||||
- infra-prod-service-static
|
- infra-prod-service-static
|
||||||
- infra-prod-service-backup
|
|
||||||
- infra-prod-service-borg-backup
|
- infra-prod-service-borg-backup
|
||||||
- infra-prod-service-registry
|
- infra-prod-service-registry
|
||||||
- infra-prod-service-refstack
|
- infra-prod-service-refstack
|
||||||
@ -326,7 +324,6 @@
|
|||||||
- infra-prod-service-mirror
|
- infra-prod-service-mirror
|
||||||
- infra-prod-service-static
|
- infra-prod-service-static
|
||||||
- infra-prod-service-borg-backup
|
- infra-prod-service-borg-backup
|
||||||
- infra-prod-service-backup
|
|
||||||
- infra-prod-service-zookeeper
|
- infra-prod-service-zookeeper
|
||||||
- infra-prod-service-review
|
- infra-prod-service-review
|
||||||
- infra-prod-service-review-dev
|
- infra-prod-service-review-dev
|
||||||
|
|||||||
@ -305,30 +305,6 @@
|
|||||||
- testinfra/test_adns.py
|
- testinfra/test_adns.py
|
||||||
- testinfra/test_ns.py
|
- testinfra/test_ns.py
|
||||||
|
|
||||||
- job:
|
|
||||||
name: system-config-run-backup
|
|
||||||
parent: system-config-run
|
|
||||||
description: |
|
|
||||||
Run the playbook for backup configuration
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: bridge.openstack.org
|
|
||||||
label: ubuntu-bionic
|
|
||||||
- name: backup01.region.provider.opendev.org
|
|
||||||
label: ubuntu-bionic
|
|
||||||
- name: backup-test01.opendev.org
|
|
||||||
label: ubuntu-bionic
|
|
||||||
- name: backup-test02.opendev.org
|
|
||||||
label: ubuntu-xenial
|
|
||||||
vars:
|
|
||||||
run_playbooks:
|
|
||||||
- playbooks/service-backup.yaml
|
|
||||||
files:
|
|
||||||
- playbooks/install-ansible.yaml
|
|
||||||
- playbooks/roles/backup
|
|
||||||
- playbooks/zuul/templates/host_vars/backup
|
|
||||||
- testinfra/test_backups.py
|
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: system-config-run-borg-backup
|
name: system-config-run-borg-backup
|
||||||
parent: system-config-run
|
parent: system-config-run
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user