Merge "Allow for periodic afs releases from mirror-update"
This commit is contained in:
Zuul
Gerrit Code Review
commit
3c13f523a1
|
|
@ -0,0 +1,9 @@
|
|||
afs-release
|
||||
|
||||
Install the script and related bits and pieces for periodic release of
|
||||
various AFS volumes. This role is really only intended to be run on
|
||||
the `mirror-update` host, as it uses the ssh-key installed by that
|
||||
host to run `vos release` under `-localauth` on the remote AFS
|
||||
servers.
|
||||
|
||||
**Role Variables**
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2020 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
#
|
||||
# This script is intended to run on mirror-update.opendev.org
|
||||
# periodically called from a cron job.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import statsd
|
||||
import subprocess
|
||||
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from paramiko import SSHClient
|
||||
|
||||
|
||||
VOLUMES = ['docs',
|
||||
'docs.dev',
|
||||
'project.governance',
|
||||
'project.opendev',
|
||||
'project.security',
|
||||
'project.service-types',
|
||||
'project.specs',
|
||||
'project.starlingx',
|
||||
'project.zuul',
|
||||
]
|
||||
|
||||
STATSD_PREFIX='afs.release'
|
||||
|
||||
UPDATE_RE = re.compile("^\s+Last Update (.*)$")
|
||||
|
||||
log = logging.getLogger("release")
|
||||
|
||||
|
||||
def get_last_update(volume):
|
||||
ret = []
|
||||
out = subprocess.check_output(['vos', 'examine', volume],
|
||||
stderr=subprocess.STDOUT).decode('utf-8')
|
||||
state = 0
|
||||
for line in out.split('\n'):
|
||||
if state == 0 and line.startswith(volume):
|
||||
state = 1
|
||||
site = None
|
||||
elif state == 1:
|
||||
site = line.strip()
|
||||
state = 0
|
||||
m = UPDATE_RE.match(line)
|
||||
if m:
|
||||
ret.append(dict(site=site,
|
||||
volume=volume,
|
||||
updated=datetime.strptime(m.group(1),
|
||||
'%a %b %d %H:%M:%S %Y')))
|
||||
return ret
|
||||
|
||||
|
||||
def release(volume, host, key, stats):
|
||||
log.info("Releasing %s" % volume)
|
||||
|
||||
vn = volume.replace('.','_')
|
||||
|
||||
with stats.timer('%s.%s' % (STATSD_PREFIX, vn)):
|
||||
client = SSHClient()
|
||||
client.load_host_keys(key)
|
||||
client.connect(host)
|
||||
stdin, stdout, stderr = client.exec_command('vos release %s' % volume)
|
||||
for line in stdout.readlines():
|
||||
log.debug(line)
|
||||
client.close()
|
||||
logging.info("Release of %s successful" % volume)
|
||||
|
||||
|
||||
def check_release(volume):
|
||||
'''Check if a volume needs release.
|
||||
|
||||
Return TRUE if it needs to be released, FALSE if not
|
||||
'''
|
||||
log.info("Checking %s" % volume)
|
||||
rw = get_last_update(volume)[0]
|
||||
log.debug(" %s %s %s" % (rw['site'], rw['updated'], rw['volume']))
|
||||
ros = get_last_update(volume + '.readonly')
|
||||
update = False
|
||||
for ro in ros:
|
||||
log.debug(" %s %s %s" % (ro['site'], ro['updated'], ro['volume']))
|
||||
if ro['updated'] < rw['updated']:
|
||||
update = True
|
||||
if update:
|
||||
return True
|
||||
else:
|
||||
log.info("... no release required")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_lock(path):
|
||||
with open(path, 'w') as f:
|
||||
try:
|
||||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except IOError:
|
||||
print("Unable to get lockfile!")
|
||||
sys.exit(1)
|
||||
f.write("%s\n" % os.getpid())
|
||||
f.flush()
|
||||
log.debug("Acquired release lock")
|
||||
yield
|
||||
logging.debug("Release lock")
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Periodically release various OpenDev AFS volumes')
|
||||
parser.add_argument('-d', '--debug', action='store_true')
|
||||
parser.add_argument('--lockfile', action='store',
|
||||
default='/var/run/release-volumes.lock',
|
||||
help='volume release lockfile')
|
||||
parser.add_argument('--force-release', action='store_true',
|
||||
help="Force vos release, even if not required")
|
||||
parser.add_argument('--skip-release', action='store_true',
|
||||
help="Skip vos release, even if required")
|
||||
parser.add_argument('--ssh-identity', action='store',
|
||||
default='/root/.ssh/id_vos_release',
|
||||
help="SSH identify file for remote vos release")
|
||||
parser.add_argument('--ssh-server', action='store',
|
||||
default='afs01.dfw.openstack.org',
|
||||
help="Remote host to run vos release")
|
||||
parser.add_argument('--statsd-host', action='store',
|
||||
default='graphite.opendev.org',
|
||||
help='Remote host to send stats to')
|
||||
parser.add_argument('--statsd-port', action='store',
|
||||
default=8125,
|
||||
help='Remote port to send stats to')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
level = logging.DEBUG if args.debug else logging.INFO
|
||||
logging.basicConfig(level=level,
|
||||
format='%(asctime)s %(name)s '
|
||||
'%(levelname)-8s %(message)s')
|
||||
|
||||
log.debug("--- Starting %s ---" % datetime.now())
|
||||
log.debug("Sending stats to %s:%s" % (args.statsd_host, args.statsd_port))
|
||||
if args.force_release:
|
||||
log.info("Forcing release of all volumes")
|
||||
|
||||
stats = statsd.StatsClient(host=args.statsd_host,
|
||||
port=args.statsd_port)
|
||||
|
||||
with get_lock(args.lockfile):
|
||||
for volume in VOLUMES:
|
||||
if check_release(volume) or args.force_release:
|
||||
if args.skip_release:
|
||||
log.info("Force skipping release")
|
||||
else:
|
||||
release(volume, args.ssh_server, args.ssh_identity,
|
||||
stats)
|
||||
|
||||
log.debug("--- Complete %s ---" % datetime.now())
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
paramiko # LGPL 2.1
|
||||
statsd>=3.2.1 # MIT
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
- name: Ensure release script directory
|
||||
file:
|
||||
path: '/opt/afs-release'
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
|
||||
- name: Ensure log directory
|
||||
file:
|
||||
path: '/var/log/afs-release/'
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
|
||||
- name: Copy script
|
||||
copy:
|
||||
src: release-volumes.py
|
||||
dest: '/opt/afs-release'
|
||||
mode: '0755'
|
||||
|
||||
- name: Install script dependency packages
|
||||
package:
|
||||
name:
|
||||
- python3-statsd
|
||||
- python3-paramiko
|
||||
state: present
|
||||
|
||||
- name: Install release cron job
|
||||
cron:
|
||||
name: 'Release afs volumes'
|
||||
state: present
|
||||
# NOTE(ianw) : for initial testing purposes while we migrate,
|
||||
# don't actually release
|
||||
job: '/opt/afs-release/release-volumes.py -d --skip-release 2&>1 >> /var/log/afs-release/afs-release.log'
|
||||
hour: '*'
|
||||
minute: '5'
|
||||
|
||||
- name: Install logrotate rules
|
||||
include_role:
|
||||
name: logrotate
|
||||
vars:
|
||||
logrotate_file_name: '/var/log/afs-release/afs-release.log'
|
||||
|
|
@ -19,6 +19,7 @@ K5START="k5start -t -f /etc/logs.keytab service/logs-mirror --"
|
|||
RSYNC="rsync -avz --no-perms --no-owner --no-group"
|
||||
|
||||
$K5START $RSYNC /var/log/rsync-mirrors $DEST
|
||||
$K5START $RSYNC /var/log/afs-release $DEST
|
||||
|
||||
# NOTE(ianw) : r/w volume as this is infrequently accessed; thus no
|
||||
# replications and no "vos release" etc required.
|
||||
|
|
|
|||
|
|
@ -33,4 +33,8 @@
|
|||
- name: Setup rsync mirror scripts
|
||||
include_tasks: rsync.yaml
|
||||
|
||||
- name: Setup periodic AFS release script
|
||||
include_role:
|
||||
name: afs-release
|
||||
|
||||
# TODO: reprepro and other mirror components
|
||||
|
|
|
|||
|
|
@ -53,3 +53,13 @@ def test_keytabs(host):
|
|||
assert f.sha256sum == KEYTAB_SHA256
|
||||
assert f.mode == 0o400
|
||||
|
||||
def test_afs_release_script(host):
|
||||
f = host.file('/opt/afs-release/release-volumes.py')
|
||||
assert f.exists
|
||||
|
||||
def test_afs_release_script_run(host):
|
||||
# This will just run the command in a no-op mode to make sure deps
|
||||
# are installed, etc.
|
||||
cmd = host.run('/opt/afs-release/release-volumes.py '
|
||||
'--debug --skip-release')
|
||||
assert cmd.succeeded
|
||||
|
|
|
|||
Loading…
Reference in New Issue