openstack/openstack-ansible-repo_server
Code Issues Proposed changes
openstack-ansible-repo_server/templates/lsyncd.lua.j2
Jesse Pretorius 6663637374 Remove apt-cacher-ng 
The repo container's package cache causes quite a bit of confusion
given that it's a 'hidden' feature which catches deployers off-guard
when they already have their own cache configured. This is really
the kind of service which people should manage outside of OSA. It
also makes no sense if the deployer is using their own local mirror
which is a fairly common practise. Adding to that, it seems that it
is broken in bionic, causing massive delays in package installs.
Finally, it also adds to quite a bit of complexity due to the fact
that it's in a container - so in the playbooks prior to the container's
existence we have to detect whether it's there and add/remove the config
accordingly.

Let's just remove it and let deployers managing their own caching
infrastructure if they want it.

Change-Id: I829b9cfa16fbd1f9f4d33b5943f1e46623e1b157
2018-10-08 14:48:32 +01:00

619 lines
13 KiB
Django/Jinja
Raw Blame History

--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- default-rsync.lua
--
-- Syncs with rsync ("classic" Lsyncd)
-- A (Layer 1) configuration.
--
-- License: GPLv2 (see COPYING) or any later version
-- Authors: Axel Kittenberger <axkibe@gmail.com>
-- Pre/Post Command additions: Hugh Saunders <hugh@wherenow.org>
--
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- needed for executing pre/post tasks. Can't use the lsync spawn method as
-- that requires an event, and an event isn't available for the initial
-- recursive sync.
require ('os')
if not default
then
error( 'default not loaded' )
end
local rsync = { }
default.rsync = rsync
-- uses default collect
--
-- used to ensure there aren't typos in the keys
--
rsync.checkgauge = {
-- unsets default user action handlers
onCreate = false,
onModify = false,
onDelete = false,
onStartup = false,
onMove = false,
delete = true,
exclude = true,
excludeFrom = true,
target = true,
rsync = {
acls = true,
archive = true,
binary = true,
bwlimit = true,
checksum = true,
compress = true,
copy_links = true,
cvs_exclude = true,
dry_run = true,
executability = true,
group = true,
hard_links = true,
ignore_times = true,
inplace = true,
ipv4 = true,
ipv6 = true,
keep_dirlinks = true,
links = true,
one_file_system = true,
owner = true,
password_file = true,
perms = true,
protect_args = true,
prune_empty_dirs = true,
quiet = true,
rsh = true,
rsync_path = true,
sparse = true,
temp_dir = true,
timeout = true,
times = true,
update = true,
verbose = true,
whole_file = true,
xattrs = true,
_extra = true,
precmd = true,
postcmd = true,
},
}
--
-- Execute Pre/Post Command
--
prepost = function (config, prepost)
local cmd_string = (config.rsync.rsh .. ' '
.. string.gsub(config.target, ':.*$', '')
.. ' "' .. config.rsync[prepost] .. '"' )
log('Normal', 'Executing ' .. prepost .. ': ' .. cmd_string)
os.execute(cmd_string)
end
--
-- Spawns rsync for a list of events
--
-- Exlcusions are already handled by not having
-- events for them.
--
rsync.action = function( inlet )
--
-- gets all events ready for syncing
--
local elist = inlet.getEvents(
function(event)
return event.etype ~= 'Init' and event.etype ~= 'Blanket'
end
)
--
-- Replaces what rsync would consider filter rules by literals
--
local function sub( p )
if not p then
return
end
return p:
gsub( '%?', '\\?' ):
gsub( '%*', '\\*' ):
gsub( '%[', '\\[' ):
gsub( '%]', '\\]' )
end
--
-- Gets the list of paths for the event list
--
-- Deletes create multi match patterns
--
local paths = elist.getPaths(
function( etype, path1, path2 )
if string.byte( path1, -1 ) == 47 and etype == 'Delete' then
return sub( path1 )..'***', sub( path2 )
else
return sub( path1 ), sub( path2 )
end
end
)
--
-- stores all filters by integer index
--
local filterI = { }
--
-- Stores all filters with path index
--
local filterP = { }
--
-- Adds one path to the filter
--
local function addToFilter( path )
if filterP[ path ] then
return
end
filterP[ path ] = true
table.insert( filterI, path )
end
--
-- Adds a path to the filter.
--
-- Rsync needs to have entries for all steps in the path,
-- so the file for example d1/d2/d3/f1 needs following filters:
-- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1'
--
for _, path in ipairs( paths ) do
if path and path ~= '' then
addToFilter(path)
local pp = string.match( path, '^(.*/)[^/]+/?' )
while pp do
addToFilter(pp)
pp = string.match( pp, '^(.*/)[^/]+/?' )
end
end
end
local filterS = table.concat( filterI, '\n' )
local filter0 = table.concat( filterI, '\000' )
log(
'Normal',
'Calling rsync with filter-list of new/modified files/dirs\n',
filterS
)
local config = inlet.getConfig( )
local delete = nil
if config.delete == true or config.delete == 'running' then
delete = { '--delete', '--ignore-errors' }
end
prepost(config, 'precmd')
spawn(
elist,
config.rsync.binary,
'<', filter0,
config.rsync._computed,
'-r',
delete,
'--force',
'--from0',
'--include-from=-',
'--exclude=*',
config.source,
config.target
)
end
--
-- Spawns the recursive startup sync
--
rsync.init = function(event)
local config = event.config
local inlet = event.inlet
local excludes = inlet.getExcludes( )
local delete = nil
local target = config.target
if not target then
if not config.host then
error('Internal fail, Neither target nor host is configured')
end
target = config.host .. ':' .. config.targetdir
end
if config.delete == true or config.delete == 'startup' then
delete = { '--delete', '--ignore-errors' }
end
prepost(config, 'precmd')
if #excludes == 0 then
-- start rsync without any excludes
log(
'Normal',
'recursive startup rsync: ',
config.source,
' -> ',
target
)
spawn(
event,
config.rsync.binary,
delete,
config.rsync._computed,
'-r',
config.source,
target
)
else
-- start rsync providing an exclude list
-- on stdin
local exS = table.concat( excludes, '\n' )
log(
'Normal',
'recursive startup rsync: ',
config.source,
' -> ',
target,
' excluding\n',
exS
)
spawn(
event,
config.rsync.binary,
'<', exS,
'--exclude-from=-',
delete,
config.rsync._computed,
'-r',
config.source,
target
)
end
end
--
-- Prepares and checks a syncs configuration on startup.
--
rsync.prepare =
function(
config, -- the configuration
level, -- additional error level for inherited use ( by rsyncssh )
skipTarget -- used by rsyncssh, do not check for target
)
-- First let default.prepare test the checkgauge
default.prepare( config, level + 6 )
if not skipTarget and not config.target
then
error(
'default.rsync needs "target" configured',
level
)
end
if config.rsyncOps
then
error(
'"rsyncOps" is outdated please use the new rsync = { ... } syntax.',
level
)
end
if config.rsyncOpts and config.rsync._extra
then
error(
'"rsyncOpts" is outdated in favor of the new rsync = { ... } syntax\n"' +
'for which you provided the _extra attribute as well.\n"' +
'Please remove rsyncOpts from your config.',
level
)
end
if config.rsyncOpts
then
log(
'Warn',
'"rsyncOpts" is outdated. Please use the new rsync = { ... } syntax."'
)
config.rsync._extra = config.rsyncOpts
config.rsyncOpts = nil
end
if config.rsyncBinary and config.rsync.binary
then
error(
'"rsyncBinary is outdated in favor of the new rsync = { ... } syntax\n"'+
'for which you provided the binary attribute as well.\n"' +
"Please remove rsyncBinary from your config.'",
level
)
end
if config.rsyncBinary
then
log(
'Warn',
'"rsyncBinary" is outdated. Please use the new rsync = { ... } syntax."'
)
config.rsync.binary = config.rsyncBinary
config.rsyncOpts = nil
end
-- checks if the _computed argument exists already
if config.rsync._computed
then
error(
'please do not use the internal rsync._computed parameter',
level
)
end
-- computes the rsync arguments into one list
local crsync = config.rsync;
-- everything implied by archive = true
local archiveFlags = {
recursive = true,
links = true,
perms = true,
times = true,
group = true,
owner = true,
devices = true,
specials = true,
hard_links = false,
acls = false,
xattrs = false,
}
-- if archive is given the implications are filled in
if crsync.archive
then
for k, v in pairs( archiveFlags )
do
if crsync[ k ] == nil
then
crsync[ k ] = v
end
end
end
crsync._computed = { true }
local computed = crsync._computed
local computedN = 2
local shortFlags = {
acls = 'A',
checksum = 'c',
compress = 'z',
copy_links = 'L',
cvs_exclude = 'C',
dry_run = 'n',
executability = 'E',
group = 'g',
hard_links = 'H',
ignore_times = 'I',
ipv4 = '4',
ipv6 = '6',
keep_dirlinks = 'K',
links = 'l',
one_file_system = 'x',
owner = 'o',
perms = 'p',
protect_args = 's',
prune_empty_dirs = 'm',
quiet = 'q',
sparse = 'S',
times = 't',
update = 'u',
verbose = 'v',
whole_file = 'W',
xattrs = 'X',
}
local shorts = { '-' }
local shortsN = 2
if crsync._extra
then
for k, v in ipairs( crsync._extra )
do
computed[ computedN ] = v
computedN = computedN + 1
end
end
for k, flag in pairs( shortFlags )
do
if crsync[ k ]
then
shorts[ shortsN ] = flag
shortsN = shortsN + 1
end
end
if crsync.devices and crsync.specials
then
shorts[ shortsN ] = 'D'
shortsN = shortsN + 1
else
if crsync.devices
then
computed[ computedN ] = '--devices'
computedN = computedN + 1
end
if crsync.specials
then
computed[ computedN ] = '--specials'
computedN = computedN + 1
end
end
if crsync.bwlimit
then
computed[ computedN ] = '--bwlimit=' .. crsync.bwlimit
computedN = computedN + 1
end
if crsync.inplace
then
computed[ computedN ] = '--inplace'
computedN = computedN + 1
end
if crsync.password_file
then
computed[ computedN ] = '--password-file=' .. crsync.password_file
computedN = computedN + 1
end
if crsync.rsh
then
computed[ computedN ] = '--rsh=' .. crsync.rsh
computedN = computedN + 1
end
if crsync.rsync_path
then
computed[ computedN ] = '--rsync-path=' .. crsync.rsync_path
computedN = computedN + 1
end
if crsync.temp_dir
then
computed[ computedN ] = '--temp-dir=' .. crsync.temp_dir
computedN = computedN + 1
end
if crsync.timeout
then
computed[ computedN ] = '--timeout=' .. crsync.timeout
computedN = computedN + 1
end
if shortsN ~= 2
then
computed[ 1 ] = table.concat( shorts, '' )
else
computed[ 1 ] = { }
end
-- appends a / to target if not present
if not skipTarget and string.sub(config.target, -1) ~= '/'
then
config.target = config.target..'/'
end
end
---
--- Collect events and run postcmd after successful rsync
---
rsync.collect = function(agent, exitcode)
log('Normal', 'Collecting event, exitcode: ', exitcode)
local rc = default.collect(agent, exitcode)
if rc ~= 'again'
then
prepost(agent.config, 'postcmd')
end
return rc
end
--
-- By default do deletes.
--
rsync.delete = true
--
-- Rsyncd exitcodes
--
rsync.exitcodes = default.rsyncExitCodes
--
-- Calls rsync with this default options
--
rsync.rsync =
{
-- The rsync binary to be called.
binary = '/usr/bin/rsync',
links = true,
times = true,
protect_args = true
}
--
-- Default delay
--
rsync.delay = 15
settings {
logfile = "/var/log/lsyncd/lsyncd.log",
statusFile = "/var/log/lsyncd/lsyncd-status.log",
statusInterval = 20
}
{% for node in groups['repo_all'] %}
{% if groups['repo_all'][0] != node %}
sync {
rsync,
source = "{{ repo_service_home_folder }}/repo",
target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_service_home_folder }}/repo",
exclude = {"{{ repo_git_cache_dir }}"},
rsync = {
compress = true,
acls = true,
rsh = "/usr/bin/ssh -l {{ repo_service_user_name }} -i {{ repo_service_home_folder }}/.ssh/id_rsa -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=5",
precmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh pre-www",
postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-www"
}
}
sync {
rsync,
source = "{{ repo_git_cache_dir }}",
target = "{{ hostvars[node]['ansible_host'] }}:{{ repo_git_cache_dir }}",
rsync = {
compress = true,
acls = true,
rsh = "/usr/bin/ssh -l {{ repo_service_user_name }} -i {{ repo_service_home_folder }}/.ssh/id_rsa -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=5",
precmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh pre-git",
postcmd = "sudo {{ repo_service_home_folder }}/repo/repo_prepost_cmd.sh post-git"
}
}
{% endif %}
{% endfor %}
View Git Blame Copy Permalink