Add possibilities to skip subdirs in snapshots
* added tests * modified snapshot settings to exclude files from atop and copy only atop_current file * add posibilities to skip dirs and files using unix syntax now files can be skiped by construction eg **/*.gz when we want to skip all gz. files in all subdirs Change-Id: I73fdc932d53d75bafd7386555f95e88536d29a25 Closes-Bug: #1429112 DocImpact
This commit is contained in:
Kamil Sambor
parent
508c652f2e
commit
1ae695b968
@ -737,6 +737,10 @@ DUMP:
|
||||
path: /etc/cobbler*
|
||||
- type: dir
|
||||
path: /var/log
|
||||
exclude:
|
||||
- atop/
|
||||
- type: file
|
||||
path: /var/log/atop/atop_current
|
||||
|
||||
controller:
|
||||
hosts: []
|
||||
|
||||
@ -40,7 +40,6 @@ def parse_args():
|
||||
'--config',
|
||||
help='configuration file',
|
||||
required=True)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@ -24,12 +24,8 @@ import tempfile
|
||||
import xmlrpclib
|
||||
|
||||
import fabric.api
|
||||
import fabric.exceptions
|
||||
|
||||
from shotgun.utils import CCStringIO
|
||||
from shotgun.utils import execute
|
||||
from shotgun.utils import is_local
|
||||
from shotgun.utils import remove_matched_files
|
||||
from shotgun import utils
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -67,7 +63,7 @@ class Driver(object):
|
||||
self.data = data
|
||||
self.host = self.data.get("host", {}).get("address", "localhost")
|
||||
self.ssh_key = self.data.get("host", {}).get("ssh-key")
|
||||
self.local = is_local(self.host)
|
||||
self.local = utils.is_local(self.host)
|
||||
self.conf = conf
|
||||
|
||||
def snapshot(self):
|
||||
@ -76,7 +72,7 @@ class Driver(object):
|
||||
def command(self, command):
|
||||
out = CommandOut()
|
||||
|
||||
raw_stdout = CCStringIO(writers=sys.stdout)
|
||||
raw_stdout = utils.CCStringIO(writers=sys.stdout)
|
||||
try:
|
||||
if not self.local:
|
||||
with fabric.api.settings(
|
||||
@ -96,7 +92,8 @@ class Driver(object):
|
||||
out.return_code = output.return_code
|
||||
else:
|
||||
logger.debug("Running local command: %s", command)
|
||||
out.return_code, out.stdout, out.stderr = execute(command)
|
||||
out.return_code, out.stdout, out.stderr = utils.execute(
|
||||
command)
|
||||
except Exception as e:
|
||||
logger.error("Error occured: %s", str(e))
|
||||
out.stdout = raw_stdout.getvalue()
|
||||
@ -116,13 +113,14 @@ class Driver(object):
|
||||
):
|
||||
logger.debug("Getting remote file: %s %s",
|
||||
path, target_path)
|
||||
execute('mkdir -p "{0}"'.format(target_path))
|
||||
utils.execute('mkdir -p "{0}"'.format(target_path))
|
||||
return fabric.api.get(path, target_path)
|
||||
else:
|
||||
logger.debug("Getting local file: cp -r %s %s",
|
||||
path, target_path)
|
||||
execute('mkdir -p "{0}"'.format(target_path))
|
||||
return execute('cp -r "{0}" "{1}"'.format(path, target_path))
|
||||
utils.execute('mkdir -p "{0}"'.format(target_path))
|
||||
return utils.execute('cp -r "{0}" "{1}"'.format(path,
|
||||
target_path))
|
||||
except Exception as e:
|
||||
logger.error("Error occured: %s", str(e))
|
||||
|
||||
@ -136,6 +134,9 @@ class File(Driver):
|
||||
self.target_path = str(os.path.join(
|
||||
self.conf.target, self.host,
|
||||
os.path.dirname(self.path).lstrip("/")))
|
||||
self.full_dst_path = os.path.join(
|
||||
self.conf.target, self.host,
|
||||
self.path.lstrip("/"))
|
||||
logger.debug("File to save: %s", self.target_path)
|
||||
|
||||
def snapshot(self):
|
||||
@ -148,8 +149,7 @@ class File(Driver):
|
||||
self.get(self.path, self.target_path)
|
||||
|
||||
if self.exclude:
|
||||
remove_matched_files(self.target_path, self.exclude)
|
||||
|
||||
utils.remove(self.full_dst_path, self.exclude)
|
||||
|
||||
Dir = File
|
||||
|
||||
@ -186,7 +186,7 @@ class Subs(File):
|
||||
"sed -f {0}".format(sedscript.name),
|
||||
self.compress(from_filename),
|
||||
]))
|
||||
execute(command, to_filename=to_filename)
|
||||
utils.execute(command, to_filename=to_filename)
|
||||
sedscript.close()
|
||||
|
||||
def snapshot(self):
|
||||
@ -219,9 +219,10 @@ class Subs(File):
|
||||
match_orig_path = os.path.join("/", rel_tgt_host)
|
||||
if not fnmatch.fnmatch(match_orig_path, self.path):
|
||||
continue
|
||||
tempfilename = execute("mktemp")[1].strip()
|
||||
tempfilename = utils.execute("mktemp")[1].strip()
|
||||
self.sed(fullfilename, tempfilename)
|
||||
execute('mv -f "{0}" "{1}"'.format(tempfilename, fullfilename))
|
||||
utils.execute('mv -f "{0}" "{1}"'.format(tempfilename,
|
||||
fullfilename))
|
||||
|
||||
|
||||
class Postgres(Driver):
|
||||
@ -257,10 +258,10 @@ class Postgres(Driver):
|
||||
"-f {file} {dbname}".format(
|
||||
dbhost=self.dbhost, username=self.username,
|
||||
file=temp, dbname=self.dbname))
|
||||
execute('mkdir -p "{0}"'.format(self.target_path))
|
||||
utils.execute('mkdir -p "{0}"'.format(self.target_path))
|
||||
dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname)
|
||||
|
||||
execute('mv -f "{0}" "{1}"'.format(
|
||||
utils.execute('mv -f "{0}" "{1}"'.format(
|
||||
temp,
|
||||
os.path.join(self.target_path, dump_basename)))
|
||||
|
||||
@ -277,7 +278,8 @@ class XmlRpc(Driver):
|
||||
self.conf.target, self.host, "xmlrpc", self.to_file)
|
||||
|
||||
def snapshot(self):
|
||||
execute('mkdir -p "{0}"'.format(os.path.dirname(self.target_path)))
|
||||
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
|
||||
self.target_path)))
|
||||
|
||||
server = xmlrpclib.Server(self.server)
|
||||
with open(self.target_path, "w") as f:
|
||||
@ -301,7 +303,8 @@ class Command(Driver):
|
||||
|
||||
def snapshot(self):
|
||||
out = self.command(self.cmdname)
|
||||
execute('mkdir -p "{0}"'.format(os.path.dirname(self.target_path)))
|
||||
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
|
||||
self.target_path)))
|
||||
with open(self.target_path, "w") as f:
|
||||
f.write("===== COMMAND =====: {0}\n".format(self.cmdname))
|
||||
f.write("===== RETURN CODE =====: {0}\n".format(
|
||||
|
||||
@ -48,7 +48,7 @@ class TestDriver(base.BaseTestCase):
|
||||
shotgun.driver.Driver.getDriver({"type": t}, None)
|
||||
mocked.assert_called_with({"type": t}, None)
|
||||
|
||||
@mock.patch('shotgun.driver.CCStringIO')
|
||||
@mock.patch('shotgun.driver.utils.CCStringIO')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.run')
|
||||
def test_driver_remote_command(self, mfabrun, mfabset, mccstring):
|
||||
@ -74,7 +74,7 @@ class TestDriver(base.BaseTestCase):
|
||||
warn_only=True, key_filename=None)
|
||||
self.assertEqual(result, out)
|
||||
|
||||
@mock.patch('shotgun.driver.execute')
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
def test_driver_local_command(self, mexecute):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
|
||||
@ -86,10 +86,10 @@ class TestDriver(base.BaseTestCase):
|
||||
command = "COMMAND"
|
||||
driver = shotgun.driver.Driver({}, None)
|
||||
result = driver.command(command)
|
||||
shotgun.driver.execute.assert_called_with(command)
|
||||
shotgun.driver.utils.execute.assert_called_with(command)
|
||||
self.assertEqual(result, out)
|
||||
|
||||
@mock.patch('shotgun.driver.CCStringIO')
|
||||
@mock.patch('shotgun.driver.utils.CCStringIO')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.run')
|
||||
def test_command_timeout(self, mfabrun, mfabset, mstringio):
|
||||
@ -111,7 +111,7 @@ class TestDriver(base.BaseTestCase):
|
||||
mfabrun.assert_called_with(command, stdout=mstdout)
|
||||
self.assertEqual(result.stdout, 'FULL STDOUT')
|
||||
|
||||
@mock.patch('shotgun.driver.execute')
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.get')
|
||||
def test_driver_get(self, mfabget, mfabset, mexecute):
|
||||
@ -160,7 +160,7 @@ class TestFile(base.BaseTestCase):
|
||||
|
||||
mget.assert_called_with(data["path"], target_path)
|
||||
|
||||
@mock.patch('shotgun.driver.remove_matched_files')
|
||||
@mock.patch('shotgun.driver.utils.remove')
|
||||
@mock.patch('shotgun.driver.Driver.get')
|
||||
def test_dir_exclude_called(self, mget, mremove):
|
||||
data = {
|
||||
@ -179,7 +179,7 @@ class TestFile(base.BaseTestCase):
|
||||
dir_driver.snapshot()
|
||||
|
||||
mget.assert_called_with(data["path"], target_path)
|
||||
mremove.assert_called_with(target_path, data['exclude'])
|
||||
mremove.assert_called_with(dir_driver.full_dst_path, data['exclude'])
|
||||
|
||||
|
||||
class TestSubs(base.BaseTestCase):
|
||||
@ -205,7 +205,7 @@ class TestSubs(base.BaseTestCase):
|
||||
|
||||
@mock.patch('shotgun.driver.tempfile.NamedTemporaryFile')
|
||||
@mock.patch('shotgun.driver.Driver.get')
|
||||
@mock.patch('shotgun.driver.execute')
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
def test_sed(self, mexecute, mget, mntemp):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
mntemp.return_value = self.sedscript
|
||||
@ -215,23 +215,23 @@ class TestSubs(base.BaseTestCase):
|
||||
self.assertEqual(self.sedscript.write.mock_calls, [
|
||||
mock.call("s/{0}/{1}/g\n".format(old, new))
|
||||
for old, new in self.data["subs"].iteritems()])
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
shotgun.driver.utils.execute.assert_called_with(
|
||||
"cat from_file | sed -f SEDSCRIPT", to_filename="to_file")
|
||||
|
||||
subs_driver.sed("from_file.gz", "to_file.gz")
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
shotgun.driver.utils.execute.assert_called_with(
|
||||
"cat from_file.gz | gunzip -c | sed -f SEDSCRIPT | gzip -c",
|
||||
to_filename="to_file.gz")
|
||||
|
||||
subs_driver.sed("from_file.bz2", "to_file.bz2")
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
shotgun.driver.utils.execute.assert_called_with(
|
||||
"cat from_file.bz2 | bunzip2 -c | sed -f SEDSCRIPT | bzip2 -c",
|
||||
to_filename="to_file.bz2")
|
||||
|
||||
@mock.patch('shotgun.driver.os.walk')
|
||||
@mock.patch('shotgun.driver.Subs.sed')
|
||||
@mock.patch('shotgun.driver.Driver.get')
|
||||
@mock.patch('shotgun.driver.execute')
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
def test_snapshot(self, mexecute, mdriverget, msed, mwalk):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
|
||||
|
||||
@ -14,8 +14,7 @@
|
||||
|
||||
import StringIO
|
||||
|
||||
from mock import call
|
||||
from mock import patch
|
||||
import mock
|
||||
|
||||
from shotgun.test import base
|
||||
from shotgun import utils
|
||||
@ -23,24 +22,14 @@ from shotgun import utils
|
||||
|
||||
class TestUtils(base.BaseTestCase):
|
||||
|
||||
@patch('shotgun.utils.iterfiles')
|
||||
@patch('shotgun.utils.os.unlink')
|
||||
def test_remove_matched_files(self, munlink, mfiles):
|
||||
mfiles.return_value = ['/file1.good', '/file2.bad']
|
||||
utils.remove_matched_files('/', ['*.good'])
|
||||
@mock.patch('shotgun.utils.execute')
|
||||
def test_remove_subdir(self, mexecute):
|
||||
utils.remove('/', ['good', '**/*.py'])
|
||||
mexecute.assert_has_calls([
|
||||
mock.call('shopt -s globstar; rm -rf /good', shell=True),
|
||||
mock.call('shopt -s globstar; rm -rf /**/*.py', shell=True)])
|
||||
|
||||
munlink.assert_called_once_with('/file1.good')
|
||||
|
||||
@patch('shotgun.utils.iterfiles')
|
||||
@patch('shotgun.utils.os.unlink')
|
||||
def test_remove_several_files(self, munlink, mfiles):
|
||||
mfiles.return_value = ['/file1.good', '/file2.good']
|
||||
utils.remove_matched_files('/', ['*.good'])
|
||||
|
||||
munlink.assert_has_calls(
|
||||
[call('/file1.good'), call('/file2.good')])
|
||||
|
||||
@patch('shotgun.utils.os.walk')
|
||||
@mock.patch('shotgun.utils.os.walk')
|
||||
def test_iterfiles(self, mwalk):
|
||||
path = '/root'
|
||||
mwalk.return_value = [
|
||||
@ -53,7 +42,7 @@ class TestUtils(base.BaseTestCase):
|
||||
self.assertEqual(
|
||||
result, ['/root/file1', '/root/file2', '/root/sub/file3'])
|
||||
|
||||
@patch('shotgun.utils.execute')
|
||||
@mock.patch('shotgun.utils.execute')
|
||||
def test_compress(self, mexecute):
|
||||
target = '/path/target'
|
||||
level = '-3'
|
||||
|
||||
@ -13,7 +13,6 @@
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@ -52,18 +51,17 @@ def iterfiles(path):
|
||||
yield os.path.join(root, filename)
|
||||
|
||||
|
||||
def remove_matched_files(path, patterns):
|
||||
"""Removes files that are matched by provided unix patterns
|
||||
def remove(full_dst_path, excludes):
|
||||
"""Removes subdirs/files using unixs syntax.
|
||||
full_dst_path is treated as root directory for remove
|
||||
|
||||
:param path: str
|
||||
:param patterns: list with unix file patterns
|
||||
:param full_dst_path: str
|
||||
:param excludes: list with excludes paths/files
|
||||
"""
|
||||
for file_path in iterfiles(path):
|
||||
for pattern in patterns:
|
||||
if fnmatch.fnmatch(file_path, pattern):
|
||||
logger.debug('Deleting file %s', file_path)
|
||||
os.unlink(file_path)
|
||||
break
|
||||
for exclude in excludes:
|
||||
path = os.path.join(full_dst_path, exclude.lstrip('/'))
|
||||
logger.debug('Deleting %s', path)
|
||||
execute("shopt -s globstar; rm -rf {0}".format(path), shell=True)
|
||||
|
||||
|
||||
def compress(target, level, keep_target=False):
|
||||
@ -83,7 +81,7 @@ def compress(target, level, keep_target=False):
|
||||
execute("rm -r {0}".format(target))
|
||||
|
||||
|
||||
def execute(command, to_filename=None, env=None):
|
||||
def execute(command, to_filename=None, env=None, shell=False):
|
||||
logger.debug("Trying to execute command: %s", command)
|
||||
commands = [c.strip() for c in re.split(ur'\|', command)]
|
||||
env = env or os.environ
|
||||
@ -100,15 +98,15 @@ def execute(command, to_filename=None, env=None):
|
||||
# We have to convert to ascii before shlex'ing the command.
|
||||
# http://bugs.python.org/issue6988
|
||||
encoded_command = c.encode('ascii')
|
||||
|
||||
process.append(subprocess.Popen(
|
||||
shlex.split(encoded_command),
|
||||
shlex.split(encoded_command) if not shell else encoded_command,
|
||||
env=env,
|
||||
stdin=(process[-1].stdout if process else None),
|
||||
stdout=(to_file
|
||||
if (len(process) == len(commands) - 1) and to_file
|
||||
else subprocess.PIPE),
|
||||
stderr=(subprocess.PIPE)
|
||||
stderr=(subprocess.PIPE),
|
||||
shell=shell
|
||||
))
|
||||
except OSError as e:
|
||||
return (1, "", "{0}\n".format(e))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user