diff --git a/nailgun/nailgun/settings.yaml b/nailgun/nailgun/settings.yaml index bc4be14196..7b48d1ef25 100644 --- a/nailgun/nailgun/settings.yaml +++ b/nailgun/nailgun/settings.yaml @@ -737,6 +737,10 @@ DUMP: path: /etc/cobbler* - type: dir path: /var/log + exclude: + - atop/ + - type: file + path: /var/log/atop/atop_current controller: hosts: [] diff --git a/shotgun/shotgun/cli.py b/shotgun/shotgun/cli.py index e7462e646b..14de225a77 100644 --- a/shotgun/shotgun/cli.py +++ b/shotgun/shotgun/cli.py @@ -40,7 +40,6 @@ def parse_args(): '--config', help='configuration file', required=True) - return parser.parse_args() diff --git a/shotgun/shotgun/driver.py b/shotgun/shotgun/driver.py index dd53b283e5..5f85cf1334 100644 --- a/shotgun/shotgun/driver.py +++ b/shotgun/shotgun/driver.py @@ -24,12 +24,8 @@ import tempfile import xmlrpclib import fabric.api -import fabric.exceptions -from shotgun.utils import CCStringIO -from shotgun.utils import execute -from shotgun.utils import is_local -from shotgun.utils import remove_matched_files +from shotgun import utils logger = logging.getLogger(__name__) @@ -67,7 +63,7 @@ class Driver(object): self.data = data self.host = self.data.get("host", {}).get("address", "localhost") self.ssh_key = self.data.get("host", {}).get("ssh-key") - self.local = is_local(self.host) + self.local = utils.is_local(self.host) self.conf = conf def snapshot(self): @@ -76,7 +72,7 @@ class Driver(object): def command(self, command): out = CommandOut() - raw_stdout = CCStringIO(writers=sys.stdout) + raw_stdout = utils.CCStringIO(writers=sys.stdout) try: if not self.local: with fabric.api.settings( @@ -96,7 +92,8 @@ class Driver(object): out.return_code = output.return_code else: logger.debug("Running local command: %s", command) - out.return_code, out.stdout, out.stderr = execute(command) + out.return_code, out.stdout, out.stderr = utils.execute( + command) except Exception as e: logger.error("Error occured: %s", str(e)) out.stdout = raw_stdout.getvalue() @@ -116,13 +113,14 @@ class Driver(object): ): logger.debug("Getting remote file: %s %s", path, target_path) - execute('mkdir -p "{0}"'.format(target_path)) + utils.execute('mkdir -p "{0}"'.format(target_path)) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) - execute('mkdir -p "{0}"'.format(target_path)) - return execute('cp -r "{0}" "{1}"'.format(path, target_path)) + utils.execute('mkdir -p "{0}"'.format(target_path)) + return utils.execute('cp -r "{0}" "{1}"'.format(path, + target_path)) except Exception as e: logger.error("Error occured: %s", str(e)) @@ -136,6 +134,9 @@ class File(Driver): self.target_path = str(os.path.join( self.conf.target, self.host, os.path.dirname(self.path).lstrip("/"))) + self.full_dst_path = os.path.join( + self.conf.target, self.host, + self.path.lstrip("/")) logger.debug("File to save: %s", self.target_path) def snapshot(self): @@ -148,8 +149,7 @@ class File(Driver): self.get(self.path, self.target_path) if self.exclude: - remove_matched_files(self.target_path, self.exclude) - + utils.remove(self.full_dst_path, self.exclude) Dir = File @@ -186,7 +186,7 @@ class Subs(File): "sed -f {0}".format(sedscript.name), self.compress(from_filename), ])) - execute(command, to_filename=to_filename) + utils.execute(command, to_filename=to_filename) sedscript.close() def snapshot(self): @@ -219,9 +219,10 @@ class Subs(File): match_orig_path = os.path.join("/", rel_tgt_host) if not fnmatch.fnmatch(match_orig_path, self.path): continue - tempfilename = execute("mktemp")[1].strip() + tempfilename = utils.execute("mktemp")[1].strip() self.sed(fullfilename, tempfilename) - execute('mv -f "{0}" "{1}"'.format(tempfilename, fullfilename)) + utils.execute('mv -f "{0}" "{1}"'.format(tempfilename, + fullfilename)) class Postgres(Driver): @@ -257,10 +258,10 @@ class Postgres(Driver): "-f {file} {dbname}".format( dbhost=self.dbhost, username=self.username, file=temp, dbname=self.dbname)) - execute('mkdir -p "{0}"'.format(self.target_path)) + utils.execute('mkdir -p "{0}"'.format(self.target_path)) dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname) - execute('mv -f "{0}" "{1}"'.format( + utils.execute('mv -f "{0}" "{1}"'.format( temp, os.path.join(self.target_path, dump_basename))) @@ -277,7 +278,8 @@ class XmlRpc(Driver): self.conf.target, self.host, "xmlrpc", self.to_file) def snapshot(self): - execute('mkdir -p "{0}"'.format(os.path.dirname(self.target_path))) + utils.execute('mkdir -p "{0}"'.format(os.path.dirname( + self.target_path))) server = xmlrpclib.Server(self.server) with open(self.target_path, "w") as f: @@ -301,7 +303,8 @@ class Command(Driver): def snapshot(self): out = self.command(self.cmdname) - execute('mkdir -p "{0}"'.format(os.path.dirname(self.target_path))) + utils.execute('mkdir -p "{0}"'.format(os.path.dirname( + self.target_path))) with open(self.target_path, "w") as f: f.write("===== COMMAND =====: {0}\n".format(self.cmdname)) f.write("===== RETURN CODE =====: {0}\n".format( diff --git a/shotgun/shotgun/test/test_driver.py b/shotgun/shotgun/test/test_driver.py index 85c1628a6c..5379c17fdc 100644 --- a/shotgun/shotgun/test/test_driver.py +++ b/shotgun/shotgun/test/test_driver.py @@ -48,7 +48,7 @@ class TestDriver(base.BaseTestCase): shotgun.driver.Driver.getDriver({"type": t}, None) mocked.assert_called_with({"type": t}, None) - @mock.patch('shotgun.driver.CCStringIO') + @mock.patch('shotgun.driver.utils.CCStringIO') @mock.patch('shotgun.driver.fabric.api.settings') @mock.patch('shotgun.driver.fabric.api.run') def test_driver_remote_command(self, mfabrun, mfabset, mccstring): @@ -74,7 +74,7 @@ class TestDriver(base.BaseTestCase): warn_only=True, key_filename=None) self.assertEqual(result, out) - @mock.patch('shotgun.driver.execute') + @mock.patch('shotgun.driver.utils.execute') def test_driver_local_command(self, mexecute): mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") @@ -86,10 +86,10 @@ class TestDriver(base.BaseTestCase): command = "COMMAND" driver = shotgun.driver.Driver({}, None) result = driver.command(command) - shotgun.driver.execute.assert_called_with(command) + shotgun.driver.utils.execute.assert_called_with(command) self.assertEqual(result, out) - @mock.patch('shotgun.driver.CCStringIO') + @mock.patch('shotgun.driver.utils.CCStringIO') @mock.patch('shotgun.driver.fabric.api.settings') @mock.patch('shotgun.driver.fabric.api.run') def test_command_timeout(self, mfabrun, mfabset, mstringio): @@ -111,7 +111,7 @@ class TestDriver(base.BaseTestCase): mfabrun.assert_called_with(command, stdout=mstdout) self.assertEqual(result.stdout, 'FULL STDOUT') - @mock.patch('shotgun.driver.execute') + @mock.patch('shotgun.driver.utils.execute') @mock.patch('shotgun.driver.fabric.api.settings') @mock.patch('shotgun.driver.fabric.api.get') def test_driver_get(self, mfabget, mfabset, mexecute): @@ -160,7 +160,7 @@ class TestFile(base.BaseTestCase): mget.assert_called_with(data["path"], target_path) - @mock.patch('shotgun.driver.remove_matched_files') + @mock.patch('shotgun.driver.utils.remove') @mock.patch('shotgun.driver.Driver.get') def test_dir_exclude_called(self, mget, mremove): data = { @@ -179,7 +179,7 @@ class TestFile(base.BaseTestCase): dir_driver.snapshot() mget.assert_called_with(data["path"], target_path) - mremove.assert_called_with(target_path, data['exclude']) + mremove.assert_called_with(dir_driver.full_dst_path, data['exclude']) class TestSubs(base.BaseTestCase): @@ -205,7 +205,7 @@ class TestSubs(base.BaseTestCase): @mock.patch('shotgun.driver.tempfile.NamedTemporaryFile') @mock.patch('shotgun.driver.Driver.get') - @mock.patch('shotgun.driver.execute') + @mock.patch('shotgun.driver.utils.execute') def test_sed(self, mexecute, mget, mntemp): mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") mntemp.return_value = self.sedscript @@ -215,23 +215,23 @@ class TestSubs(base.BaseTestCase): self.assertEqual(self.sedscript.write.mock_calls, [ mock.call("s/{0}/{1}/g\n".format(old, new)) for old, new in self.data["subs"].iteritems()]) - shotgun.driver.execute.assert_called_with( + shotgun.driver.utils.execute.assert_called_with( "cat from_file | sed -f SEDSCRIPT", to_filename="to_file") subs_driver.sed("from_file.gz", "to_file.gz") - shotgun.driver.execute.assert_called_with( + shotgun.driver.utils.execute.assert_called_with( "cat from_file.gz | gunzip -c | sed -f SEDSCRIPT | gzip -c", to_filename="to_file.gz") subs_driver.sed("from_file.bz2", "to_file.bz2") - shotgun.driver.execute.assert_called_with( + shotgun.driver.utils.execute.assert_called_with( "cat from_file.bz2 | bunzip2 -c | sed -f SEDSCRIPT | bzip2 -c", to_filename="to_file.bz2") @mock.patch('shotgun.driver.os.walk') @mock.patch('shotgun.driver.Subs.sed') @mock.patch('shotgun.driver.Driver.get') - @mock.patch('shotgun.driver.execute') + @mock.patch('shotgun.driver.utils.execute') def test_snapshot(self, mexecute, mdriverget, msed, mwalk): mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") diff --git a/shotgun/shotgun/test/test_utils.py b/shotgun/shotgun/test/test_utils.py index e7496fead3..e425b58944 100644 --- a/shotgun/shotgun/test/test_utils.py +++ b/shotgun/shotgun/test/test_utils.py @@ -14,8 +14,7 @@ import StringIO -from mock import call -from mock import patch +import mock from shotgun.test import base from shotgun import utils @@ -23,24 +22,14 @@ from shotgun import utils class TestUtils(base.BaseTestCase): - @patch('shotgun.utils.iterfiles') - @patch('shotgun.utils.os.unlink') - def test_remove_matched_files(self, munlink, mfiles): - mfiles.return_value = ['/file1.good', '/file2.bad'] - utils.remove_matched_files('/', ['*.good']) + @mock.patch('shotgun.utils.execute') + def test_remove_subdir(self, mexecute): + utils.remove('/', ['good', '**/*.py']) + mexecute.assert_has_calls([ + mock.call('shopt -s globstar; rm -rf /good', shell=True), + mock.call('shopt -s globstar; rm -rf /**/*.py', shell=True)]) - munlink.assert_called_once_with('/file1.good') - - @patch('shotgun.utils.iterfiles') - @patch('shotgun.utils.os.unlink') - def test_remove_several_files(self, munlink, mfiles): - mfiles.return_value = ['/file1.good', '/file2.good'] - utils.remove_matched_files('/', ['*.good']) - - munlink.assert_has_calls( - [call('/file1.good'), call('/file2.good')]) - - @patch('shotgun.utils.os.walk') + @mock.patch('shotgun.utils.os.walk') def test_iterfiles(self, mwalk): path = '/root' mwalk.return_value = [ @@ -53,7 +42,7 @@ class TestUtils(base.BaseTestCase): self.assertEqual( result, ['/root/file1', '/root/file2', '/root/sub/file3']) - @patch('shotgun.utils.execute') + @mock.patch('shotgun.utils.execute') def test_compress(self, mexecute): target = '/path/target' level = '-3' diff --git a/shotgun/shotgun/utils.py b/shotgun/shotgun/utils.py index d2db1475f3..0399cb2e14 100644 --- a/shotgun/shotgun/utils.py +++ b/shotgun/shotgun/utils.py @@ -13,7 +13,6 @@ # under the License. import copy -import fnmatch import logging import os import re @@ -52,18 +51,17 @@ def iterfiles(path): yield os.path.join(root, filename) -def remove_matched_files(path, patterns): - """Removes files that are matched by provided unix patterns +def remove(full_dst_path, excludes): + """Removes subdirs/files using unixs syntax. + full_dst_path is treated as root directory for remove - :param path: str - :param patterns: list with unix file patterns + :param full_dst_path: str + :param excludes: list with excludes paths/files """ - for file_path in iterfiles(path): - for pattern in patterns: - if fnmatch.fnmatch(file_path, pattern): - logger.debug('Deleting file %s', file_path) - os.unlink(file_path) - break + for exclude in excludes: + path = os.path.join(full_dst_path, exclude.lstrip('/')) + logger.debug('Deleting %s', path) + execute("shopt -s globstar; rm -rf {0}".format(path), shell=True) def compress(target, level, keep_target=False): @@ -83,7 +81,7 @@ def compress(target, level, keep_target=False): execute("rm -r {0}".format(target)) -def execute(command, to_filename=None, env=None): +def execute(command, to_filename=None, env=None, shell=False): logger.debug("Trying to execute command: %s", command) commands = [c.strip() for c in re.split(ur'\|', command)] env = env or os.environ @@ -100,15 +98,15 @@ def execute(command, to_filename=None, env=None): # We have to convert to ascii before shlex'ing the command. # http://bugs.python.org/issue6988 encoded_command = c.encode('ascii') - process.append(subprocess.Popen( - shlex.split(encoded_command), + shlex.split(encoded_command) if not shell else encoded_command, env=env, stdin=(process[-1].stdout if process else None), stdout=(to_file if (len(process) == len(commands) - 1) and to_file else subprocess.PIPE), - stderr=(subprocess.PIPE) + stderr=(subprocess.PIPE), + shell=shell )) except OSError as e: return (1, "", "{0}\n".format(e))