diff --git a/nailgun/nailgun/settings.yaml b/nailgun/nailgun/settings.yaml index 1c5b6c0617..9c27f692b5 100644 --- a/nailgun/nailgun/settings.yaml +++ b/nailgun/nailgun/settings.yaml @@ -324,6 +324,30 @@ DUMP: slave: - type: file path: /etc/astute.yaml + - type: file + path: /root/ceph* + - type: file + path: /root/anaconda* + - type: file + path: /root/*.log + - type: file + path: /root/*.ks + - type: file + path: /etc/ceph* + - type: file + path: /etc/keystone* + - type: file + path: /etc/nova* + - type: file + path: /etc/horizon* + - type: file + path: /etc/cinder* + - type: file + path: /etc/glance* + - type: file + path: /etc/swift* + - type: file + path: /var/log/ceph - type: command command: df -h to_file: df.txt diff --git a/shotgun/run_tests.sh b/shotgun/run_tests.sh deleted file mode 100755 index b952cb6bbf..0000000000 --- a/shotgun/run_tests.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -function usage { - echo "Usage: $0 [OPTION]..." - echo "Run tests" - echo "" - echo " -p, --flake8 Just run flake8 and HACKING compliance check" - echo " -f, --fail-first Nosetests will stop on first error" - echo " -u, --unit Just run unit tests" - echo " -x, --xunit Generate reports (useful in Jenkins environment)" - echo " -P, --no-flake8 Don't run static code checks" - echo " -c, --clean Only clean *.log, *.json, *.pyc, *.pid files, doesn't run tests" - echo " -h, --help Print this usage message" - echo "" - echo "By default it runs tests and flake8 check." - exit -} - -function process_option { - case "$1" in - -h|--help) usage;; - -p|--flake8) just_flake8=1;; - -f|--fail-first) fail_first=1;; - -P|--no-flake8) no_flake8=1;; - -u|--unit) unit_tests=1;; - -x|--xunit) xunit=1;; - -c|--clean) clean=1;; - -*) noseopts="$noseopts $1";; - *) noseargs="$noseargs $1" - esac -} - -just_flake8=0 -no_flake8=0 -fail_first=0 -unit_tests=0 -xunit=0 -clean=0 -default_noseargs="" -noseargs="$default_noseargs" -noseopts= - -for arg in "$@"; do - process_option $arg -done - -function clean { - echo "cleaning *.pyc, *.json, *.log, *.pid files" - find . -type f -name "*.pyc" -delete - rm -f *.json - rm -f *.log - rm -f *.pid -} - -if [ $clean -eq 1 ]; then - clean - exit 0 -fi - -# If enabled, tell nose to create xunit report -if [ $xunit -eq 1 ]; then - noseopts=${noseopts}" --with-xunit" -fi - -if [ $fail_first -eq 1 ]; then - noseopts=${noseopts}" --stop" -fi - - -function run_flake8 { - # H302 - "import only modules. does not import a module" requires to import only modules and not functions - # H802 - first line of git commit commentary should be less than 50 characters - # __init__.py - excluded because it doesn't comply with pep8 standard - flake8 --exclude=__init__.py --ignore=H302,H802 --show-source --show-pep8 --count . || return 1 - echo "Flake8 check passed successfully." -} - -if [ $just_flake8 -eq 1 ]; then - run_flake8 || exit 1 - exit -fi - - -function run_tests { - clean - [ -z "$noseargs" ] && test_args=. || test_args="$noseargs" - stderr=$(nosetests $noseopts $test_args --verbosity=2 3>&1 1>&2 2>&3 | tee /dev/stderr) -} - - -function run_unit_tests { - noseargs="shotgun/test" - run_tests -} - -if [ $unit_tests -eq 1 ]; then - run_unit_tests || exit 1 - exit -fi - -errors='' - -run_tests || errors+=' unittests' - -if [ "$noseargs" == "$default_noseargs" ]; then - if [ $no_flake8 -eq 0 ]; then - run_flake8 || errors+=' flake8' - fi -fi - -if [ -n "$errors" ]; then - echo Failed tests: $errors - exit 1 -fi diff --git a/shotgun/shotgun/driver.py b/shotgun/shotgun/driver.py index 6000246704..da7551bc3f 100644 --- a/shotgun/shotgun/driver.py +++ b/shotgun/shotgun/driver.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import fnmatch import os import re import stat @@ -81,17 +82,21 @@ class Driver(object): return out def get(self, path, target_path): + """target_path must be the directory where to put + copied files or directories + """ try: if not self.local: with fabric.api.settings(host_string=self.host, timeout=2, warn_only=True): logger.debug("Getting remote file: %s %s", path, target_path) + execute("mkdir -p %s" % target_path) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) - execute("mkdir -p %s" % os.path.dirname(target_path)) + execute("mkdir -p %s" % target_path) return execute("cp -r %s %s" % (path, target_path)) except Exception as e: logger.error("Error occured: %s", str(e)) @@ -102,11 +107,18 @@ class File(Driver): super(File, self).__init__(data, conf) self.path = self.data["path"] logger.debug("File to get: %s", self.path) - self.target_path = os.path.join( - self.conf.target, self.host, self.path.lstrip("/")) + self.target_path = str(os.path.join( + self.conf.target, self.host, + os.path.dirname(self.path).lstrip("/"))) logger.debug("File to save: %s", self.target_path) def snapshot(self): + """Example: + self.conf.target IS /target + self.host IS host.domain.tld + self.path IS /var/log/somedir + self.target_path IS /target/host.domain.tld/var/log + """ self.get(self.path, self.target_path) @@ -149,16 +161,38 @@ class Subs(File): sedscript.close() def snapshot(self): + """Example: + self.conf.target IS /target + self.host IS host.domain.tld + self.path IS /var/log/somedir (it can be /var/log/somedir*) + self.target_path IS /target/host.domain.tld/var/log + + 1. we get remote directory host.domain.tld:/var/log/somedir + 2. we put it into /target/host.domain.tld/var/log + 3. we walk through /target/host.domain.tld/var/log + 4. we check fnmatch(/var/log/*, /var/log/somedir) + """ + # 1. + # 2. super(Subs, self).snapshot() + # 3. walk = os.walk(self.target_path) - if not os.path.isdir(self.target_path): - walk = (("/", [], [self.target_path]),) for root, _, files in walk: for filename in files: + # /target/host.domain.tld/var/log/somedir/1/2 fullfilename = os.path.join(root, filename) - tempfilename = self.command("mktemp").stdout.strip() + # 4. + # /target/host.domain.tld + tgt_host = os.path.join(self.conf.target, self.host) + # var/log/somedir/1/2 + rel_tgt_host = os.path.relpath(fullfilename, tgt_host) + # /var/log/somedir/1/2 + match_orig_path = os.path.join("/", rel_tgt_host) + if not fnmatch.fnmatch(match_orig_path, self.path): + continue + tempfilename = execute("mktemp")[1].strip() self.sed(fullfilename, tempfilename) - execute("mv %s %s" % (tempfilename, fullfilename)) + execute("mv -f %s %s" % (tempfilename, fullfilename)) class Postgres(Driver): @@ -168,6 +202,8 @@ class Postgres(Driver): self.dbname = self.data["dbname"] self.username = self.data.get("username", "postgres") self.password = self.data.get("password") + self.target_path = str(os.path.join(self.conf.target, + self.host, "pg_dump")) def snapshot(self): if self.password: @@ -191,9 +227,10 @@ class Postgres(Driver): "-f {file} {dbname}".format( dbhost=self.dbhost, username=self.username, file=temp, dbname=self.dbname)) - self.get(temp, os.path.join( - self.conf.target, self.host, "postgres_dump_%s.sql" % self.dbname)) - self.command("rm -f %s" % temp) + execute("mkdir -p %s" % self.target_path) + dump_basename = "%s_%s.sql" % (self.dbhost, self.dbname) + execute("mv -f %s %s" % + (temp, os.path.join(self.target_path, dump_basename))) class Command(Driver): diff --git a/shotgun/shotgun/test/test_driver.py b/shotgun/shotgun/test/test_driver.py index e00a51a62c..ebfe911ac9 100644 --- a/shotgun/shotgun/test/test_driver.py +++ b/shotgun/shotgun/test/test_driver.py @@ -12,16 +12,20 @@ # License for the specific language governing permissions and limitations # under the License. +import fnmatch import os try: from unittest.case import TestCase except ImportError: # Runing unit-tests in production environment from unittest2.case import TestCase +from mock import call from mock import MagicMock -from mock import Mock +from mock import patch +import shotgun.config import shotgun.driver +import shotgun.settings class RunOut(object): @@ -34,12 +38,6 @@ class RunOut(object): class TestDriver(TestCase): - def constructorMock(self, name): - instance = Mock() - instance._name_of_parent_class = name - constructor = Mock(return_value=instance) - return constructor - def test_driver_factory(self): types = { "file": "File", @@ -49,11 +47,14 @@ class TestDriver(TestCase): "command": "Command" } for t, n in types.iteritems(): - setattr(shotgun.driver, n, self.constructorMock(n)) - self.assertEquals(shotgun.driver.Driver.getDriver( - {"type": t}, None)._name_of_parent_class, n) + with patch("shotgun.driver.%s" % n) as mocked: + shotgun.driver.Driver.getDriver({"type": t}, None) + mocked.assert_called_with({"type": t}, None) - def test_driver_command(self): + @patch('shotgun.driver.execute') + @patch('shotgun.driver.fabric.api.settings') + @patch('shotgun.driver.fabric.api.run') + def test_driver_command(self, mfabrun, mfabset, mexecute): out = shotgun.driver.CommandOut() out.stdout = "STDOUT" out.return_code = "RETURN_CODE" @@ -64,41 +65,163 @@ class TestDriver(TestCase): runout.return_code = "RETURN_CODE" runout.stderr = "STDERR" - shotgun.driver.fabric.api.run = MagicMock(return_value=runout) - shotgun.driver.fabric.api.settings = MagicMock() - shotgun.driver.execute = MagicMock( - return_value=("RETURN_CODE", "STDOUT", "STDERR")) + mfabrun.return_value = runout + mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") command = "COMMAND" - driver = shotgun.driver.Driver({"host": "remote"}, None) + driver = shotgun.driver.Driver({"host": "remote_host"}, None) result = driver.command(command) shotgun.driver.fabric.api.run.assert_called_with(command, pty=True) self.assertEquals(result, out) shotgun.driver.fabric.api.settings.assert_called_with( - host_string="remote", timeout=2, warn_only=True) + host_string="remote_host", timeout=2, warn_only=True) driver = shotgun.driver.Driver({}, None) result = driver.command(command) shotgun.driver.execute.assert_called_with(command) self.assertEquals(result, out) - def test_driver_get(self): - shotgun.driver.fabric.api.get = MagicMock() - shotgun.driver.fabric.api.settings = MagicMock() - shotgun.driver.execute = MagicMock( - return_value=("RETURN_CODE", "STDOUT", "STDERR")) - path = "PATH" - target_path = "/tmp/TARGET_PATH" + @patch('shotgun.driver.execute') + @patch('shotgun.driver.fabric.api.settings') + @patch('shotgun.driver.fabric.api.get') + def test_driver_get(self, mfabget, mfabset, mexecute): + mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") + remote_path = "/remote_dir/remote_file" + target_path = "/target_dir" - driver = shotgun.driver.Driver({"host": "remote"}, None) - driver.get(path, target_path) - shotgun.driver.fabric.api.get.assert_called_with(path, target_path) - shotgun.driver.fabric.api.settings.assert_called_with( - host_string="remote", timeout=2, warn_only=True) + driver = shotgun.driver.Driver({"host": "remote_host"}, None) + driver.get(remote_path, target_path) + mexecute.assert_called_with("mkdir -p %s" % target_path) + mfabget.assert_called_with(remote_path, target_path) + mfabset.assert_called_with( + host_string="remote_host", timeout=2, warn_only=True) + mexecute.reset_mock() driver = shotgun.driver.Driver({}, None) - driver.get(path, target_path) - shotgun.driver.execute.assert_any_call( - "mkdir -p %s" % os.path.dirname(target_path)) - shotgun.driver.execute.assert_any_call( - "cp -r %s %s" % (path, target_path)) + driver.get(remote_path, target_path) + assert mexecute.mock_calls == [ + call("mkdir -p %s" % target_path), + call("cp -r %s %s" % (remote_path, target_path)) + ] + + +class TestFile(TestCase): + + @patch('shotgun.driver.Driver.get') + def test_snapshot(self, mget): + data = { + "type": "file", + "path": "/remote_dir/remote_file", + "host": "remote_host" + } + conf = MagicMock() + conf.target = "/target" + file_driver = shotgun.driver.File(data, conf) + + target_path = "/target/remote_host/remote_dir" + file_driver.snapshot() + + mget.assert_called_with(data["path"], target_path) + + +class TestSubs(TestCase): + def setUp(self): + self.data = { + "type": "subs", + "path": "/remote_dir/remote_file", + "host": "remote_host", + "subs": { + "line0": "LINE0", + "line1": "LINE1" + } + } + + self.conf = MagicMock() + self.conf.target = "/target" + + self.sedscript = MagicMock() + self.sedscript.name = "SEDSCRIPT" + self.sedscript.write = MagicMock() + + @patch('shotgun.driver.tempfile.NamedTemporaryFile') + @patch('shotgun.driver.Driver.get') + @patch('shotgun.driver.execute') + def test_sed(self, mexecute, mget, mntemp): + mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") + mntemp.return_value = self.sedscript + + subs_driver = shotgun.driver.Subs(self.data, self.conf) + subs_driver.sed("from_file", "to_file") + assert self.sedscript.write.mock_calls == [ + call("s/%s/%s/g\n" % (old, new)) + for old, new in self.data["subs"].iteritems()] + shotgun.driver.execute.assert_called_with( + "cat from_file | sed -f SEDSCRIPT", to_filename="to_file") + + subs_driver.sed("from_file.gz", "to_file.gz") + shotgun.driver.execute.assert_called_with( + "cat from_file.gz | gunzip -c | sed -f SEDSCRIPT | gzip -c", + to_filename="to_file.gz") + + subs_driver.sed("from_file.bz2", "to_file.bz2") + shotgun.driver.execute.assert_called_with( + "cat from_file.bz2 | bunzip2 -c | sed -f SEDSCRIPT | bzip2 -c", + to_filename="to_file.bz2") + + @patch('shotgun.driver.os.walk') + @patch('shotgun.driver.Subs.sed') + @patch('shotgun.driver.Driver.get') + @patch('shotgun.driver.execute') + def test_snapshot(self, mexecute, mdriverget, msed, mwalk): + mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") + + """ 1. Should get remote (or local) file (or directory) + 2. Should put it into /target/host.domain.tld + 3. Should walk through and check if files match given path pattern + 4. If matched, sed them + """ + + """this return_value corresponds to the following structure + /target/remote_host/remote_dir/ + /target/remote_host/remote_dir/remote_file + /target/remote_host/remote_dir/1 + /target/remote_host/remote_dir/2 + /target/remote_host/remote_dir/3/ + /target/remote_host/remote_dir/3/4 + /target/remote_host/remote_dir/3/5 + /target/remote_host/remote_dir/3/6/ + """ + mock_walk = [ + ( + '/target/remote_host/remote_dir', + ['3'], + ['1', '2', 'remote_file'] + ), + ('/target/remote_host/remote_dir/3', ['6'], ['5', '4']), + ('/target/remote_host/remote_dir/3/6', [], []) + ] + mwalk.return_value = mock_walk + + subs_driver = shotgun.driver.Subs(self.data, self.conf) + subs_driver.snapshot() + + sed_calls = [] + execute_calls = [] + for root, _, files in mock_walk: + for filename in files: + fullfilename = os.path.join(root, filename) + # /target/remote_host + tgt_host = os.path.join(self.conf.target, self.data["host"]) + rel_tgt_host = os.path.relpath(fullfilename, tgt_host) + # /remote_dir/remote_file + match_orig_path = os.path.join("/", rel_tgt_host) + if not fnmatch.fnmatch(match_orig_path, self.data["path"]): + continue + tempfilename = "STDOUT" + execute_calls.append(call("mktemp")) + sed_calls.append(call(fullfilename, tempfilename)) + execute_calls.append( + call("mv %s %s" % (tempfilename, fullfilename))) + + assert msed.mock_calls == sed_calls + assert mexecute.mock_calls == execute_calls