Modified some methods a bit and added a couple of tests
Added snapshot files into settings.yaml Change-Id: Id1a53503483c85e4effbed3ada180c787f686eb1
This commit is contained in:
Vladimir Kozhukalov
parent
86a0f13dfc
commit
378fe4f3d4
|
|
@ -324,6 +324,30 @@ DUMP:
|
|||
slave:
|
||||
- type: file
|
||||
path: /etc/astute.yaml
|
||||
- type: file
|
||||
path: /root/ceph*
|
||||
- type: file
|
||||
path: /root/anaconda*
|
||||
- type: file
|
||||
path: /root/*.log
|
||||
- type: file
|
||||
path: /root/*.ks
|
||||
- type: file
|
||||
path: /etc/ceph*
|
||||
- type: file
|
||||
path: /etc/keystone*
|
||||
- type: file
|
||||
path: /etc/nova*
|
||||
- type: file
|
||||
path: /etc/horizon*
|
||||
- type: file
|
||||
path: /etc/cinder*
|
||||
- type: file
|
||||
path: /etc/glance*
|
||||
- type: file
|
||||
path: /etc/swift*
|
||||
- type: file
|
||||
path: /var/log/ceph
|
||||
- type: command
|
||||
command: df -h
|
||||
to_file: df.txt
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
function usage {
|
||||
echo "Usage: $0 [OPTION]..."
|
||||
echo "Run tests"
|
||||
echo ""
|
||||
echo " -p, --flake8 Just run flake8 and HACKING compliance check"
|
||||
echo " -f, --fail-first Nosetests will stop on first error"
|
||||
echo " -u, --unit Just run unit tests"
|
||||
echo " -x, --xunit Generate reports (useful in Jenkins environment)"
|
||||
echo " -P, --no-flake8 Don't run static code checks"
|
||||
echo " -c, --clean Only clean *.log, *.json, *.pyc, *.pid files, doesn't run tests"
|
||||
echo " -h, --help Print this usage message"
|
||||
echo ""
|
||||
echo "By default it runs tests and flake8 check."
|
||||
exit
|
||||
}
|
||||
|
||||
function process_option {
|
||||
case "$1" in
|
||||
-h|--help) usage;;
|
||||
-p|--flake8) just_flake8=1;;
|
||||
-f|--fail-first) fail_first=1;;
|
||||
-P|--no-flake8) no_flake8=1;;
|
||||
-u|--unit) unit_tests=1;;
|
||||
-x|--xunit) xunit=1;;
|
||||
-c|--clean) clean=1;;
|
||||
-*) noseopts="$noseopts $1";;
|
||||
*) noseargs="$noseargs $1"
|
||||
esac
|
||||
}
|
||||
|
||||
just_flake8=0
|
||||
no_flake8=0
|
||||
fail_first=0
|
||||
unit_tests=0
|
||||
xunit=0
|
||||
clean=0
|
||||
default_noseargs=""
|
||||
noseargs="$default_noseargs"
|
||||
noseopts=
|
||||
|
||||
for arg in "$@"; do
|
||||
process_option $arg
|
||||
done
|
||||
|
||||
function clean {
|
||||
echo "cleaning *.pyc, *.json, *.log, *.pid files"
|
||||
find . -type f -name "*.pyc" -delete
|
||||
rm -f *.json
|
||||
rm -f *.log
|
||||
rm -f *.pid
|
||||
}
|
||||
|
||||
if [ $clean -eq 1 ]; then
|
||||
clean
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If enabled, tell nose to create xunit report
|
||||
if [ $xunit -eq 1 ]; then
|
||||
noseopts=${noseopts}" --with-xunit"
|
||||
fi
|
||||
|
||||
if [ $fail_first -eq 1 ]; then
|
||||
noseopts=${noseopts}" --stop"
|
||||
fi
|
||||
|
||||
|
||||
function run_flake8 {
|
||||
# H302 - "import only modules. does not import a module" requires to import only modules and not functions
|
||||
# H802 - first line of git commit commentary should be less than 50 characters
|
||||
# __init__.py - excluded because it doesn't comply with pep8 standard
|
||||
flake8 --exclude=__init__.py --ignore=H302,H802 --show-source --show-pep8 --count . || return 1
|
||||
echo "Flake8 check passed successfully."
|
||||
}
|
||||
|
||||
if [ $just_flake8 -eq 1 ]; then
|
||||
run_flake8 || exit 1
|
||||
exit
|
||||
fi
|
||||
|
||||
|
||||
function run_tests {
|
||||
clean
|
||||
[ -z "$noseargs" ] && test_args=. || test_args="$noseargs"
|
||||
stderr=$(nosetests $noseopts $test_args --verbosity=2 3>&1 1>&2 2>&3 | tee /dev/stderr)
|
||||
}
|
||||
|
||||
|
||||
function run_unit_tests {
|
||||
noseargs="shotgun/test"
|
||||
run_tests
|
||||
}
|
||||
|
||||
if [ $unit_tests -eq 1 ]; then
|
||||
run_unit_tests || exit 1
|
||||
exit
|
||||
fi
|
||||
|
||||
errors=''
|
||||
|
||||
run_tests || errors+=' unittests'
|
||||
|
||||
if [ "$noseargs" == "$default_noseargs" ]; then
|
||||
if [ $no_flake8 -eq 0 ]; then
|
||||
run_flake8 || errors+=' flake8'
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$errors" ]; then
|
||||
echo Failed tests: $errors
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -12,6 +12,7 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import fnmatch
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
|
|
@ -81,17 +82,21 @@ class Driver(object):
|
|||
return out
|
||||
|
||||
def get(self, path, target_path):
|
||||
"""target_path must be the directory where to put
|
||||
copied files or directories
|
||||
"""
|
||||
try:
|
||||
if not self.local:
|
||||
with fabric.api.settings(host_string=self.host,
|
||||
timeout=2, warn_only=True):
|
||||
logger.debug("Getting remote file: %s %s",
|
||||
path, target_path)
|
||||
execute("mkdir -p %s" % target_path)
|
||||
return fabric.api.get(path, target_path)
|
||||
else:
|
||||
logger.debug("Getting local file: cp -r %s %s",
|
||||
path, target_path)
|
||||
execute("mkdir -p %s" % os.path.dirname(target_path))
|
||||
execute("mkdir -p %s" % target_path)
|
||||
return execute("cp -r %s %s" % (path, target_path))
|
||||
except Exception as e:
|
||||
logger.error("Error occured: %s", str(e))
|
||||
|
|
@ -102,11 +107,18 @@ class File(Driver):
|
|||
super(File, self).__init__(data, conf)
|
||||
self.path = self.data["path"]
|
||||
logger.debug("File to get: %s", self.path)
|
||||
self.target_path = os.path.join(
|
||||
self.conf.target, self.host, self.path.lstrip("/"))
|
||||
self.target_path = str(os.path.join(
|
||||
self.conf.target, self.host,
|
||||
os.path.dirname(self.path).lstrip("/")))
|
||||
logger.debug("File to save: %s", self.target_path)
|
||||
|
||||
def snapshot(self):
|
||||
"""Example:
|
||||
self.conf.target IS /target
|
||||
self.host IS host.domain.tld
|
||||
self.path IS /var/log/somedir
|
||||
self.target_path IS /target/host.domain.tld/var/log
|
||||
"""
|
||||
self.get(self.path, self.target_path)
|
||||
|
||||
|
||||
|
|
@ -149,16 +161,38 @@ class Subs(File):
|
|||
sedscript.close()
|
||||
|
||||
def snapshot(self):
|
||||
"""Example:
|
||||
self.conf.target IS /target
|
||||
self.host IS host.domain.tld
|
||||
self.path IS /var/log/somedir (it can be /var/log/somedir*)
|
||||
self.target_path IS /target/host.domain.tld/var/log
|
||||
|
||||
1. we get remote directory host.domain.tld:/var/log/somedir
|
||||
2. we put it into /target/host.domain.tld/var/log
|
||||
3. we walk through /target/host.domain.tld/var/log
|
||||
4. we check fnmatch(/var/log/*, /var/log/somedir)
|
||||
"""
|
||||
# 1.
|
||||
# 2.
|
||||
super(Subs, self).snapshot()
|
||||
# 3.
|
||||
walk = os.walk(self.target_path)
|
||||
if not os.path.isdir(self.target_path):
|
||||
walk = (("/", [], [self.target_path]),)
|
||||
for root, _, files in walk:
|
||||
for filename in files:
|
||||
# /target/host.domain.tld/var/log/somedir/1/2
|
||||
fullfilename = os.path.join(root, filename)
|
||||
tempfilename = self.command("mktemp").stdout.strip()
|
||||
# 4.
|
||||
# /target/host.domain.tld
|
||||
tgt_host = os.path.join(self.conf.target, self.host)
|
||||
# var/log/somedir/1/2
|
||||
rel_tgt_host = os.path.relpath(fullfilename, tgt_host)
|
||||
# /var/log/somedir/1/2
|
||||
match_orig_path = os.path.join("/", rel_tgt_host)
|
||||
if not fnmatch.fnmatch(match_orig_path, self.path):
|
||||
continue
|
||||
tempfilename = execute("mktemp")[1].strip()
|
||||
self.sed(fullfilename, tempfilename)
|
||||
execute("mv %s %s" % (tempfilename, fullfilename))
|
||||
execute("mv -f %s %s" % (tempfilename, fullfilename))
|
||||
|
||||
|
||||
class Postgres(Driver):
|
||||
|
|
@ -168,6 +202,8 @@ class Postgres(Driver):
|
|||
self.dbname = self.data["dbname"]
|
||||
self.username = self.data.get("username", "postgres")
|
||||
self.password = self.data.get("password")
|
||||
self.target_path = str(os.path.join(self.conf.target,
|
||||
self.host, "pg_dump"))
|
||||
|
||||
def snapshot(self):
|
||||
if self.password:
|
||||
|
|
@ -191,9 +227,10 @@ class Postgres(Driver):
|
|||
"-f {file} {dbname}".format(
|
||||
dbhost=self.dbhost, username=self.username,
|
||||
file=temp, dbname=self.dbname))
|
||||
self.get(temp, os.path.join(
|
||||
self.conf.target, self.host, "postgres_dump_%s.sql" % self.dbname))
|
||||
self.command("rm -f %s" % temp)
|
||||
execute("mkdir -p %s" % self.target_path)
|
||||
dump_basename = "%s_%s.sql" % (self.dbhost, self.dbname)
|
||||
execute("mv -f %s %s" %
|
||||
(temp, os.path.join(self.target_path, dump_basename)))
|
||||
|
||||
|
||||
class Command(Driver):
|
||||
|
|
|
|||
|
|
@ -12,16 +12,20 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import fnmatch
|
||||
import os
|
||||
try:
|
||||
from unittest.case import TestCase
|
||||
except ImportError:
|
||||
# Runing unit-tests in production environment
|
||||
from unittest2.case import TestCase
|
||||
from mock import call
|
||||
from mock import MagicMock
|
||||
from mock import Mock
|
||||
from mock import patch
|
||||
|
||||
import shotgun.config
|
||||
import shotgun.driver
|
||||
import shotgun.settings
|
||||
|
||||
|
||||
class RunOut(object):
|
||||
|
|
@ -34,12 +38,6 @@ class RunOut(object):
|
|||
|
||||
|
||||
class TestDriver(TestCase):
|
||||
def constructorMock(self, name):
|
||||
instance = Mock()
|
||||
instance._name_of_parent_class = name
|
||||
constructor = Mock(return_value=instance)
|
||||
return constructor
|
||||
|
||||
def test_driver_factory(self):
|
||||
types = {
|
||||
"file": "File",
|
||||
|
|
@ -49,11 +47,14 @@ class TestDriver(TestCase):
|
|||
"command": "Command"
|
||||
}
|
||||
for t, n in types.iteritems():
|
||||
setattr(shotgun.driver, n, self.constructorMock(n))
|
||||
self.assertEquals(shotgun.driver.Driver.getDriver(
|
||||
{"type": t}, None)._name_of_parent_class, n)
|
||||
with patch("shotgun.driver.%s" % n) as mocked:
|
||||
shotgun.driver.Driver.getDriver({"type": t}, None)
|
||||
mocked.assert_called_with({"type": t}, None)
|
||||
|
||||
def test_driver_command(self):
|
||||
@patch('shotgun.driver.execute')
|
||||
@patch('shotgun.driver.fabric.api.settings')
|
||||
@patch('shotgun.driver.fabric.api.run')
|
||||
def test_driver_command(self, mfabrun, mfabset, mexecute):
|
||||
out = shotgun.driver.CommandOut()
|
||||
out.stdout = "STDOUT"
|
||||
out.return_code = "RETURN_CODE"
|
||||
|
|
@ -64,41 +65,163 @@ class TestDriver(TestCase):
|
|||
runout.return_code = "RETURN_CODE"
|
||||
runout.stderr = "STDERR"
|
||||
|
||||
shotgun.driver.fabric.api.run = MagicMock(return_value=runout)
|
||||
shotgun.driver.fabric.api.settings = MagicMock()
|
||||
shotgun.driver.execute = MagicMock(
|
||||
return_value=("RETURN_CODE", "STDOUT", "STDERR"))
|
||||
mfabrun.return_value = runout
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
command = "COMMAND"
|
||||
|
||||
driver = shotgun.driver.Driver({"host": "remote"}, None)
|
||||
driver = shotgun.driver.Driver({"host": "remote_host"}, None)
|
||||
result = driver.command(command)
|
||||
shotgun.driver.fabric.api.run.assert_called_with(command, pty=True)
|
||||
self.assertEquals(result, out)
|
||||
shotgun.driver.fabric.api.settings.assert_called_with(
|
||||
host_string="remote", timeout=2, warn_only=True)
|
||||
host_string="remote_host", timeout=2, warn_only=True)
|
||||
|
||||
driver = shotgun.driver.Driver({}, None)
|
||||
result = driver.command(command)
|
||||
shotgun.driver.execute.assert_called_with(command)
|
||||
self.assertEquals(result, out)
|
||||
|
||||
def test_driver_get(self):
|
||||
shotgun.driver.fabric.api.get = MagicMock()
|
||||
shotgun.driver.fabric.api.settings = MagicMock()
|
||||
shotgun.driver.execute = MagicMock(
|
||||
return_value=("RETURN_CODE", "STDOUT", "STDERR"))
|
||||
path = "PATH"
|
||||
target_path = "/tmp/TARGET_PATH"
|
||||
@patch('shotgun.driver.execute')
|
||||
@patch('shotgun.driver.fabric.api.settings')
|
||||
@patch('shotgun.driver.fabric.api.get')
|
||||
def test_driver_get(self, mfabget, mfabset, mexecute):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
remote_path = "/remote_dir/remote_file"
|
||||
target_path = "/target_dir"
|
||||
|
||||
driver = shotgun.driver.Driver({"host": "remote"}, None)
|
||||
driver.get(path, target_path)
|
||||
shotgun.driver.fabric.api.get.assert_called_with(path, target_path)
|
||||
shotgun.driver.fabric.api.settings.assert_called_with(
|
||||
host_string="remote", timeout=2, warn_only=True)
|
||||
driver = shotgun.driver.Driver({"host": "remote_host"}, None)
|
||||
driver.get(remote_path, target_path)
|
||||
mexecute.assert_called_with("mkdir -p %s" % target_path)
|
||||
mfabget.assert_called_with(remote_path, target_path)
|
||||
mfabset.assert_called_with(
|
||||
host_string="remote_host", timeout=2, warn_only=True)
|
||||
|
||||
mexecute.reset_mock()
|
||||
driver = shotgun.driver.Driver({}, None)
|
||||
driver.get(path, target_path)
|
||||
shotgun.driver.execute.assert_any_call(
|
||||
"mkdir -p %s" % os.path.dirname(target_path))
|
||||
shotgun.driver.execute.assert_any_call(
|
||||
"cp -r %s %s" % (path, target_path))
|
||||
driver.get(remote_path, target_path)
|
||||
assert mexecute.mock_calls == [
|
||||
call("mkdir -p %s" % target_path),
|
||||
call("cp -r %s %s" % (remote_path, target_path))
|
||||
]
|
||||
|
||||
|
||||
class TestFile(TestCase):
|
||||
|
||||
@patch('shotgun.driver.Driver.get')
|
||||
def test_snapshot(self, mget):
|
||||
data = {
|
||||
"type": "file",
|
||||
"path": "/remote_dir/remote_file",
|
||||
"host": "remote_host"
|
||||
}
|
||||
conf = MagicMock()
|
||||
conf.target = "/target"
|
||||
file_driver = shotgun.driver.File(data, conf)
|
||||
|
||||
target_path = "/target/remote_host/remote_dir"
|
||||
file_driver.snapshot()
|
||||
|
||||
mget.assert_called_with(data["path"], target_path)
|
||||
|
||||
|
||||
class TestSubs(TestCase):
|
||||
def setUp(self):
|
||||
self.data = {
|
||||
"type": "subs",
|
||||
"path": "/remote_dir/remote_file",
|
||||
"host": "remote_host",
|
||||
"subs": {
|
||||
"line0": "LINE0",
|
||||
"line1": "LINE1"
|
||||
}
|
||||
}
|
||||
|
||||
self.conf = MagicMock()
|
||||
self.conf.target = "/target"
|
||||
|
||||
self.sedscript = MagicMock()
|
||||
self.sedscript.name = "SEDSCRIPT"
|
||||
self.sedscript.write = MagicMock()
|
||||
|
||||
@patch('shotgun.driver.tempfile.NamedTemporaryFile')
|
||||
@patch('shotgun.driver.Driver.get')
|
||||
@patch('shotgun.driver.execute')
|
||||
def test_sed(self, mexecute, mget, mntemp):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
mntemp.return_value = self.sedscript
|
||||
|
||||
subs_driver = shotgun.driver.Subs(self.data, self.conf)
|
||||
subs_driver.sed("from_file", "to_file")
|
||||
assert self.sedscript.write.mock_calls == [
|
||||
call("s/%s/%s/g\n" % (old, new))
|
||||
for old, new in self.data["subs"].iteritems()]
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
"cat from_file | sed -f SEDSCRIPT", to_filename="to_file")
|
||||
|
||||
subs_driver.sed("from_file.gz", "to_file.gz")
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
"cat from_file.gz | gunzip -c | sed -f SEDSCRIPT | gzip -c",
|
||||
to_filename="to_file.gz")
|
||||
|
||||
subs_driver.sed("from_file.bz2", "to_file.bz2")
|
||||
shotgun.driver.execute.assert_called_with(
|
||||
"cat from_file.bz2 | bunzip2 -c | sed -f SEDSCRIPT | bzip2 -c",
|
||||
to_filename="to_file.bz2")
|
||||
|
||||
@patch('shotgun.driver.os.walk')
|
||||
@patch('shotgun.driver.Subs.sed')
|
||||
@patch('shotgun.driver.Driver.get')
|
||||
@patch('shotgun.driver.execute')
|
||||
def test_snapshot(self, mexecute, mdriverget, msed, mwalk):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
|
||||
""" 1. Should get remote (or local) file (or directory)
|
||||
2. Should put it into /target/host.domain.tld
|
||||
3. Should walk through and check if files match given path pattern
|
||||
4. If matched, sed them
|
||||
"""
|
||||
|
||||
"""this return_value corresponds to the following structure
|
||||
/target/remote_host/remote_dir/
|
||||
/target/remote_host/remote_dir/remote_file
|
||||
/target/remote_host/remote_dir/1
|
||||
/target/remote_host/remote_dir/2
|
||||
/target/remote_host/remote_dir/3/
|
||||
/target/remote_host/remote_dir/3/4
|
||||
/target/remote_host/remote_dir/3/5
|
||||
/target/remote_host/remote_dir/3/6/
|
||||
"""
|
||||
mock_walk = [
|
||||
(
|
||||
'/target/remote_host/remote_dir',
|
||||
['3'],
|
||||
['1', '2', 'remote_file']
|
||||
),
|
||||
('/target/remote_host/remote_dir/3', ['6'], ['5', '4']),
|
||||
('/target/remote_host/remote_dir/3/6', [], [])
|
||||
]
|
||||
mwalk.return_value = mock_walk
|
||||
|
||||
subs_driver = shotgun.driver.Subs(self.data, self.conf)
|
||||
subs_driver.snapshot()
|
||||
|
||||
sed_calls = []
|
||||
execute_calls = []
|
||||
for root, _, files in mock_walk:
|
||||
for filename in files:
|
||||
fullfilename = os.path.join(root, filename)
|
||||
# /target/remote_host
|
||||
tgt_host = os.path.join(self.conf.target, self.data["host"])
|
||||
rel_tgt_host = os.path.relpath(fullfilename, tgt_host)
|
||||
# /remote_dir/remote_file
|
||||
match_orig_path = os.path.join("/", rel_tgt_host)
|
||||
if not fnmatch.fnmatch(match_orig_path, self.data["path"]):
|
||||
continue
|
||||
tempfilename = "STDOUT"
|
||||
execute_calls.append(call("mktemp"))
|
||||
sed_calls.append(call(fullfilename, tempfilename))
|
||||
execute_calls.append(
|
||||
call("mv %s %s" % (tempfilename, fullfilename)))
|
||||
|
||||
assert msed.mock_calls == sed_calls
|
||||
assert mexecute.mock_calls == execute_calls
|
||||
|
|
|
|||
Loading…
Reference in New Issue