From bec94c555423e1c41b267805065b87d31ce1b5ee Mon Sep 17 00:00:00 2001 From: Tom Viner Date: Sat, 26 Nov 2016 18:09:02 +0000 Subject: [PATCH 1/6] Allow <@ syntax to safely include subdirectories --- gabbi/case.py | 8 +++- gabbi/tests/test_load_data_file.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 gabbi/tests/test_load_data_file.py diff --git a/gabbi/case.py b/gabbi/case.py index 63dfb82..f145e7d 100644 --- a/gabbi/case.py +++ b/gabbi/case.py @@ -243,7 +243,13 @@ class HTTPTestCase(testtools.TestCase): def _load_data_file(self, filename): """Read a file from the current test directory.""" - path = os.path.join(self.test_directory, os.path.basename(filename)) + path = os.path.join(self.test_directory, filename) + has_dir_traversal = os.path.relpath( + path, start=self.test_directory).startswith(os.pardir) + if has_dir_traversal: + raise ValueError( + 'Attempted loading of data file outside test directory: %s' + % filename) with open(path, mode='rb') as data_file: return data_file.read() diff --git a/gabbi/tests/test_load_data_file.py b/gabbi/tests/test_load_data_file.py new file mode 100644 index 0000000..83f9451 --- /dev/null +++ b/gabbi/tests/test_load_data_file.py @@ -0,0 +1,76 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Test loading data from files with <@. +""" + +import unittest + +from gabbi import case +from six.moves import mock + + +@mock.patch( + 'gabbi.case.open', + new_callable=mock.mock_open, + read_data='dummy content' +) +class DataFileTest(unittest.TestCase): + """Reading from local file is only allowed at or below the + test_directory level. + """ + + def setUp(self): + self.http_case = case.HTTPTestCase('test_request') + + def _assert_content_read(self, filepath): + self.assertEqual( + 'dummy content', self.http_case._load_data_file(filepath)) + + def test_load_file(self, m_open): + self.http_case.test_directory = '.' + self._assert_content_read('data.json') + m_open.assert_called_with('./data.json', mode='rb') + + def test_load_file_in_directory(self, m_open): + self.http_case.test_directory = '.' + self._assert_content_read('a/b/c/data.json') + m_open.assert_called_with('./a/b/c/data.json', mode='rb') + + def test_load_file_in_root(self, m_open): + self.http_case.test_directory = '.' + filepath = '/top-level.private' + + with self.assertRaises(ValueError): + self.http_case._load_data_file(filepath) + self.assertFalse(m_open.called) + + def test_load_file_in_parent_dir(self, m_open): + self.http_case.test_directory = '.' + filepath = '../file-in-parent-dir.txt' + + with self.assertRaises(ValueError): + self.http_case._load_data_file(filepath) + self.assertFalse(m_open.called) + + def test_load_file_within_test_directory(self, m_open): + self.http_case.test_directory = '/a/b/c' + self._assert_content_read('../../b/c/file-in-test-dir.txt') + m_open.assert_called_with( + '/a/b/c/../../b/c/file-in-test-dir.txt', mode='rb') + + def test_load_file_not_within_test_directory(self, m_open): + self.http_case.test_directory = '/a/b/c' + filepath = '../../b/E/file-in-test-dir.txt' + with self.assertRaises(ValueError): + self.http_case._load_data_file(filepath) + self.assertFalse(m_open.called) From 7b6346daacacab97a332e6dddaf9d5ae0f7d21ca Mon Sep 17 00:00:00 2001 From: Tom Viner Date: Sun, 27 Nov 2016 13:42:26 +0000 Subject: [PATCH 2/6] enable mock_open to work on py3.3/3.4 --- gabbi/tests/test_load_data_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gabbi/tests/test_load_data_file.py b/gabbi/tests/test_load_data_file.py index 83f9451..99536d4 100644 --- a/gabbi/tests/test_load_data_file.py +++ b/gabbi/tests/test_load_data_file.py @@ -22,7 +22,8 @@ from six.moves import mock @mock.patch( 'gabbi.case.open', new_callable=mock.mock_open, - read_data='dummy content' + read_data='dummy content', + create=True, ) class DataFileTest(unittest.TestCase): """Reading from local file is only allowed at or below the From 968c11b3da791721e69b1b36536aedcd4117ed55 Mon Sep 17 00:00:00 2001 From: Tom Viner Date: Sun, 27 Nov 2016 13:47:47 +0000 Subject: [PATCH 3/6] update docs to mention filepath rather than file --- docs/source/format.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/format.rst b/docs/source/format.rst index 1487616..bd5209a 100644 --- a/docs/source/format.rst +++ b/docs/source/format.rst @@ -267,8 +267,8 @@ flexibility when doing a ``POST`` or ``PUT``. If the value is not a string (that is, it is a sequence or structure) it is treated as a data structure which is turned into a JSON string. If the value is a string that begins with ``<@`` then the rest of the string is treated -as the name of a file to be loaded from the same directory as the YAML -file. If the value is an undecorated string, that's the value. +as a filepath to the loaded. The path is relative to the test directory. +If the value is an undecorated string, that's the value. When reading from a file care should be taken to ensure that a reasonable content-type is set for the data as this will control if any From 398318ee17b37260226bd4598c8f3f370a956b5d Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Sun, 27 Nov 2016 14:34:17 +0000 Subject: [PATCH 4/6] Use the dirname of each runner testfile as the test_dir When gabbi-run is passed one or more files, set the working directory for the associated test suite as the directory of each test file. Doing so ensures that `<@file` syntax in the YAML has a fixed point from which to find the data files. If data comes from STDIN then the test directory is '.' (as it has always been). Fixes #184 --- gabbi/runner.py | 8 ++++--- gabbi/tests/gabbits_runner/subdir/sample.json | 1 + gabbi/tests/gabbits_runner/test_data.yaml | 8 +++++++ gabbi/tests/test_runner.py | 23 +++++++++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 gabbi/tests/gabbits_runner/subdir/sample.json create mode 100644 gabbi/tests/gabbits_runner/test_data.yaml diff --git a/gabbi/runner.py b/gabbi/runner.py index ac98dea..9f45351 100644 --- a/gabbi/runner.py +++ b/gabbi/runner.py @@ -14,6 +14,7 @@ import argparse from importlib import import_module +import os import sys import unittest @@ -84,8 +85,9 @@ def run(): else: for input_file in input_files: with open(input_file, 'r') as fh: + data_dir = os.path.dirname(input_file) success = run_suite(fh, handler_objects, host, port, - prefix, force_ssl, failfast) + prefix, force_ssl, failfast, data_dir) if not failure: # once failed, this is considered immutable failure = not success if failure and failfast: @@ -95,7 +97,7 @@ def run(): def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, - failfast=False): + failfast=False, data_dir='.'): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle) if force_ssl: @@ -106,7 +108,7 @@ def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( - loader, 'input', data, '.', host, port, None, None, prefix=prefix, + loader, 'input', data, data_dir, host, port, None, None, prefix=prefix, handlers=handler_objects) result = ConciseTestRunner( diff --git a/gabbi/tests/gabbits_runner/subdir/sample.json b/gabbi/tests/gabbits_runner/subdir/sample.json new file mode 100644 index 0000000..ddbce20 --- /dev/null +++ b/gabbi/tests/gabbits_runner/subdir/sample.json @@ -0,0 +1 @@ +{"items": {"house": "blue"}} diff --git a/gabbi/tests/gabbits_runner/test_data.yaml b/gabbi/tests/gabbits_runner/test_data.yaml new file mode 100644 index 0000000..35d056a --- /dev/null +++ b/gabbi/tests/gabbits_runner/test_data.yaml @@ -0,0 +1,8 @@ +tests: + +- name: POST data from file + verbose: true + POST: / + request_headers: + content-type: application/json + data: <@subdir/sample.json diff --git a/gabbi/tests/test_runner.py b/gabbi/tests/test_runner.py index bf882ab..1b86235 100644 --- a/gabbi/tests/test_runner.py +++ b/gabbi/tests/test_runner.py @@ -22,6 +22,7 @@ from wsgi_intercept.interceptor import Urllib3Interceptor from gabbi import exception from gabbi.handlers import base +from gabbi.handlers.jsonhandler import JSONHandler from gabbi import runner from gabbi.tests.simple_wsgi import SimpleWsgi @@ -249,6 +250,28 @@ class RunnerTest(unittest.TestCase): self.assertIn('{\n', output) self.assertIn('}\n', output) + def test_data_dir_good(self): + """Confirm that data dir is the test file's dir.""" + sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)] + + sys.argv.append('--') + sys.argv.append('gabbi/tests/gabbits_runner/test_data.yaml') + with self.server(): + try: + runner.run() + except SystemExit as err: + self.assertSuccess(err) + + # Compare the verbose output of tests with pretty printed + # data. + with open('gabbi/tests/gabbits_runner/subdir/sample.json') as data: + data = JSONHandler.loads(data.read()) + expected_string = JSONHandler.dumps(data, pretty=True) + + sys.stdout.seek(0) + output = sys.stdout.read() + self.assertIn(expected_string, output) + def assertSuccess(self, exitError): errors = exitError.args[0] if errors: From d181dae2296f42e4f2821b8e338ba7111f635717 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Sun, 27 Nov 2016 15:13:03 +0000 Subject: [PATCH 5/6] Add a -v or --verbose flag to gabbi-run A value of 'all', 'headers' or 'body' will have the expected effect on all the tests being run in the current testing session. This is done by manipulating the 'defaults' of each test suite. Fixes: #185 --- docs/source/runner.rst | 3 ++ gabbi/runner.py | 21 ++++++++-- gabbi/tests/gabbits_runner/verbosity.yaml | 8 ++++ gabbi/tests/test_runner.py | 47 +++++++++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 gabbi/tests/gabbits_runner/verbosity.yaml diff --git a/docs/source/runner.rst b/docs/source/runner.rst index df17db4..a468bf6 100644 --- a/docs/source/runner.rst +++ b/docs/source/runner.rst @@ -50,3 +50,6 @@ YAML will default to ``ssl: True``. If a ``-x`` or ``--failfast`` argument is provided then ``gabbi-run`` will exit after the first test failure. + +Use ``-v`` or ``--verbose`` with a value of ``all``, ``headers`` or ``body`` +to turn on :ref:`verbosity ` for all tests being run. diff --git a/gabbi/runner.py b/gabbi/runner.py index ac98dea..184869f 100644 --- a/gabbi/runner.py +++ b/gabbi/runner.py @@ -57,6 +57,9 @@ def run(): gabbi-run -x example.com:9999 /mountpoint < mytest.yaml + Use `-v` or `--verbose` with a value of `all`, `headers` or `body` to + turn on verbosity for all tests being run. + Multiple files may be named as arguments, separated from other arguments by a ``--``. Each file will be run as a separate test suite:: @@ -74,18 +77,19 @@ def run(): handler_objects = initialize_handlers(args.response_handlers) + verbosity = args.verbosity failfast = args.failfast failure = False if not input_files: success = run_suite(sys.stdin, handler_objects, host, port, - prefix, force_ssl, failfast) + prefix, force_ssl, failfast, verbosity) failure = not success else: for input_file in input_files: with open(input_file, 'r') as fh: success = run_suite(fh, handler_objects, host, port, - prefix, force_ssl, failfast) + prefix, force_ssl, failfast, verbosity) if not failure: # once failed, this is considered immutable failure = not success if failure and failfast: @@ -95,7 +99,7 @@ def run(): def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, - failfast=False): + failfast=False, verbosity=False): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle) if force_ssl: @@ -103,6 +107,11 @@ def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, data['defaults']['ssl'] = True else: data['defaults'] = {'ssl': True} + if verbosity: + if 'defaults' in data: + data['defaults']['verbose'] = verbosity + else: + data['defaults'] = {'verbose': verbosity} loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( @@ -194,6 +203,12 @@ def _make_argparser(): help='Custom response handler. Should be an import path of the ' 'form package.module or package.module:class.' ) + parser.add_argument( + '-v', '--verbose', + dest='verbosity', + choices=['all', 'body', 'headers'], + help='Turn on test verbosity for all tests run in this session.' + ) return parser diff --git a/gabbi/tests/gabbits_runner/verbosity.yaml b/gabbi/tests/gabbits_runner/verbosity.yaml new file mode 100644 index 0000000..4522211 --- /dev/null +++ b/gabbi/tests/gabbits_runner/verbosity.yaml @@ -0,0 +1,8 @@ +tests: + +- name: simple data post + POST: / + request_headers: + content-type: application/json + data: + cat: poppy diff --git a/gabbi/tests/test_runner.py b/gabbi/tests/test_runner.py index bf882ab..97ea391 100644 --- a/gabbi/tests/test_runner.py +++ b/gabbi/tests/test_runner.py @@ -249,6 +249,53 @@ class RunnerTest(unittest.TestCase): self.assertIn('{\n', output) self.assertIn('}\n', output) + def _run_verbosity_arg(self): + sys.argv.append('--') + sys.argv.append('gabbi/tests/gabbits_runner/verbosity.yaml') + with self.server(): + try: + runner.run() + except SystemExit as err: + self.assertSuccess(err) + + sys.stdout.seek(0) + output = sys.stdout.read() + return output + + def test_verbosity_arg_none(self): + """Confirm --verbose handling.""" + sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)] + + output = self._run_verbosity_arg() + self.assertEqual('', output) + + def test_verbosity_arg_body(self): + """Confirm --verbose handling.""" + sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port), + '--verbose=body'] + + output = self._run_verbosity_arg() + self.assertIn('{\n "cat": "poppy"\n}', output) + self.assertNotIn('application/json', output) + + def test_verbosity_arg_headers(self): + """Confirm --verbose handling.""" + sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port), + '--verbose=headers'] + + output = self._run_verbosity_arg() + self.assertNotIn('{\n "cat": "poppy"\n}', output) + self.assertIn('application/json', output) + + def test_verbosity_arg_all(self): + """Confirm --verbose handling.""" + sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port), + '--verbose=all'] + + output = self._run_verbosity_arg() + self.assertIn('{\n "cat": "poppy"\n}', output) + self.assertIn('application/json', output) + def assertSuccess(self, exitError): errors = exitError.args[0] if errors: From 59ed73d75f77492d72bd215bcbc3df75de354680 Mon Sep 17 00:00:00 2001 From: Tom Viner Date: Sun, 27 Nov 2016 19:16:52 +0000 Subject: [PATCH 6/6] clarify wording of docs --- docs/source/format.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/format.rst b/docs/source/format.rst index bd5209a..3ea2cd7 100644 --- a/docs/source/format.rst +++ b/docs/source/format.rst @@ -267,8 +267,9 @@ flexibility when doing a ``POST`` or ``PUT``. If the value is not a string (that is, it is a sequence or structure) it is treated as a data structure which is turned into a JSON string. If the value is a string that begins with ``<@`` then the rest of the string is treated -as a filepath to the loaded. The path is relative to the test directory. -If the value is an undecorated string, that's the value. +as a filepath to be loaded. The path is relative to the test directory +and may not traverse up into parent directories. If the value is an +undecorated string, that's the value. When reading from a file care should be taken to ensure that a reasonable content-type is set for the data as this will control if any