From 1566b9541a60944c2bc3f4f4dcb947801864e944 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Sat, 7 Apr 2018 14:53:48 -0400 Subject: [PATCH] Clean up functional test directory and entrypoint script This PS simply reorganizes Deckhand's functional test directory to make it more maintainable and readable as right now it is hard to figure out what is covered by a functional test and what isn't. Additionally, the entrypoint for these tests in tools/functional-tests.sh has also been refactored slightly. Change-Id: I262c7e1f7cbce248c12ee013a9bab4e32b89adee --- deckhand/tests/functional/README.rst | 30 ++++++++++++ .../document-crud-error-bucket-conflict.yaml | 0 .../document-crud-success-multi-bucket.yaml | 0 ...document-crud-success-owned-documents.yaml | 0 .../document-crud-success-single-bucket.yaml | 0 ...cument-crud-success-unusual-documents.yaml | 0 .../layering-multiple-bucket.yaml} | 0 .../layering-policy-conflict.yaml} | 0 .../layering-single-bucket.yaml} | 0 ...ering-with-replacement-single-bucket.yaml} | 0 ...ring-with-substitution-single-bucket.yaml} | 0 .../revision-diff-success.yaml | 0 .../revision-documents-filters-negative.yaml | 0 .../revision-documents-filters.yaml | 0 .../revision-documents-multiple-filters.yaml | 0 .../rollback-success-single-bucket.yaml | 0 .../revision-tag-success.yaml | 0 .../revision-crud-success-single-bucket.yaml | 0 .../{ => revision}/revision-filters.yaml | 0 .../schema-validation-success-ucp-sample.yaml | 0 .../schema-validation-success.yaml | 0 ...dation-with-validation-policy-success.yaml | 0 .../substitution-chained-single-bucket.yaml} | 0 .../substitution-multiple-bucket.yaml} | 0 .../substitution-single-bucket-generic.yaml} | 0 .../substitution-single-bucket.yaml} | 0 deckhand/tests/functional/test_gabbi.py | 46 +++++++++++++++++-- tools/common-tests.sh | 1 + tools/functional-tests.sh | 4 +- 29 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 deckhand/tests/functional/README.rst rename deckhand/tests/functional/gabbits/{ => document}/document-crud-error-bucket-conflict.yaml (100%) rename deckhand/tests/functional/gabbits/{ => document}/document-crud-success-multi-bucket.yaml (100%) rename deckhand/tests/functional/gabbits/{ => document}/document-crud-success-owned-documents.yaml (100%) rename deckhand/tests/functional/gabbits/{ => document}/document-crud-success-single-bucket.yaml (100%) rename deckhand/tests/functional/gabbits/{ => document}/document-crud-success-unusual-documents.yaml (100%) rename deckhand/tests/functional/gabbits/{document-render-success-multiple-bucket-with-layering.yaml => layering/layering-multiple-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-error-layering-policy-conflict.yaml => layering/layering-policy-conflict.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-single-bucket-with-layering.yaml => layering/layering-single-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-single-bucket-with-replacement.yaml => layering/layering-with-replacement-single-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-single-bucket-with-layering-and-substitution.yaml => layering/layering-with-substitution-single-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{ => revision-diff}/revision-diff-success.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision-documents}/revision-documents-filters-negative.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision-documents}/revision-documents-filters.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision-documents}/revision-documents-multiple-filters.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision-rollback}/rollback-success-single-bucket.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision-tag}/revision-tag-success.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision}/revision-crud-success-single-bucket.yaml (100%) rename deckhand/tests/functional/gabbits/{ => revision}/revision-filters.yaml (100%) rename deckhand/tests/functional/gabbits/{ => schema-validation}/schema-validation-success-ucp-sample.yaml (100%) rename deckhand/tests/functional/gabbits/{ => schema-validation}/schema-validation-success.yaml (100%) rename deckhand/tests/functional/gabbits/{ => schema-validation}/schema-validation-with-validation-policy-success.yaml (100%) rename deckhand/tests/functional/gabbits/{chained-substitution.yaml => substitution/substitution-chained-single-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-multiple-bucket-with-substitution.yaml => substitution/substitution-multiple-bucket.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-single-bucket-with-substitution-generic.yaml => substitution/substitution-single-bucket-generic.yaml} (100%) rename deckhand/tests/functional/gabbits/{document-render-success-single-bucket-with-substitution.yaml => substitution/substitution-single-bucket.yaml} (100%) diff --git a/deckhand/tests/functional/README.rst b/deckhand/tests/functional/README.rst new file mode 100644 index 00000000..2bd329f2 --- /dev/null +++ b/deckhand/tests/functional/README.rst @@ -0,0 +1,30 @@ +Functional Tests +================ + +Deckhand uses `gabbi`_ to drive its functional tests. The entry point for +these tests is ``functional-tests.sh`` under ``tools`` directory. + +Directory Test Layout +--------------------- + +Tests are contained in intuitively named subdirectories nested under +``deckhand/tests/functional/gabbits``. For example, layering tests are +contained under the ``layering`` subdirectory. This pattern should be strictly +followed. + +Because `gabbi`_ does not support loading tests from subdirectories, logic +is included in ``test_gabbi.py`` to: + +#. Create a temporary directory. +#. Create a symlink between all the test files in the nested subdirectories + and the temporary directory. + +However, the test directory can still be modified: + +* New subdirectories under ``gabbits`` can be added. +* New tests under any of those subdirectories can be added. +* New resource files under ``gabits/resources`` can be added. This directory + name should never be renamed. +* All other subdirectories, test files, and resources may be renamed. + +.. _gabbi: https://gabbi.readthedocs.io/en/latest/gabbi.html diff --git a/deckhand/tests/functional/gabbits/document-crud-error-bucket-conflict.yaml b/deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-error-bucket-conflict.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-multi-bucket.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-multi-bucket.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-owned-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-owned-documents.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-unusual-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-unusual-documents.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-layering.yaml b/deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-layering.yaml rename to deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-error-layering-policy-conflict.yaml b/deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-error-layering-policy-conflict.yaml rename to deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering.yaml b/deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering.yaml rename to deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-replacement.yaml b/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-replacement.yaml rename to deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering-and-substitution.yaml b/deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering-and-substitution.yaml rename to deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-diff-success.yaml b/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-diff-success.yaml rename to deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-filters-negative.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-filters-negative.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-filters.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-filters.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-multiple-filters.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-multiple-filters.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml diff --git a/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-tag-success.yaml b/deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-tag-success.yaml rename to deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml diff --git a/deckhand/tests/functional/gabbits/revision-crud-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-crud-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-filters.yaml b/deckhand/tests/functional/gabbits/revision/revision-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-filters.yaml rename to deckhand/tests/functional/gabbits/revision/revision-filters.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-success-ucp-sample.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-success-ucp-sample.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-success.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-with-validation-policy-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-with-validation-policy-success.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml diff --git a/deckhand/tests/functional/gabbits/chained-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/chained-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution-generic.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution-generic.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml diff --git a/deckhand/tests/functional/test_gabbi.py b/deckhand/tests/functional/test_gabbi.py index 871d682e..465e8fcc 100644 --- a/deckhand/tests/functional/test_gabbi.py +++ b/deckhand/tests/functional/test_gabbi.py @@ -12,14 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit import os +import shutil +import tempfile import yaml from gabbi import driver from gabbi.driver import test_pytest # noqa from gabbi.handlers import jsonhandler -TESTS_DIR = 'gabbits' +TEST_DIR = tempfile.mkdtemp(prefix='deckhand') + + +def __create_temp_test_dir(): + """Hack around the fact that gabbi doesn't support loading tests contained + in subdirectories. This inconvenience leads to poor test directory layout + in which all the test files are contained in one directory. + + """ + root_test_dir = os.path.join(os.path.dirname(__file__), 'gabbits') + test_files = [] + + for root, dirs, files in os.walk(root_test_dir): + is_test_file = ( + 'gabbits' in root and not root.endswith('gabbits') + ) + if is_test_file: + test_files.extend([os.path.abspath(os.path.join(root, f)) + for f in files]) + + resources_dir = os.path.join(TEST_DIR, 'resources') + if not os.path.exists(resources_dir): + os.makedirs(resources_dir) + + for test_file in test_files: + basename = os.path.basename(test_file) + if 'resources' in test_file: + os.symlink(test_file, os.path.join(resources_dir, basename)) + else: + os.symlink(test_file, os.path.join(TEST_DIR, basename)) + + +__create_temp_test_dir() + + +@atexit.register +def __remove_temp_test_dir(): + if os.path.exists(TEST_DIR): + shutil.rmtree(TEST_DIR) # This is quite similar to the existing JSONHandler, so use it as the base @@ -47,12 +88,11 @@ class MultidocJsonpaths(jsonhandler.JSONHandler): def pytest_generate_tests(metafunc): - test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR) # NOTE(fmontei): While only `url` or `host` is needed, strangely both # are needed because we use `pytest-html` which throws an error without # `host`. driver.py_test_generator( - test_dir, url=os.environ['DECKHAND_TEST_URL'], host='localhost', + TEST_DIR, url=os.environ['DECKHAND_TEST_URL'], host='localhost', # NOTE(fmontei): When there are multiple handlers listed that accept # the same content-type, the one that is earliest in the list will be # used. Thus, we cannot specify multiple content handlers for handling diff --git a/tools/common-tests.sh b/tools/common-tests.sh index 4c17103b..0de0cf0c 100644 --- a/tools/common-tests.sh +++ b/tools/common-tests.sh @@ -154,6 +154,7 @@ function gen_paste { local disable_keystone=$1 if $disable_keystone; then + log_section Disabling Keystone authentication. sed 's/authtoken api/api/' etc/deckhand/deckhand-paste.ini &> $CONF_DIR/deckhand-paste.ini else cp etc/deckhand/deckhand-paste.ini $CONF_DIR/deckhand-paste.ini diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh index 5755b6ef..f1d3cacb 100755 --- a/tools/functional-tests.sh +++ b/tools/functional-tests.sh @@ -51,7 +51,7 @@ function deploy_deckhand { gen_policy if [ -z "$DECKHAND_IMAGE" ]; then - log_section "Running Deckhand via uwsgi" + log_section "Running Deckhand via uwsgi." alembic upgrade head # NOTE(fmontei): Deckhand's database is not configured to work with @@ -63,7 +63,7 @@ function deploy_deckhand { export DECKHAND_API_THREADS=4 source $ROOTDIR/../entrypoint.sh server & else - log_section "Running Deckhand via Docker" + log_section "Running Deckhand via Docker." sudo docker run \ --rm \ --net=host \