291 lines
9.4 KiB
Python
291 lines
9.4 KiB
Python
"""
|
|
Test runner for the JSON Schema official test suite
|
|
|
|
Tests comprehensive correctness of each draft's validator.
|
|
|
|
See https://github.com/json-schema/JSON-Schema-Test-Suite for details.
|
|
|
|
"""
|
|
|
|
from contextlib import closing
|
|
from decimal import Decimal
|
|
import glob
|
|
import json
|
|
import io
|
|
import itertools
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
try:
|
|
from sys import pypy_version_info
|
|
except ImportError:
|
|
pypy_version_info = None
|
|
|
|
from jsonschema import (
|
|
FormatError, SchemaError, ValidationError, Draft3Validator,
|
|
Draft4Validator, FormatChecker, draft3_format_checker,
|
|
draft4_format_checker, validate,
|
|
)
|
|
from jsonschema.compat import PY3
|
|
from jsonschema.tests.compat import mock, unittest
|
|
import jsonschema
|
|
|
|
|
|
REPO_ROOT = os.path.join(os.path.dirname(jsonschema.__file__), os.path.pardir)
|
|
SUITE = os.getenv("JSON_SCHEMA_TEST_SUITE", os.path.join(REPO_ROOT, "json"))
|
|
|
|
if not os.path.isdir(SUITE):
|
|
raise ValueError(
|
|
"Can't find the JSON-Schema-Test-Suite directory. Set the "
|
|
"'JSON_SCHEMA_TEST_SUITE' environment variable or run the tests from "
|
|
"alongside a checkout of the suite."
|
|
)
|
|
|
|
TESTS_DIR = os.path.join(SUITE, "tests")
|
|
JSONSCHEMA_SUITE = os.path.join(SUITE, "bin", "jsonschema_suite")
|
|
|
|
remotes_stdout = subprocess.Popen(
|
|
["python", JSONSCHEMA_SUITE, "remotes"], stdout=subprocess.PIPE,
|
|
).stdout
|
|
|
|
with closing(remotes_stdout):
|
|
if PY3:
|
|
remotes_stdout = io.TextIOWrapper(remotes_stdout)
|
|
REMOTES = json.load(remotes_stdout)
|
|
|
|
|
|
def make_case(schema, data, valid, name):
|
|
if valid:
|
|
def test_case(self):
|
|
kwargs = getattr(self, "validator_kwargs", {})
|
|
validate(data, schema, cls=self.validator_class, **kwargs)
|
|
else:
|
|
def test_case(self):
|
|
kwargs = getattr(self, "validator_kwargs", {})
|
|
with self.assertRaises(ValidationError):
|
|
validate(data, schema, cls=self.validator_class, **kwargs)
|
|
|
|
if not PY3:
|
|
name = name.encode("utf-8")
|
|
test_case.__name__ = name
|
|
|
|
return test_case
|
|
|
|
|
|
def maybe_skip(skip, test_case, case, test):
|
|
if skip is not None:
|
|
reason = skip(case, test)
|
|
if reason is not None:
|
|
test_case = unittest.skip(reason)(test_case)
|
|
return test_case
|
|
|
|
|
|
def load_json_cases(tests_glob, ignore_glob="", basedir=TESTS_DIR, skip=None):
|
|
if ignore_glob:
|
|
ignore_glob = os.path.join(basedir, ignore_glob)
|
|
|
|
def add_test_methods(test_class):
|
|
ignored = set(glob.iglob(ignore_glob))
|
|
|
|
for filename in glob.iglob(os.path.join(basedir, tests_glob)):
|
|
if filename in ignored:
|
|
continue
|
|
|
|
validating, _ = os.path.splitext(os.path.basename(filename))
|
|
id = itertools.count(1)
|
|
|
|
with open(filename) as test_file:
|
|
for case in json.load(test_file):
|
|
for test in case["tests"]:
|
|
name = "test_%s_%s_%s" % (
|
|
validating,
|
|
next(id),
|
|
re.sub(r"[\W ]+", "_", test["description"]),
|
|
)
|
|
assert not hasattr(test_class, name), name
|
|
|
|
test_case = make_case(
|
|
data=test["data"],
|
|
schema=case["schema"],
|
|
valid=test["valid"],
|
|
name=name,
|
|
)
|
|
test_case = maybe_skip(skip, test_case, case, test)
|
|
setattr(test_class, name, test_case)
|
|
|
|
return test_class
|
|
return add_test_methods
|
|
|
|
|
|
class TypesMixin(object):
|
|
@unittest.skipIf(PY3, "In Python 3 json.load always produces unicode")
|
|
def test_string_a_bytestring_is_a_string(self):
|
|
self.validator_class({"type" : "string"}).validate(b"foo")
|
|
|
|
|
|
class DecimalMixin(object):
|
|
def test_it_can_validate_with_decimals(self):
|
|
schema = {"type" : "number"}
|
|
validator = self.validator_class(
|
|
schema, types={"number" : (int, float, Decimal)}
|
|
)
|
|
|
|
for valid in [1, 1.1, Decimal(1) / Decimal(8)]:
|
|
validator.validate(valid)
|
|
|
|
for invalid in ["foo", {}, [], True, None]:
|
|
with self.assertRaises(ValidationError):
|
|
validator.validate(invalid)
|
|
|
|
|
|
def missing_format(checker):
|
|
def missing_format(case, test):
|
|
format = case["schema"].get("format")
|
|
if format not in checker.checkers:
|
|
return "Format checker {0!r} not found.".format(format)
|
|
elif (
|
|
format == "date-time" and
|
|
pypy_version_info is not None and
|
|
pypy_version_info[:2] <= (1, 9)
|
|
):
|
|
# datetime.datetime is overzealous about typechecking in <=1.9
|
|
return "datetime.datetime is broken on this version of PyPy."
|
|
return missing_format
|
|
|
|
|
|
class FormatMixin(object):
|
|
def test_it_returns_true_for_formats_it_does_not_know_about(self):
|
|
validator = self.validator_class(
|
|
{"format" : "carrot"}, format_checker=FormatChecker(),
|
|
)
|
|
validator.validate("bugs")
|
|
|
|
def test_it_does_not_validate_formats_by_default(self):
|
|
validator = self.validator_class({})
|
|
self.assertIsNone(validator.format_checker)
|
|
|
|
def test_it_validates_formats_if_a_checker_is_provided(self):
|
|
checker = mock.Mock(spec=FormatChecker)
|
|
validator = self.validator_class(
|
|
{"format" : "foo"}, format_checker=checker,
|
|
)
|
|
|
|
validator.validate("bar")
|
|
|
|
checker.check.assert_called_once_with("bar", "foo")
|
|
|
|
cause = ValueError()
|
|
checker.check.side_effect = FormatError('aoeu', cause=cause)
|
|
|
|
with self.assertRaises(ValidationError) as cm:
|
|
validator.validate("bar")
|
|
# Make sure original cause is attached
|
|
self.assertIs(cm.exception.cause, cause)
|
|
|
|
def test_it_validates_formats_of_any_type(self):
|
|
checker = mock.Mock(spec=FormatChecker)
|
|
validator = self.validator_class(
|
|
{"format" : "foo"}, format_checker=checker,
|
|
)
|
|
|
|
validator.validate([1, 2, 3])
|
|
|
|
checker.check.assert_called_once_with([1, 2, 3], "foo")
|
|
|
|
cause = ValueError()
|
|
checker.check.side_effect = FormatError('aoeu', cause=cause)
|
|
|
|
with self.assertRaises(ValidationError) as cm:
|
|
validator.validate([1, 2, 3])
|
|
# Make sure original cause is attached
|
|
self.assertIs(cm.exception.cause, cause)
|
|
|
|
|
|
if sys.maxunicode == 2 ** 16 - 1: # This is a narrow build.
|
|
def narrow_unicode_build(case, test):
|
|
if "supplementary Unicode" in test["description"]:
|
|
return "Not running surrogate Unicode case, this Python is narrow."
|
|
else:
|
|
def narrow_unicode_build(case, test): # This isn't, skip nothing.
|
|
return
|
|
|
|
|
|
@load_json_cases(
|
|
"draft3/*.json",
|
|
skip=narrow_unicode_build,
|
|
ignore_glob="draft3/refRemote.json",
|
|
)
|
|
@load_json_cases(
|
|
"draft3/optional/format.json", skip=missing_format(draft3_format_checker)
|
|
)
|
|
@load_json_cases("draft3/optional/bignum.json")
|
|
@load_json_cases("draft3/optional/zeroTerminatedFloats.json")
|
|
class TestDraft3(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin):
|
|
validator_class = Draft3Validator
|
|
validator_kwargs = {"format_checker" : draft3_format_checker}
|
|
|
|
def test_any_type_is_valid_for_type_any(self):
|
|
validator = self.validator_class({"type" : "any"})
|
|
validator.validate(mock.Mock())
|
|
|
|
# TODO: we're in need of more meta schema tests
|
|
def test_invalid_properties(self):
|
|
with self.assertRaises(SchemaError):
|
|
validate({}, {"properties": {"test": True}},
|
|
cls=self.validator_class)
|
|
|
|
def test_minItems_invalid_string(self):
|
|
with self.assertRaises(SchemaError):
|
|
# needs to be an integer
|
|
validate([1], {"minItems" : "1"}, cls=self.validator_class)
|
|
|
|
|
|
@load_json_cases(
|
|
"draft4/*.json",
|
|
skip=narrow_unicode_build,
|
|
ignore_glob="draft4/refRemote.json",
|
|
)
|
|
@load_json_cases(
|
|
"draft4/optional/format.json", skip=missing_format(draft4_format_checker)
|
|
)
|
|
@load_json_cases("draft4/optional/bignum.json")
|
|
@load_json_cases("draft4/optional/zeroTerminatedFloats.json")
|
|
class TestDraft4(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin):
|
|
validator_class = Draft4Validator
|
|
validator_kwargs = {"format_checker" : draft4_format_checker}
|
|
|
|
# TODO: we're in need of more meta schema tests
|
|
def test_invalid_properties(self):
|
|
with self.assertRaises(SchemaError):
|
|
validate({}, {"properties": {"test": True}},
|
|
cls=self.validator_class)
|
|
|
|
def test_minItems_invalid_string(self):
|
|
with self.assertRaises(SchemaError):
|
|
# needs to be an integer
|
|
validate([1], {"minItems" : "1"}, cls=self.validator_class)
|
|
|
|
|
|
class RemoteRefResolutionMixin(object):
|
|
def setUp(self):
|
|
patch = mock.patch("jsonschema.validators.requests")
|
|
requests = patch.start()
|
|
requests.get.side_effect = self.resolve
|
|
self.addCleanup(patch.stop)
|
|
|
|
def resolve(self, reference):
|
|
_, _, reference = reference.partition("http://localhost:1234/")
|
|
return mock.Mock(**{"json.return_value" : REMOTES.get(reference)})
|
|
|
|
|
|
@load_json_cases("draft3/refRemote.json")
|
|
class Draft3RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase):
|
|
validator_class = Draft3Validator
|
|
|
|
|
|
@load_json_cases("draft4/refRemote.json")
|
|
class Draft4RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase):
|
|
validator_class = Draft4Validator
|