Schema validation

Adds JSON schema validation to Spyglass.

Change-Id: Ib29bbf9fa02cd6623c75db37a4c8d6f510b52831

spyglass/cli.py

@@ -22,6 +22,7 @@ import yaml
 
 from spyglass.parser.engine import ProcessDataSource
 from spyglass.site_processors.site_processor import SiteProcessor
+from spyglass.validators.json_validator import JSONSchemaValidator
 
 LOG = logging.getLogger(__name__)
 
@@ -133,3 +134,28 @@ def generate_manifests_using_intermediary(
     LOG.info("Generating site Manifests")
     processor_engine = SiteProcessor(intermediary_yaml, manifest_dir)
     processor_engine.render_template(template_dir)
+
+
+@main.command(
+    'validate',
+    short_help='validates pegleg documents',
+    help='Validates pegleg documents against their schema.')
+@click.option(
+    '-d',
+    '--document-path',
+    'document_path',
+    type=click.Path(exists=True, readable=True),
+    required=True,
+    help='Path to the documents to validate.')
+@click.option(
+    '-p',
+    '--schema-path',
+    'schema_path',
+    type=click.Path(exists=True, readable=True),
+    required=True,
+    help=(
+        'Path to a schema file or directory of schema files used to '
+        'validate documents.'))
+def validate_manifests_against_schemas(document_path, schema_path):
+    validator = JSONSchemaValidator(document_path, schema_path)
+    validator.validate()

spyglass/schemas/data_schema.json

@@ -1,5 +1,8 @@
 {
   "$schema": "http://json-schema.org/schema#",
+  "metadata": {
+    "name": "spyglass/Intermediary/v1"
+  },
   "title": "All",
   "description": "All information",
   "type": "object",

spyglass/validators/exceptions.py

@@ -0,0 +1,32 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class PathDoesNotExistError(OSError):
+    """Exception that occurs when the document or schema path does not exist"""
+    pass
+
+
+class UnexpectedFileType(OSError):
+    """Exception that occurs when an unexpected file type is given"""
+    pass
+
+
+class DirectoryEmptyError(OSError):
+    """Exception for when a directory is empty
+
+    This exception can occur when either a directory is empty or if a directory
+    does not have any files with the correct file extension.
+    """
+    pass

spyglass/validators/json_validator.py

@@ -0,0 +1,164 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from glob import glob
+import logging
+import os
+
+from jsonschema import Draft7Validator
+import yaml
+
+from spyglass.validators import exceptions
+from spyglass.validators.validator import BaseDocumentValidator
+
+LOG = logging.getLogger(__name__)
+
+LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:' \
+             '%(funcName)s [%(lineno)3d] %(message)s'
+
+
+class JSONSchemaValidator(BaseDocumentValidator):
+    """Validator for validating documents using jsonschema"""
+
+    def __init__(
+            self,
+            document_path,
+            schema_path,
+            document_extension='.yaml',
+            schema_extension='.yaml',
+            document_loader=yaml.safe_load,
+            schema_loader=yaml.safe_load):
+        super().__init__()
+
+        # Check that given paths are valid
+        if not os.path.exists(document_path):
+            LOG.error('Document path: %s does not exist.', document_path)
+            raise exceptions.PathDoesNotExistError()
+        if not os.path.exists(schema_path):
+            LOG.error('Schema path: %s does not exist.', document_path)
+            raise exceptions.PathDoesNotExistError()
+
+        # Extract list of document file paths from path
+        if os.path.isdir(document_path):
+            # Create match string and use glob to generate list of file paths
+            match = os.path.join(document_path, '**', '*' + document_extension)
+            self.documents = glob(match, recursive=True)
+
+            # Directory should not be empty
+            if not self.documents:
+                LOG.error(
+                    'No files with %s extension found in document path '
+                    '%s', document_extension, document_path)
+                raise exceptions.DirectoryEmptyError()
+        elif os.path.splitext(document_path) == document_extension:
+            # Single files can just be appended to the list to process the same
+            # so long as the extension matches
+            self.documents.append(document_path)
+        else:
+            # Throw error if unexpected file type given
+            raise exceptions.UnexpectedFileType()
+
+        # Extract list of schema file paths from path
+        if os.path.isdir(schema_path):
+            # Create match string and use glob to generate list of file paths
+            match = os.path.join(schema_path, '**', '*' + schema_extension)
+            self.schemas = glob(match, recursive=True)
+
+            # Directory should not be empty
+            if not self.schemas:
+                LOG.error(
+                    'No files with %s extension found in document path '
+                    '%s', document_extension, document_path)
+                raise exceptions.DirectoryEmptyError()
+        elif os.path.splitext(schema_path) == schema_extension:
+            # Single files can just be appended to the list to process the same
+            self.schemas.append(schema_path)
+        else:
+            # Throw error if unexpected file type given
+            raise exceptions.UnexpectedFileType()
+
+        # Initialize pairs list for next step
+        self.document_schema_pairs = []
+
+        self.document_loader = document_loader
+        self.schema_loader = schema_loader
+        self._match_documents_to_schemas()
+
+    def _match_documents_to_schemas(self):
+        """Pairs documents to their schemas for easier processing
+
+        Loops through all documents and finds its associated schema using the
+        "schema" key from documents and the "metadata:name" key from schemas.
+        Matching document/schema pairs are added to document_schema_pairs. Any
+        unmatched documents will display a warning.
+        """
+        if not self.documents:
+            LOG.warning('No documents found.')
+
+        if not self.schemas:
+            LOG.warning('No schemas found.')
+
+        for document in self.documents:
+            pair_found = False
+            with open(document, 'r') as f_doc:
+                loaded_doc = self.document_loader(f_doc)
+            if 'schema' in loaded_doc:
+                schema_name = loaded_doc['schema']
+                for schema in self.schemas:
+                    with open(schema, 'r') as f_schema:
+                        loaded_schema = self.schema_loader(f_schema)
+                    if schema_name == loaded_schema['metadata']['name']:
+                        self.document_schema_pairs.append((document, schema))
+                        pair_found = True
+                        break
+            else:
+                LOG.warning('No schema entry found for file %s', document)
+            if not pair_found:
+                LOG.warning(
+                    'No matching schema found for file %s, '
+                    'data will not be validated.', document)
+
+    def _validate_file(self, document, schema):
+        """Validate a document against a schema using JSON Schema Draft 7
+
+        :param document: File path to the document to validate
+        :param schema: File path to the schema used to validate document
+        :return: A list of errors from the validator
+        """
+        with open(document, 'r') as f_doc:
+            loaded_doc = self.document_loader(f_doc)
+        with open(schema, 'r') as f_schema:
+            loaded_schema = self.schema_loader(f_schema)
+        validator = Draft7Validator(loaded_schema)
+        return sorted(validator.iter_errors(loaded_doc), key=lambda e: e.path)
+
+    def validate(self):
+        """Validates document against its schema
+
+        Loops through document_schema_pairs list and validates each pair. Any
+        errors are logged and returned in a dictionary by file.
+
+        :return: A dictionary of filenames and their list of validation errors
+        """
+        error_list = {}
+        for document, schema in self.document_schema_pairs:
+            LOG.info(
+                'Validating document %s using schema %s', document, schema)
+            errors = self._validate_file(document, schema)
+            if errors:
+                for error in errors:
+                    LOG.error(error.message)
+                error_list[document] = errors
+
+        return error_list

spyglass/validators/validator.py

@@ -0,0 +1,33 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+
+class BaseDocumentValidator(metaclass=abc.ABCMeta):
+    """Abstract class for document validation"""
+
+    def __init__(self):
+        self.documents = []
+        self.schemas = []
+
+    @abc.abstractmethod
+    def validate(self):
+        """Validate documents against schemas.
+
+         Runs a validation method on documents, comparing them to schemas for
+         valid data structure and types.
+        """
+
+        return

test-requirements.txt

@@ -1,3 +1,8 @@
+# Testing
+pytest==4.4.1
+pytest-xdist==1.28.0
+pytest-cov==2.6.1
+
 # Formatting
 yapf==0.27.0
 

tests/shared/documents/invalid/invalid.yaml

@@ -0,0 +1,10 @@
+---
+schema: InvalidSchema
+foo: Not a number
+bar: "Doesn't equal constant"
+baz:
+  staticProperty:
+  - This array needs at least one number
+  property1: The propertyNames keyword is an alternative to patternProperties
+  pr()perty2: "All property names must match supplied conditions (in this"
+              "case, it's a regex)"

tests/shared/documents/valid/PKICatalogue/pki-catalogue.yaml

@@ -0,0 +1,342 @@
+---
+schema: promenade/PKICatalog/v1
+metadata:
+  schema: metadata/Document/v1
+  name: cluster-certificates
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data:
+  certificate_authorities:
+    kubernetes:
+      description: CA for Kubernetes components
+      certificates:
+        - document_name: apiserver
+          description: Service certificate for Kubernetes apiserver
+          common_name: apiserver
+          hosts:
+            - localhost
+            - 127.0.0.1
+            - 10.96.0.1
+          kubernetes_service_names:
+            - kubernetes.default.svc.cluster.local
+
+        - document_name: kubelet-genesis
+          common_name: system:node:cab2r72c16
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c12
+          common_name: system:node:cab2r72c12
+          hosts:
+            - cab2r72c12
+            - 10.0.220.12
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c13
+          common_name: system:node:cab2r72c13
+          hosts:
+            - cab2r72c13
+            - 10.0.220.13
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c14
+          common_name: system:node:cab2r72c14
+          hosts:
+            - cab2r72c14
+            - 10.0.220.14
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c15
+          common_name: system:node:cab2r72c15
+          hosts:
+            - cab2r72c15
+            - 10.0.220.15
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c16
+          common_name: system:node:cab2r72c16
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r72c17
+          common_name: system:node:cab2r72c17
+          hosts:
+            - cab2r72c17
+            - 10.0.220.17
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c12
+          common_name: system:node:cab2r73c12
+          hosts:
+            - cab2r73c12
+            - 10.0.220.18
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c13
+          common_name: system:node:cab2r73c13
+          hosts:
+            - cab2r73c13
+            - 10.0.220.19
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c14
+          common_name: system:node:cab2r73c14
+          hosts:
+            - cab2r73c14
+            - 10.0.220.20
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c15
+          common_name: system:node:cab2r73c15
+          hosts:
+            - cab2r73c15
+            - 10.0.220.21
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c16
+          common_name: system:node:cab2r73c16
+          hosts:
+            - cab2r73c16
+            - 10.0.220.22
+            -
+          groups:
+            - system:nodes
+        - document_name: kubelet-cab2r73c17
+          common_name: system:node:cab2r73c17
+          hosts:
+            - cab2r73c17
+            - 10.0.220.23
+            -
+          groups:
+            - system:nodes
+        - document_name: scheduler
+          description: Service certificate for Kubernetes scheduler
+          common_name: system:kube-scheduler
+        - document_name: controller-manager
+          description: certificate for controller-manager
+          common_name: system:kube-controller-manager
+        - document_name: admin
+          common_name: admin
+          groups:
+            - system:masters
+        - document_name: armada
+          common_name: armada
+          groups:
+            - system:masters
+    kubernetes-etcd:
+      description: Certificates for Kubernetes's etcd servers
+      certificates:
+        - document_name: apiserver-etcd
+          description: etcd client certificate for use by Kubernetes apiserver
+          common_name: apiserver
+        - document_name: kubernetes-etcd-anchor
+          description: anchor
+          common_name: anchor
+        - document_name: kubernetes-etcd-genesis
+          common_name: kubernetes-etcd-genesis
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r72c16
+          common_name: kubernetes-etcd-cab2r72c16
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r72c17
+          common_name: kubernetes-etcd-cab2r72c17
+          hosts:
+            - cab2r72c17
+            - 10.0.220.17
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r73c16
+          common_name: kubernetes-etcd-cab2r73c16
+          hosts:
+            - cab2r73c16
+            - 10.0.220.22
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r73c17
+          common_name: kubernetes-etcd-cab2r73c17
+          hosts:
+            - cab2r73c17
+            - 10.0.220.23
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+    kubernetes-etcd-peer:
+      certificates:
+        - document_name: kubernetes-etcd-genesis-peer
+          common_name: kubernetes-etcd-genesis-peer
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r72c16-peer
+          common_name: kubernetes-etcd-cab2r72c16-peer
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r72c17-peer
+          common_name: kubernetes-etcd-cab2r72c17-peer
+          hosts:
+            - cab2r72c17
+            - 10.0.220.17
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r73c16-peer
+          common_name: kubernetes-etcd-cab2r73c16-peer
+          hosts:
+            - cab2r73c16
+            - 10.0.220.22
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+        - document_name: kubernetes-etcd-cab2r73c17-peer
+          common_name: kubernetes-etcd-cab2r73c17-peer
+          hosts:
+            - cab2r73c17
+            - 10.0.220.23
+            -
+            - 127.0.0.1
+            - localhost
+            - kubernetes-etcd.kube-system.svc.cluster.local
+            - 10.96.0.2
+    ksn-etcd:
+      description: Certificates for Calico etcd client traffic
+      certificates:
+        - document_name: ksn-etcd-anchor
+          description: anchor
+          common_name: anchor
+        - document_name: ksn-etcd-cab2r72c16
+          common_name: ksn-etcd-cab2r72c16
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r72c17
+          common_name: ksn-etcd-cab2r72c17
+          hosts:
+            - cab2r72c17
+            - 10.0.220.17
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r73c16
+          common_name: ksn-etcd-cab2r73c16
+          hosts:
+            - cab2r73c16
+            - 10.0.220.22
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r73c17
+          common_name: ksn-etcd-cab2r73c17
+          hosts:
+            - cab2r73c17
+            - 10.0.220.23
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-node
+          common_name: calcico-node
+    ksn-etcd-peer:
+      description: Certificates for Calico etcd clients
+      certificates:
+        - document_name: ksn-etcd-cab2r72c16-peer
+          common_name: ksn-etcd-cab2r72c16-peer
+          hosts:
+            - cab2r72c16
+            - 10.0.220.16
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r72c17-peer
+          common_name: ksn-etcd-cab2r72c17-peer
+          hosts:
+            - cab2r72c17
+            - 10.0.220.17
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r73c16-peer
+          common_name: ksn-etcd-cab2r73c16-peer
+          hosts:
+            - cab2r73c16
+            - 10.0.220.22
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-etcd-cab2r73c17-peer
+          common_name: ksn-etcd-cab2r73c17-peer
+          hosts:
+            - cab2r73c17
+            - 10.0.220.23
+            -
+            - 127.0.0.1
+            - localhost
+            - 10.96.232.136
+        - document_name: ksn-node-peer
+          common_name: calico-node-peer
+  keypairs:
+    - name: service-account
+      description: Service account signing key for use by Kubernetes controller-manager.
+...

tests/shared/documents/valid/SiteDefinition/site-definition.yaml

@@ -0,0 +1,12 @@
+---
+schema: pegleg/SiteDefinition/v1
+metadata:
+  schema: metadata/Document/v1
+  layeringDefinition:
+    abstract: false
+    layer: site
+  name: airship-seaworthy
+  storagePolicy: cleartext
+data:
+  site_type: foundry
+...

tests/shared/schemas/InvalidSchema/invalid-schema.yaml

@@ -0,0 +1,20 @@
+---
+metadata:
+  name: InvalidSchema
+type: object
+properties:
+  foo:
+    type: number
+  bar:
+    const: Must equal this value
+  baz:
+    type: object
+    properties:
+      staticProperty:
+        type: array
+        contains:
+          type: number
+    propertyNames:
+      pattern: "^([0-9a-zA-Z]*)$"
+    additionalProperties:
+      type: string

tests/shared/schemas/PKICatalogue/pki-catalogue-schema.yaml

@@ -0,0 +1,42 @@
+---
+schema: deckhand/DataSchema/v1
+metadata:
+  schema: metadata/Control/v1
+  name: promenade/PKICatalog/v1
+  labels:
+    application: promenade
+data:
+  certificate_authorities:
+    type: array
+    items:
+      type: object
+      properties:
+        description:
+          type: string
+        certificates:
+          type: array
+          items:
+            type: object
+            properties:
+              document_name:
+                type: string
+              description:
+                type: string
+              common_name:
+                type: string
+              hosts:
+                type: array
+                items: string
+              groups:
+                type: array
+                items: string
+  keypairs:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        description:
+          type: string
+...

tests/shared/schemas/SiteDefinition/site-definition-schema.yaml

@@ -0,0 +1,27 @@
+---
+schema: deckhand/DataSchema/v1
+metadata:
+  schema: metadata/Control/v1
+  name: pegleg/SiteDefinition/v1
+data:
+  type: object
+  properties:
+    repositories:
+      type: object
+      additionalProperties:
+        type: object
+        properties:
+          revision:
+            type: string
+          url:
+            type: string
+        required:
+          - revision
+          - url
+
+    site_type:
+      type: string
+  required:
+    - site_type
+  additionalProperties: false
+...

tests/unit/validators/test_json_validator.py

@@ -0,0 +1,89 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import pytest
+
+from spyglass.validators.exceptions import PathDoesNotExistError
+from spyglass.validators.json_validator import JSONSchemaValidator
+
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared')
+
+DOCUMENT_DIR = os.path.join(FIXTURE_DIR, 'documents')
+
+VALID_DOCUMENTS_DIR = os.path.join(DOCUMENT_DIR, 'valid')
+
+INVALID_DOCUMENTS_DIR = os.path.join(DOCUMENT_DIR, 'invalid')
+
+SCHEMA_DIR = os.path.join(FIXTURE_DIR, 'schemas')
+
+
+def test_bad_document_path():
+    """Tests that an invalid document path raises a PathDoesNotExistError"""
+    bad_path = os.path.join(FIXTURE_DIR, 'not_documents')
+    with pytest.raises(PathDoesNotExistError):
+        JSONSchemaValidator(bad_path, SCHEMA_DIR)
+
+
+def test_bad_schema_path():
+    """Tests that an invalid schema path raises a PathDoesNotExistError"""
+    bad_path = os.path.join(FIXTURE_DIR, 'not_schemas')
+    with pytest.raises(PathDoesNotExistError):
+        JSONSchemaValidator(DOCUMENT_DIR, bad_path)
+
+
+def test_document_schema_matching():
+    """Tests that documents and schema are correctly paired up"""
+    expected_pairs = [
+        ('site-definition.yaml', 'site-definition-schema.yaml'),
+        ('pki-catalogue.yaml', 'pki-catalogue-schema.yaml')
+    ]
+    validator = JSONSchemaValidator(VALID_DOCUMENTS_DIR, SCHEMA_DIR)
+    no_path_pairs = []
+    for pair in validator.document_schema_pairs:
+        no_path_pairs.append(
+            (os.path.split(pair[0])[1], os.path.split(pair[1])[1]))
+    assert no_path_pairs == expected_pairs
+
+
+def test_document_schema_matching_no_files():
+    """Tests that document and schema are not paired if there are no matches"""
+    site_definition_doc_dir = os.path.join(
+        VALID_DOCUMENTS_DIR, 'SiteDefinition')
+    site_definition_schema_dir = os.path.join(SCHEMA_DIR, 'PKICatalogue')
+
+    expected_pairs = []
+    validator = JSONSchemaValidator(
+        site_definition_doc_dir, site_definition_schema_dir)
+    no_path_pairs = []
+    for pair in validator.document_schema_pairs:
+        no_path_pairs.append(
+            (os.path.split(pair[0])[1], os.path.split(pair[1])[1]))
+    assert no_path_pairs == expected_pairs
+
+
+def test_validate():
+    """Tests that validation of correct files yields no errors"""
+    validator = JSONSchemaValidator(VALID_DOCUMENTS_DIR, SCHEMA_DIR)
+    errors = validator.validate()
+    assert not errors
+
+
+def test_validate_with_errors():
+    """Tests that correct errors are generated for an invalid document"""
+    validator = JSONSchemaValidator(INVALID_DOCUMENTS_DIR, SCHEMA_DIR)
+    errors = validator.validate()
+    assert errors

tools/gate/run-unit-tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+posargs=$@
+# cross-platform way to derive the number of logical cores
+readonly num_cores=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+if [ ${#posargs} -ge 1 ]; then
+    pytest -k ${posargs} -n $num_cores
+else
+    pytest -n $num_cores
+fi
+set +e

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = pep8, docs
+envlist = py36, py37, pep8, docs, cover
 minversion = 2.3.1
 skipsdist = True
 
@@ -17,13 +17,14 @@ whitelist_externals =
   find
 commands =
   find . -type f -name "*.pyc" -delete
+  {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'
 
 [testenv:fmt]
 basepython = python3
 deps =
     -r{toxinidir}/test-requirements.txt
 commands =
-    yapf -ir {toxinidir}/spyglass
+    yapf -ir {toxinidir}/spyglass {toxinidir}/tests
 
 [testenv:pep8]
 basepython = python3
@@ -31,8 +32,8 @@ deps =
   -r{toxinidir}/test-requirements.txt
 commands =
   bash -c {toxinidir}/tools/gate/whitespace-linter.sh
-  yapf -dr {toxinidir}/spyglass {toxinidir}/setup.py
+  yapf -dr {toxinidir}/spyglass {toxinidir}/setup.py {toxinidir}/tests
-  flake8 {toxinidir}/spyglass
+  flake8 {toxinidir}/spyglass {toxinidir}/tests
   bandit -r spyglass -n 5
   safety check -r requirements.txt --bare
 whitelist_externals =
@@ -62,3 +63,15 @@ commands =
     rm -rf doc/build
     sphinx-build -b html doc/source doc/build/html -n -W -v
 whitelist_externals = rm
+
+[testenv:cover]
+basepython = python3
+deps =
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+commands =
+  bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass --cov-report \
+      html:cover --cov-report xml:cover/coverage.xml --cov-report term \
+      --cov-fail-under 10 tests/'
+whitelist_externals =
+  bash