Merge "Validate existence of "deployment-version" doc on configdocs commit"

charts/shipyard/values.yaml

@@ -416,6 +416,10 @@ conf:
       # deny_all rule is hard coded in the policy.py code to allow no access.
       policy_default_rule: deny_all
     document_info:
+      # The name of the deployment version document that Shipyard validates
+      deployment_version_name: deployment-version
+      # The schema of the deployment version document that Shipyard validates
+      deployment_version_schema: pegleg/DeploymentData/v1
       # The name of the deployment configuration document that Shipyard expects
       # and validates
       deployment_configuration_name: deployment-configuration
@@ -425,6 +429,11 @@ conf:
       # The schema of the deployment strategy document that Shipyard expects
       # and validates.
       deployment_strategy_schema: shipyard/DeploymentStrategy/v1
+    validations:
+      # Control the severity of the deployment-version document validation
+      # that Shipyard performs during commit configdocs.
+      # Possible values are Skip, Info, Warning, and Error
+      deployment_version_commit: Skip
   airflow_config_file:
     path: /usr/local/airflow/airflow.cfg
   airflow:

doc/source/API.rst

@@ -203,6 +203,7 @@ Responses
     with no contents.
   - The request has no new/changed contents for the collection.
   - The request is missing a Content-Length header.
+  - The provided document(s) fail Shipyard/Deckhand validations.
 
 409 Conflict
   A condition in the system is blocking this document ingestion

doc/source/_static/shipyard.conf.sample

@@ -90,6 +90,14 @@
 # From shipyard_api
 #
 
+# The name of the deployment version document that Shipyard validates (string
+# value)
+#deployment_version_name = deployment-version
+
+# The schema of the deployment version document that Shipyard validates (string
+# value)
+#deployment_version_schema = pegleg/DeploymentData/v1
+
 # The name of the deployment-configuration document that Shipyard expects and
 # validates (string value)
 #deployment_configuration_name = deployment-configuration
@@ -425,3 +433,20 @@
 # type is used to perform the service lookup in the Keystone service catalog.
 # (string value)
 #service_type = shipyard
+
+
+[validations]
+
+#
+# From shipyard_api
+#
+
+# Control the severity of the deployment-version validation validation during
+# commit configdocs.  (string value)
+# Possible values:
+# Skip - Skip the validation altogether
+# Info - Print an Info level message if the validation fails
+# Warning - Print a Warning level message if the validation fails
+# Error - Return an error when the validation fails and prevent the commit from
+# proceeding
+#deployment_version_commit = Skip

doc/source/site-definition-documents.rst

@@ -38,7 +38,7 @@ of the Armada manifest that will be used during the deployment/update.
 A `sample deployment-configuration`_ shows a completely specified example.
 
 Note that the name and schema Shipyard expects the deployment configuration
-document to have is conifgurable via the document_info section in the
+document to have is configurable via the document_info section in the
 :ref:`Shipyard configuration <sample-configuration>`, but should be left
 defaulted in most cases.
 
@@ -185,7 +185,7 @@ document for the site. Example::
    the group a success.
 
 Note that the schema Shipyard expects the deployment strategy document to have
-is conifgurable via the document_info section in the
+is configurable via the document_info section in the
 :ref:`Shipyard configuration <sample-configuration>`, but should be left
 defaulted in most cases.
 
@@ -625,6 +625,43 @@ An Example of CLI `describe action` command output, with failed processing::
     step/01BZZK07NF04XPC5F4SCTHNPKN/drydock_nodes                      6            failed
 
 
+Deployment Version
+-------------------
+A deployment version document is a Pegleg_-generated document that captures
+information about the repositories used to generate the site defintion. The
+presence of this document is optional by default, but Shipyard can be
+:ref:`configured <sample-configuration>` to ensure this document exists, and
+issue a warning or error if it is absent from a configdocs collection.
+Document example::
+
+  ---
+  schema: pegleg/DeploymentData/v1
+  metadata:
+  schema: metadata/Document/v1
+  name: deployment-version
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+  data:
+    documents:
+      site-repository:
+        commit: 37260deff6a213e30897fc284a993c791336a99d
+        tag: master
+        dirty: false
+      repository-of-secrets:
+        commit: 23e7265aee4843301807d649036f8e860fda0cda
+        tag: master
+        dirty: false
+
+Currently, Shipyard does not use this document for anything. Use of this
+document's data will be added to a future version of Shipyard/Airship.
+
+Note, the name and schema Shipyard expects this document to have can be
+configured via the document_info section in the
+:ref:`Shipyard configuration <sample-configuration>`.
+
+.. _Pegleg: https://git.airshipit.org/cgit/airship-pegleg
 .. _`Armada manifest document`: https://airship-armada.readthedocs.io/en/latest/operations/guide-build-armada-yaml.html?highlight=manifest
 .. _`Default configuration values`: https://git.airshipit.org/cgit/airship-shipyard/tree/src/bin/shipyard_airflow/shipyard_airflow/plugins/deployment_configuration_operator.py
 .. _DeploymentConfiguration: https://git.airshipit.org/cgit/airship-shipyard/tree/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -90,6 +90,14 @@
 # From shipyard_api
 #
 
+# The name of the deployment version document that Shipyard validates (string
+# value)
+#deployment_version_name = deployment-version
+
+# The schema of the deployment version document that Shipyard validates (string
+# value)
+#deployment_version_schema = pegleg/DeploymentData/v1
+
 # The name of the deployment-configuration document that Shipyard expects and
 # validates (string value)
 #deployment_configuration_name = deployment-configuration
@@ -425,3 +433,20 @@
 # type is used to perform the service lookup in the Keystone service catalog.
 # (string value)
 #service_type = shipyard
+
+
+[validations]
+
+#
+# From shipyard_api
+#
+
+# Control the severity of the deployment-version validation validation during
+# commit configdocs.  (string value)
+# Possible values:
+# Skip - Skip the validation altogether
+# Info - Print an Info level message if the validation fails
+# Warning - Print a Warning level message if the validation fails
+# Error - Return an error when the validation fails and prevent the commit from
+# proceeding
+#deployment_version_commit = Skip

src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@@ -291,6 +291,18 @@ SECTIONS = [
         title=('Information about some of the documents Shipyard needs to '
                'handle'),
         options=[
+            cfg.StrOpt(
+                'deployment_version_name',
+                default='deployment-version',
+                help=('The name of the deployment version document that '
+                      'Shipyard validates')
+            ),
+            cfg.StrOpt(
+                'deployment_version_schema',
+                default='pegleg/DeploymentData/v1',
+                help=('The schema of the deployment version document that '
+                      'Shipyard validates')
+            ),
             cfg.StrOpt(
                 'deployment_configuration_name',
                 default='deployment-configuration',
@@ -314,6 +326,26 @@ SECTIONS = [
             ),
         ]
     ),
+    ConfigSection(
+        name='validations',
+        title='Validation Configurations',
+        options=[
+            cfg.StrOpt(
+                'deployment_version_commit',
+                default='Skip',
+                help=('Control the severity of the deployment-version '
+                      'validation validation during commit configdocs. '),
+                ignore_case=True,
+                choices=[('Skip', 'Skip the validation altogether'),
+                         ('Info', 'Print an Info level message if the '
+                                  'validation fails'),
+                         ('Warning', 'Print a Warning level message if the '
+                                     'validation fails'),
+                         ('Error', 'Return an error when the validation fails '
+                                   'and prevent the commit from proceeding')]
+            ),
+        ]
+    ),
 ]
 
 

src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py

@@ -24,6 +24,7 @@ from shipyard_airflow.common.document_validators.document_validator import (
     DocumentValidator
 )
 from .validate_deployment_strategy import ValidateDeploymentStrategy
+from .validate_deployment_version import ValidateDeploymentVersion
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
@@ -50,7 +51,7 @@ class ValidateDeploymentConfigurationFull(
         ValidateDeploymentConfigurationBasic):
     """Validates the DeploymentConfiguration
 
-    Includes a triggered check for DeploymentStrategy
+    Includes triggered checks for DeploymentStrategy and DeploymentVersion
     """
     def do_validate(self):
         try:
@@ -76,4 +77,9 @@ class ValidateDeploymentConfigurationFull(
                      "'all-at-once' is assumed, and deployment strategy will "
                      "not be further validated")
 
+        if CONF.validations.deployment_version_commit.lower() != 'skip':
+            self.add_triggered_validation(
+                ValidateDeploymentVersion,
+                CONF.document_info.deployment_version_name)
+
         super().do_validate()

src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_version.py

@@ -0,0 +1,45 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Classes and functions to support Shipyard specific document validation
+
+Set up a validator for the DeploymentData document generated by pegleg
+Only validates that which is not already covered by schema validation, which
+is performed by Deckhand on Shipyard's behalf.
+"""
+from shipyard_airflow.common.document_validators.document_validator import (
+    DocumentValidator
+)
+import logging
+from oslo_config import cfg
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class ValidateDeploymentVersion(DocumentValidator):
+    """Validates the existence of the deployment data document"""
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    schema = CONF.document_info.deployment_version_schema
+    # Just capitalize the "missing_severity", and then the base class will take
+    # care of whether or not the value is actually valid
+    missing_severity = CONF.validations.deployment_version_commit.capitalize()
+
+    def do_validate(self):
+        # Need to define this since it is abstract in the base class, but all
+        # we need to check for this document is its existence, which is taken
+        # care of by the validate() function in the base class
+        pass

src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py

@@ -62,6 +62,8 @@ def get_doc_returner(style, ds_name):
                 return [dc]
             elif doc == 'dep-strat':
                 return [strategy]
+            elif doc == 'deployment-version':
+                return [MagicMock()]
         return []
     return doc_returner
 

tools/resources/shipyard.conf

@@ -48,4 +48,8 @@ policy_default_rule = deny_all
 [document_info]
 deployment_configuration_name = deployment-configuration
 deployment_configuration_schema = shipyard/DeploymentConfiguration/v1
-deployment_strategy_schema = shipyard/DeploymentStrategy/v1
+deployment_strategy_schema = shipyard/DeploymentStrategy/v1
+deployment_version_name = deployment-version
+deployment_version_schema = pegleg/DeploymentData/v1
+[validations]
+deployment_version_commit=Skip