From 64171aacf415c9bdf9c858b61e7e1059505aebe9 Mon Sep 17 00:00:00 2001 From: "Carter, Matthew (mc981n)" Date: Tue, 30 Apr 2019 10:11:54 -0500 Subject: [PATCH] Validate existence of "deployment-version" doc on configdocs commit This PS adds funtionality to Shipyard to validate the existence of the Pegleg-generated "deployment-version" document (Pegleg change id: I7919b02d70c9797f689cdad85066d3953b978901). As implied, this new validation only checks for the presence of the document (by name and schema) and currently does not care about any of the document's contents under "data". The severity of a failed validation is configurable through the new "validations" configuration section in shipyard.conf, and is defaulted to skip the validation altogether. This means that by default, this patch set does not alter the functionality of Shipyard Note that with the default configuration of this new validation, Shipyard functionality should be unchanged. Change-Id: I5e7269066f769804710a0fd1f2c8d0aece0d3314 --- charts/shipyard/values.yaml | 9 ++++ doc/source/API.rst | 1 + doc/source/_static/shipyard.conf.sample | 25 +++++++++++ doc/source/site-definition-documents.rst | 41 ++++++++++++++++- .../etc/shipyard/shipyard.conf.sample | 25 +++++++++++ .../shipyard_airflow/conf/config.py | 32 +++++++++++++ .../validate_deployment_configuration.py | 8 +++- .../validators/validate_deployment_version.py | 45 +++++++++++++++++++ .../unit/control/test_action_validators.py | 2 + tools/resources/shipyard.conf | 6 ++- 10 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_version.py diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 199a54df..47535832 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -416,6 +416,10 @@ conf: # deny_all rule is hard coded in the policy.py code to allow no access. policy_default_rule: deny_all document_info: + # The name of the deployment version document that Shipyard validates + deployment_version_name: deployment-version + # The schema of the deployment version document that Shipyard validates + deployment_version_schema: pegleg/DeploymentData/v1 # The name of the deployment configuration document that Shipyard expects # and validates deployment_configuration_name: deployment-configuration @@ -425,6 +429,11 @@ conf: # The schema of the deployment strategy document that Shipyard expects # and validates. deployment_strategy_schema: shipyard/DeploymentStrategy/v1 + validations: + # Control the severity of the deployment-version document validation + # that Shipyard performs during commit configdocs. + # Possible values are Skip, Info, Warning, and Error + deployment_version_commit: Skip airflow_config_file: path: /usr/local/airflow/airflow.cfg airflow: diff --git a/doc/source/API.rst b/doc/source/API.rst index 42bab95d..99720174 100644 --- a/doc/source/API.rst +++ b/doc/source/API.rst @@ -203,6 +203,7 @@ Responses with no contents. - The request has no new/changed contents for the collection. - The request is missing a Content-Length header. + - The provided document(s) fail Shipyard/Deckhand validations. 409 Conflict A condition in the system is blocking this document ingestion diff --git a/doc/source/_static/shipyard.conf.sample b/doc/source/_static/shipyard.conf.sample index 8c67a399..8a07171c 100644 --- a/doc/source/_static/shipyard.conf.sample +++ b/doc/source/_static/shipyard.conf.sample @@ -90,6 +90,14 @@ # From shipyard_api # +# The name of the deployment version document that Shipyard validates (string +# value) +#deployment_version_name = deployment-version + +# The schema of the deployment version document that Shipyard validates (string +# value) +#deployment_version_schema = pegleg/DeploymentData/v1 + # The name of the deployment-configuration document that Shipyard expects and # validates (string value) #deployment_configuration_name = deployment-configuration @@ -425,3 +433,20 @@ # type is used to perform the service lookup in the Keystone service catalog. # (string value) #service_type = shipyard + + +[validations] + +# +# From shipyard_api +# + +# Control the severity of the deployment-version validation validation during +# commit configdocs. (string value) +# Possible values: +# Skip - Skip the validation altogether +# Info - Print an Info level message if the validation fails +# Warning - Print a Warning level message if the validation fails +# Error - Return an error when the validation fails and prevent the commit from +# proceeding +#deployment_version_commit = Skip diff --git a/doc/source/site-definition-documents.rst b/doc/source/site-definition-documents.rst index 65b66eb7..09dd2b87 100644 --- a/doc/source/site-definition-documents.rst +++ b/doc/source/site-definition-documents.rst @@ -38,7 +38,7 @@ of the Armada manifest that will be used during the deployment/update. A `sample deployment-configuration`_ shows a completely specified example. Note that the name and schema Shipyard expects the deployment configuration -document to have is conifgurable via the document_info section in the +document to have is configurable via the document_info section in the :ref:`Shipyard configuration `, but should be left defaulted in most cases. @@ -185,7 +185,7 @@ document for the site. Example:: the group a success. Note that the schema Shipyard expects the deployment strategy document to have -is conifgurable via the document_info section in the +is configurable via the document_info section in the :ref:`Shipyard configuration `, but should be left defaulted in most cases. @@ -625,6 +625,43 @@ An Example of CLI `describe action` command output, with failed processing:: step/01BZZK07NF04XPC5F4SCTHNPKN/drydock_nodes 6 failed +Deployment Version +------------------- +A deployment version document is a Pegleg_-generated document that captures +information about the repositories used to generate the site defintion. The +presence of this document is optional by default, but Shipyard can be +:ref:`configured ` to ensure this document exists, and +issue a warning or error if it is absent from a configdocs collection. +Document example:: + + --- + schema: pegleg/DeploymentData/v1 + metadata: + schema: metadata/Document/v1 + name: deployment-version + layeringDefinition: + abstract: false + layer: global + storagePolicy: cleartext + data: + documents: + site-repository: + commit: 37260deff6a213e30897fc284a993c791336a99d + tag: master + dirty: false + repository-of-secrets: + commit: 23e7265aee4843301807d649036f8e860fda0cda + tag: master + dirty: false + +Currently, Shipyard does not use this document for anything. Use of this +document's data will be added to a future version of Shipyard/Airship. + +Note, the name and schema Shipyard expects this document to have can be +configured via the document_info section in the +:ref:`Shipyard configuration `. + +.. _Pegleg: https://git.airshipit.org/cgit/airship-pegleg .. _`Armada manifest document`: https://airship-armada.readthedocs.io/en/latest/operations/guide-build-armada-yaml.html?highlight=manifest .. _`Default configuration values`: https://git.airshipit.org/cgit/airship-shipyard/tree/src/bin/shipyard_airflow/shipyard_airflow/plugins/deployment_configuration_operator.py .. _DeploymentConfiguration: https://git.airshipit.org/cgit/airship-shipyard/tree/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index 8c67a399..8a07171c 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -90,6 +90,14 @@ # From shipyard_api # +# The name of the deployment version document that Shipyard validates (string +# value) +#deployment_version_name = deployment-version + +# The schema of the deployment version document that Shipyard validates (string +# value) +#deployment_version_schema = pegleg/DeploymentData/v1 + # The name of the deployment-configuration document that Shipyard expects and # validates (string value) #deployment_configuration_name = deployment-configuration @@ -425,3 +433,20 @@ # type is used to perform the service lookup in the Keystone service catalog. # (string value) #service_type = shipyard + + +[validations] + +# +# From shipyard_api +# + +# Control the severity of the deployment-version validation validation during +# commit configdocs. (string value) +# Possible values: +# Skip - Skip the validation altogether +# Info - Print an Info level message if the validation fails +# Warning - Print a Warning level message if the validation fails +# Error - Return an error when the validation fails and prevent the commit from +# proceeding +#deployment_version_commit = Skip diff --git a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py index fda345b8..8a5bf4cc 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py @@ -291,6 +291,18 @@ SECTIONS = [ title=('Information about some of the documents Shipyard needs to ' 'handle'), options=[ + cfg.StrOpt( + 'deployment_version_name', + default='deployment-version', + help=('The name of the deployment version document that ' + 'Shipyard validates') + ), + cfg.StrOpt( + 'deployment_version_schema', + default='pegleg/DeploymentData/v1', + help=('The schema of the deployment version document that ' + 'Shipyard validates') + ), cfg.StrOpt( 'deployment_configuration_name', default='deployment-configuration', @@ -314,6 +326,26 @@ SECTIONS = [ ), ] ), + ConfigSection( + name='validations', + title='Validation Configurations', + options=[ + cfg.StrOpt( + 'deployment_version_commit', + default='Skip', + help=('Control the severity of the deployment-version ' + 'validation validation during commit configdocs. '), + ignore_case=True, + choices=[('Skip', 'Skip the validation altogether'), + ('Info', 'Print an Info level message if the ' + 'validation fails'), + ('Warning', 'Print a Warning level message if the ' + 'validation fails'), + ('Error', 'Return an error when the validation fails ' + 'and prevent the commit from proceeding')] + ), + ] + ), ] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py index 762f8de2..6369ab47 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py @@ -24,6 +24,7 @@ from shipyard_airflow.common.document_validators.document_validator import ( DocumentValidator ) from .validate_deployment_strategy import ValidateDeploymentStrategy +from .validate_deployment_version import ValidateDeploymentVersion CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -50,7 +51,7 @@ class ValidateDeploymentConfigurationFull( ValidateDeploymentConfigurationBasic): """Validates the DeploymentConfiguration - Includes a triggered check for DeploymentStrategy + Includes triggered checks for DeploymentStrategy and DeploymentVersion """ def do_validate(self): try: @@ -76,4 +77,9 @@ class ValidateDeploymentConfigurationFull( "'all-at-once' is assumed, and deployment strategy will " "not be further validated") + if CONF.validations.deployment_version_commit.lower() != 'skip': + self.add_triggered_validation( + ValidateDeploymentVersion, + CONF.document_info.deployment_version_name) + super().do_validate() diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_version.py b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_version.py new file mode 100644 index 00000000..0d1b7822 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_version.py @@ -0,0 +1,45 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Classes and functions to support Shipyard specific document validation + +Set up a validator for the DeploymentData document generated by pegleg +Only validates that which is not already covered by schema validation, which +is performed by Deckhand on Shipyard's behalf. +""" +from shipyard_airflow.common.document_validators.document_validator import ( + DocumentValidator +) +import logging +from oslo_config import cfg + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class ValidateDeploymentVersion(DocumentValidator): + """Validates the existence of the deployment data document""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + + schema = CONF.document_info.deployment_version_schema + # Just capitalize the "missing_severity", and then the base class will take + # care of whether or not the value is actually valid + missing_severity = CONF.validations.deployment_version_commit.capitalize() + + def do_validate(self): + # Need to define this since it is abstract in the base class, but all + # we need to check for this document is its existence, which is taken + # care of by the validate() function in the base class + pass diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py index 58298aee..7b6bf0cc 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py @@ -62,6 +62,8 @@ def get_doc_returner(style, ds_name): return [dc] elif doc == 'dep-strat': return [strategy] + elif doc == 'deployment-version': + return [MagicMock()] return [] return doc_returner diff --git a/tools/resources/shipyard.conf b/tools/resources/shipyard.conf index dee194e3..eede00a6 100644 --- a/tools/resources/shipyard.conf +++ b/tools/resources/shipyard.conf @@ -48,4 +48,8 @@ policy_default_rule = deny_all [document_info] deployment_configuration_name = deployment-configuration deployment_configuration_schema = shipyard/DeploymentConfiguration/v1 -deployment_strategy_schema = shipyard/DeploymentStrategy/v1 \ No newline at end of file +deployment_strategy_schema = shipyard/DeploymentStrategy/v1 +deployment_version_name = deployment-version +deployment_version_schema = pegleg/DeploymentData/v1 +[validations] +deployment_version_commit=Skip \ No newline at end of file