Browse Source

Merge "Be configuration driven when referencing document names/schemas"

changes/30/656530/8
Zuul 2 years ago
committed by Gerrit Code Review
parent
commit
e2a87a1830
  1. 10
      charts/shipyard/values.yaml
  2. 21
      doc/source/_static/shipyard.conf.sample
  3. 1
      doc/source/sampleconf.rst
  4. 10
      doc/source/site-definition-documents.rst
  5. 21
      src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
  6. 28
      src/bin/shipyard_airflow/shipyard_airflow/conf/config.py
  7. 2
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
  8. 6
      src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py
  9. 5
      src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py
  10. 5
      src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py
  11. 39
      src/bin/shipyard_airflow/shipyard_airflow/plugins/deployment_configuration_operator.py
  12. 11
      src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py
  13. 5
      src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py
  14. 37
      src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_configuration_operator.py
  15. 4
      tools/resources/shipyard.conf

10
charts/shipyard/values.yaml

@ -415,6 +415,16 @@ conf:
# If non-existent rule is used, the request should be denied. The
# deny_all rule is hard coded in the policy.py code to allow no access.
policy_default_rule: deny_all
document_info:
# The name of the deployment configuration document that Shipyard expects
# and validates
deployment_configuration_name: deployment-configuration
# The schema of the deployment configuration document that Shipyard
# expects and validates
deployment_configuration_schema: shipyard/DeploymentConfiguration/v1
# The schema of the deployment strategy document that Shipyard expects
# and validates.
deployment_strategy_schema: shipyard/DeploymentStrategy/v1
airflow_config_file:
path: /usr/local/airflow/airflow.cfg
airflow:

21
doc/source/_static/shipyard.conf.sample

@ -84,6 +84,27 @@
#service_type = deckhand
[document_info]
#
# From shipyard_api
#
# The name of the deployment-configuration document that Shipyard expects and
# validates (string value)
#deployment_configuration_name = deployment-configuration
# The schema of the deployment-configuration document that Shipyard expects and
# validates (string value)
#deployment_configuration_schema = shipyard/DeploymentConfiguration/v1
# The schema of the deployment strategy document that Shipyard expects and
# validates. Note that the name of this document is not configurable, because
# it is controlled by a field in the deployment configuration document. (string
# value)
#deployment_strategy_schema = shipyard/DeploymentStrategy/v1
[drydock]
#

1
doc/source/sampleconf.rst

@ -12,6 +12,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
.. _sample-configuration:
Sample Configuration File
==========================

10
doc/source/site-definition-documents.rst

@ -37,6 +37,11 @@ of the Armada manifest that will be used during the deployment/update.
A `sample deployment-configuration`_ shows a completely specified example.
Note that the name and schema Shipyard expects the deployment configuration
document to have is conifgurable via the document_info section in the
:ref:`Shipyard configuration <sample-configuration>`, but should be left
defaulted in most cases.
`Default configuration values`_ are provided for most values.
Supported values
@ -179,6 +184,11 @@ document for the site. Example::
- The success criteria indicates that all nodes must be succssful to consider
the group a success.
Note that the schema Shipyard expects the deployment strategy document to have
is conifgurable via the document_info section in the
:ref:`Shipyard configuration <sample-configuration>`, but should be left
defaulted in most cases.
In short, the default behavior is to deploy everything all at once, and halt
if there are any failures.

21
src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@ -84,6 +84,27 @@
#service_type = deckhand
[document_info]
#
# From shipyard_api
#
# The name of the deployment-configuration document that Shipyard expects and
# validates (string value)
#deployment_configuration_name = deployment-configuration
# The schema of the deployment-configuration document that Shipyard expects and
# validates (string value)
#deployment_configuration_schema = shipyard/DeploymentConfiguration/v1
# The schema of the deployment strategy document that Shipyard expects and
# validates. Note that the name of this document is not configurable, because
# it is controlled by a field in the deployment configuration document. (string
# value)
#deployment_strategy_schema = shipyard/DeploymentStrategy/v1
[drydock]
#

28
src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@ -286,6 +286,34 @@ SECTIONS = [
),
]
),
ConfigSection(
name='document_info',
title=('Information about some of the documents Shipyard needs to '
'handle'),
options=[
cfg.StrOpt(
'deployment_configuration_name',
default='deployment-configuration',
help=('The name of the deployment-configuration document that '
'Shipyard expects and validates')
),
cfg.StrOpt(
'deployment_configuration_schema',
default='shipyard/DeploymentConfiguration/v1',
help=('The schema of the deployment-configuration document '
'that Shipyard expects and validates')
),
cfg.StrOpt(
'deployment_strategy_schema',
default='shipyard/DeploymentStrategy/v1',
help=('The schema of the deployment strategy document that '
'Shipyard expects and validates. Note that the name of '
'this document is not configurable, because it is '
'controlled by a field in the deployment configuration '
'document.')
),
]
),
]

2
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py

@ -525,7 +525,7 @@ class ConfigdocsHelper(object):
service_clients.deckhand_client(),
revision_id,
[(ValidateDeploymentConfigurationFull,
'deployment-configuration')]
CONF.document_info.deployment_configuration_name)]
)
return sy_val_mgr.validate()
except Exception as ex:

6
src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py

@ -14,6 +14,7 @@
import logging
import falcon
from oslo_config import cfg
from .validate_deployment_configuration \
import ValidateDeploymentConfigurationBasic
@ -24,6 +25,7 @@ from shipyard_airflow.common.document_validators.document_validator_manager \
from shipyard_airflow.errors import ApiError
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class ValidateDeploymentAction:
@ -40,7 +42,7 @@ class ValidateDeploymentAction:
dh_client,
self.doc_revision,
[(ValidateDeploymentConfigurationFull,
'deployment-configuration')]
CONF.document_info.deployment_configuration_name)]
)
else:
# Perform a basic validation only
@ -48,7 +50,7 @@ class ValidateDeploymentAction:
dh_client,
self.doc_revision,
[(ValidateDeploymentConfigurationBasic,
'deployment-configuration')]
CONF.document_info.deployment_configuration_name)]
)
def validate(self):

5
src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py

@ -18,11 +18,14 @@ is performed by Deckhand on Shipyard's behalf.
"""
import logging
from oslo_config import cfg
from shipyard_airflow.common.document_validators.document_validator import (
DocumentValidator
)
from .validate_deployment_strategy import ValidateDeploymentStrategy
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -36,7 +39,7 @@ class ValidateDeploymentConfigurationBasic(DocumentValidator):
def __init__(self, **kwargs):
super().__init__(**kwargs)
schema = "shipyard/DeploymentConfiguration/v1"
schema = CONF.document_info.deployment_configuration_schema
missing_severity = "Error"
def do_validate(self):

5
src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py

@ -18,6 +18,8 @@ is performed by Deckhand on Shipyard's behalf.
"""
import logging
from oslo_config import cfg
from shipyard_airflow.common.deployment_group.deployment_group_manager import (
DeploymentGroupManager
)
@ -35,6 +37,7 @@ from shipyard_airflow.control.helpers.design_reference_helper import (
DesignRefHelper
)
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def _get_node_lookup(revision_id):
@ -56,7 +59,7 @@ class ValidateDeploymentStrategy(DocumentValidator):
def __init__(self, **kwargs):
super().__init__(**kwargs)
schema = "shipyard/DeploymentStrategy/v1"
schema = CONF.document_info.deployment_strategy_schema
missing_severity = "Error"
def do_validate(self):

39
src/bin/shipyard_airflow/shipyard_airflow/plugins/deployment_configuration_operator.py

@ -13,9 +13,10 @@
# limitations under the License.
"""Deployment Configuration
Retrieves the deployment configuration from Deckhand and places the values
Retrieves the deployment-configuration from Deckhand and places the values
retrieved into a dictionary
"""
import configparser
import logging
from airflow.exceptions import AirflowException
@ -34,12 +35,13 @@ except ImportError:
from shipyard_airflow.shipyard_const import CustomHeaders
LOG = logging.getLogger(__name__)
DOCUMENT_INFO = 'document_info'
class DeploymentConfigurationOperator(BaseOperator):
"""Deployment Configuration Operator
Retrieve the deployment configuration from Deckhand for use throughout
Retrieve the deployment-configuration from Deckhand for use throughout
the workflow. Put the configuration into a dictionary.
Failures are raised:
@ -89,18 +91,23 @@ class DeploymentConfigurationOperator(BaseOperator):
:param main_dag_name: Parent Dag
:param shipyard_conf: Location of shipyard.conf
"""
super(DeploymentConfigurationOperator, self).__init__(*args, **kwargs)
self.main_dag_name = main_dag_name
self.shipyard_conf = shipyard_conf
self.action_info = {}
def _read_config(self):
"""Read in and parse the shipyard config"""
self.config = configparser.ConfigParser()
self.config.read(self.shipyard_conf)
def execute(self, context):
"""Perform Deployment Configuration extraction"""
self._read_config()
revision_id = self.get_revision_id(context.get('task_instance'))
doc = self.get_doc(revision_id)
converted = self.map_config_keys(doc)
# return the mapped configuration so that it can be placed on xcom
return converted
@ -116,7 +123,7 @@ class DeploymentConfigurationOperator(BaseOperator):
revision_id = self.action_info['committed_rev_id']
if revision_id:
LOG.info("Revision is set to: %s for deployment configuration",
LOG.info("Revision is set to: %s for deployment-configuration",
revision_id)
return revision_id
# either revision id was not on xcom, or the task_instance is messed
@ -127,14 +134,16 @@ class DeploymentConfigurationOperator(BaseOperator):
def get_doc(self, revision_id):
"""Get the DeploymentConfiguration document dictionary from Deckhand"""
LOG.info(
"Attempting to retrieve shipyard/DeploymentConfiguration/v1, "
"deployment-configuration from Deckhand"
)
filters = {
"schema": "shipyard/DeploymentConfiguration/v1",
"metadata.name": "deployment-configuration"
}
schema_fallback = 'shipyard/DeploymentConfiguration/v1'
schema = self.config.get(DOCUMENT_INFO,
'deployment_configuration_schema',
fallback=schema_fallback)
name = self.config.get(DOCUMENT_INFO,
'deployment_configuration_name',
fallback='deployment-configuration')
LOG.info("Attempting to retrieve {}, {} from Deckhand".format(schema,
name))
filters = {"schema": schema, "metadata.name": name}
# Create additional headers dict to pass context marker
# and end user
@ -160,7 +169,7 @@ class DeploymentConfigurationOperator(BaseOperator):
except AttributeError:
failed_url = "No URL generated"
LOG.exception(ex)
raise AirflowException("Failed to retrieve deployment "
raise AirflowException("Failed to retrieve deployment-"
"configuration yaml using url: "
"{}".format(failed_url))
@ -178,7 +187,7 @@ class DeploymentConfigurationOperator(BaseOperator):
Converts to a more simple map of key-value pairs
"""
LOG.info("Mapping keys from deployment configuration")
LOG.info("Mapping keys from deployment-configuration")
return {
cfg_key: self.get_cfg_value(cfg_data, cfg_key, cfg_default)
for cfg_key, cfg_default in

11
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py

@ -53,6 +53,7 @@ except ImportError:
)
LOG = logging.getLogger(__name__)
DOCUMENT_INFO = 'document_info'
class DrydockNodesOperator(DrydockBaseOperator):
@ -287,10 +288,12 @@ class DrydockNodesOperator(DrydockBaseOperator):
strat_name = self.dc['physical_provisioner.deployment_strategy']
if strat_name:
# if there is a deployment strategy specified, use it
strategy = self.get_unique_doc(
name=strat_name,
schema="shipyard/DeploymentStrategy/v1"
)
schema_fallback = 'shipyard/DeploymentStrategy/v1'
schema = self.config.get(DOCUMENT_INFO,
'deployment_strategy_schema',
fallback=schema_fallback)
strategy = self.get_unique_doc(name=strat_name, schema=schema)
else:
# The default behavior is to deploy all nodes, and fail if
# any nodes fail to deploy.

5
src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py

@ -253,10 +253,7 @@ class UcpBaseOperator(BaseOperator):
if revision_id is None:
revision_id = self.revision_id
LOG.info(
"Retrieve shipyard/DeploymentConfiguration/v1, "
"deployment-configuration from Deckhand"
)
LOG.info("Retrieve {}, {} from Deckhand".format(schema, name))
try:
return self.doc_utils.get_unique_doc(revision_id=revision_id,
name=name,

37
src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_configuration_operator.py

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from configparser import ConfigParser
from unittest import mock
import pytest
@ -40,11 +41,13 @@ ACTION_INFO_NO_COMMIT = {
try:
from deployment_configuration_operator import (
DeploymentConfigurationOperator
DeploymentConfigurationOperator,
DOCUMENT_INFO
)
except ImportError:
from shipyard_airflow.plugins.deployment_configuration_operator import (
DeploymentConfigurationOperator
DeploymentConfigurationOperator,
DOCUMENT_INFO
)
try:
@ -55,6 +58,21 @@ except ImportError:
)
def make_fake_config():
"""Make/return a fake config using configparser that we can use for testing
:returns: A fake configuration object
:rtype: ConfigParser
"""
cfg = ConfigParser()
cfg.add_section(DOCUMENT_INFO)
cfg.set(DOCUMENT_INFO, 'deployment_configuration_name',
'deployment-configuration')
cfg.set(DOCUMENT_INFO, 'deployment_configuration_schema',
'shipyard/DeploymentConfiguration/v1')
return cfg
def test_execute_exception():
"""Test that execute results in a failure with bad context"""
@ -67,18 +85,21 @@ def test_execute_exception():
assert ("Design_revision is not set. Cannot proceed with retrieval"
" of the design configuration") in str(expected_exc)
@mock.patch.object(DeploymentConfigurationOperator, '_read_config')
@mock.patch.object(DeploymentConfigurationOperator, 'get_revision_id',
return_value=99)
def test_execute_no_client(*args):
def test_execute_no_client(get_revision_id, read_config):
# no keystone authtoken present in configuration
dco = DeploymentConfigurationOperator(main_dag_name="main",
shipyard_conf="shipyard.conf",
task_id="t1")
dco.config = make_fake_config()
with pytest.raises(AirflowException) as expected_exc:
dco.execute(context={})
assert ("Failed to retrieve deployment configuration yaml") in str(
dco.execute(context={'task_instance': 'asdf'})
assert ("Failed to retrieve deployment-configuration yaml") in str(
expected_exc)
get_revision_id.assert_called_once_with('asdf')
read_config.assert_called_once_with()
@mock.patch.object(airflow.models.TaskInstance, 'xcom_pull',
@ -113,6 +134,7 @@ def test_get_doc_no_deckhand():
dco = DeploymentConfigurationOperator(main_dag_name="main",
shipyard_conf="shipyard.conf",
task_id="t1")
dco.config = make_fake_config()
with pytest.raises(AirflowException) as expected_exc:
dco.get_doc(99)
assert "Failed to retrieve deployment" in str(expected_exc)
@ -134,7 +156,7 @@ def test_get_doc_mock_deckhand(*args):
dco = DeploymentConfigurationOperator(main_dag_name="main",
shipyard_conf="shipyard.conf",
task_id="t1")
dco.config = make_fake_config()
doc = dco.get_doc(99)
assert doc == 'abcdefg'
@ -146,6 +168,7 @@ def test_get_doc_mock_deckhand_invalid(*args):
dco = DeploymentConfigurationOperator(main_dag_name="main",
shipyard_conf="shipyard.conf",
task_id="t1")
dco.config = make_fake_config()
with pytest.raises(AirflowException) as airflow_ex:
dco.get_doc(99)

4
tools/resources/shipyard.conf

@ -45,3 +45,7 @@ service_type = shipyard
[oslo_policy]
policy_file = /etc/shipyard/policy.yaml
policy_default_rule = deny_all
[document_info]
deployment_configuration_name = deployment-configuration
deployment_configuration_schema = shipyard/DeploymentConfiguration/v1
deployment_strategy_schema = shipyard/DeploymentStrategy/v1
Loading…
Cancel
Save