[WIP]: Improve wait API and semantics in v2 docs

See the the v1-v2 migration guide updates in this commit for details.

Change-Id: I6a8a69f8392e8065eda039597278c7dfe593a4fd
This commit is contained in:
Sean Eagan 2019-02-12 12:51:16 -06:00
parent 8a50591dbf
commit 150f0a05d6
7 changed files with 402 additions and 131 deletions

View File

@ -63,7 +63,7 @@ class ChartDeploy(object):
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
chart,
ch,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,

View File

@ -13,6 +13,7 @@
# limitations under the License.
from abc import ABC, abstractmethod
import copy
import collections
import math
import re
@ -21,6 +22,7 @@ import time
from oslo_log import log as logging
from armada import const
from armada.handlers.schema import get_schema_info
from armada.utils.helm import is_test_pod
from armada.utils.release import label_selectors
from armada.exceptions import k8s_exceptions
@ -46,36 +48,52 @@ class ChartWait():
self.k8s = k8s
self.release_name = release_name
self.chart = chart
self.wait_config = chart.get('wait', {})
chart_data = self.chart[const.KEYWORD_DATA]
self.chart_data = chart_data
self.wait_config = self.chart_data.get('wait', {})
self.namespace = namespace
self.k8s_wait_attempts = max(k8s_wait_attempts, 1)
self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1)
resources = self.wait_config.get('resources')
labels = get_wait_labels(self.chart)
schema_info = get_schema_info(self.chart['schema'])
if resources is not None:
waits = []
for resource_config in resources:
# Initialize labels
resource_config.setdefault('labels', {})
# Add base labels
resource_config['labels'].update(labels)
waits.append(self.get_resource_wait(resource_config))
resources = self.wait_config.get('resources')
if isinstance(resources, list):
# Explicit resource config list provided.
resources_list = resources
else:
waits = [
JobWait('job', self, labels, skip_if_none_found=True),
PodWait('pod', self, labels)
]
self.waits = waits
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
resources_list = [{
'type': 'job',
'skip_if_none_found': True
}, {
'type': 'pod'
}]
else:
resources_list = self.get_resources_list(resources)
chart_labels = get_wait_labels(self.chart_data)
for resource_config in resources_list:
# Use chart labels as base labels for each config.
labels = dict(chart_labels)
resource_labels = resource_config.get('labels', {})
# Merge in any resource-specific labels.
if resource_labels:
labels.update(resource_labels)
resource_config['labels'] = labels
LOG.debug('Resolved `wait.resources` list: %s', resources_list)
self.waits = [self.get_resource_wait(conf) for conf in resources_list]
# Calculate timeout
wait_timeout = timeout
if wait_timeout is None:
wait_timeout = self.wait_config.get('timeout')
# TODO(MarshM): Deprecated, remove `timeout` key.
deprecated_timeout = self.chart.get('timeout')
# TODO: Remove when v1 doc support is removed.
deprecated_timeout = self.chart_data.get('timeout')
if deprecated_timeout is not None:
LOG.warn('The `timeout` key is deprecated and support '
'for this will be removed soon. Use '
@ -90,12 +108,19 @@ class ChartWait():
self.timeout = wait_timeout
# Determine whether to enable native wait.
native = self.wait_config.get('native', {})
# TODO: Remove when v1 doc support is removed.
default_native = schema_info.version < 2
self.native_enabled = native.get('enabled', default_native)
def get_timeout(self):
return self.timeout
def is_native_enabled(self):
native_wait = self.wait_config.get('native', {})
return native_wait.get('enabled', True)
return self.native_enabled
def wait(self, timeout):
deadline = time.time() + timeout
@ -104,6 +129,54 @@ class ChartWait():
wait.wait(timeout=timeout)
timeout = int(round(deadline - time.time()))
def get_resources_list(self, resources):
# Use default resource configs, with any provided resource type
# overrides merged in.
# By default, wait on all supported resource types.
resource_order = [
# Jobs may perform initialization so add them first.
'job',
'daemonset',
'statefulset',
'deployment',
'pod'
]
base_resource_config = {
# By default, skip if none found so we don't fail on charts
# which don't contain resources of a given type.
'skip_if_none_found': True
}
# Create a map of resource types to default configs.
resource_configs = collections.OrderedDict(
[(type, base_resource_config) for type in resource_order])
# Handle any overrides and/or removals of resource type configs.
if resources:
for type, v in resources.items():
if v is False:
# Remove this type.
resource_configs.pop(type)
else:
# Override config for this type.
resource_configs[type] = v
resources_list = []
# Convert the resource type map to a list of fully baked resource
# configs with type included.
for type, config in resource_configs.items():
if isinstance(config, list):
configs = config
else:
configs = [config]
for conf in configs:
resource_config = copy.deepcopy(conf)
resource_config['type'] = type
resources_list.append(resource_config)
return resources_list
def get_resource_wait(self, resource_config):
kwargs = dict(resource_config)
@ -174,19 +247,19 @@ class ResourceWait(ABC):
def handle_resource(self, resource):
resource_name = resource.metadata.name
resource_desc = '{} {}'.format(self.resource_type, resource_name)
try:
message, resource_ready = self.is_resource_ready(resource)
if resource_ready:
LOG.debug('Resource %s is ready!', resource_name)
LOG.debug('%s is ready!', resource_desc)
else:
LOG.debug('Resource %s not ready: %s', resource_name, message)
LOG.debug('%s not ready: %s', resource_desc, message)
return resource_ready
except armada_exceptions.WaitException as e:
LOG.warn('Resource %s unlikely to become ready: %s', resource_name,
e)
LOG.warn('%s unlikely to become ready: %s', resource_desc, e)
return False
def wait(self, timeout):
@ -194,10 +267,14 @@ class ResourceWait(ABC):
:param timeout: time before disconnecting ``Watch`` stream
'''
min_ready_msg = ', min_ready={}'.format(
self.min_ready.source) if isinstance(self, ControllerWait) else ''
LOG.info(
"Waiting for resource type=%s, namespace=%s labels=%s for %ss "
"Waiting for resource type=%s, namespace=%s labels=%s "
"skip_if_none_found=%s%s for %ss "
"(k8s wait %s times, sleep %ss)", self.resource_type,
self.chart_wait.namespace, self.label_selector, timeout,
self.chart_wait.namespace, self.label_selector,
self.skip_if_none_found, min_ready_msg, timeout,
self.chart_wait.k8s_wait_attempts,
self.chart_wait.k8s_wait_attempt_sleep)
if not self.label_selector:
@ -207,60 +284,73 @@ class ResourceWait(ABC):
# Track the overall deadline for timing out during waits
deadline = time.time() + timeout
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new resources appear after our watch exits.
successes = 0
while True:
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
error = (
"Timed out waiting for resource type={}, namespace={}, "
"labels={}".format(self.resource_type,
self.chart_wait.namespace,
self.label_selector))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
timed_out, modified, unready, found_resources = (
self._watch_resource_completions(timeout=deadline_remaining))
if (not found_resources) and self.skip_if_none_found:
return
if timed_out:
if not found_resources:
details = (
'None found! Are `wait.labels` correct? Does '
'`wait.resources` need to exclude `type: {}`?'.format(
self.resource_type))
schema_info = get_schema_info(self.chart_wait.chart['schema'])
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new resources appear after our watch exits.
successes = 0
while True:
modified = self._wait(deadline)
if modified is None:
break
if modified:
successes = 0
LOG.debug('Found modified resources: %s', sorted(modified))
else:
details = ('These {}s were not ready={}'.format(
self.resource_type, sorted(unready)))
error = (
'Timed out waiting for {}s (namespace={}, labels=({})). {}'
.format(self.resource_type, self.chart_wait.namespace,
self.label_selector, details))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
successes += 1
LOG.debug('Found no modified resources.')
if modified:
successes = 0
LOG.debug('Found modified resources: %s', sorted(modified))
if successes >= self.chart_wait.k8s_wait_attempts:
return
LOG.debug(
'Continuing to wait: %s consecutive attempts without '
'modified resources of %s required.', successes,
self.chart_wait.k8s_wait_attempts)
time.sleep(self.chart_wait.k8s_wait_attempt_sleep)
else:
self._wait(deadline)
def _wait(self, deadline):
'''
Waits for resources to become ready.
Returns whether resources were modified, or `None` if that is to be
ignored.
'''
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
error = ("Timed out waiting for resource type={}, namespace={}, "
"labels={}".format(self.resource_type,
self.chart_wait.namespace,
self.label_selector))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
timed_out, modified, unready, found_resources = (
self._watch_resource_completions(timeout=deadline_remaining))
if (not found_resources) and self.skip_if_none_found:
return None
if timed_out:
if not found_resources:
details = (
'None found! Are `wait.labels` correct? Does '
'`wait.resources` need to exclude `type: {}`?'.format(
self.resource_type))
else:
successes += 1
LOG.debug('Found no modified resources.')
details = ('These {}s were not ready={}'.format(
self.resource_type, sorted(unready)))
error = (
'Timed out waiting for {}s (namespace={}, labels=({})). {}'.
format(self.resource_type, self.chart_wait.namespace,
self.label_selector, details))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
if successes >= self.chart_wait.k8s_wait_attempts:
break
LOG.debug(
'Continuing to wait: %s consecutive attempts without '
'modified resources of %s required.', successes,
self.chart_wait.k8s_wait_attempts)
time.sleep(self.chart_wait.k8s_wait_attempt_sleep)
return True
return modified
def _watch_resource_completions(self, timeout):
'''
@ -370,11 +460,19 @@ class PodWait(ResourceWait):
if is_test_pod(pod):
return 'helm test pod'
# Exclude job pods
# TODO: Once controller-based waits are enabled by default, ignore
# controller-owned pods as well.
if has_owner(pod, 'Job'):
return 'generated by job (wait on that instead if not already)'
schema_info = get_schema_info(self.chart_wait.chart['schema'])
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
# Exclude job pods
if has_owner(pod, 'Job'):
return 'owned by job'
else:
# Exclude all pods with an owner (only include raw pods)
# TODO: In helm 3, all resources will likely have the release CR as
# an owner, so this will need to be updated to not exclude pods
# directly owned by the release.
if has_owner(pod):
return 'owned by another resource'
return None
@ -409,7 +507,7 @@ class JobWait(ResourceWait):
# Exclude cronjob jobs
if has_owner(job, 'CronJob'):
return 'generated by cronjob (not part of release)'
return 'owned by cronjob (not part of release)'
return None
@ -493,10 +591,13 @@ class DeploymentWait(ControllerWait):
name = deployment.metadata.name
spec = deployment.spec
status = deployment.status
gen = deployment.metadata.generation or 0
observed_gen = status.observed_generation or 0
if gen <= observed_gen:
# TODO: Don't fail for lack of progress if `min_ready` is met.
# TODO: Consider continuing after `min_ready` is met, so long as
# progress is being made.
cond = self._get_resource_condition(status.conditions,
'Progressing')
if cond and (cond.reason or '') == 'ProgressDeadlineExceeded':
@ -541,20 +642,23 @@ class DaemonSetWait(ControllerWait):
name = daemon.metadata.name
spec = daemon.spec
status = daemon.status
gen = daemon.metadata.generation or 0
observed_gen = status.observed_generation or 0
is_update = observed_gen > 1
if spec.update_strategy.type != ROLLING_UPDATE_STRATEGY_TYPE:
if (is_update and
spec.update_strategy.type != ROLLING_UPDATE_STRATEGY_TYPE):
msg = ("Assuming non-readiness for strategy type {}, can only "
"determine for {}")
raise armada_exceptions.WaitException(
msg.format(spec.update_strategy.type,
ROLLING_UPDATE_STRATEGY_TYPE))
gen = daemon.metadata.generation or 0
observed_gen = status.observed_generation or 0
updated_number_scheduled = status.updated_number_scheduled or 0
desired_number_scheduled = status.desired_number_scheduled or 0
number_available = status.number_available or 0
if gen <= observed_gen:
updated_number_scheduled = status.updated_number_scheduled or 0
desired_number_scheduled = status.desired_number_scheduled or 0
number_available = status.number_available or 0
if (updated_number_scheduled < desired_number_scheduled):
msg = ("Waiting for daemon set {} rollout to finish: {} out "
"of {} new pods have been updated...")
@ -588,17 +692,18 @@ class StatefulSetWait(ControllerWait):
name = sts.metadata.name
spec = sts.spec
status = sts.status
gen = sts.metadata.generation or 0
observed_gen = status.observed_generation or 0
is_update = observed_gen > 1
update_strategy_type = spec.update_strategy.type or ''
if update_strategy_type != ROLLING_UPDATE_STRATEGY_TYPE:
if is_update and update_strategy_type != ROLLING_UPDATE_STRATEGY_TYPE:
msg = ("Assuming non-readiness for strategy type {}, can only "
"determine for {}")
raise armada_exceptions.WaitException(
msg.format(update_strategy_type, ROLLING_UPDATE_STRATEGY_TYPE))
gen = sts.metadata.generation or 0
observed_gen = status.observed_generation or 0
if (observed_gen == 0 or gen > observed_gen):
msg = "Waiting for statefulset spec update to be observed..."
return (msg, False)
@ -614,7 +719,8 @@ class StatefulSetWait(ControllerWait):
return (msg.format(name, ready_replicas, replicas,
self.min_ready.source), False)
if (update_strategy_type == ROLLING_UPDATE_STRATEGY_TYPE and
if (is_update and
update_strategy_type == ROLLING_UPDATE_STRATEGY_TYPE and
spec.update_strategy.rolling_update):
if replicas and spec.update_strategy.rolling_update.partition:
msg = ("Waiting on partitioned rollout not supported, "

View File

@ -36,6 +36,16 @@ data:
required:
- type
additionalProperties: false
wait_resource_type_config:
properties:
labels:
$ref: '#/definitions/labels'
min_ready:
anyOf:
- type: integer
- type: string
skip_if_none_found:
type: boolean
type: object
properties:
release:
@ -76,20 +86,22 @@ data:
timeout:
type: integer
resources:
type: array
items:
properties:
type:
type: string
labels:
$ref: '#/definitions/labels'
min_ready:
anyOf:
- additionalProperties:
anyOf:
- type: integer
- type: string
required:
- type
additionalProperties: false
- $ref: '#/definitions/wait_resource_type_config'
- type: array
items:
$ref: '#/definitions/wait_resource_type_config'
- type: array
items:
allOf:
- $ref: '#/definitions/wait_resource_type_config'
- properties:
type:
type: string
required:
- type
labels:
$ref: "#/definitions/labels"
# Config for helm's native `--wait` param.

View File

@ -140,7 +140,7 @@ data:
wait:
timeout: 10
native:
enabled: false
enabled: true
test:
enabled: true
"""
@ -195,7 +195,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
'wait': {
'timeout': 10,
'native': {
'enabled': False
'enabled': True
}
},
'test': {

View File

@ -24,7 +24,14 @@ test_chart = {'wait': {'timeout': 10, 'native': {'enabled': False}}}
class ChartWaitTestCase(base.ArmadaTestCase):
def get_unit(self, chart, timeout=None):
def get_unit(self, chart_data, timeout=None, version=2):
chart = {
'schema': 'armada/Chart/v{}'.format(str(version)),
'metadata': {
'name': 'test'
},
const.KEYWORD_DATA: chart_data
}
return wait.ChartWait(
k8s=mock.MagicMock(),
release_name='test-test',
@ -44,7 +51,7 @@ class ChartWaitTestCase(base.ArmadaTestCase):
def test_get_timeout_override(self):
unit = self.get_unit(
timeout=20, chart={
timeout=20, chart_data={
'timeout': 5,
'wait': {
'timeout': 10
@ -57,9 +64,9 @@ class ChartWaitTestCase(base.ArmadaTestCase):
unit = self.get_unit({'timeout': 5})
self.assertEquals(unit.get_timeout(), 5)
def test_is_native_enabled_default_true(self):
def test_is_native_enabled_default_false(self):
unit = self.get_unit({})
self.assertEquals(unit.is_native_enabled(), True)
self.assertEquals(unit.is_native_enabled(), False)
def test_is_native_enabled_true(self):
unit = self.get_unit({'wait': {'native': {'enabled': True}}})
@ -188,9 +195,11 @@ class ChartWaitTestCase(base.ArmadaTestCase):
class PodWaitTestCase(base.ArmadaTestCase):
def get_unit(self, labels):
def get_unit(self, labels, version=2):
return wait.PodWait(
resource_type='pod', chart_wait=mock.MagicMock(), labels=labels)
resource_type='pod',
chart_wait=ChartWaitTestCase.get_unit(None, {}, version=version),
labels=labels)
def test_include_resource(self):
@ -223,7 +232,7 @@ class PodWaitTestCase(base.ArmadaTestCase):
mock_resource(owner_references=[mock.Mock(kind='NotAJob')])
]
unit = self.get_unit({})
unit = self.get_unit({}, version=1)
# Validate test pods excluded
for pod in test_pods:

View File

@ -53,6 +53,65 @@ Chart
| ``source.subpath`` | |
| now optional | |
+--------------------------------+------------------------------------------------------------+
| ``wait`` improvements | See `Wait Improvements`_. |
+--------------------------------+------------------------------------------------------------+
Wait Improvements
^^^^^^^^^^^^^^^^^
The :ref:`v2 wait API <wait_v2>` includes the following changes.
Breaking changes
****************
1. ``wait.resources`` now defaults to all supported resource ``type`` s,
currently ``job``, ``daemonset``, ``statefulset``, ``deployment``, and ``pod``, with
``skip_if_none_found``_ (a new option) set to ``true``. The previous default was
the equivalent of pods with ``skip_if_none_found=false``, and jobs with
``skip_if_none_found=true``.
2. ``type: pod`` waits now exclude pods owned by other resources, such
as controllers, as one should instead wait directly on the controller itself,
which per 1. is now the default.
3. Waits are no longer retried multiple times to attempt to wait for
modifications to pods to quiet down. This crutch was only mildly useful
before, and now even less so due to 1. and 2. above. If the ``min_ready`` for a
controller is achieved, there should be no reason not to move on.
4. ``wait.native.enabled`` is now disabled by default. With the above changes,
this is no longer useful as a backup mechanism. Having both enabled leads to
ambiguity in which wait would fail in each case. More importantly, this must
be disabled in order to use the ``min_ready`` functionality, otherwise tiller
will wait for 100% anyway. So this prevents accidentally leaving it enabled
in that case. Also when the tiller native wait times out, this caused the
release to be marked FAILED by tiller, which caused it to be purged and
re-installed (unless protected), even though the wait criteria may have
eventually succeeded, which is already validated by armada on a retry.
New features
************
Per-resource-type overrides
+++++++++++++++++++++++++++
``wait.resources`` can now be a dict, mapping individual resource types to
wait configurations (or lists thereof), such that one can keep the default
configuration for the other resource types, and also disable a given resource
type, by mapping it to ``false``.
The ability to provide the entire explicit list for ``wait.resources`` remains in
place as well.
skip_if_none_found
++++++++++++++++++
A ``skip_if_none_found`` field is also exposed for items/values in
``wait.resources`` which causes them to be skipped if no matching resources are
found, which is useful for things which may only be dynamically included in the
release depending on configuration. When explicitly overriding ``wait.resources``
items, this defaults to ``false``, as it is assumed the user likely intends for
them to exist in that case.
ChartGroup
----------

View File

@ -124,6 +124,8 @@ Chart
| dependencies | object | (optional) reference any chart dependencies before install |
+-----------------+----------+---------------------------------------------------------------------------------------+
.. _wait_v2:
Wait
^^^^
@ -132,8 +134,25 @@ Wait
+=============+==========+====================================================================+
| timeout | int | time (in seconds) to wait for chart to deploy |
+-------------+----------+--------------------------------------------------------------------+
| resources | array | Array of `Wait Resource`_ to wait on, with ``labels`` added to each|
| | | item. Defaults to pods and jobs (if any exist) matching ``labels``.|
| resources | dict \| | `Wait Resource`_ s to wait on. Defaults to all supported resource |
| | array | types (see `Wait Resource`_ ``.type``), with |
| | | ``skip_if_none_found: true``. |
| | | |
| | | **dict** - Maps resource types to one of: |
| | | |
| | | - `Wait Resource`_ : config for resource type. |
| | | |
| | | - list[ `Wait Resource`_ ] - multiple configs for resource type. |
| | | |
| | | - ``false`` - disable waiting for resource type. |
| | | |
| | | Any resource types not overridden retain the default config. |
| | | |
| | | **array** - Lists all `Wait Resource`_ s to use, completely |
| | | overriding the default. Can be set to ``[]`` to disable all |
| | | resource types. |
| | | |
| | | See examples `Wait Resources Examples`_. |
+-------------+----------+--------------------------------------------------------------------+
| labels | object | Base mapping of labels to wait on. They are added to any labels in |
| | | each item in the ``resources`` array. |
@ -143,18 +162,84 @@ Wait
Wait Resource
^^^^^^^^^^^^^
+-------------+----------+--------------------------------------------------------------------+
| keyword | type | action |
+=============+==========+====================================================================+
| type | string | k8s resource type, supports: controllers ('deployment', |
| | | 'daemonset', 'statefulset'), 'pod', 'job' |
+-------------+----------+--------------------------------------------------------------------+
| labels | object | mapping of kubernetes resource labels |
+-------------+----------+--------------------------------------------------------------------+
| min\_ready | int | Only for controller ``type``s. Amount of pods in a controller |
| | string | which must be ready. Can be integer or percent string e.g. ``80%``.|
| | | Default ``100%``. |
+-------------+----------+--------------------------------------------------------------------+
+--------------------+----------+--------------------------------------------------------------------+
| keyword | type | action |
+====================+==========+====================================================================+
| type | string | K8s resource type, supports: 'deployment', 'daemonset', |
| | | 'statefulset', 'pod', 'job'. |
| | | |
| | | NOTE: Omit when Wait_ ``.resources`` is a dict, as then the dict |
| | | key is used instead. |
+--------------------+----------+--------------------------------------------------------------------+
| labels | object | Kubernetes labels specific to this resource. |
| | | Wait_``.labels`` are included with these, so only define this if |
| | | additional labels are required to identify the targeted resources. |
+--------------------+----------+--------------------------------------------------------------------+
| min\_ready | int \| | Only for controller ``type`` s. Amount of pods in a controller |
| | string | which must be ready. Can be integer or percent string e.g. ``80%``.|
| | | Default ``100%``. |
+--------------------+----------+--------------------------------------------------------------------+
| skip_if_none_found | boolean | Whether to skip waiting if no matching resources are found, |
| | | Default ``false``. |
+--------------------+----------+--------------------------------------------------------------------+
Wait Resources Examples
^^^^^^^^^^^^^^^^^^^^^^^
.. code-block:: yaml
wait:
# ...
# Disable all waiting.
native:
enabled: false
resources: []
.. code-block:: yaml
wait:
# ...
# Disable waiting for a given type (job).
resources:
job: false
.. code-block:: yaml
wait:
# ...
# Use min_ready < 100%.
resources:
daemonset:
min_ready: 80%
.. code-block:: yaml
wait:
resources:
# Multiple configs for same type.
daemonset:
- labels:
component: one
min_ready: 80%
- labels:
component: two
min_ready: 50%
.. code-block:: yaml
wait:
# ...
resources:
- type: daemonset
labels:
component: critical
min_ready: 100%
- type: daemonset
labels:
component: best_effort
min_ready: 80%
# ... (re-include any other resource types needed when using list)
Wait Native
^^^^^^^^^^^
@ -164,7 +249,7 @@ Config for the native ``helm (install|upgrade) --wait`` flag.
+-------------+----------+--------------------------------------------------------------------+
| keyword | type | action |
+=============+==========+====================================================================+
| enabled | boolean | defaults to true |
| enabled | boolean | defaults to false |
+-------------+----------+--------------------------------------------------------------------+
.. _test_v2: