Add support for batched upgrades to composable upgrades
Some services (e.g ceph mon) require upgrading in batches (the old upgrade architecture did the ceph mon upgrade one controller at a time). This interface enables doing the same, and over time we can probably move more services into this interface (e.g when services support rolling upgrades) to reduce downtime. Change-Id: If581f301a5493ef33ac1386bdc22f9fca4f2544e Partially-Implements: blueprint overcloud-upgrades-per-service
This commit is contained in:
parent
7dbd771a35
commit
1cdc514871
|
@ -1,3 +1,4 @@
|
||||||
|
{% set upgrade_steps_max = 8 -%}
|
||||||
heat_template_version: ocata
|
heat_template_version: ocata
|
||||||
description: 'Upgrade steps for all roles'
|
description: 'Upgrade steps for all roles'
|
||||||
|
|
||||||
|
@ -17,12 +18,12 @@ parameters:
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
|
|
||||||
# Upgrade Steps for all roles
|
# Upgrade Steps for all roles, batched updates
|
||||||
# FIXME(shardy): would be nice to make the number of steps configurable
|
# FIXME(shardy): would be nice to make the number of steps configurable
|
||||||
{% for step in range(0, 8) %}
|
{% for step in range(0, upgrade_steps_max) %}
|
||||||
{% for role in roles %}
|
{% for role in roles %}
|
||||||
# Step {{step}} resources
|
# Step {{step}} resources
|
||||||
{{role.name}}UpgradeConfig_Step{{step}}:
|
{{role.name}}UpgradeBatchConfig_Step{{step}}:
|
||||||
type: OS::TripleO::UpgradeConfig
|
type: OS::TripleO::UpgradeConfig
|
||||||
# The UpgradeConfig resources could actually be created without
|
# The UpgradeConfig resources could actually be created without
|
||||||
# serialization, but the event output is easier to follow if we
|
# serialization, but the event output is easier to follow if we
|
||||||
|
@ -30,11 +31,59 @@ resources:
|
||||||
# config is cheap compared to the time to apply the deployment).
|
# config is cheap compared to the time to apply the deployment).
|
||||||
{% if step > 0 %}
|
{% if step > 0 %}
|
||||||
depends_on:
|
depends_on:
|
||||||
|
{% for dep in roles %}
|
||||||
|
- {{dep.name}}UpgradeBatch_Step{{step -1}}
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
properties:
|
||||||
|
UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_batch_tasks]}
|
||||||
|
step: {{step}}
|
||||||
|
|
||||||
|
{{role.name}}UpgradeBatch_Step{{step}}:
|
||||||
|
type: OS::Heat::StructuredDeploymentGroup
|
||||||
|
{% if step > 0 %}
|
||||||
|
depends_on:
|
||||||
|
{% for dep in roles %}
|
||||||
|
- {{dep.name}}UpgradeBatch_Step{{step -1}}
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
update_policy:
|
||||||
|
batch_create:
|
||||||
|
max_batch_size: {{role.upgrade_batch_size|default(1)}}
|
||||||
|
rolling_update:
|
||||||
|
max_batch_size: {{role.upgrade_batch_size|default(1)}}
|
||||||
|
properties:
|
||||||
|
name: {{role.name}}UpgradeBatch_Step{{step}}
|
||||||
|
servers: {get_param: [servers, {{role.name}}]}
|
||||||
|
config: {get_resource: {{role.name}}UpgradeBatchConfig_Step{{step}}}
|
||||||
|
input_values:
|
||||||
|
role: {{role.name}}
|
||||||
|
update_identifier: {get_param: UpdateIdentifier}
|
||||||
|
{% endfor %}
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
# Upgrade Steps for all roles
|
||||||
|
# FIXME(shardy): would be nice to make the number of steps configurable
|
||||||
|
{% for step in range(0, upgrade_steps_max) %}
|
||||||
|
{% for role in roles %}
|
||||||
|
# Step {{step}} resources
|
||||||
|
{{role.name}}UpgradeConfig_Step{{step}}:
|
||||||
|
type: OS::TripleO::UpgradeConfig
|
||||||
|
# The UpgradeConfig resources could actually be created without
|
||||||
|
# serialization, but the event output is easier to follow if we
|
||||||
|
# do, and there should be minimal performance hit (creating the
|
||||||
|
# config is cheap compared to the time to apply the deployment).
|
||||||
|
depends_on:
|
||||||
|
{% if step > 0 %}
|
||||||
{% for dep in roles %}
|
{% for dep in roles %}
|
||||||
{% if not dep.disable_upgrade_deployment|default(false) %}
|
{% if not dep.disable_upgrade_deployment|default(false) %}
|
||||||
- {{dep.name}}Upgrade_Step{{step -1}}
|
- {{dep.name}}Upgrade_Step{{step -1}}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
{% for dep in roles %}
|
||||||
|
- {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
|
||||||
|
{% endfor %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
properties:
|
properties:
|
||||||
UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]}
|
UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]}
|
||||||
|
@ -42,13 +91,17 @@ resources:
|
||||||
{% if not role.disable_upgrade_deployment|default(false) %}
|
{% if not role.disable_upgrade_deployment|default(false) %}
|
||||||
{{role.name}}Upgrade_Step{{step}}:
|
{{role.name}}Upgrade_Step{{step}}:
|
||||||
type: OS::Heat::StructuredDeploymentGroup
|
type: OS::Heat::StructuredDeploymentGroup
|
||||||
{% if step > 0 %}
|
|
||||||
depends_on:
|
depends_on:
|
||||||
|
{% if step > 0 %}
|
||||||
{% for dep in roles %}
|
{% for dep in roles %}
|
||||||
{% if not dep.disable_upgrade_deployment|default(false) %}
|
{% if not dep.disable_upgrade_deployment|default(false) %}
|
||||||
- {{dep.name}}Upgrade_Step{{step -1}}
|
- {{dep.name}}Upgrade_Step{{step -1}}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
{% for dep in roles %}
|
||||||
|
- {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
|
||||||
|
{% endfor %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
properties:
|
properties:
|
||||||
name: {{role.name}}Upgrade_Step{{step}}
|
name: {{role.name}}Upgrade_Step{{step}}
|
||||||
|
|
|
@ -49,6 +49,28 @@ are re-asserted when applying latter ones.
|
||||||
|
|
||||||
5) Service activation (Pacemaker)
|
5) Service activation (Pacemaker)
|
||||||
|
|
||||||
|
Batch Upgrade Steps
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Each service template may optionally define a `upgrade_batch_tasks` key, which
|
||||||
|
is a list of ansible tasks to be performed during the upgrade process.
|
||||||
|
|
||||||
|
Similar to the step_config, we allow a series of steps for the per-service
|
||||||
|
upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first
|
||||||
|
step, "step2" for the second, etc. Note that each step is performed in batches,
|
||||||
|
then we move on to the next step which is also performed in batches (we don't
|
||||||
|
perform all steps on one node, then move on to the next one which means you
|
||||||
|
can sequence rolling upgrades of dependent services via the step value).
|
||||||
|
|
||||||
|
The tasks performed at each step is service specific, but note that all batch
|
||||||
|
upgrade steps are performed before the `upgrade_tasks` described below. This
|
||||||
|
means that all services that support rolling upgrades can be upgraded without
|
||||||
|
downtime during `upgrade_batch_tasks`, then any remaining services are stopped
|
||||||
|
and upgraded during `upgrade_tasks`
|
||||||
|
|
||||||
|
The default batch size is 1, but this can be overridden for each role via the
|
||||||
|
`upgrade_batch_size` option in roles_data.yaml
|
||||||
|
|
||||||
Upgrade Steps
|
Upgrade Steps
|
||||||
-------------
|
-------------
|
||||||
|
|
||||||
|
|
|
@ -118,4 +118,9 @@ outputs:
|
||||||
# Note we use distinct() here to filter any identical tasks, e.g yum update for all services
|
# Note we use distinct() here to filter any identical tasks, e.g yum update for all services
|
||||||
expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
|
expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
|
||||||
data: {get_attr: [ServiceChain, role_data]}
|
data: {get_attr: [ServiceChain, role_data]}
|
||||||
|
upgrade_batch_tasks:
|
||||||
|
yaql:
|
||||||
|
# Note we use distinct() here to filter any identical tasks, e.g yum update for all services
|
||||||
|
expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
|
||||||
|
data: {get_attr: [ServiceChain, role_data]}
|
||||||
service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
|
service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
|
||||||
|
|
|
@ -17,8 +17,9 @@
|
||||||
# disable_constraints: (boolean) optional, whether to disable Nova and Glance
|
# disable_constraints: (boolean) optional, whether to disable Nova and Glance
|
||||||
# constraints for each role specified in the templates.
|
# constraints for each role specified in the templates.
|
||||||
#
|
#
|
||||||
# disable_upgrade_deployment: (boolean) optional, whether to run the composable upgrade
|
# upgrade_batch_size: (number): batch size for upgrades where tasks are
|
||||||
# steps for all services that are deployed on the particular role.
|
# specified by services to run in batches vs all nodes at once.
|
||||||
|
# This defaults to 1, but larger batches may be specified here.
|
||||||
#
|
#
|
||||||
# ServicesDefault: (list) optional default list of services to be deployed
|
# ServicesDefault: (list) optional default list of services to be deployed
|
||||||
# on the role, defaults to an empty list. Sets the default for the
|
# on the role, defaults to an empty list. Sets the default for the
|
||||||
|
|
Loading…
Reference in New Issue