[TRAIN ONLY] Ensure interim db migration containers work properly

During the upgrade from Queens to Train we currently fail on non-bootstrap
controllers due to a script failure.

This patch does the following:

1. Implements a better bootstrap node check to ensure that the containers
   only start on the bootstrap nodes.
2. Improves the db migration check to ensure idempotence if the interim
   container is re-run for whatever reason.

Change-Id: I3b610ad2a91ca5deee301c338e4239c25166f41e
This commit is contained in:
Lukas Bezdicka 2020-04-16 16:19:47 +02:00
parent abf6c28cfa
commit 5babfe0021
13 changed files with 280 additions and 69 deletions

View File

@ -0,0 +1,42 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
show_usage() {
echo "Usage: cinder_ffu_db_sync.sh <sync|online>"
}
if [ $# -lt 1 ]
then
show_usage
exit 1
fi
DB_VERSION=$( sudo -u cinder cinder-manage db version )
_RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/cinder/db/sqlalchemy/migrate_repo/versions/ | grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1)
RPM_VERSION=$(expr $_RPM_VERSION + 0)
if (( $RPM_VERSION >= $DB_VERSION )); then
if [[ "$1" == "online" ]]; then
sudo -u cinder cinder-manage service list | grep -v Binary| tr '@' ' ' | awk '{print $1 " " $2}' | while read i ; do
sudo -u cinder cinder-manage service remove $i
done
sudo -u cinder cinder-manage db online_data_migrations
elif [[ "$1" == "sync" ]]; then
sudo -u cinder cinder-manage db sync --bump-versions
fi
else
echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION;
fi

View File

@ -0,0 +1,21 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
OUT=$( sudo -u glance -E /usr/local/bin/kolla_start 2>&1 ) || if [[ $OUT =~ train ]]; then
echo 'DB is already up to date'
else
echo $OUT
exit 1
fi

View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
DB_VERSION=$( /usr/local/bin/kolla_set_configs && /usr/bin/keystone-manage --config-file /etc/keystone/keystone.conf db_version )
_RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/keystone/common/sql/migrate_repo/versions | grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1)
RPM_VERSION=$(expr $_RPM_VERSION + 0);
if (( $RPM_VERSION > $DB_VERSION )); then
/usr/local/bin/kolla_start
else
echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION
fi

View File

@ -0,0 +1,20 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Here we sadly just expect that if we can't run 'mistral-db-manage current'
# then it means we already updated. If there is any issue with the service the
# upgrade will continue regardless...
sudo -u mistral mistral-db-manage --config-file /etc/mistral/mistral.conf current && sudo -u mistral mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head
exit 0

View File

@ -0,0 +1,20 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Here we sadly just expect that if we can't run 'neutron-db-manage current'
# then it means we already updated. If there is any issue with the service the
# upgrade will continue regardless...
neutron-db-manage --subproject neutron current && neutron-db-manage upgrade heads
exit 0

View File

@ -0,0 +1,45 @@
#!/bin/bash
# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
show_usage() {
echo "Usage: nova_ffu_db_sync.sh <api_db|db>"
}
if [ $# -lt 1 ]
then
show_usage
exit 1
fi
if [[ "$1" == "api_db" ]]; then
RPM_MIGRATIONS_PATH="/usr/lib/python3.6/site-packages/nova/db/sqlalchemy/api_migrations/migrate_repo/versions"
elif [[ "$1" == "db" ]]; then
RPM_MIGRATIONS_PATH="/usr/lib/python3.6/site-packages/nova/db/sqlalchemy/migrate_repo/versions"
fi
DB_VERSION=$( sudo -u nova /usr/bin/nova-manage $1 version )
_RPM_VERSION=$(ls ${RPM_MIGRATIONS_PATH} | grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1)
RPM_VERSION=$(expr $_RPM_VERSION + 0)
if (( $RPM_VERSION >= $DB_VERSION )); then
sudo -u nova /usr/bin/nova-manage $1 sync
if [[ "$1" == "db" ]]; then
sudo -u nova /usr/bin/nova-manage db online_data_migrations
fi
else
echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION;
fi

View File

@ -282,6 +282,12 @@ outputs:
- path: /var/log/cinder
owner: cinder:cinder
recurse: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- cinder_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/cinder_ffu_db_sync.sh }
docker_config:
step_2:
cinder_api_init_logs:
@ -325,14 +331,16 @@ outputs:
privileged: false
detach: false
user: root
volumes: *cinder_volumes
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
volumes:
list_concat:
- *cinder_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml cinder_api_short_bootstrap_node_name | grep `/bin/hostname -s` && DB_VERSION=$( sudo -u cinder cinder-manage db version ) _RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/cinder/db/sqlalchemy/migrate_repo/versions/ |grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1) RPM_VERSION=$(expr $_RPM_VERSION + 0) && if (( $RPM_VERSION >= $DB_VERSION )); then sudo -u cinder cinder-manage service list | grep -v Binary| tr '@' ' ' | awk '{print $1 \" \" $2}' | while read i ; do sudo -u cinder cinder-manage service remove $i ; done ; sudo -u cinder cinder-manage db online_data_migrations; else echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION; fi"
- '/usr/bin/bootstrap_host_exec'
- 'cinder_api'
- '/container-config-scripts/cinder_ffu_db_sync.sh'
- 'online'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
cinder_api_db_sync_stein:
@ -342,11 +350,16 @@ outputs:
privileged: false
detach: false
user: root
volumes: *cinder_volumes
volumes:
list_concat:
- *cinder_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml cinder_api_short_bootstrap_node_name | grep `/bin/hostname -s` && DB_VERSION=$( sudo -u cinder cinder-manage db version ) _RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/cinder/db/sqlalchemy/migrate_repo/versions/ |grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1) RPM_VERSION=$(expr $_RPM_VERSION + 0) && if (( $RPM_VERSION >= $DB_VERSION )); then sudo -u cinder cinder-manage db sync --bump-versions; else echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION; fi"
- '/usr/bin/bootstrap_host_exec'
- 'cinder_api'
- '/container-config-scripts/cinder_ffu_db_sync.sh'
- 'sync'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
- {}

View File

@ -587,6 +587,12 @@ outputs:
dest: "/"
merge: true
preserve_properties: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- glance_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/glance_ffu_db_sync.sh }
docker_config:
step_2:
get_attr: [GlanceLogging, docker_config, step_2]
@ -636,18 +642,19 @@ outputs:
privileged: false
detach: false
user: root
volumes: *glance_volumes
volumes:
list_concat:
- *glance_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
environment:
KOLLA_BOOTSTRAP: true
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml glance_api_short_bootstrap_node_name | grep `/bin/hostname -s` && OUT=$( sudo -u glance -E /usr/local/bin/kolla_start 2>&1 ) || if [[ $OUT =~ train ]]; then echo 'DB is already up to date' ; else echo $OUT; exit 1; fi"
- '/usr/bin/bootstrap_host_exec'
- 'glance_api'
- '/container-config-scripts/glance_ffu_db_sync.sh'
- {}
step_4:
map_merge:

View File

@ -623,6 +623,12 @@ outputs:
dest: "/"
merge: true
preserve_properties: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- keystone_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/keystone_ffu_db_sync.sh }
docker_config:
# Kolla_bootstrap/db sync runs before permissions set by kolla_config
step_2:
@ -695,20 +701,21 @@ outputs:
user: root
privileged: false
detach: false
volumes: *keystone_volumes
volumes:
list_concat:
- *keystone_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
environment:
map_merge:
- {get_attr: [KeystoneLogging, environment]}
- KOLLA_BOOTSTRAP: true
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml keystone_short_bootstrap_node_name | grep `/bin/hostname -s` && DB_VERSION=$( /usr/local/bin/kolla_set_configs && /usr/bin/keystone-manage --config-file /etc/keystone/keystone.conf db_version ) _RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/keystone/common/sql/migrate_repo/versions |grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1) RPM_VERSION=$(expr $_RPM_VERSION + 0) && if (( $RPM_VERSION > $DB_VERSION )); then /usr/local/bin/kolla_start ; else echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION; fi"
- '/usr/bin/bootstrap_host_exec'
- 'keystone'
- '/container-config-scripts/keystone_ffu_db_sync.sh'
- {}
step_4:
# There are cases where we need to refresh keystone after the resource provisioning,

View File

@ -184,6 +184,12 @@ outputs:
- path: /var/log/mistral
owner: mistral:mistral
recurse: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- mistral_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/mistral_ffu_db_sync.sh }
docker_config:
# db sync runs before permissions set by kolla_config
step_2:
@ -222,17 +228,15 @@ outputs:
privileged: false
detach: false
user: root
volumes: *mistral_api_volumes
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
# Here we sadly just expect that if we can't run mistral-db-manage curent
# it means we already updated. If there is any issue with the service the
# upgrade will continue regardless...
volumes:
list_concat:
- *mistral_api_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml mistral_api_short_bootstrap_node_name | grep `/bin/hostname -s` && sudo -u mistral mistral-db-manage --config-file /etc/mistral/mistral.conf current && sudo -u mistral mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head ; exit 0"
- '/usr/bin/bootstrap_host_exec'
- 'mistral_api'
- '/container-config-scripts/mistral_ffu_db_sync.sh'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
- {}

View File

@ -450,6 +450,12 @@ outputs:
dest: "/etc/httpd/conf.d"
merge: false
preserve_properties: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- neutron_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/neutron_ffu_db_sync.sh }
docker_config:
step_2:
get_attr: [NeutronLogging, docker_config, step_2]
@ -484,17 +490,15 @@ outputs:
privileged: false
detach: false
user: root
volumes: *neutron_api_volumes
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
# Here we sadly just expect that if we can't run neutron-db-manage curent
# it means we already updated. If there is any issue with the service the
# upgrade will continue regardless...
volumes:
list_concat:
- *neutron_api_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml neutron_api_short_bootstrap_node_name | grep `/bin/hostname -s` && neutron-db-manage --subproject neutron current && neutron-db-manage upgrade heads ; exit 0 "
- '/usr/bin/bootstrap_host_exec'
- 'neutron_api'
- '/container-config-scripts/neutron_ffu_db_sync.sh'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
- {}

View File

@ -291,6 +291,9 @@ outputs:
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- nova_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/nova_ffu_db_sync.sh }
- nova_wait_for_api_service.py:
mode: "0755"
content: { get_file: ../../container_config_scripts/nova_wait_for_api_service.py }
@ -413,24 +416,16 @@ outputs:
net: host
detach: false
user: root
volumes: *nova_api_bootstrap_volumes
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
volumes:
list_concat:
- *nova_api_bootstrap_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml nova_api_short_bootstrap_node_name | grep `/bin/hostname -s` && DB_VERSION=$( sudo -u nova /usr/bin/nova-manage api_db version ) _RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/nova/db/sqlalchemy/api_migrations/migrate_repo/versions |grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1) RPM_VERSION=$(expr $_RPM_VERSION + 0) && if (( $RPM_VERSION >= $DB_VERSION )); then sudo -u nova /usr/bin/nova-manage api_db sync; else echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION; fi"
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
nova_api_online_data_migrations_stein:
start_order: 2 # Runs after nova-conductor dbsync Stein
image: *nova_api_image_stein
net: host
detach: false
user: root
volumes: *nova_api_bootstrap_volumes
command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage db online_data_migrations'"
- '/usr/bin/bootstrap_host_exec'
- 'nova_api'
- '/container-config-scripts/nova_ffu_db_sync.sh'
- 'api_db'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
- {}

View File

@ -165,6 +165,12 @@ outputs:
- path: /var/log/nova
owner: nova:nova
recurse: true
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- nova_ffu_db_sync.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/nova_ffu_db_sync.sh }
docker_config:
step_2:
get_attr: [NovaLogging, docker_config, step_2]
@ -193,15 +199,17 @@ outputs:
start_order: 1 # Runs after nova-api db sync Stein
net: host
detach: false
volumes: *nova_conductor_bootstrap_volumes
volumes:
list_concat:
- *nova_conductor_bootstrap_volumes
-
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
user: root
# TODO FIXME: we need LP/BZ for this. Each service should provide db
# version and version provided by package so we can check if we need
# update or not.
command:
- '/bin/bash'
- '-c'
- "/bin/hiera -c /etc/puppet/hiera.yaml nova_conductor_short_bootstrap_node_name | grep `/bin/hostname -s` && DB_VERSION=$( sudo -u nova /usr/bin/nova-manage db version ) _RPM_VERSION=$(ls /usr/lib/python3.6/site-packages/nova/db/sqlalchemy/migrate_repo/versions | grep -e '[0-9]_.*.py' | cut -d '_' -f1 | sort | tail -n1) RPM_VERSION=$(expr $_RPM_VERSION + 0) && if (( $RPM_VERSION >= $DB_VERSION )); then sudo -u nova /usr/bin/nova-manage db sync; else echo DB_VERSION: $DB_VERSION RPM_VERSION: $RPM_VERSION; fi"
- '/usr/bin/bootstrap_host_exec'
- 'nova_conductor'
- '/container-config-scripts/nova_ffu_db_sync.sh'
- 'db'
environment:
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
- {}