heat/heat_integrationtests/functional/test_reload_on_sighup.py
Peter Razumovsky 48387e92a0 Fix race condition in reload_on_sighup functional
Race condition appears in next situation:

  1. First thread calls _set_config_value for one section.
  2. Second thread calls _set_config_value for another section.
  3. First thread. Config option value set, calls
     open(self.config_file, 'wb'), which erases all file content.
  4. Second thread. In previous point moment second thread tries to
     set config option value to self.config_file, which is empty (see 3).
     So, NoSectionError exception raised.

This patch adds ten retries for setting option value, if NoSectionError
raised, i.e. try to wait until self.config_file is busy.

Change-Id: Ic54ea287ebe4724511f75d42677cae5dfdec4e57
Closes-bug: #1535766
2016-03-15 17:58:31 +00:00

127 lines
4.9 KiB
Python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
import eventlet
from oslo_concurrency import processutils
from six.moves import configparser
from heat_integrationtests.functional import functional_base
class ReloadOnSighupTest(functional_base.FunctionalTestsBase):
def setUp(self):
self.config_file = "/etc/heat/heat.conf"
super(ReloadOnSighupTest, self).setUp()
def _set_config_value(self, service, key, value):
config = configparser.ConfigParser()
# NOTE(prazumovsky): If there are several workers, there can be
# situation, when one thread opens self.config_file for writing
# (so config_file erases with opening), in that moment other thread
# intercepts to this file and try to set config option value, i.e.
# write to file, which is already erased by first thread, so,
# NoSectionError raised. So, should wait until first thread writes to
# config_file.
retries_count = self.conf.sighup_config_edit_retries
while True:
config.read(self.config_file)
try:
config.set(service, key, value)
except configparser.NoSectionError:
if retries_count <= 0:
raise
retries_count -= 1
eventlet.sleep(1)
else:
break
with open(self.config_file, 'wb') as f:
config.write(f)
def _get_config_value(self, service, key):
config = configparser.ConfigParser()
config.read(self.config_file)
val = config.get(service, key)
return val
def _get_heat_api_pids(self, service):
# get the pids of all heat-api processes
if service == "heat_api":
process = "heat-api|grep -Ev 'grep|cloudwatch|cfn'"
else:
process = "%s|grep -Ev 'grep'" % service.replace('_', '-')
cmd = "ps -ef|grep %s|awk '{print $2}'" % process
out, err = processutils.execute(cmd, shell=True)
self.assertIsNotNone(out, "heat-api service not running. %s" % err)
pids = filter(None, out.split('\n'))
# get the parent pids of all heat-api processes
cmd = "ps -ef|grep %s|awk '{print $3}'" % process
out, _ = processutils.execute(cmd, shell=True)
parent_pids = filter(None, out.split('\n'))
heat_api_parent = list(set(pids) & set(parent_pids))[0]
heat_api_children = list(set(pids) - set(parent_pids))
return heat_api_parent, heat_api_children
def _change_config(self, service, old_workers, new_workers):
pre_reload_parent, pre_reload_children = self._get_heat_api_pids(
service)
self.assertEqual(old_workers, len(pre_reload_children))
# change the config values
self._set_config_value(service, 'workers', new_workers)
cmd = "kill -HUP %s" % pre_reload_parent
processutils.execute(cmd, shell=True)
# wait till heat-api reloads
start_time = time.time()
while time.time() - start_time < self.conf.sighup_timeout:
post_reload_parent, post_reload_children = self._get_heat_api_pids(
service)
intersect = set(post_reload_children) & set(pre_reload_children)
if (new_workers == len(post_reload_children)
and pre_reload_parent == post_reload_parent
and intersect == set()):
break
eventlet.sleep(1)
self.assertEqual(pre_reload_parent, post_reload_parent)
self.assertEqual(new_workers, len(post_reload_children))
# test if all child processes are newly created
self.assertEqual(set(post_reload_children) & set(pre_reload_children),
set())
def _reload(self, service):
old_workers = int(self._get_config_value(service, 'workers'))
new_workers = old_workers + 1
self.addCleanup(self._set_config_value, service, 'workers',
old_workers)
self._change_config(service, old_workers, new_workers)
# revert all the changes made
self._change_config(service, new_workers, old_workers)
def test_api_reload_on_sighup(self):
self._reload('heat_api')
def test_api_cfn_reload_on_sighup(self):
self._reload('heat_api_cfn')
def test_api_cloudwatch_on_sighup(self):
self._reload('heat_api_cloudwatch')