openstack/fuel-agent
Code Issues Proposed changes

Fix bootstrap apt-get update issues

Browse Source 
Sometimes 'apt-get update' fail with strange error:

  Get:22 http://mirror.fuel-infra.org/mos-repos/ubuntu/10.0 mos10.0-updates Release.gpg [473 B]
  Err:22 http://mirror.fuel-infra.org/mos-repos/ubuntu/10.0 mos10.0-updates Release.gpg
    Writing more data than expected (473 > 20) [IP: 208.78.244.194 80]

This is only reproduced during running bootstrap, and never reproduced manually.

APT::http::Pipeline-Depth=0 fixes the problem.

By default pipeline depth is 10 in apt 1.2.10 (Ubuntu Xenial) [1], [2].

[1] https://github.com/Debian/apt/blob/1.2.10/doc/apt.conf.5.xml#L402-L407
[2] https://github.com/Debian/apt/blob/1.2.10/methods/server.cc#L754-L756

Related-bug: #1612731
Related-bug: #1470468

Change-Id: Iac8c22d87f18718ae9a318dda56266f8ed57f866
This commit is contained in:
Dmitry Teselkin
2016-08-09 16:58:43 +03:00
parent b23fe46336
commit 426d2a39f4
2 changed files with 36 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
fuel_agent/tests/test_build_utils.py
View File

@@ -159,12 +159,14 @@ class BuildUtilsTestCase(unittest2.TestCase):
@mock.patch.object(bu, 'remove_files')
@mock.patch.object(bu, 'clean_dirs')
def test_clean_apt_settings(self, mock_dirs, mock_files):
bu.clean_apt_settings('chroot', 'unsigned', 'force_ipv4')
bu.clean_apt_settings('chroot', 'unsigned', 'force_ipv4',
'pipeline_depth')
mock_dirs.assert_called_once_with(
'chroot', ['etc/apt/preferences.d', 'etc/apt/sources.list.d'])
files = set(['etc/apt/sources.list', 'etc/apt/preferences',
'etc/apt/apt.conf.d/%s' % 'force_ipv4',
'etc/apt/apt.conf.d/%s' % 'unsigned',
'etc/apt/apt.conf.d/%s' % 'pipeline_depth',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-ftp',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-http',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-https'])
@@ -191,7 +193,8 @@ class BuildUtilsTestCase(unittest2.TestCase):
bu.do_post_inst('chroot',
hashed_root_password=password,
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
force_ipv4_file='fake_force_ipv4',
pipeline_depth_file='fake_pipeline_depth')
file_handle_mock = mock_open.return_value.__enter__.return_value
file_handle_mock.write.assert_called_once_with('manual\n')
@@ -209,9 +212,11 @@ class BuildUtilsTestCase(unittest2.TestCase):
self.assertEqual([mock.call('chroot', ['usr/sbin/policy-rc.d']),
mock.call('chroot', [bu.GRUB2_DMRAID_SETTINGS])],
mock_files.call_args_list)
mock_clean.assert_called_once_with('chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
mock_clean.assert_called_once_with(
'chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4',
pipeline_depth_file='fake_pipeline_depth')
mock_path_join_expected_calls = [
mock.call('chroot', 'etc/shadow'),
mock.call('chroot', 'etc/init.d/puppet'),
@@ -496,20 +501,26 @@ class BuildUtilsTestCase(unittest2.TestCase):
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
file_handle_mock = mock_open.return_value.__enter__.return_value
bu.pre_apt_get('chroot', allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
force_ipv4_file='fake_force_ipv4',
pipeline_depth_file='fake_pipeline_depth')
expected_calls = [
mock.call('APT::Get::AllowUnauthenticated 1;\n'),
mock.call('Acquire::ForceIPv4 "true";\n')]
mock.call('Acquire::ForceIPv4 "true";\n'),
mock.call('Acquire::http::Pipeline-Depth 0;\n')]
self.assertEqual(expected_calls,
file_handle_mock.write.call_args_list)
mock_clean.assert_called_once_with('chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
mock_clean.assert_called_once_with(
'chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4',
pipeline_depth_file='fake_pipeline_depth')
expected_join_calls = [
mock.call('chroot', 'etc/apt/apt.conf.d',
'fake_unsigned'),
mock.call('chroot', 'etc/apt/apt.conf.d',
'fake_force_ipv4')]
'fake_force_ipv4'),
mock.call('chroot', 'etc/apt/apt.conf.d',
'fake_pipeline_depth')]
self.assertEqual(expected_join_calls, mock_path.join.call_args_list)
@mock.patch.object(bu.utils, 'execute')

19
fuel_agent/utils/build.py
View File

@@ -160,12 +160,14 @@ def remove_files(chroot, files):
def clean_apt_settings(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
force_ipv4_file='force_ipv4',
pipeline_depth_file='pipeline_depth'):
"""Cleans apt settings such as package sources and repo pinning."""
files = [DEFAULT_APT_PATH['sources_file'],
DEFAULT_APT_PATH['preferences_file'],
os.path.join(DEFAULT_APT_PATH['conf_dir'], force_ipv4_file),
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file)]
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file),
os.path.join(DEFAULT_APT_PATH['conf_dir'], pipeline_depth_file)]
# also remove proxies
for p_file in six.itervalues(PROXY_PROTOCOLS):
files.append(os.path.join(DEFAULT_APT_PATH['conf_dir'], p_file))
@@ -177,7 +179,8 @@ def clean_apt_settings(chroot, allow_unsigned_file='allow_unsigned_packages',
def do_post_inst(chroot, hashed_root_password,
allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
force_ipv4_file='force_ipv4',
pipeline_depth_file='pipeline_depth'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i',
's%root:[\*,\!]%root:' + hashed_root_password + '%',
@@ -215,7 +218,8 @@ def do_post_inst(chroot, hashed_root_password,
# remove cached apt files
utils.execute('chroot', chroot, 'apt-get', 'clean')
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
force_ipv4_file=force_ipv4_file,
pipeline_depth_file=pipeline_depth_file)
def stop_chrooted_processes(chroot, signal=sig.SIGTERM,
@@ -513,10 +517,12 @@ def set_apt_proxy(chroot, proxies, direct_repo_addr=None):
def pre_apt_get(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4',
pipeline_depth_file='pipeline_depth',
proxies=None, direct_repo_addr=None):
"""It must be called prior run_apt_get."""
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
force_ipv4_file=force_ipv4_file,
pipeline_depth_file=pipeline_depth_file)
# NOTE(agordeev): allow to install packages without gpg digest
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
allow_unsigned_file), 'w') as f:
@@ -524,6 +530,9 @@ def pre_apt_get(chroot, allow_unsigned_file='allow_unsigned_packages',
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
force_ipv4_file), 'w') as f:
f.write('Acquire::ForceIPv4 "true";\n')
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
pipeline_depth_file), 'w') as f:
f.write('Acquire::http::Pipeline-Depth 0;\n')
if proxies:
set_apt_proxy(chroot, proxies, direct_repo_addr)