Implement advanced apt pinning

Current approach is an error prone, because Debian repos may contain
metadata that doesn't fit our pinning template. Yes, in some cases
our template will work (and it does with upstream Ubuntu and Fuel),
but we want to be flexible and cover all cases.

In order to set corrent pinning, we have to use repo's metadata and
that means download repo's Release file, parse it and create pinning
based on it. In that case we'll use all available pinning rules, so
all possible issues will be up to repo maintainer and not because
we're using error prone approach.

Related-Bug: #1435892
Related-Bug: #1438783
Closes-Bug:  #1446686

Change-Id: I5279b7f414f831503fd993109dd5c91bef51a9ef
Signed-off-by: Igor Kalnitsky <igor@kalnitsky.org>
This commit is contained in:
Igor Kalnitsky 2015-04-15 18:05:45 +03:00
parent 69fca3bada
commit 46cd6a8d8e
9 changed files with 514 additions and 64 deletions

View File

@ -151,13 +151,17 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer):
elif operating_system == consts.RELEASE_OS.ubuntu:
repo = self.get_ubuntu_repo(plugin)
repo_tasks.extend([
repo_tasks.append(
self.serialize_task(
plugin,
templates.make_ubuntu_sources_task(uids, repo)),
self.serialize_task(
plugin,
templates.make_ubuntu_preferences_task(uids, repo))])
templates.make_ubuntu_sources_task(uids, repo)))
# do not add preferences task to task list if we can't
# complete it (e.g. can't retrieve or parse Release file)
task = self.serialize_task(
plugin, templates.make_ubuntu_preferences_task(uids, repo))
if task is not None:
repo_tasks.append(task)
# apt-get update executed after every additional source.list
# to be able understand what plugin source.list caused error

View File

@ -186,7 +186,11 @@ class UploadMOSRepo(GenericRolesHook):
yield templates.make_ubuntu_sources_task(uids, repo)
if repo.get('priority'):
yield templates.make_ubuntu_preferences_task(uids, repo)
# do not add preferences task to task list if we can't
# complete it (e.g. can't retrieve or parse Release file)
task = templates.make_ubuntu_preferences_task(uids, repo)
if task is not None:
yield task
yield templates.make_apt_update_task(uids)

View File

@ -17,8 +17,12 @@
import os
from oslo.serialization import jsonutils
import requests
import six
from nailgun.logger import logger
from nailgun.settings import settings
from nailgun.utils import debian
def make_upload_task(uids, data, path):
@ -38,34 +42,48 @@ def make_ubuntu_sources_task(uids, repo):
def make_ubuntu_preferences_task(uids, repo):
# TODO(ikalnitsky):
# Research how to add host condition to the current pinning.
# NOTE(ikalnitsky): In order to implement the proper pinning,
# we have to download and parse the repo's "Release" file.
# Generally, that's not a good idea to make some HTTP request
# from Nailgun, but taking into account that this task
# will be executed in uWSGI's mule worker we can skip this
# rule, because proper pinning is more valuable thing right now.
template = '\n'.join([
'Package: *',
'Pin: release a={suite},c={section}',
'Pin-Priority: {priority}', ])
template_flat = '\n'.join([
'Package: *',
'Pin: release a={suite}',
'Pin-Priority: {priority}', ])
'Pin: release {conditions}',
'Pin-Priority: {priority}'])
preferences_content = []
try:
release = debian.get_release_file(repo, retries=3)
release = debian.parse_release_file(release)
pin = debian.get_apt_preferences_line(release)
except requests.exceptions.HTTPError as exc:
logger.error("Failed to fetch 'Release' file due to '%s'. "
"The apt preferences won't be applied for repo '%s'.",
six.text_type(exc), repo['name'])
return None
except Exception:
logger.exception("Failed to parse 'Release' file.")
return None
# if sections are detected (non-flat repo) create pinning per
# sections; otherwise - create just one pin for the entire repo
if repo['section']:
for section in repo['section'].split():
preferences_content.append(template.format(
suite=repo['suite'],
section=section,
conditions='{0},c={1}'.format(pin, section),
priority=repo['priority']))
else:
preferences_content.append(template_flat.format(
suite=repo['suite'],
preferences_content.append(template.format(
conditions=pin,
priority=repo['priority']))
preferences_content = '\n\n'.join(preferences_content)
preferences_path = os.path.join('/etc/apt/preferences.d', repo['name'])
preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name'])
return make_upload_task(uids, preferences_content, preferences_path)

View File

@ -224,6 +224,12 @@ class TestPrePostHooks(BasePluginTest):
def setUp(self):
super(TestPrePostHooks, self).setUp()
self._requests_mock = mock.patch(
'nailgun.utils.debian.requests.get',
return_value=mock.Mock(text='Archive: test'))
self._requests_mock.start()
resp = self.create_plugin()
self.plugin = attr_plugin.wrap_plugin(
objects.Plugin.get_by_uid(resp.json['id']))
@ -232,6 +238,10 @@ class TestPrePostHooks(BasePluginTest):
{'roles': ['compute'], 'pending_addition': True}])
self.enable_plugin(self.cluster, self.sample_plugin['name'])
def tearDown(self):
self._requests_mock.stop()
super(TestPrePostHooks, self).tearDown()
def test_generate_pre_hooks(self):
tasks = self.get_pre_hooks(self.cluster).json
plugins_tasks = [t for t in tasks if t.get('diagnostic_name')]

View File

@ -55,6 +55,12 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase):
def setUp(self):
super(BaseTaskSerializationTestUbuntu, self).setUp()
self._requests_mock = mock.patch(
'nailgun.utils.debian.requests.get',
return_value=mock.Mock(text='Archive: test'))
self._requests_mock.start()
self.release = self.env.create_release(
api=False, attributes_metadata=self.env.read_fixtures(
['openstack'])[1]['fields']['attributes_metadata'])
@ -70,6 +76,10 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase):
self.all_uids = [n.uid for n in self.nodes]
self.cluster.deployment_tasks = yaml.load(self.TASKS)
def tearDown(self):
self._requests_mock.stop()
super(BaseTaskSerializationTestUbuntu, self).tearDown()
class TestHooksSerializersUbuntu(BaseTaskSerializationTestUbuntu):
def test_create_repo_ubuntu(self):

View File

@ -14,7 +14,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
import mock
import requests
from oslo.serialization import jsonutils
from six.moves import zip_longest
from nailgun.test import base
@ -44,48 +50,6 @@ class TestMakeTask(base.BaseTestCase):
'type': 'upload_file',
'uids': [1, 2, 3]})
def test_make_ubuntu_preferences_task(self):
result = tasks_templates.make_ubuntu_preferences_task(
[1, 2, 3],
{
'name': 'plugin_name',
'type': 'deb',
'uri': 'http://url',
'suite': '/',
'section': '',
'priority': 1001
})
self.assertEqual(
result,
{'parameters': {
'data': 'Package: *\nPin: release a=/\nPin-Priority: 1001',
'path': '/etc/apt/preferences.d/plugin_name'},
'type': 'upload_file',
'uids': [1, 2, 3]})
result = tasks_templates.make_ubuntu_preferences_task(
[1, 2, 3],
{
'name': 'plugin_name',
'type': 'deb',
'uri': 'http://url',
'suite': 'jessie',
'section': 'main universe',
'priority': 1004
})
self.assertEqual(
result,
{'parameters': {
'data': ('Package: *\n'
'Pin: release a=jessie,c=main\n'
'Pin-Priority: 1004\n\n'
'Package: *\n'
'Pin: release a=jessie,c=universe\n'
'Pin-Priority: 1004'),
'path': '/etc/apt/preferences.d/plugin_name'},
'type': 'upload_file',
'uids': [1, 2, 3]})
def test_make_ubuntu_unauth_repos_task(self):
result = tasks_templates.make_ubuntu_unauth_repos_task([1, 2, 3])
self.assertEqual(
@ -238,3 +202,110 @@ class TestMakeTask(base.BaseTestCase):
'interval': 1,
'cwd': '/',
}})
class TestMakeUbuntuPreferencesTask(base.BaseTestCase):
_fake_debian_release = '''
Origin: TestOrigin
Label: TestLabel
Archive: test-archive
Codename: testcodename
'''
_re_pin = re.compile('Pin: release (.*)')
def _check_apt_preferences(self, data, sections, priority):
pins = data.split('\n\n')
# in non-flat repo we have one pin per section
if sections:
self.assertEqual(len(pins), len(sections))
# we should have one pin per section
for pin, section in zip_longest(pins, sections):
conditions = self._re_pin.search(pin).group(1).split(',')
# check general template
self.assertRegexpMatches(
data, (
'Package: \*\n'
'Pin: release .*\n'
'Pin-Priority: {0}'.format(priority)
))
# check pin
expected_conditions = [
'a=test-archive',
'l=TestLabel',
'n=testcodename',
'o=TestOrigin',
]
if section:
expected_conditions.append('c={0}'.format(section))
self.assertItemsEqual(conditions, expected_conditions)
@mock.patch('nailgun.utils.debian.requests.get',
return_value=mock.Mock(text=_fake_debian_release))
def test_make_ubuntu_preferences_task(self, _):
result = tasks_templates.make_ubuntu_preferences_task(
[1, 2, 3],
{
'name': 'plugin_name',
'type': 'deb',
'uri': 'http://url',
'suite': 'test-archive',
'section': 'main universe',
'priority': 1004
})
data = result['parameters'].pop('data')
self.assertEqual(
result,
{'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'},
'type': 'upload_file',
'uids': [1, 2, 3]})
self._check_apt_preferences(data, ['main', 'universe'], 1004)
@mock.patch('nailgun.utils.debian.requests.get',
return_value=mock.Mock(text=_fake_debian_release))
def test_make_ubuntu_preferences_task_flat(self, _):
result = tasks_templates.make_ubuntu_preferences_task(
[1, 2, 3],
{
'name': 'plugin_name',
'type': 'deb',
'uri': 'http://url',
'suite': '/',
'section': '',
'priority': 1004
})
data = result['parameters'].pop('data')
self.assertEqual(
result,
{'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'},
'type': 'upload_file',
'uids': [1, 2, 3]})
self._check_apt_preferences(data, [], 1004)
@mock.patch('nailgun.utils.debian.requests.get')
def test_make_ubuntu_preferences_task_returns_none_if_errors(self, m_get):
r = requests.Response()
r.status_code = 404
m_get.return_value = r
result = tasks_templates.make_ubuntu_preferences_task(
[1, 2, 3],
{
'name': 'plugin_name',
'type': 'deb',
'uri': 'http://url',
'suite': 'test-archive',
'section': 'main universe',
'priority': 1004
})
self.assertIsNone(result)

View File

@ -19,6 +19,8 @@ from mock import patch
import os
import tempfile
import requests
from nailgun.test import base
from nailgun.utils import camel_to_snake_case
from nailgun.utils import compact
@ -26,8 +28,13 @@ from nailgun.utils import dict_merge
from nailgun.utils import extract_env_version
from nailgun.utils import flatten
from nailgun.utils import get_fuel_release_versions
from nailgun.utils import grouper
from nailgun.utils import traverse
from nailgun.utils.debian import get_apt_preferences_line
from nailgun.utils.debian import get_release_file
from nailgun.utils.debian import parse_release_file
class TestUtils(base.BaseIntegrationTest):
@ -97,6 +104,19 @@ class TestUtils(base.BaseIntegrationTest):
flatten([7, 5, [3, [4, 5], 1], 2]),
[7, 5, 3, [4, 5], 1, 2])
def test_grouper(self):
self.assertEqual(
list(grouper([0, 1, 2, 3], 2)), [(0, 1), (2, 3)])
self.assertEqual(
list(grouper([0, 1, 2, 3, 4, 5], 3)), [(0, 1, 2), (3, 4, 5)])
self.assertEqual(
list(grouper([0, 1, 2, 3, 4], 3)), [(0, 1, 2), (3, 4, None)])
self.assertEqual(
list(grouper([0, 1, 2, 3, 4], 3, 'x')), [(0, 1, 2), (3, 4, 'x')])
class TestTraverse(base.BaseUnitTest):
@ -165,3 +185,197 @@ class TestTraverse(base.BaseUnitTest):
'y': 'b 42 b',
}
]})
class TestGetDebianReleaseFile(base.BaseUnitTest):
@patch('nailgun.utils.debian.requests.get')
def test_normal_ubuntu_repo(self, m_get):
get_release_file({
'name': 'myrepo',
'uri': 'http://some-uri.com/path',
'suite': 'mysuite',
'section': 'main university',
})
m_get.assert_called_with(
'http://some-uri.com/path/dists/mysuite/Release')
@patch('nailgun.utils.debian.requests.get')
def test_flat_ubuntu_repo(self, m_get):
testcases = [
# (suite, uri)
('/', 'http://some-uri.com/deb/Release'),
('/path', 'http://some-uri.com/deb/path/Release'),
('path', 'http://some-uri.com/deb/path/Release'),
]
for suite, uri in testcases:
get_release_file({
'name': 'myrepo',
'uri': 'http://some-uri.com/deb',
'suite': suite,
'section': '',
})
m_get.assert_called_with(uri)
@patch('nailgun.utils.debian.requests.get')
def test_do_not_silence_http_errors(self, m_get):
r = requests.Response()
r.status_code = 404
m_get.return_value = r
self.assertRaises(requests.exceptions.HTTPError, get_release_file, {
'name': 'myrepo',
'uri': 'http://some-uri.com/path',
'suite': 'mysuite',
'section': 'main university',
})
@patch('nailgun.utils.debian.requests.get')
def test_do_not_retry_on_404(self, m_get):
r = requests.Response()
r.status_code = 404
m_get.return_value = r
self.assertRaises(requests.exceptions.HTTPError, get_release_file, {
'name': 'myrepo',
'uri': 'http://some-uri.com/path',
'suite': 'mysuite',
'section': 'main university',
}, retries=3)
self.assertEqual(m_get.call_count, 1)
@patch('nailgun.utils.debian.requests.get')
def test_do_retry_on_error(self, m_get):
r = requests.Response()
r.status_code = 500
m_get.return_value = r
self.assertRaises(requests.exceptions.HTTPError, get_release_file, {
'name': 'myrepo',
'uri': 'http://some-uri.com/path',
'suite': 'mysuite',
'section': 'main university',
}, retries=3)
self.assertEqual(m_get.call_count, 3)
@patch('nailgun.utils.debian.requests.get')
def test_returns_content_if_http_ok(self, m_get):
r = requests.Response()
r._content = 'content'
r.status_code = 200
m_get.return_value = r
content = get_release_file({
'name': 'myrepo',
'uri': 'http://some-uri.com/path',
'suite': 'mysuite',
'section': 'main university',
})
self.assertEqual(content, 'content')
class TestParseDebianReleaseFile(base.BaseUnitTest):
_deb_release_info = '''
Origin: TestOrigin
Label: TestLabel
Archive: test-archive
Codename: testcodename
Date: Thu, 08 May 2014 14:19:09 UTC
Architectures: amd64 i386
Components: main restricted universe multiverse
Description: Test Description
MD5Sum:
ead1cbf42ed119c50bf3aab28b5b6351 934 main/binary-amd64/Packages
52d605b4217be64f461751f233dd9a8f 96 main/binary-amd64/Release
SHA1:
28c4460e3aaf1b93f11911fdc4ff23c28809af89 934 main/binary-amd64/Packages
d03d716bceaba35f91726c096e2a9a8c23ccc766 96 main/binary-amd64/Release
'''
def test_parse(self):
deb_release = parse_release_file(self._deb_release_info)
self.assertEqual(deb_release, {
'Origin': 'TestOrigin',
'Label': 'TestLabel',
'Archive': 'test-archive',
'Codename': 'testcodename',
'Date': 'Thu, 08 May 2014 14:19:09 UTC',
'Architectures': 'amd64 i386',
'Components': 'main restricted universe multiverse',
'Description': 'Test Description',
'MD5Sum': [
{
'md5sum': 'ead1cbf42ed119c50bf3aab28b5b6351',
'size': '934',
'name': 'main/binary-amd64/Packages',
},
{
'md5sum': '52d605b4217be64f461751f233dd9a8f',
'size': '96',
'name': 'main/binary-amd64/Release',
}
],
'SHA1': [
{
'sha1': '28c4460e3aaf1b93f11911fdc4ff23c28809af89',
'size': '934',
'name': 'main/binary-amd64/Packages',
},
{
'sha1': 'd03d716bceaba35f91726c096e2a9a8c23ccc766',
'size': '96',
'name': 'main/binary-amd64/Release',
}
],
})
class TestGetAptPreferencesLine(base.BaseUnitTest):
_deb_release = {
'Origin': 'TestOrigin',
'Label': 'TestLabel',
'Archive': 'test-archive',
'Version': '42.42',
'Codename': 'testcodename',
'Date': 'Thu, 08 May 2014 14:19:09 UTC',
'Architectures': 'amd64 i386',
'Components': 'main restricted universe multiverse',
'Description': 'Test Description',
}
def test_all_rules(self):
pin = get_apt_preferences_line(self._deb_release)
self.assertItemsEqual(pin.split(','), [
'o=TestOrigin',
'l=TestLabel',
'a=test-archive',
'v=42.42',
'n=testcodename',
])
def test_not_all_rules(self):
deb_release = self._deb_release.copy()
del deb_release['Codename']
del deb_release['Label']
del deb_release['Version']
pin = get_apt_preferences_line(deb_release)
self.assertItemsEqual(pin.split(','), [
'o=TestOrigin',
'a=test-archive',
])
def test_suite_is_synonym_for_archive(self):
deb_release = self._deb_release.copy()
deb_release['Suite'] = 'test-suite'
del deb_release['Archive']
pin = get_apt_preferences_line(deb_release)
conditions = pin.split(',')
self.assertIn('a=test-suite', conditions)

View File

@ -24,6 +24,8 @@ from copy import deepcopy
from itertools import chain
from random import choice
from six.moves import zip_longest
from nailgun.logger import logger
from nailgun.settings import settings
@ -191,3 +193,10 @@ def flatten(array):
check = lambda x: x if isinstance(x, list) else [x]
return list(chain.from_iterable(check(x) for x in array))
def grouper(iterable, n, fillvalue=None):
"""Collect data into fixed-length chunks or blocks
"""
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)

View File

@ -0,0 +1,110 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import requests
import six
import yaml
from nailgun.utils import grouper
def get_release_file(repo, retries=1):
"""Get Release content of a given repo.
:param repo: a repo as dict
:returns: a release's content as string
"""
if repo['section']:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(
repo['uri'], 'dists', repo['suite'], 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(
repo['uri'], repo['suite'].lstrip('/'), 'Release')
for _ in six.moves.range(0, retries):
response = requests.get(download_uri)
# do not perform retries if release is not found
if response.status_code == 404:
break
response.raise_for_status()
return response.text
def parse_release_file(content):
"""Parse Debian repo's Release file content.
:param content: a Debian's Release file content
:returns: a dict with repo's attributes
"""
# TODO(ikalnitsky): Consider to use some existing library for
# parsing debian's release file (e.g. python-debian).
_multivalued_fields = {
'SHA1': ['sha1', 'size', 'name'],
'SHA256': ['sha256', 'size', 'name'],
'SHA512': ['sha512', 'size', 'name'],
'MD5Sum': ['md5sum', 'size', 'name'],
}
# debian data format is very similiar to yaml, except
# multivalued field. so we can parse it just like yaml
# and then perform additional transformation for those
# fields (we know which ones are multivalues).
data = yaml.load(content)
for attr, columns in six.iteritems(_multivalued_fields):
if attr not in data:
continue
values = data[attr].split()
data[attr] = []
for group in grouper(values, len(columns)):
data[attr].append(dict(zip(columns, group)))
return data
def get_apt_preferences_line(deb_release):
"""Get an APT Preferences line from repo's release information.
:param deb_release: a Debian's Release content as dict
:returns: an apt pinning line as string
"""
_transformations = {
'Archive': 'a',
'Suite': 'a', # suite is a synonym for archive
'Codename': 'n',
'Version': 'v',
'Origin': 'o',
'Label': 'l',
}
conditions = set()
for field, condition in six.iteritems(_transformations):
if field in deb_release:
conditions.add('{0}={1}'.format(condition, deb_release[field]))
return ','.join(conditions)