From 46cd6a8d8e943669e4cb7557551a3e4eeae0019c Mon Sep 17 00:00:00 2001 From: Igor Kalnitsky Date: Wed, 15 Apr 2015 18:05:45 +0300 Subject: [PATCH] Implement advanced apt pinning Current approach is an error prone, because Debian repos may contain metadata that doesn't fit our pinning template. Yes, in some cases our template will work (and it does with upstream Ubuntu and Fuel), but we want to be flexible and cover all cases. In order to set corrent pinning, we have to use repo's metadata and that means download repo's Release file, parse it and create pinning based on it. In that case we'll use all available pinning rules, so all possible issues will be up to repo maintainer and not because we're using error prone approach. Related-Bug: #1435892 Related-Bug: #1438783 Closes-Bug: #1446686 Change-Id: I5279b7f414f831503fd993109dd5c91bef51a9ef Signed-off-by: Igor Kalnitsky --- .../orchestrator/plugins_serializers.py | 14 +- .../nailgun/orchestrator/tasks_serializer.py | 6 +- .../nailgun/orchestrator/tasks_templates.py | 50 ++-- .../test/integration/test_plugins_api.py | 10 + .../unit/test_stages_task_serialization.py | 10 + .../nailgun/test/unit/test_tasks_templates.py | 155 +++++++++---- nailgun/nailgun/test/unit/test_utils.py | 214 ++++++++++++++++++ nailgun/nailgun/utils/__init__.py | 9 + nailgun/nailgun/utils/debian.py | 110 +++++++++ 9 files changed, 514 insertions(+), 64 deletions(-) create mode 100644 nailgun/nailgun/utils/debian.py diff --git a/nailgun/nailgun/orchestrator/plugins_serializers.py b/nailgun/nailgun/orchestrator/plugins_serializers.py index f8d483710d..6b67d15a52 100644 --- a/nailgun/nailgun/orchestrator/plugins_serializers.py +++ b/nailgun/nailgun/orchestrator/plugins_serializers.py @@ -151,13 +151,17 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer): elif operating_system == consts.RELEASE_OS.ubuntu: repo = self.get_ubuntu_repo(plugin) - repo_tasks.extend([ + repo_tasks.append( self.serialize_task( plugin, - templates.make_ubuntu_sources_task(uids, repo)), - self.serialize_task( - plugin, - templates.make_ubuntu_preferences_task(uids, repo))]) + templates.make_ubuntu_sources_task(uids, repo))) + + # do not add preferences task to task list if we can't + # complete it (e.g. can't retrieve or parse Release file) + task = self.serialize_task( + plugin, templates.make_ubuntu_preferences_task(uids, repo)) + if task is not None: + repo_tasks.append(task) # apt-get update executed after every additional source.list # to be able understand what plugin source.list caused error diff --git a/nailgun/nailgun/orchestrator/tasks_serializer.py b/nailgun/nailgun/orchestrator/tasks_serializer.py index acf33fa5ce..daa6e20e05 100644 --- a/nailgun/nailgun/orchestrator/tasks_serializer.py +++ b/nailgun/nailgun/orchestrator/tasks_serializer.py @@ -186,7 +186,11 @@ class UploadMOSRepo(GenericRolesHook): yield templates.make_ubuntu_sources_task(uids, repo) if repo.get('priority'): - yield templates.make_ubuntu_preferences_task(uids, repo) + # do not add preferences task to task list if we can't + # complete it (e.g. can't retrieve or parse Release file) + task = templates.make_ubuntu_preferences_task(uids, repo) + if task is not None: + yield task yield templates.make_apt_update_task(uids) diff --git a/nailgun/nailgun/orchestrator/tasks_templates.py b/nailgun/nailgun/orchestrator/tasks_templates.py index 199398e43e..c313f66a49 100644 --- a/nailgun/nailgun/orchestrator/tasks_templates.py +++ b/nailgun/nailgun/orchestrator/tasks_templates.py @@ -17,8 +17,12 @@ import os from oslo.serialization import jsonutils +import requests +import six +from nailgun.logger import logger from nailgun.settings import settings +from nailgun.utils import debian def make_upload_task(uids, data, path): @@ -38,34 +42,48 @@ def make_ubuntu_sources_task(uids, repo): def make_ubuntu_preferences_task(uids, repo): - # TODO(ikalnitsky): - # Research how to add host condition to the current pinning. + # NOTE(ikalnitsky): In order to implement the proper pinning, + # we have to download and parse the repo's "Release" file. + # Generally, that's not a good idea to make some HTTP request + # from Nailgun, but taking into account that this task + # will be executed in uWSGI's mule worker we can skip this + # rule, because proper pinning is more valuable thing right now. template = '\n'.join([ 'Package: *', - 'Pin: release a={suite},c={section}', - 'Pin-Priority: {priority}', ]) - - template_flat = '\n'.join([ - 'Package: *', - 'Pin: release a={suite}', - 'Pin-Priority: {priority}', ]) - + 'Pin: release {conditions}', + 'Pin-Priority: {priority}']) preferences_content = [] + + try: + release = debian.get_release_file(repo, retries=3) + release = debian.parse_release_file(release) + pin = debian.get_apt_preferences_line(release) + + except requests.exceptions.HTTPError as exc: + logger.error("Failed to fetch 'Release' file due to '%s'. " + "The apt preferences won't be applied for repo '%s'.", + six.text_type(exc), repo['name']) + return None + + except Exception: + logger.exception("Failed to parse 'Release' file.") + return None + + # if sections are detected (non-flat repo) create pinning per + # sections; otherwise - create just one pin for the entire repo if repo['section']: for section in repo['section'].split(): preferences_content.append(template.format( - suite=repo['suite'], - section=section, + conditions='{0},c={1}'.format(pin, section), priority=repo['priority'])) else: - preferences_content.append(template_flat.format( - suite=repo['suite'], + preferences_content.append(template.format( + conditions=pin, priority=repo['priority'])) preferences_content = '\n\n'.join(preferences_content) - preferences_path = os.path.join('/etc/apt/preferences.d', repo['name']) - + preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name']) return make_upload_task(uids, preferences_content, preferences_path) diff --git a/nailgun/nailgun/test/integration/test_plugins_api.py b/nailgun/nailgun/test/integration/test_plugins_api.py index 14c0e1a24e..7c4d14e2a5 100644 --- a/nailgun/nailgun/test/integration/test_plugins_api.py +++ b/nailgun/nailgun/test/integration/test_plugins_api.py @@ -224,6 +224,12 @@ class TestPrePostHooks(BasePluginTest): def setUp(self): super(TestPrePostHooks, self).setUp() + + self._requests_mock = mock.patch( + 'nailgun.utils.debian.requests.get', + return_value=mock.Mock(text='Archive: test')) + self._requests_mock.start() + resp = self.create_plugin() self.plugin = attr_plugin.wrap_plugin( objects.Plugin.get_by_uid(resp.json['id'])) @@ -232,6 +238,10 @@ class TestPrePostHooks(BasePluginTest): {'roles': ['compute'], 'pending_addition': True}]) self.enable_plugin(self.cluster, self.sample_plugin['name']) + def tearDown(self): + self._requests_mock.stop() + super(TestPrePostHooks, self).tearDown() + def test_generate_pre_hooks(self): tasks = self.get_pre_hooks(self.cluster).json plugins_tasks = [t for t in tasks if t.get('diagnostic_name')] diff --git a/nailgun/nailgun/test/unit/test_stages_task_serialization.py b/nailgun/nailgun/test/unit/test_stages_task_serialization.py index e1c0c12495..74f80165a3 100644 --- a/nailgun/nailgun/test/unit/test_stages_task_serialization.py +++ b/nailgun/nailgun/test/unit/test_stages_task_serialization.py @@ -55,6 +55,12 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase): def setUp(self): super(BaseTaskSerializationTestUbuntu, self).setUp() + + self._requests_mock = mock.patch( + 'nailgun.utils.debian.requests.get', + return_value=mock.Mock(text='Archive: test')) + self._requests_mock.start() + self.release = self.env.create_release( api=False, attributes_metadata=self.env.read_fixtures( ['openstack'])[1]['fields']['attributes_metadata']) @@ -70,6 +76,10 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase): self.all_uids = [n.uid for n in self.nodes] self.cluster.deployment_tasks = yaml.load(self.TASKS) + def tearDown(self): + self._requests_mock.stop() + super(BaseTaskSerializationTestUbuntu, self).tearDown() + class TestHooksSerializersUbuntu(BaseTaskSerializationTestUbuntu): def test_create_repo_ubuntu(self): diff --git a/nailgun/nailgun/test/unit/test_tasks_templates.py b/nailgun/nailgun/test/unit/test_tasks_templates.py index 9e485c700c..d276fdf5a8 100644 --- a/nailgun/nailgun/test/unit/test_tasks_templates.py +++ b/nailgun/nailgun/test/unit/test_tasks_templates.py @@ -14,7 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +import re + +import mock +import requests + from oslo.serialization import jsonutils +from six.moves import zip_longest from nailgun.test import base @@ -44,48 +50,6 @@ class TestMakeTask(base.BaseTestCase): 'type': 'upload_file', 'uids': [1, 2, 3]}) - def test_make_ubuntu_preferences_task(self): - result = tasks_templates.make_ubuntu_preferences_task( - [1, 2, 3], - { - 'name': 'plugin_name', - 'type': 'deb', - 'uri': 'http://url', - 'suite': '/', - 'section': '', - 'priority': 1001 - }) - self.assertEqual( - result, - {'parameters': { - 'data': 'Package: *\nPin: release a=/\nPin-Priority: 1001', - 'path': '/etc/apt/preferences.d/plugin_name'}, - 'type': 'upload_file', - 'uids': [1, 2, 3]}) - - result = tasks_templates.make_ubuntu_preferences_task( - [1, 2, 3], - { - 'name': 'plugin_name', - 'type': 'deb', - 'uri': 'http://url', - 'suite': 'jessie', - 'section': 'main universe', - 'priority': 1004 - }) - self.assertEqual( - result, - {'parameters': { - 'data': ('Package: *\n' - 'Pin: release a=jessie,c=main\n' - 'Pin-Priority: 1004\n\n' - 'Package: *\n' - 'Pin: release a=jessie,c=universe\n' - 'Pin-Priority: 1004'), - 'path': '/etc/apt/preferences.d/plugin_name'}, - 'type': 'upload_file', - 'uids': [1, 2, 3]}) - def test_make_ubuntu_unauth_repos_task(self): result = tasks_templates.make_ubuntu_unauth_repos_task([1, 2, 3]) self.assertEqual( @@ -238,3 +202,110 @@ class TestMakeTask(base.BaseTestCase): 'interval': 1, 'cwd': '/', }}) + + +class TestMakeUbuntuPreferencesTask(base.BaseTestCase): + + _fake_debian_release = ''' + Origin: TestOrigin + Label: TestLabel + Archive: test-archive + Codename: testcodename + ''' + + _re_pin = re.compile('Pin: release (.*)') + + def _check_apt_preferences(self, data, sections, priority): + pins = data.split('\n\n') + + # in non-flat repo we have one pin per section + if sections: + self.assertEqual(len(pins), len(sections)) + + # we should have one pin per section + for pin, section in zip_longest(pins, sections): + conditions = self._re_pin.search(pin).group(1).split(',') + + # check general template + self.assertRegexpMatches( + data, ( + 'Package: \*\n' + 'Pin: release .*\n' + 'Pin-Priority: {0}'.format(priority) + )) + + # check pin + expected_conditions = [ + 'a=test-archive', + 'l=TestLabel', + 'n=testcodename', + 'o=TestOrigin', + ] + if section: + expected_conditions.append('c={0}'.format(section)) + self.assertItemsEqual(conditions, expected_conditions) + + @mock.patch('nailgun.utils.debian.requests.get', + return_value=mock.Mock(text=_fake_debian_release)) + def test_make_ubuntu_preferences_task(self, _): + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': 'test-archive', + 'section': 'main universe', + 'priority': 1004 + }) + + data = result['parameters'].pop('data') + self.assertEqual( + result, + {'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'}, + 'type': 'upload_file', + 'uids': [1, 2, 3]}) + + self._check_apt_preferences(data, ['main', 'universe'], 1004) + + @mock.patch('nailgun.utils.debian.requests.get', + return_value=mock.Mock(text=_fake_debian_release)) + def test_make_ubuntu_preferences_task_flat(self, _): + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': '/', + 'section': '', + 'priority': 1004 + }) + + data = result['parameters'].pop('data') + self.assertEqual( + result, + {'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'}, + 'type': 'upload_file', + 'uids': [1, 2, 3]}) + + self._check_apt_preferences(data, [], 1004) + + @mock.patch('nailgun.utils.debian.requests.get') + def test_make_ubuntu_preferences_task_returns_none_if_errors(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': 'test-archive', + 'section': 'main universe', + 'priority': 1004 + }) + + self.assertIsNone(result) diff --git a/nailgun/nailgun/test/unit/test_utils.py b/nailgun/nailgun/test/unit/test_utils.py index d8a0f85b37..9ef647414c 100644 --- a/nailgun/nailgun/test/unit/test_utils.py +++ b/nailgun/nailgun/test/unit/test_utils.py @@ -19,6 +19,8 @@ from mock import patch import os import tempfile +import requests + from nailgun.test import base from nailgun.utils import camel_to_snake_case from nailgun.utils import compact @@ -26,8 +28,13 @@ from nailgun.utils import dict_merge from nailgun.utils import extract_env_version from nailgun.utils import flatten from nailgun.utils import get_fuel_release_versions +from nailgun.utils import grouper from nailgun.utils import traverse +from nailgun.utils.debian import get_apt_preferences_line +from nailgun.utils.debian import get_release_file +from nailgun.utils.debian import parse_release_file + class TestUtils(base.BaseIntegrationTest): @@ -97,6 +104,19 @@ class TestUtils(base.BaseIntegrationTest): flatten([7, 5, [3, [4, 5], 1], 2]), [7, 5, 3, [4, 5], 1, 2]) + def test_grouper(self): + self.assertEqual( + list(grouper([0, 1, 2, 3], 2)), [(0, 1), (2, 3)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4, 5], 3)), [(0, 1, 2), (3, 4, 5)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4], 3)), [(0, 1, 2), (3, 4, None)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4], 3, 'x')), [(0, 1, 2), (3, 4, 'x')]) + class TestTraverse(base.BaseUnitTest): @@ -165,3 +185,197 @@ class TestTraverse(base.BaseUnitTest): 'y': 'b 42 b', } ]}) + + +class TestGetDebianReleaseFile(base.BaseUnitTest): + + @patch('nailgun.utils.debian.requests.get') + def test_normal_ubuntu_repo(self, m_get): + get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + m_get.assert_called_with( + 'http://some-uri.com/path/dists/mysuite/Release') + + @patch('nailgun.utils.debian.requests.get') + def test_flat_ubuntu_repo(self, m_get): + testcases = [ + # (suite, uri) + ('/', 'http://some-uri.com/deb/Release'), + ('/path', 'http://some-uri.com/deb/path/Release'), + ('path', 'http://some-uri.com/deb/path/Release'), + ] + + for suite, uri in testcases: + get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/deb', + 'suite': suite, + 'section': '', + }) + m_get.assert_called_with(uri) + + @patch('nailgun.utils.debian.requests.get') + def test_do_not_silence_http_errors(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + + @patch('nailgun.utils.debian.requests.get') + def test_do_not_retry_on_404(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }, retries=3) + self.assertEqual(m_get.call_count, 1) + + @patch('nailgun.utils.debian.requests.get') + def test_do_retry_on_error(self, m_get): + r = requests.Response() + r.status_code = 500 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }, retries=3) + self.assertEqual(m_get.call_count, 3) + + @patch('nailgun.utils.debian.requests.get') + def test_returns_content_if_http_ok(self, m_get): + r = requests.Response() + r._content = 'content' + r.status_code = 200 + m_get.return_value = r + + content = get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + self.assertEqual(content, 'content') + + +class TestParseDebianReleaseFile(base.BaseUnitTest): + + _deb_release_info = ''' + Origin: TestOrigin + Label: TestLabel + Archive: test-archive + Codename: testcodename + Date: Thu, 08 May 2014 14:19:09 UTC + Architectures: amd64 i386 + Components: main restricted universe multiverse + Description: Test Description + MD5Sum: + ead1cbf42ed119c50bf3aab28b5b6351 934 main/binary-amd64/Packages + 52d605b4217be64f461751f233dd9a8f 96 main/binary-amd64/Release + SHA1: + 28c4460e3aaf1b93f11911fdc4ff23c28809af89 934 main/binary-amd64/Packages + d03d716bceaba35f91726c096e2a9a8c23ccc766 96 main/binary-amd64/Release + ''' + + def test_parse(self): + deb_release = parse_release_file(self._deb_release_info) + + self.assertEqual(deb_release, { + 'Origin': 'TestOrigin', + 'Label': 'TestLabel', + 'Archive': 'test-archive', + 'Codename': 'testcodename', + 'Date': 'Thu, 08 May 2014 14:19:09 UTC', + 'Architectures': 'amd64 i386', + 'Components': 'main restricted universe multiverse', + 'Description': 'Test Description', + 'MD5Sum': [ + { + 'md5sum': 'ead1cbf42ed119c50bf3aab28b5b6351', + 'size': '934', + 'name': 'main/binary-amd64/Packages', + }, + { + 'md5sum': '52d605b4217be64f461751f233dd9a8f', + 'size': '96', + 'name': 'main/binary-amd64/Release', + } + ], + 'SHA1': [ + { + 'sha1': '28c4460e3aaf1b93f11911fdc4ff23c28809af89', + 'size': '934', + 'name': 'main/binary-amd64/Packages', + }, + { + 'sha1': 'd03d716bceaba35f91726c096e2a9a8c23ccc766', + 'size': '96', + 'name': 'main/binary-amd64/Release', + } + ], + }) + + +class TestGetAptPreferencesLine(base.BaseUnitTest): + + _deb_release = { + 'Origin': 'TestOrigin', + 'Label': 'TestLabel', + 'Archive': 'test-archive', + 'Version': '42.42', + 'Codename': 'testcodename', + 'Date': 'Thu, 08 May 2014 14:19:09 UTC', + 'Architectures': 'amd64 i386', + 'Components': 'main restricted universe multiverse', + 'Description': 'Test Description', + } + + def test_all_rules(self): + pin = get_apt_preferences_line(self._deb_release) + self.assertItemsEqual(pin.split(','), [ + 'o=TestOrigin', + 'l=TestLabel', + 'a=test-archive', + 'v=42.42', + 'n=testcodename', + ]) + + def test_not_all_rules(self): + deb_release = self._deb_release.copy() + + del deb_release['Codename'] + del deb_release['Label'] + del deb_release['Version'] + + pin = get_apt_preferences_line(deb_release) + self.assertItemsEqual(pin.split(','), [ + 'o=TestOrigin', + 'a=test-archive', + ]) + + def test_suite_is_synonym_for_archive(self): + deb_release = self._deb_release.copy() + deb_release['Suite'] = 'test-suite' + del deb_release['Archive'] + + pin = get_apt_preferences_line(deb_release) + conditions = pin.split(',') + + self.assertIn('a=test-suite', conditions) diff --git a/nailgun/nailgun/utils/__init__.py b/nailgun/nailgun/utils/__init__.py index 81c042fd35..c223fad466 100644 --- a/nailgun/nailgun/utils/__init__.py +++ b/nailgun/nailgun/utils/__init__.py @@ -24,6 +24,8 @@ from copy import deepcopy from itertools import chain from random import choice +from six.moves import zip_longest + from nailgun.logger import logger from nailgun.settings import settings @@ -191,3 +193,10 @@ def flatten(array): check = lambda x: x if isinstance(x, list) else [x] return list(chain.from_iterable(check(x) for x in array)) + + +def grouper(iterable, n, fillvalue=None): + """Collect data into fixed-length chunks or blocks + """ + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) diff --git a/nailgun/nailgun/utils/debian.py b/nailgun/nailgun/utils/debian.py new file mode 100644 index 0000000000..375a991660 --- /dev/null +++ b/nailgun/nailgun/utils/debian.py @@ -0,0 +1,110 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import requests +import six +import yaml + +from nailgun.utils import grouper + + +def get_release_file(repo, retries=1): + """Get Release content of a given repo. + + :param repo: a repo as dict + :returns: a release's content as string + """ + if repo['section']: + # We can't use urljoin here because it works pretty bad in + # cases when 'uri' doesn't have a trailing slash. + download_uri = os.path.join( + repo['uri'], 'dists', repo['suite'], 'Release') + else: + # Well, we have a flat repo case, so we should download Release + # file from a different place. Please note, we have to strip + # a leading slash from suite because otherwise the download + # link will be wrong. + download_uri = os.path.join( + repo['uri'], repo['suite'].lstrip('/'), 'Release') + + for _ in six.moves.range(0, retries): + response = requests.get(download_uri) + + # do not perform retries if release is not found + if response.status_code == 404: + break + + response.raise_for_status() + return response.text + + +def parse_release_file(content): + """Parse Debian repo's Release file content. + + :param content: a Debian's Release file content + :returns: a dict with repo's attributes + """ + + # TODO(ikalnitsky): Consider to use some existing library for + # parsing debian's release file (e.g. python-debian). + + _multivalued_fields = { + 'SHA1': ['sha1', 'size', 'name'], + 'SHA256': ['sha256', 'size', 'name'], + 'SHA512': ['sha512', 'size', 'name'], + 'MD5Sum': ['md5sum', 'size', 'name'], + } + + # debian data format is very similiar to yaml, except + # multivalued field. so we can parse it just like yaml + # and then perform additional transformation for those + # fields (we know which ones are multivalues). + data = yaml.load(content) + + for attr, columns in six.iteritems(_multivalued_fields): + if attr not in data: + continue + + values = data[attr].split() + data[attr] = [] + + for group in grouper(values, len(columns)): + data[attr].append(dict(zip(columns, group))) + + return data + + +def get_apt_preferences_line(deb_release): + """Get an APT Preferences line from repo's release information. + + :param deb_release: a Debian's Release content as dict + :returns: an apt pinning line as string + """ + _transformations = { + 'Archive': 'a', + 'Suite': 'a', # suite is a synonym for archive + 'Codename': 'n', + 'Version': 'v', + 'Origin': 'o', + 'Label': 'l', + } + + conditions = set() + for field, condition in six.iteritems(_transformations): + if field in deb_release: + conditions.add('{0}={1}'.format(condition, deb_release[field])) + + return ','.join(conditions)