diff --git a/nailgun/nailgun/orchestrator/plugins_serializers.py b/nailgun/nailgun/orchestrator/plugins_serializers.py index f8d483710d..6b67d15a52 100644 --- a/nailgun/nailgun/orchestrator/plugins_serializers.py +++ b/nailgun/nailgun/orchestrator/plugins_serializers.py @@ -151,13 +151,17 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer): elif operating_system == consts.RELEASE_OS.ubuntu: repo = self.get_ubuntu_repo(plugin) - repo_tasks.extend([ + repo_tasks.append( self.serialize_task( plugin, - templates.make_ubuntu_sources_task(uids, repo)), - self.serialize_task( - plugin, - templates.make_ubuntu_preferences_task(uids, repo))]) + templates.make_ubuntu_sources_task(uids, repo))) + + # do not add preferences task to task list if we can't + # complete it (e.g. can't retrieve or parse Release file) + task = self.serialize_task( + plugin, templates.make_ubuntu_preferences_task(uids, repo)) + if task is not None: + repo_tasks.append(task) # apt-get update executed after every additional source.list # to be able understand what plugin source.list caused error diff --git a/nailgun/nailgun/orchestrator/tasks_serializer.py b/nailgun/nailgun/orchestrator/tasks_serializer.py index acf33fa5ce..daa6e20e05 100644 --- a/nailgun/nailgun/orchestrator/tasks_serializer.py +++ b/nailgun/nailgun/orchestrator/tasks_serializer.py @@ -186,7 +186,11 @@ class UploadMOSRepo(GenericRolesHook): yield templates.make_ubuntu_sources_task(uids, repo) if repo.get('priority'): - yield templates.make_ubuntu_preferences_task(uids, repo) + # do not add preferences task to task list if we can't + # complete it (e.g. can't retrieve or parse Release file) + task = templates.make_ubuntu_preferences_task(uids, repo) + if task is not None: + yield task yield templates.make_apt_update_task(uids) diff --git a/nailgun/nailgun/orchestrator/tasks_templates.py b/nailgun/nailgun/orchestrator/tasks_templates.py index 199398e43e..c313f66a49 100644 --- a/nailgun/nailgun/orchestrator/tasks_templates.py +++ b/nailgun/nailgun/orchestrator/tasks_templates.py @@ -17,8 +17,12 @@ import os from oslo.serialization import jsonutils +import requests +import six +from nailgun.logger import logger from nailgun.settings import settings +from nailgun.utils import debian def make_upload_task(uids, data, path): @@ -38,34 +42,48 @@ def make_ubuntu_sources_task(uids, repo): def make_ubuntu_preferences_task(uids, repo): - # TODO(ikalnitsky): - # Research how to add host condition to the current pinning. + # NOTE(ikalnitsky): In order to implement the proper pinning, + # we have to download and parse the repo's "Release" file. + # Generally, that's not a good idea to make some HTTP request + # from Nailgun, but taking into account that this task + # will be executed in uWSGI's mule worker we can skip this + # rule, because proper pinning is more valuable thing right now. template = '\n'.join([ 'Package: *', - 'Pin: release a={suite},c={section}', - 'Pin-Priority: {priority}', ]) - - template_flat = '\n'.join([ - 'Package: *', - 'Pin: release a={suite}', - 'Pin-Priority: {priority}', ]) - + 'Pin: release {conditions}', + 'Pin-Priority: {priority}']) preferences_content = [] + + try: + release = debian.get_release_file(repo, retries=3) + release = debian.parse_release_file(release) + pin = debian.get_apt_preferences_line(release) + + except requests.exceptions.HTTPError as exc: + logger.error("Failed to fetch 'Release' file due to '%s'. " + "The apt preferences won't be applied for repo '%s'.", + six.text_type(exc), repo['name']) + return None + + except Exception: + logger.exception("Failed to parse 'Release' file.") + return None + + # if sections are detected (non-flat repo) create pinning per + # sections; otherwise - create just one pin for the entire repo if repo['section']: for section in repo['section'].split(): preferences_content.append(template.format( - suite=repo['suite'], - section=section, + conditions='{0},c={1}'.format(pin, section), priority=repo['priority'])) else: - preferences_content.append(template_flat.format( - suite=repo['suite'], + preferences_content.append(template.format( + conditions=pin, priority=repo['priority'])) preferences_content = '\n\n'.join(preferences_content) - preferences_path = os.path.join('/etc/apt/preferences.d', repo['name']) - + preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name']) return make_upload_task(uids, preferences_content, preferences_path) diff --git a/nailgun/nailgun/test/integration/test_plugins_api.py b/nailgun/nailgun/test/integration/test_plugins_api.py index 14c0e1a24e..7c4d14e2a5 100644 --- a/nailgun/nailgun/test/integration/test_plugins_api.py +++ b/nailgun/nailgun/test/integration/test_plugins_api.py @@ -224,6 +224,12 @@ class TestPrePostHooks(BasePluginTest): def setUp(self): super(TestPrePostHooks, self).setUp() + + self._requests_mock = mock.patch( + 'nailgun.utils.debian.requests.get', + return_value=mock.Mock(text='Archive: test')) + self._requests_mock.start() + resp = self.create_plugin() self.plugin = attr_plugin.wrap_plugin( objects.Plugin.get_by_uid(resp.json['id'])) @@ -232,6 +238,10 @@ class TestPrePostHooks(BasePluginTest): {'roles': ['compute'], 'pending_addition': True}]) self.enable_plugin(self.cluster, self.sample_plugin['name']) + def tearDown(self): + self._requests_mock.stop() + super(TestPrePostHooks, self).tearDown() + def test_generate_pre_hooks(self): tasks = self.get_pre_hooks(self.cluster).json plugins_tasks = [t for t in tasks if t.get('diagnostic_name')] diff --git a/nailgun/nailgun/test/unit/test_stages_task_serialization.py b/nailgun/nailgun/test/unit/test_stages_task_serialization.py index e1c0c12495..74f80165a3 100644 --- a/nailgun/nailgun/test/unit/test_stages_task_serialization.py +++ b/nailgun/nailgun/test/unit/test_stages_task_serialization.py @@ -55,6 +55,12 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase): def setUp(self): super(BaseTaskSerializationTestUbuntu, self).setUp() + + self._requests_mock = mock.patch( + 'nailgun.utils.debian.requests.get', + return_value=mock.Mock(text='Archive: test')) + self._requests_mock.start() + self.release = self.env.create_release( api=False, attributes_metadata=self.env.read_fixtures( ['openstack'])[1]['fields']['attributes_metadata']) @@ -70,6 +76,10 @@ class BaseTaskSerializationTestUbuntu(base.BaseTestCase): self.all_uids = [n.uid for n in self.nodes] self.cluster.deployment_tasks = yaml.load(self.TASKS) + def tearDown(self): + self._requests_mock.stop() + super(BaseTaskSerializationTestUbuntu, self).tearDown() + class TestHooksSerializersUbuntu(BaseTaskSerializationTestUbuntu): def test_create_repo_ubuntu(self): diff --git a/nailgun/nailgun/test/unit/test_tasks_templates.py b/nailgun/nailgun/test/unit/test_tasks_templates.py index 9e485c700c..d276fdf5a8 100644 --- a/nailgun/nailgun/test/unit/test_tasks_templates.py +++ b/nailgun/nailgun/test/unit/test_tasks_templates.py @@ -14,7 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +import re + +import mock +import requests + from oslo.serialization import jsonutils +from six.moves import zip_longest from nailgun.test import base @@ -44,48 +50,6 @@ class TestMakeTask(base.BaseTestCase): 'type': 'upload_file', 'uids': [1, 2, 3]}) - def test_make_ubuntu_preferences_task(self): - result = tasks_templates.make_ubuntu_preferences_task( - [1, 2, 3], - { - 'name': 'plugin_name', - 'type': 'deb', - 'uri': 'http://url', - 'suite': '/', - 'section': '', - 'priority': 1001 - }) - self.assertEqual( - result, - {'parameters': { - 'data': 'Package: *\nPin: release a=/\nPin-Priority: 1001', - 'path': '/etc/apt/preferences.d/plugin_name'}, - 'type': 'upload_file', - 'uids': [1, 2, 3]}) - - result = tasks_templates.make_ubuntu_preferences_task( - [1, 2, 3], - { - 'name': 'plugin_name', - 'type': 'deb', - 'uri': 'http://url', - 'suite': 'jessie', - 'section': 'main universe', - 'priority': 1004 - }) - self.assertEqual( - result, - {'parameters': { - 'data': ('Package: *\n' - 'Pin: release a=jessie,c=main\n' - 'Pin-Priority: 1004\n\n' - 'Package: *\n' - 'Pin: release a=jessie,c=universe\n' - 'Pin-Priority: 1004'), - 'path': '/etc/apt/preferences.d/plugin_name'}, - 'type': 'upload_file', - 'uids': [1, 2, 3]}) - def test_make_ubuntu_unauth_repos_task(self): result = tasks_templates.make_ubuntu_unauth_repos_task([1, 2, 3]) self.assertEqual( @@ -238,3 +202,110 @@ class TestMakeTask(base.BaseTestCase): 'interval': 1, 'cwd': '/', }}) + + +class TestMakeUbuntuPreferencesTask(base.BaseTestCase): + + _fake_debian_release = ''' + Origin: TestOrigin + Label: TestLabel + Archive: test-archive + Codename: testcodename + ''' + + _re_pin = re.compile('Pin: release (.*)') + + def _check_apt_preferences(self, data, sections, priority): + pins = data.split('\n\n') + + # in non-flat repo we have one pin per section + if sections: + self.assertEqual(len(pins), len(sections)) + + # we should have one pin per section + for pin, section in zip_longest(pins, sections): + conditions = self._re_pin.search(pin).group(1).split(',') + + # check general template + self.assertRegexpMatches( + data, ( + 'Package: \*\n' + 'Pin: release .*\n' + 'Pin-Priority: {0}'.format(priority) + )) + + # check pin + expected_conditions = [ + 'a=test-archive', + 'l=TestLabel', + 'n=testcodename', + 'o=TestOrigin', + ] + if section: + expected_conditions.append('c={0}'.format(section)) + self.assertItemsEqual(conditions, expected_conditions) + + @mock.patch('nailgun.utils.debian.requests.get', + return_value=mock.Mock(text=_fake_debian_release)) + def test_make_ubuntu_preferences_task(self, _): + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': 'test-archive', + 'section': 'main universe', + 'priority': 1004 + }) + + data = result['parameters'].pop('data') + self.assertEqual( + result, + {'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'}, + 'type': 'upload_file', + 'uids': [1, 2, 3]}) + + self._check_apt_preferences(data, ['main', 'universe'], 1004) + + @mock.patch('nailgun.utils.debian.requests.get', + return_value=mock.Mock(text=_fake_debian_release)) + def test_make_ubuntu_preferences_task_flat(self, _): + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': '/', + 'section': '', + 'priority': 1004 + }) + + data = result['parameters'].pop('data') + self.assertEqual( + result, + {'parameters': {'path': '/etc/apt/preferences.d/plugin_name.pref'}, + 'type': 'upload_file', + 'uids': [1, 2, 3]}) + + self._check_apt_preferences(data, [], 1004) + + @mock.patch('nailgun.utils.debian.requests.get') + def test_make_ubuntu_preferences_task_returns_none_if_errors(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + result = tasks_templates.make_ubuntu_preferences_task( + [1, 2, 3], + { + 'name': 'plugin_name', + 'type': 'deb', + 'uri': 'http://url', + 'suite': 'test-archive', + 'section': 'main universe', + 'priority': 1004 + }) + + self.assertIsNone(result) diff --git a/nailgun/nailgun/test/unit/test_utils.py b/nailgun/nailgun/test/unit/test_utils.py index d8a0f85b37..9ef647414c 100644 --- a/nailgun/nailgun/test/unit/test_utils.py +++ b/nailgun/nailgun/test/unit/test_utils.py @@ -19,6 +19,8 @@ from mock import patch import os import tempfile +import requests + from nailgun.test import base from nailgun.utils import camel_to_snake_case from nailgun.utils import compact @@ -26,8 +28,13 @@ from nailgun.utils import dict_merge from nailgun.utils import extract_env_version from nailgun.utils import flatten from nailgun.utils import get_fuel_release_versions +from nailgun.utils import grouper from nailgun.utils import traverse +from nailgun.utils.debian import get_apt_preferences_line +from nailgun.utils.debian import get_release_file +from nailgun.utils.debian import parse_release_file + class TestUtils(base.BaseIntegrationTest): @@ -97,6 +104,19 @@ class TestUtils(base.BaseIntegrationTest): flatten([7, 5, [3, [4, 5], 1], 2]), [7, 5, 3, [4, 5], 1, 2]) + def test_grouper(self): + self.assertEqual( + list(grouper([0, 1, 2, 3], 2)), [(0, 1), (2, 3)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4, 5], 3)), [(0, 1, 2), (3, 4, 5)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4], 3)), [(0, 1, 2), (3, 4, None)]) + + self.assertEqual( + list(grouper([0, 1, 2, 3, 4], 3, 'x')), [(0, 1, 2), (3, 4, 'x')]) + class TestTraverse(base.BaseUnitTest): @@ -165,3 +185,197 @@ class TestTraverse(base.BaseUnitTest): 'y': 'b 42 b', } ]}) + + +class TestGetDebianReleaseFile(base.BaseUnitTest): + + @patch('nailgun.utils.debian.requests.get') + def test_normal_ubuntu_repo(self, m_get): + get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + m_get.assert_called_with( + 'http://some-uri.com/path/dists/mysuite/Release') + + @patch('nailgun.utils.debian.requests.get') + def test_flat_ubuntu_repo(self, m_get): + testcases = [ + # (suite, uri) + ('/', 'http://some-uri.com/deb/Release'), + ('/path', 'http://some-uri.com/deb/path/Release'), + ('path', 'http://some-uri.com/deb/path/Release'), + ] + + for suite, uri in testcases: + get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/deb', + 'suite': suite, + 'section': '', + }) + m_get.assert_called_with(uri) + + @patch('nailgun.utils.debian.requests.get') + def test_do_not_silence_http_errors(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + + @patch('nailgun.utils.debian.requests.get') + def test_do_not_retry_on_404(self, m_get): + r = requests.Response() + r.status_code = 404 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }, retries=3) + self.assertEqual(m_get.call_count, 1) + + @patch('nailgun.utils.debian.requests.get') + def test_do_retry_on_error(self, m_get): + r = requests.Response() + r.status_code = 500 + m_get.return_value = r + + self.assertRaises(requests.exceptions.HTTPError, get_release_file, { + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }, retries=3) + self.assertEqual(m_get.call_count, 3) + + @patch('nailgun.utils.debian.requests.get') + def test_returns_content_if_http_ok(self, m_get): + r = requests.Response() + r._content = 'content' + r.status_code = 200 + m_get.return_value = r + + content = get_release_file({ + 'name': 'myrepo', + 'uri': 'http://some-uri.com/path', + 'suite': 'mysuite', + 'section': 'main university', + }) + self.assertEqual(content, 'content') + + +class TestParseDebianReleaseFile(base.BaseUnitTest): + + _deb_release_info = ''' + Origin: TestOrigin + Label: TestLabel + Archive: test-archive + Codename: testcodename + Date: Thu, 08 May 2014 14:19:09 UTC + Architectures: amd64 i386 + Components: main restricted universe multiverse + Description: Test Description + MD5Sum: + ead1cbf42ed119c50bf3aab28b5b6351 934 main/binary-amd64/Packages + 52d605b4217be64f461751f233dd9a8f 96 main/binary-amd64/Release + SHA1: + 28c4460e3aaf1b93f11911fdc4ff23c28809af89 934 main/binary-amd64/Packages + d03d716bceaba35f91726c096e2a9a8c23ccc766 96 main/binary-amd64/Release + ''' + + def test_parse(self): + deb_release = parse_release_file(self._deb_release_info) + + self.assertEqual(deb_release, { + 'Origin': 'TestOrigin', + 'Label': 'TestLabel', + 'Archive': 'test-archive', + 'Codename': 'testcodename', + 'Date': 'Thu, 08 May 2014 14:19:09 UTC', + 'Architectures': 'amd64 i386', + 'Components': 'main restricted universe multiverse', + 'Description': 'Test Description', + 'MD5Sum': [ + { + 'md5sum': 'ead1cbf42ed119c50bf3aab28b5b6351', + 'size': '934', + 'name': 'main/binary-amd64/Packages', + }, + { + 'md5sum': '52d605b4217be64f461751f233dd9a8f', + 'size': '96', + 'name': 'main/binary-amd64/Release', + } + ], + 'SHA1': [ + { + 'sha1': '28c4460e3aaf1b93f11911fdc4ff23c28809af89', + 'size': '934', + 'name': 'main/binary-amd64/Packages', + }, + { + 'sha1': 'd03d716bceaba35f91726c096e2a9a8c23ccc766', + 'size': '96', + 'name': 'main/binary-amd64/Release', + } + ], + }) + + +class TestGetAptPreferencesLine(base.BaseUnitTest): + + _deb_release = { + 'Origin': 'TestOrigin', + 'Label': 'TestLabel', + 'Archive': 'test-archive', + 'Version': '42.42', + 'Codename': 'testcodename', + 'Date': 'Thu, 08 May 2014 14:19:09 UTC', + 'Architectures': 'amd64 i386', + 'Components': 'main restricted universe multiverse', + 'Description': 'Test Description', + } + + def test_all_rules(self): + pin = get_apt_preferences_line(self._deb_release) + self.assertItemsEqual(pin.split(','), [ + 'o=TestOrigin', + 'l=TestLabel', + 'a=test-archive', + 'v=42.42', + 'n=testcodename', + ]) + + def test_not_all_rules(self): + deb_release = self._deb_release.copy() + + del deb_release['Codename'] + del deb_release['Label'] + del deb_release['Version'] + + pin = get_apt_preferences_line(deb_release) + self.assertItemsEqual(pin.split(','), [ + 'o=TestOrigin', + 'a=test-archive', + ]) + + def test_suite_is_synonym_for_archive(self): + deb_release = self._deb_release.copy() + deb_release['Suite'] = 'test-suite' + del deb_release['Archive'] + + pin = get_apt_preferences_line(deb_release) + conditions = pin.split(',') + + self.assertIn('a=test-suite', conditions) diff --git a/nailgun/nailgun/utils/__init__.py b/nailgun/nailgun/utils/__init__.py index 81c042fd35..c223fad466 100644 --- a/nailgun/nailgun/utils/__init__.py +++ b/nailgun/nailgun/utils/__init__.py @@ -24,6 +24,8 @@ from copy import deepcopy from itertools import chain from random import choice +from six.moves import zip_longest + from nailgun.logger import logger from nailgun.settings import settings @@ -191,3 +193,10 @@ def flatten(array): check = lambda x: x if isinstance(x, list) else [x] return list(chain.from_iterable(check(x) for x in array)) + + +def grouper(iterable, n, fillvalue=None): + """Collect data into fixed-length chunks or blocks + """ + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) diff --git a/nailgun/nailgun/utils/debian.py b/nailgun/nailgun/utils/debian.py new file mode 100644 index 0000000000..375a991660 --- /dev/null +++ b/nailgun/nailgun/utils/debian.py @@ -0,0 +1,110 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import requests +import six +import yaml + +from nailgun.utils import grouper + + +def get_release_file(repo, retries=1): + """Get Release content of a given repo. + + :param repo: a repo as dict + :returns: a release's content as string + """ + if repo['section']: + # We can't use urljoin here because it works pretty bad in + # cases when 'uri' doesn't have a trailing slash. + download_uri = os.path.join( + repo['uri'], 'dists', repo['suite'], 'Release') + else: + # Well, we have a flat repo case, so we should download Release + # file from a different place. Please note, we have to strip + # a leading slash from suite because otherwise the download + # link will be wrong. + download_uri = os.path.join( + repo['uri'], repo['suite'].lstrip('/'), 'Release') + + for _ in six.moves.range(0, retries): + response = requests.get(download_uri) + + # do not perform retries if release is not found + if response.status_code == 404: + break + + response.raise_for_status() + return response.text + + +def parse_release_file(content): + """Parse Debian repo's Release file content. + + :param content: a Debian's Release file content + :returns: a dict with repo's attributes + """ + + # TODO(ikalnitsky): Consider to use some existing library for + # parsing debian's release file (e.g. python-debian). + + _multivalued_fields = { + 'SHA1': ['sha1', 'size', 'name'], + 'SHA256': ['sha256', 'size', 'name'], + 'SHA512': ['sha512', 'size', 'name'], + 'MD5Sum': ['md5sum', 'size', 'name'], + } + + # debian data format is very similiar to yaml, except + # multivalued field. so we can parse it just like yaml + # and then perform additional transformation for those + # fields (we know which ones are multivalues). + data = yaml.load(content) + + for attr, columns in six.iteritems(_multivalued_fields): + if attr not in data: + continue + + values = data[attr].split() + data[attr] = [] + + for group in grouper(values, len(columns)): + data[attr].append(dict(zip(columns, group))) + + return data + + +def get_apt_preferences_line(deb_release): + """Get an APT Preferences line from repo's release information. + + :param deb_release: a Debian's Release content as dict + :returns: an apt pinning line as string + """ + _transformations = { + 'Archive': 'a', + 'Suite': 'a', # suite is a synonym for archive + 'Codename': 'n', + 'Version': 'v', + 'Origin': 'o', + 'Label': 'l', + } + + conditions = set() + for field, condition in six.iteritems(_transformations): + if field in deb_release: + conditions.add('{0}={1}'.format(condition, deb_release[field])) + + return ','.join(conditions)