From b71d1c60d2026444707f6247c47a6fb5e58247d9 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 26 May 2020 11:18:29 +1000 Subject: [PATCH] package-installs : allow a list of parameters The change Ia6f10741fa6be24b11d6991c8a6b6e07951ff68d introduced having "when:" as a list of values. However, this was actually not sufficient to express the logic required for arm64/x86_64/xenial kernel matching we wanted. Because the package name is a key, we can't have multiple entires in the package-map YAML files. This means we can't do more advanced matching and thus we need to be able to match through multiple parameters. Similar to Ia6f10741fa6be24b11d6991c8a6b6e07951ff68d we modify the matching rules to allow a list. A an example of using this is provided in the README.rst, and this same example worked through by the unit tests. This also slightly updates the matching logic to be more sequential. After each check we either continue on or log the failure and continue to the next check (rather than set a list of flags then check that at the end). This makes it much easier to understand what is being matched in the logging output from the tool. Change-Id: Idff7b067ad4255e6fc4138f7eff313a81b75c8ba --- .../elements/package-installs/README.rst | 23 ++++++ .../bin/package-installs-squash | 70 +++++++++++-------- .../tests/test_package_squash.py | 49 +++++++++++++ 3 files changed, 114 insertions(+), 28 deletions(-) diff --git a/diskimage_builder/elements/package-installs/README.rst b/diskimage_builder/elements/package-installs/README.rst index c77922868..0da2b7089 100644 --- a/diskimage_builder/elements/package-installs/README.rst +++ b/diskimage_builder/elements/package-installs/README.rst @@ -96,6 +96,29 @@ packages), you can use something like:: You can also use a list of items in the ``when`` statement, which will be effectively combined with *and*. +If you need to filter multiple paths for a single package, you can +make the parameters a list. For example, if ``linux-image-generic`` +package should be installed when ``DIB_UBUNTU_KERNEL = +linux-image-generic`` is set *except* on ``arm64`` Xenial hosts, where +we would like to install ``linux-generic-hwe-16.04`` you could use the +following: + +.. code-block:: YAML + + linux-image-generic: + - not-arch: arm64 + when: DIB_UBUNTU_KERNEL = linux-image-generic + - arch: arm64 + when: + - DIB_RELEASE != xenial + - DIB_UBUNTU_KERNEL = linux-image-generic + + linux-generic-hwe-16.04: + arch: arm64 + when: + - DIB_RELEASE = xenial + - DIB_UBUNTU_KERNEL = linux-image-generic + DEPRECATED: Adding a file under your elements pre-install.d, install.d, or post-install.d directories called package-installs- will cause the list of packages in that file to be installed at the beginning of the diff --git a/diskimage_builder/elements/package-installs/bin/package-installs-squash b/diskimage_builder/elements/package-installs/bin/package-installs-squash index c59b35634..44f26c9bc 100755 --- a/diskimage_builder/elements/package-installs/bin/package-installs-squash +++ b/diskimage_builder/elements/package-installs/bin/package-installs-squash @@ -76,10 +76,8 @@ def _when(statements): # No statement means install if statements is None: return True - - if not isinstance(statements, list): + if not isinstance(statements, (list, tuple)): statements = [statements] - result = [] for s in statements: @@ -98,7 +96,7 @@ def _when(statements): if var not in os.environ: raise RuntimeError("The variable <%s> is not set" % var) - logger.debug("when eval %s%s%s against <%s>" % + logger.debug("... when eval %s%s%s against <%s>" % (var, op, val, os.environ[var])) if op == '=': @@ -121,34 +119,50 @@ def _when(statements): def collect_data(data, objs, element_name): for pkg_name, params in objs.items(): if not params: - params = {} - phase = params.get('phase', 'install.d') - installs = ["install"] - if 'uninstall' in params: - installs = ["uninstall"] - if 'build-only' in params: - installs = ["install", "uninstall"] + params = [{}] + if not isinstance(params, (list, tuple)): + params = [params] - # Filter out incorrect installtypes - installtype = params.get('installtype', None) - elem_installtype = get_element_installtype(element_name) - valid_installtype = (installtype is None or - installtype == elem_installtype) - valid_arch = _valid_for_arch(pkg_name, params.get('arch', None), - params.get('not-arch', None)) - dib_py_version = str(params.get('dib_python_version', '')) - dib_py_version_env = os.environ.get('DIB_PYTHON_VERSION', '') - valid_dib_python_version = (dib_py_version == '' or - dib_py_version == dib_py_version_env) + for param in params: + logger.debug("Considering %s/%s param:%s" % + (element_name, pkg_name, param)) + phase = param.get('phase', 'install.d') + installs = ["install"] + if 'uninstall' in param: + installs = ["uninstall"] + if 'build-only' in param: + installs = ["install", "uninstall"] - # True means install, false skip - if _when(params.get('when', None)) is False: - logger.debug("Skipped due to when: %s/%s" % - (element_name, pkg_name)) - continue + # Filter out incorrect installtypes + installtype = param.get('installtype', None) + elem_installtype = get_element_installtype(element_name) + valid_installtype = (installtype is None or + installtype == elem_installtype) + if not valid_installtype: + logger.debug("... skipping due to installtype") + continue + + valid_arch = _valid_for_arch(pkg_name, param.get('arch', None), + param.get('not-arch', None)) + if not valid_arch: + logger.debug("... skipping due to arch match") + continue + + dib_py_version = str(param.get('dib_python_version', '')) + dib_py_version_env = os.environ.get('DIB_PYTHON_VERSION', '') + valid_dib_python_version = (dib_py_version == '' or + dib_py_version == dib_py_version_env) + if not valid_dib_python_version: + logger.debug("... skipping due to python version") + continue + + # True means install, false skip + if _when(param.get('when', None)) is False: + logger.debug("... skipped due to when: failures") + continue - if valid_installtype and valid_arch and valid_dib_python_version: for install in installs: + logger.debug("... installing for '%s'" % install) data[phase][install].append((pkg_name, element_name)) return data diff --git a/diskimage_builder/elements/package-installs/tests/test_package_squash.py b/diskimage_builder/elements/package-installs/tests/test_package_squash.py index 5bebb5261..fa0915c71 100644 --- a/diskimage_builder/elements/package-installs/tests/test_package_squash.py +++ b/diskimage_builder/elements/package-installs/tests/test_package_squash.py @@ -98,6 +98,55 @@ class TestPackageInstall(base.BaseTestCase): self.assertThat(result, IsMatchingInstallList(expected)) + kernel_objs = { + 'linux-image-generic': [ + { + 'not-arch': 'arm64', + 'when': 'DIB_UBUNTU_KERNEL = linux-image-generic', + }, + { + 'arch': 'arm64', + 'when': ( + 'DIB_RELEASE != xenial', + 'DIB_UBUNTU_KERNEL = linux-image-generic', + ) + }, + ], + 'linux-generic-hwe-16.04': { + 'arch': 'arm64', + 'when': ( + 'DIB_RELEASE = xenial', + 'DIB_UBUNTU_KERNEL = linux-image-generic', + ) + }, + } + + def _test_kernel_objs_match(self, arch, release, expected): + with mock.patch.object(os, 'environ', + dict(ARCH=arch, + DIB_UBUNTU_KERNEL='linux-image-generic', + DIB_RELEASE=release, + **os.environ)): + result = installs_squash.collect_data( + self.final_dict, self.kernel_objs, 'test_element') + + expected = { + 'install.d': { + 'install': [(expected, 'test_element')] + } + } + self.assertThat(result, IsMatchingInstallList(expected)) + + def test_param_list_x86(self): + self._test_kernel_objs_match('x86_64', 'focal', 'linux-image-generic') + + def test_param_list_arm64_xenial(self): + self._test_kernel_objs_match('arm64', 'xenial', + 'linux-generic-hwe-16.04') + + def test_param_list_arm64_focal(self): + self._test_kernel_objs_match('arm64', 'focal', 'linux-image-generic') + @mock.patch.object(os, 'environ', dict(DIB_FEATURE='1', **os.environ)) def test_skip_when(self): '''Exercise the when flag'''