diff --git a/doc/source/admin/hardware_managers.rst b/doc/source/admin/hardware_managers.rst index 6649a47ba..579e6787c 100644 --- a/doc/source/admin/hardware_managers.rst +++ b/doc/source/admin/hardware_managers.rst @@ -25,6 +25,52 @@ Deploy steps and must be used through the :ironic-doc:`ironic RAID feature `. +Injecting files +~~~~~~~~~~~~~~~ + +``deploy.inject_files(node, ports, files, verify_ca=True)`` + +This optional deploy step (introduced in the Wallaby release series) allows +injecting arbitrary files into the node. The list of files is built from the +optional ``inject_files`` property of the node concatenated with the explicit +``files`` argument. Each item in the list is a dictionary with the following +fields: + +``path`` (required) + An absolute path to the file on the target partition. All missing + directories will be created. +``partition`` + Specifies the target partition in one of 3 ways: + + * A number is treated as a partition index (starting with 1) on the root + device. + * A path is treated as a block device path (e.g. ``/dev/sda1`` or + ``/dev/disk/by-partlabel/``. + * If missing, the agent will try to find a partition containing the first + component of the ``path`` on the root device. E.g. for + ``/etc/sysctl.d/my.conf``, look for a partition containing ``/etc``. +``deleted`` + If ``True``, the file is deleted, not created. + Incompatible with ``content``. +``content`` + Data to write. Incompatible with ``deleted``. Can take two forms: + + * A URL of the content. Can use Python-style formatting to build a node + specific URL, e.g. ``http://server/{node[uuid]}/{ports[0][address]}``. + * Base64 encoded binary contents. +``mode``, ``owner``, ``group`` + Numeric mode, owner ID and group ID of the file. +``dirmode`` + Numeric mode of the leaf directory if it has to be created. + +This deploy step is disabled by default and can be enabled via a deploy +template or via the ``ipa-inject-files-priority`` kernel parameter. + +Known limitations: + +* Names are not supported for ``owner`` and ``group``. +* LVM is not supported. + Clean steps ----------- diff --git a/ironic_python_agent/config.py b/ironic_python_agent/config.py index 27ebfaf83..8cfe0d58c 100644 --- a/ironic_python_agent/config.py +++ b/ironic_python_agent/config.py @@ -316,6 +316,11 @@ cli_opts = [ 'used if you are absolutely sure of what you are ' 'doing and that your hardware supports ' 'such functionality. Hint: Most hardware does not.'), + cfg.IntOpt('inject_files_priority', + default=APARAMS.get('ipa-inject-files-priority', 0), + min=0, max=99, # 100 is when IPA is booted + help='Priority of the inject_files deploy step (disabled ' + 'by default), an integer between 1 and .'), ] CONF.register_cli_opts(cli_opts) diff --git a/ironic_python_agent/hardware.py b/ironic_python_agent/hardware.py index 8d50e354e..129e58103 100644 --- a/ironic_python_agent/hardware.py +++ b/ironic_python_agent/hardware.py @@ -39,6 +39,7 @@ import yaml from ironic_python_agent import encoding from ironic_python_agent import errors from ironic_python_agent.extensions import base as ext_base +from ironic_python_agent import inject_files from ironic_python_agent import netutils from ironic_python_agent import raid_utils from ironic_python_agent import tls_utils @@ -1794,6 +1795,13 @@ class GenericHardwareManager(HardwareManager): 'interface': 'deploy', 'reboot_requested': False, }, + { + 'step': 'inject_files', + 'priority': CONF.inject_files_priority, + 'interface': 'deploy', + 'reboot_requested': False, + 'argsinfo': inject_files.ARGSINFO, + }, ] def apply_configuration(self, node, ports, raid_config, @@ -2266,6 +2274,17 @@ class GenericHardwareManager(HardwareManager): """Generate a TLS certificate for the IP address.""" return tls_utils.generate_tls_certificate(ip_address) + def inject_files(self, node, ports, files=None, verify_ca=True): + """A deploy step to inject arbitrary files. + + :param node: A dictionary of the node object + :param ports: A list of dictionaries containing information + of ports for the node (unused) + :param files: See :py:mod:`inject_files` + :param verify_ca: Whether to verify TLS certificate. + """ + return inject_files.inject_files(node, ports, files, verify_ca) + def _compare_extensions(ext1, ext2): mgr1 = ext1.obj diff --git a/ironic_python_agent/inject_files.py b/ironic_python_agent/inject_files.py new file mode 100644 index 000000000..262ec1129 --- /dev/null +++ b/ironic_python_agent/inject_files.py @@ -0,0 +1,256 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the inject_files deploy step.""" + +import base64 +import contextlib +import os + +from ironic_lib import disk_utils +from ironic_lib import utils as ironic_utils +from oslo_concurrency import processutils +from oslo_config import cfg +from oslo_log import log + +from ironic_python_agent import errors +from ironic_python_agent import hardware +from ironic_python_agent import utils + + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +ARGSINFO = { + "files": { + "description": ( + "Files to inject, a list of file structures with keys: 'path' " + "(path to the file), 'partition' (partition specifier), " + "'content' (base64 encoded string), 'mode' (new file mode) and " + "'dirmode' (mode for the leaf directory, if created). " + "Merged with the values from node.properties[inject_files]." + ), + "required": False, + }, + "verify_ca": { + "description": ( + "Whether to verify TLS certificates. Global agent options " + "are used by default." + ), + "required": False, + } +} + + +def inject_files(node, ports, files, verify_ca=True): + """A deploy step to inject arbitrary files. + + :param node: A dictionary of the node object + :param ports: A list of dictionaries containing information + of ports for the node + :param files: See ARGSINFO. + :param verify_ca: Whether to verify TLS certificate. + :raises: InvalidCommandParamsError + """ + files = _validate_files( + node['properties'].get('inject_files') or [], + files or []) + if not files: + LOG.info('No files to inject') + return + + http_get = utils.StreamingClient(verify_ca) + root_dev = hardware.dispatch_to_managers('get_os_install_device') + + for fl in files: + _inject_one(node, ports, fl, root_dev, http_get) + + +def _inject_one(node, ports, fl, root_dev, http_get): + """Inject one file. + + :param node: A dictionary of the node object + :param ports: A list of dictionaries containing information + of ports for the node + :param fl: File information. + :param root_dev: Root device used for the current node. + :param http_get: Context manager to get HTTP URLs. + """ + with _find_and_mount_path(fl['path'], fl.get('partition'), + root_dev) as path: + if fl.get('deleted'): + ironic_utils.unlink_without_raise(path) + return + + try: + dirpath = os.path.dirname(path) + try: + os.makedirs(dirpath) + except FileExistsError: + pass + else: + # Use chmod here and below to avoid relying on umask + if fl.get('dirmode'): + os.chmod(dirpath, fl['dirmode']) + + content = fl['content'] + with open(path, 'wb') as fp: + if '://' in content: + # Allow node-specific URLs to be used in a deploy template + url = content.format(node=node, ports=ports) + with http_get(url) as resp: + for chunk in resp: + fp.write(chunk) + else: + fp.write(base64.b64decode(content)) + + if fl.get('mode'): + os.chmod(path, fl['mode']) + + if fl.get('owner') is not None or fl.get('group') is not None: + # -1 means do not change + os.chown(path, fl.get('owner', -1), fl.get('group', -1)) + except Exception as exc: + LOG.exception('Failed to process file %s', fl) + raise errors.CommandExecutionError( + 'Failed to process file %s. %s: %s' + % (fl, type(exc).__class__, exc)) + + +@contextlib.contextmanager +def _find_and_mount_path(path, partition, root_dev): + """Find the specified path on a device. + + Tries to find the suitable device for the file based on the ``path`` and + ``partition``, mount the device and provides the actual full path. + + :param path: Path to the file to find. + :param partition: Device to find the file on or None. + :param root_dev: Root device from the hardware manager. + :return: Context manager that yields the full path to the file. + """ + path = os.path.normpath(path.strip('/')) # to make path joining work + if partition: + try: + part_num = int(partition) + except ValueError: + with ironic_utils.mounted(partition) as part_path: + yield os.path.join(part_path, path) + else: + # TODO(dtantsur): switch to ironic-lib instead: + # https://review.opendev.org/c/openstack/ironic-lib/+/774502 + part_template = '%s%s' + if 'nvme' in root_dev: + part_template = '%sp%s' + part_dev = part_template % (root_dev, part_num) + + with ironic_utils.mounted(part_dev) as part_path: + yield os.path.join(part_path, path) + else: + try: + # This turns e.g. etc/sysctl.d/my.conf into etc + sysctl.d/my.conf + detect_dir, rest_dir = path.split('/', 1) + except ValueError: + # Validation ensures that files in / have "partition" present, + # checking here just in case. + raise errors.InvalidCommandParamsError( + "Invalid path %s, must be an absolute path to a file" % path) + + with find_partition_with_path(detect_dir, root_dev) as part_path: + yield os.path.join(part_path, rest_dir) + + +@contextlib.contextmanager +def find_partition_with_path(path, device=None): + """Find a partition with the given path. + + :param path: Expected path. + :param device: Target device. If None, the root device is used. + :returns: A context manager that will unmount and delete the temporary + mount point on exit. + """ + if device is None: + device = hardware.dispatch_to_managers('get_os_install_device') + partitions = disk_utils.list_partitions(device) + # Make os.path.join work as expected + lookup_path = path.lstrip('/') + + for part in partitions: + if 'lvm' in part['flags']: + LOG.debug('Skipping LVM partition %s', part) + continue + + # TODO(dtantsur): switch to ironic-lib instead: + # https://review.opendev.org/c/openstack/ironic-lib/+/774502 + part_template = '%s%s' + if 'nvme' in device: + part_template = '%sp%s' + part_path = part_template % (device, part['number']) + + LOG.debug('Inspecting partition %s for path %s', part, path) + try: + with ironic_utils.mounted(part_path) as local_path: + found_path = os.path.join(local_path, lookup_path) + if not os.path.isdir(found_path): + continue + + LOG.info('Path %s has been found on partition %s', path, part) + yield found_path + return + except processutils.ProcessExecutionError as exc: + LOG.warning('Failure when inspecting partition %s: %s', part, exc) + + raise errors.DeviceNotFound("No partition found with path %s, scanned: %s" + % (path, partitions)) + + +def _validate_files(from_properties, from_args): + """Sanity check for files.""" + if not isinstance(from_properties, list): + raise errors.InvalidCommandParamsError( + "The `inject_files` node property must be a list, got %s" + % type(from_properties).__name__) + if not isinstance(from_args, list): + raise errors.InvalidCommandParamsError( + "The `files` argument must be a list, got %s" + % type(from_args).__name__) + + files = from_properties + from_args + failures = [] + + for fl in files: + unknown = set(fl) - {'path', 'partition', 'content', 'deleted', 'mode', + 'dirmode', 'owner', 'group'} + if unknown: + failures.append('unexpected fields in %s: %s' + % (fl, ', '.join(unknown))) + + if not fl.get('path'): + failures.append('expected a path in %s' % fl) + elif os.path.dirname(fl['path']) == '/' and not fl.get('partition'): + failures.append('%s in root directory requires "partition"' % fl) + elif fl['path'].endswith('/'): + failures.append('directories not supported for %s' % fl) + + if fl.get('content') and fl.get('deleted'): + failures.append('content cannot be used with deleted in %s' % fl) + + for field in ('owner', 'group', 'mode', 'dirmode'): + if field in fl and type(fl[field]) is not int: + failures.append('%s must be a number in %s' % (field, fl)) + + if failures: + raise errors.InvalidCommandParamsError( + "Validation of files failed: %s" % '; '.join(failures)) + + return files diff --git a/ironic_python_agent/tests/unit/test_inject_files.py b/ironic_python_agent/tests/unit/test_inject_files.py new file mode 100644 index 000000000..f99c7feae --- /dev/null +++ b/ironic_python_agent/tests/unit/test_inject_files.py @@ -0,0 +1,421 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import stat +import tempfile +from unittest import mock + +from ironic_python_agent import errors +from ironic_python_agent import inject_files +from ironic_python_agent.tests.unit import base + + +@mock.patch('ironic_lib.utils.mounted', autospec=True) +@mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) +@mock.patch('ironic_python_agent.hardware.dispatch_to_managers', + lambda _call: '/dev/fake') +class TestFindPartitionWithPath(base.IronicAgentTest): + + def setUp(self): + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.tempdir)) + + def test_found(self, mock_list_parts, mock_mount): + mock_list_parts.return_value = [ + {'number': 1, 'flags': 'lvm'}, + {'number': 2, 'flags': 'boot'}, + ] + mock_mount.return_value.__enter__.return_value = self.tempdir + expected = os.path.join(self.tempdir, "some/path") + os.makedirs(expected) + + with inject_files.find_partition_with_path("/some/path") as path: + self.assertEqual(expected, path) + + mock_mount.assert_called_once_with('/dev/fake2') + + def test_found_with_dev(self, mock_list_parts, mock_mount): + mock_list_parts.return_value = [ + {'number': 1, 'flags': 'lvm'}, + {'number': 2, 'flags': 'boot'}, + ] + mock_mount.return_value.__enter__.return_value = self.tempdir + expected = os.path.join(self.tempdir, "some/path") + os.makedirs(expected) + + with inject_files.find_partition_with_path("/some/path", + "/dev/nvme0n1") as path: + self.assertEqual(expected, path) + + mock_mount.assert_called_once_with('/dev/nvme0n1p2') + + def test_not_found(self, mock_list_parts, mock_mount): + mock_list_parts.return_value = [ + {'number': 1, 'flags': 'lvm'}, + {'number': 2, 'flags': 'boot'}, + {'number': 3, 'flags': ''}, + ] + mock_mount.return_value.__enter__.return_value = self.tempdir + + self.assertRaises( + errors.DeviceNotFound, + inject_files.find_partition_with_path("/some/path").__enter__) + + mock_mount.assert_has_calls([ + mock.call('/dev/fake2'), + mock.call('/dev/fake3'), + ], any_order=True) + + +class TestFindAndMountPath(base.IronicAgentTest): + + @mock.patch.object(inject_files, 'find_partition_with_path', autospec=True) + def test_without_on(self, mock_find_part): + mock_find_part.return_value.__enter__.return_value = '/mount/path' + with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf', + None, '/dev/fake') as result: + # "etc" is included in a real result of find_partition_with_path + self.assertEqual('/mount/path/sysctl.d/my.conf', result) + mock_find_part.assert_called_once_with('etc', '/dev/fake') + + def test_without_on_wrong_path(self): + self.assertRaises( + errors.InvalidCommandParamsError, + inject_files._find_and_mount_path('/etc', None, + '/dev/fake').__enter__) + + @mock.patch('ironic_lib.utils.mounted', autospec=True) + def test_with_on_as_path(self, mock_mount): + mock_mount.return_value.__enter__.return_value = '/mount/path' + with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf', + '/dev/on', + '/dev/fake') as result: + self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result) + mock_mount.assert_called_once_with('/dev/on') + + @mock.patch('ironic_lib.utils.mounted', autospec=True) + def test_with_on_as_number(self, mock_mount): + mock_mount.return_value.__enter__.return_value = '/mount/path' + with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf', + 2, '/dev/fake') as result: + self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result) + mock_mount.assert_called_once_with('/dev/fake2') + + @mock.patch('ironic_lib.utils.mounted', autospec=True) + def test_with_on_as_number_nvme(self, mock_mount): + mock_mount.return_value.__enter__.return_value = '/mount/path' + with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf', + 2, '/dev/nvme0n1') as result: + self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result) + mock_mount.assert_called_once_with('/dev/nvme0n1p2') + + +@mock.patch.object(inject_files, '_find_and_mount_path', autospec=True) +class TestInjectOne(base.IronicAgentTest): + + def setUp(self): + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.tempdir)) + self.dirpath = os.path.join(self.tempdir, 'dir1', 'dir2') + self.path = os.path.join(self.dirpath, 'file.name') + + self.http_get = mock.MagicMock() + self.http_get.return_value.__enter__.return_value = iter( + [b'con', b'tent', b'']) + + self.node = {'uuid': '1234'} + self.ports = [{'address': 'aabb'}] + + def test_delete(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True} + os.makedirs(self.dirpath) + with open(self.path, 'wb') as fp: + fp.write(b'content') + + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + self.assertFalse(os.path.exists(self.path)) + self.assertTrue(os.path.isdir(self.dirpath)) + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_not_called() + + def test_delete_not_exists(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + self.assertFalse(os.path.exists(self.path)) + self.assertFalse(os.path.isdir(self.dirpath)) + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_not_called() + + def test_plain_content(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA=='} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_not_called() + + def test_plain_content_with_on(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==', + 'partition': '/dev/sda1'} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + mock_find_and_mount.assert_called_once_with(fl['path'], '/dev/sda1', + '/dev/root') + self.http_get.assert_not_called() + + def test_plain_content_with_modes(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==', + 'mode': 0o602, 'dirmode': 0o703} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode)) + self.assertEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode)) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_not_called() + + def test_plain_content_with_modes_exists(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==', + 'mode': 0o602, 'dirmode': 0o703} + os.makedirs(self.dirpath) + with open(self.path, 'wb') as fp: + fp.write(b"I'm not a cat, I'm a lawyer") + + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode)) + # Exising directories do not change their permissions + self.assertNotEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode)) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_not_called() + + @mock.patch.object(os, 'chown', autospec=True) + def test_plain_content_with_owner(self, mock_chown, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==', + 'owner': 42} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + mock_chown.assert_called_once_with(self.path, 42, -1) + self.http_get.assert_not_called() + + @mock.patch.object(os, 'chown', autospec=True) + def test_plain_content_with_owner_and_group(self, mock_chown, + mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==', + 'owner': 0, 'group': 0} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + mock_chown.assert_called_once_with(self.path, 0, 0) + self.http_get.assert_not_called() + + def test_url(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', + 'content': 'http://example.com/path'} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_called_once_with('http://example.com/path') + + def test_url_formatting(self, mock_find_and_mount): + mock_find_and_mount.return_value.__enter__.return_value = self.path + + fl = {'path': '/etc/dir1/dir2/file.name', + 'content': 'http://example.com/{node[uuid]}/{ports[0][address]}'} + inject_files._inject_one(self.node, self.ports, fl, + '/dev/root', self.http_get) + + with open(self.path, 'rb') as fp: + self.assertEqual(b'content', fp.read()) + + mock_find_and_mount.assert_called_once_with(fl['path'], None, + '/dev/root') + self.http_get.assert_called_once_with('http://example.com/1234/aabb') + + +@mock.patch('ironic_python_agent.hardware.dispatch_to_managers', + lambda _call: '/dev/root') +@mock.patch.object(inject_files, '_inject_one', autospec=True) +class TestInjectFiles(base.IronicAgentTest): + + def test_empty(self, mock_inject): + node = { + 'properties': {} + } + + inject_files.inject_files(node, [mock.sentinel.port], []) + mock_inject.assert_not_called() + + def test_ok(self, mock_inject): + node = { + 'properties': { + 'inject_files': [ + {'path': '/etc/default/grub', 'content': 'abcdef'}, + {'path': '/etc/default/bluetooth', 'deleted': True}, + ] + } + } + files = [ + {'path': '/boot/special.conf', + 'content': 'http://example.com/data', + 'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0}, + {'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'}, + ] + + inject_files.inject_files(node, [mock.sentinel.port], files) + + mock_inject.assert_has_calls([ + mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY) + for fl in node['properties']['inject_files'] + files + ]) + http_get = mock_inject.call_args_list[0][0][4] + self.assertTrue(http_get.verify) + self.assertIsNone(http_get.cert) + + def test_verify_false(self, mock_inject): + node = { + 'properties': { + 'inject_files': [ + {'path': '/etc/default/grub', 'content': 'abcdef'}, + {'path': '/etc/default/bluetooth', 'deleted': True}, + ] + } + } + files = [ + {'path': '/boot/special.conf', + 'content': 'http://example.com/data', + 'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0}, + {'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'}, + ] + + inject_files.inject_files(node, [mock.sentinel.port], files, False) + + mock_inject.assert_has_calls([ + mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY) + for fl in node['properties']['inject_files'] + files + ]) + http_get = mock_inject.call_args_list[0][0][4] + self.assertFalse(http_get.verify) + self.assertIsNone(http_get.cert) + + def test_invalid_type_on_node(self, mock_inject): + node = { + 'properties': { + 'inject_files': 42 + } + } + self.assertRaises(errors.InvalidCommandParamsError, + inject_files.inject_files, node, [], []) + mock_inject.assert_not_called() + + def test_invalid_type_in_param(self, mock_inject): + node = { + 'properties': {} + } + self.assertRaises(errors.InvalidCommandParamsError, + inject_files.inject_files, node, [], 42) + mock_inject.assert_not_called() + + +class TestValidateFiles(base.IronicAgentTest): + + def test_missing_path(self): + fl = {'deleted': True} + self.assertRaisesRegex(errors.InvalidCommandParamsError, 'path', + inject_files._validate_files, [fl], []) + + def test_unknown_fields(self): + fl = {'path': '/etc/passwd', 'cat': 'meow'} + self.assertRaisesRegex(errors.InvalidCommandParamsError, 'cat', + inject_files._validate_files, [fl], []) + + def test_root_without_on(self): + fl = {'path': '/something', 'content': 'abcd'} + self.assertRaisesRegex(errors.InvalidCommandParamsError, 'partition', + inject_files._validate_files, [fl], []) + + def test_no_directories(self): + fl = {'path': '/something/else/', 'content': 'abcd'} + self.assertRaisesRegex(errors.InvalidCommandParamsError, 'directories', + inject_files._validate_files, [fl], []) + + def test_content_and_deleted(self): + fl = {'path': '/etc/password', 'content': 'abcd', 'deleted': True} + self.assertRaisesRegex(errors.InvalidCommandParamsError, + 'content .* with deleted', + inject_files._validate_files, [fl], []) + + def test_numeric_fields(self): + for field in ('owner', 'group', 'mode', 'dirmode'): + fl = {'path': '/etc/password', 'content': 'abcd', field: 'name'} + self.assertRaisesRegex(errors.InvalidCommandParamsError, + 'must be a number', + inject_files._validate_files, [fl], []) diff --git a/ironic_python_agent/tests/unit/test_utils.py b/ironic_python_agent/tests/unit/test_utils.py index 3dd7a5cb7..51a0291ad 100644 --- a/ironic_python_agent/tests/unit/test_utils.py +++ b/ironic_python_agent/tests/unit/test_utils.py @@ -27,6 +27,7 @@ from ironic_lib import disk_utils from ironic_lib import utils as ironic_utils from oslo_concurrency import processutils from oslo_serialization import base64 +import requests import testtools from ironic_python_agent import errors @@ -1132,3 +1133,33 @@ class TestCopyConfigFromVmedia(testtools.TestCase): mock.call(mock.ANY, '/etc/ironic-python-agent/ironic.crt'), mock.call(mock.ANY, '/etc/ironic-python-agent.d/ironic.conf'), ], any_order=True) + + +@mock.patch.object(requests, 'get', autospec=True) +class TestStreamingClient(ironic_agent_base.IronicAgentTest): + + def test_ok(self, mock_get): + client = utils.StreamingClient() + self.assertTrue(client.verify) + self.assertIsNone(client.cert) + + with client("http://url") as result: + response = mock_get.return_value.__enter__.return_value + self.assertIs(result, response.iter_content.return_value) + + mock_get.assert_called_once_with("http://url", verify=True, cert=None, + stream=True, timeout=60) + response.iter_content.assert_called_once_with(1024 * 1024) + + def test_retries(self, mock_get): + self.config(image_download_connection_retries=1, + image_download_connection_retry_interval=1) + mock_get.side_effect = requests.ConnectionError + + client = utils.StreamingClient() + self.assertRaises(errors.CommandExecutionError, + client("http://url").__enter__) + + mock_get.assert_called_with("http://url", verify=True, cert=None, + stream=True, timeout=60) + self.assertEqual(2, mock_get.call_count) diff --git a/ironic_python_agent/utils.py b/ironic_python_agent/utils.py index a8712a6a8..a5da83d68 100644 --- a/ironic_python_agent/utils.py +++ b/ironic_python_agent/utils.py @@ -35,6 +35,8 @@ from oslo_log import log as logging from oslo_serialization import base64 from oslo_serialization import jsonutils from oslo_utils import units +import requests +import tenacity from ironic_python_agent import errors @@ -811,3 +813,42 @@ def create_partition_table(dev_name, partition_table_type): msg = "Failed to create partition table on {}: {}".format( dev_name, e) raise errors.CommandExecutionError(msg) + + +class StreamingClient: + """A wrapper around HTTP client with TLS, streaming and error handling.""" + + _CHUNK_SIZE = 1 * units.Mi + + def __init__(self, verify_ca=True): + if verify_ca: + self.verify, self.cert = get_ssl_client_options(CONF) + else: + self.verify, self.cert = False, None + + @contextlib.contextmanager + def __call__(self, url): + """Execute a GET request and start streaming. + + :param url: Target URL. + :return: A generator yielding chunks of data. + """ + @tenacity.retry( + retry=tenacity.retry_if_exception_type(requests.ConnectionError), + stop=tenacity.stop_after_attempt( + CONF.image_download_connection_retries + 1), + wait=tenacity.wait_fixed( + CONF.image_download_connection_retry_interval), + reraise=True) + def _get_with_retries(): + return requests.get(url, verify=self.verify, cert=self.cert, + stream=True, + timeout=CONF.image_download_connection_timeout) + + try: + with _get_with_retries() as resp: + resp.raise_for_status() + yield resp.iter_content(self._CHUNK_SIZE) + except requests.RequestException as exc: + raise errors.CommandExecutionError( + "Unable to read data from %s: %s" % (url, exc)) diff --git a/releasenotes/notes/inject-files-b411369ce6856dac.yaml b/releasenotes/notes/inject-files-b411369ce6856dac.yaml new file mode 100644 index 000000000..bc19d365c --- /dev/null +++ b/releasenotes/notes/inject-files-b411369ce6856dac.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a new deploy step ``deploy.inject_files`` to inject arbitrary files + into the instance. See `the hardware managers documentation + `_ + for details. diff --git a/requirements.txt b/requirements.txt index 690a5b1e0..9fefc9976 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,6 @@ requests>=2.14.2 # Apache-2.0 rtslib-fb>=2.1.65 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 tenacity>=6.2.0 # Apache-2.0 -ironic-lib>=4.1.0 # Apache-2.0 +ironic-lib>=4.5.0 # Apache-2.0 Werkzeug>=1.0.1 # BSD License cryptography>=2.3 # BSD/Apache-2.0