openstack/ironic-python-agent
Code Issues Proposed changes

New deploy step for injecting arbitrary files

Browse Source 
This change adds a deploy step inject_files that adds a flexible
way to inject files into the instance.

Change-Id: I0e70a2cbc13744195c9493a48662e465ec010dbe
Story: #2008611
Task: #41794
This commit is contained in:
Dmitry Tantsur 2021-02-12 17:00:40 +01:00
parent 13c3c60ff1
commit 59cb08fd28
9 changed files with 827 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
doc/source/admin/hardware_managers.rst
View File

@ -25,6 +25,52 @@ Deploy steps
and must be used through the :ironic-doc:`ironic RAID feature
<admin/raid.html>`.
Injecting files
~~~~~~~~~~~~~~~
``deploy.inject_files(node, ports, files, verify_ca=True)``
This optional deploy step (introduced in the Wallaby release series) allows
injecting arbitrary files into the node. The list of files is built from the
optional ``inject_files`` property of the node concatenated with the explicit
``files`` argument. Each item in the list is a dictionary with the following
fields:
``path`` (required)
An absolute path to the file on the target partition. All missing
directories will be created.
``partition``
Specifies the target partition in one of 3 ways:
* A number is treated as a partition index (starting with 1) on the root
device.
* A path is treated as a block device path (e.g. ``/dev/sda1`` or
``/dev/disk/by-partlabel/<something>``.
* If missing, the agent will try to find a partition containing the first
component of the ``path`` on the root device. E.g. for
``/etc/sysctl.d/my.conf``, look for a partition containing ``/etc``.
``deleted``
If ``True``, the file is deleted, not created.
Incompatible with ``content``.
``content``
Data to write. Incompatible with ``deleted``. Can take two forms:
* A URL of the content. Can use Python-style formatting to build a node
specific URL, e.g. ``http://server/{node[uuid]}/{ports[0][address]}``.
* Base64 encoded binary contents.
``mode``, ``owner``, ``group``
Numeric mode, owner ID and group ID of the file.
``dirmode``
Numeric mode of the leaf directory if it has to be created.
This deploy step is disabled by default and can be enabled via a deploy
template or via the ``ipa-inject-files-priority`` kernel parameter.
Known limitations:
* Names are not supported for ``owner`` and ``group``.
* LVM is not supported.
Clean steps
-----------

5
ironic_python_agent/config.py
View File

@ -316,6 +316,11 @@ cli_opts = [
'used if you are absolutely sure of what you are '
'doing and that your hardware supports '
'such functionality. Hint: Most hardware does not.'),
cfg.IntOpt('inject_files_priority',
default=APARAMS.get('ipa-inject-files-priority', 0),
min=0, max=99, # 100 is when IPA is booted
help='Priority of the inject_files deploy step (disabled '
'by default), an integer between 1 and .'),
]
CONF.register_cli_opts(cli_opts)

19
ironic_python_agent/hardware.py
View File

@ -39,6 +39,7 @@ import yaml
from ironic_python_agent import encoding
from ironic_python_agent import errors
from ironic_python_agent.extensions import base as ext_base
from ironic_python_agent import inject_files
from ironic_python_agent import netutils
from ironic_python_agent import raid_utils
from ironic_python_agent import tls_utils
@ -1794,6 +1795,13 @@ class GenericHardwareManager(HardwareManager):
'interface': 'deploy',
'reboot_requested': False,
},
{
'step': 'inject_files',
'priority': CONF.inject_files_priority,
'interface': 'deploy',
'reboot_requested': False,
'argsinfo': inject_files.ARGSINFO,
},
]
def apply_configuration(self, node, ports, raid_config,
@ -2266,6 +2274,17 @@ class GenericHardwareManager(HardwareManager):
"""Generate a TLS certificate for the IP address."""
return tls_utils.generate_tls_certificate(ip_address)
def inject_files(self, node, ports, files=None, verify_ca=True):
"""A deploy step to inject arbitrary files.
:param node: A dictionary of the node object
:param ports: A list of dictionaries containing information
of ports for the node (unused)
:param files: See :py:mod:`inject_files`
:param verify_ca: Whether to verify TLS certificate.
"""
return inject_files.inject_files(node, ports, files, verify_ca)
def _compare_extensions(ext1, ext2):
mgr1 = ext1.obj

256
ironic_python_agent/inject_files.py Normal file
View File

@ -0,0 +1,256 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of the inject_files deploy step."""
import base64
import contextlib
import os
from ironic_lib import disk_utils
from ironic_lib import utils as ironic_utils
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log
from ironic_python_agent import errors
from ironic_python_agent import hardware
from ironic_python_agent import utils
CONF = cfg.CONF
LOG = log.getLogger(__name__)
ARGSINFO = {
"files": {
"description": (
"Files to inject, a list of file structures with keys: 'path' "
"(path to the file), 'partition' (partition specifier), "
"'content' (base64 encoded string), 'mode' (new file mode) and "
"'dirmode' (mode for the leaf directory, if created). "
"Merged with the values from node.properties[inject_files]."
),
"required": False,
},
"verify_ca": {
"description": (
"Whether to verify TLS certificates. Global agent options "
"are used by default."
),
"required": False,
}
}
def inject_files(node, ports, files, verify_ca=True):
"""A deploy step to inject arbitrary files.
:param node: A dictionary of the node object
:param ports: A list of dictionaries containing information
of ports for the node
:param files: See ARGSINFO.
:param verify_ca: Whether to verify TLS certificate.
:raises: InvalidCommandParamsError
"""
files = _validate_files(
node['properties'].get('inject_files') or [],
files or [])
if not files:
LOG.info('No files to inject')
return
http_get = utils.StreamingClient(verify_ca)
root_dev = hardware.dispatch_to_managers('get_os_install_device')
for fl in files:
_inject_one(node, ports, fl, root_dev, http_get)
def _inject_one(node, ports, fl, root_dev, http_get):
"""Inject one file.
:param node: A dictionary of the node object
:param ports: A list of dictionaries containing information
of ports for the node
:param fl: File information.
:param root_dev: Root device used for the current node.
:param http_get: Context manager to get HTTP URLs.
"""
with _find_and_mount_path(fl['path'], fl.get('partition'),
root_dev) as path:
if fl.get('deleted'):
ironic_utils.unlink_without_raise(path)
return
try:
dirpath = os.path.dirname(path)
try:
os.makedirs(dirpath)
except FileExistsError:
pass
else:
# Use chmod here and below to avoid relying on umask
if fl.get('dirmode'):
os.chmod(dirpath, fl['dirmode'])
content = fl['content']
with open(path, 'wb') as fp:
if '://' in content:
# Allow node-specific URLs to be used in a deploy template
url = content.format(node=node, ports=ports)
with http_get(url) as resp:
for chunk in resp:
fp.write(chunk)
else:
fp.write(base64.b64decode(content))
if fl.get('mode'):
os.chmod(path, fl['mode'])
if fl.get('owner') is not None or fl.get('group') is not None:
# -1 means do not change
os.chown(path, fl.get('owner', -1), fl.get('group', -1))
except Exception as exc:
LOG.exception('Failed to process file %s', fl)
raise errors.CommandExecutionError(
'Failed to process file %s. %s: %s'
% (fl, type(exc).__class__, exc))
@contextlib.contextmanager
def _find_and_mount_path(path, partition, root_dev):
"""Find the specified path on a device.
Tries to find the suitable device for the file based on the ``path`` and
``partition``, mount the device and provides the actual full path.
:param path: Path to the file to find.
:param partition: Device to find the file on or None.
:param root_dev: Root device from the hardware manager.
:return: Context manager that yields the full path to the file.
"""
path = os.path.normpath(path.strip('/')) # to make path joining work
if partition:
try:
part_num = int(partition)
except ValueError:
with ironic_utils.mounted(partition) as part_path:
yield os.path.join(part_path, path)
else:
# TODO(dtantsur): switch to ironic-lib instead:
# https://review.opendev.org/c/openstack/ironic-lib/+/774502
part_template = '%s%s'
if 'nvme' in root_dev:
part_template = '%sp%s'
part_dev = part_template % (root_dev, part_num)
with ironic_utils.mounted(part_dev) as part_path:
yield os.path.join(part_path, path)
else:
try:
# This turns e.g. etc/sysctl.d/my.conf into etc + sysctl.d/my.conf
detect_dir, rest_dir = path.split('/', 1)
except ValueError:
# Validation ensures that files in / have "partition" present,
# checking here just in case.
raise errors.InvalidCommandParamsError(
"Invalid path %s, must be an absolute path to a file" % path)
with find_partition_with_path(detect_dir, root_dev) as part_path:
yield os.path.join(part_path, rest_dir)
@contextlib.contextmanager
def find_partition_with_path(path, device=None):
"""Find a partition with the given path.
:param path: Expected path.
:param device: Target device. If None, the root device is used.
:returns: A context manager that will unmount and delete the temporary
mount point on exit.
"""
if device is None:
device = hardware.dispatch_to_managers('get_os_install_device')
partitions = disk_utils.list_partitions(device)
# Make os.path.join work as expected
lookup_path = path.lstrip('/')
for part in partitions:
if 'lvm' in part['flags']:
LOG.debug('Skipping LVM partition %s', part)
continue
# TODO(dtantsur): switch to ironic-lib instead:
# https://review.opendev.org/c/openstack/ironic-lib/+/774502
part_template = '%s%s'
if 'nvme' in device:
part_template = '%sp%s'
part_path = part_template % (device, part['number'])
LOG.debug('Inspecting partition %s for path %s', part, path)
try:
with ironic_utils.mounted(part_path) as local_path:
found_path = os.path.join(local_path, lookup_path)
if not os.path.isdir(found_path):
continue
LOG.info('Path %s has been found on partition %s', path, part)
yield found_path
return
except processutils.ProcessExecutionError as exc:
LOG.warning('Failure when inspecting partition %s: %s', part, exc)
raise errors.DeviceNotFound("No partition found with path %s, scanned: %s"
% (path, partitions))
def _validate_files(from_properties, from_args):
"""Sanity check for files."""
if not isinstance(from_properties, list):
raise errors.InvalidCommandParamsError(
"The `inject_files` node property must be a list, got %s"
% type(from_properties).__name__)
if not isinstance(from_args, list):
raise errors.InvalidCommandParamsError(
"The `files` argument must be a list, got %s"
% type(from_args).__name__)
files = from_properties + from_args
failures = []
for fl in files:
unknown = set(fl) - {'path', 'partition', 'content', 'deleted', 'mode',
'dirmode', 'owner', 'group'}
if unknown:
failures.append('unexpected fields in %s: %s'
% (fl, ', '.join(unknown)))
if not fl.get('path'):
failures.append('expected a path in %s' % fl)
elif os.path.dirname(fl['path']) == '/' and not fl.get('partition'):
failures.append('%s in root directory requires "partition"' % fl)
elif fl['path'].endswith('/'):
failures.append('directories not supported for %s' % fl)
if fl.get('content') and fl.get('deleted'):
failures.append('content cannot be used with deleted in %s' % fl)
for field in ('owner', 'group', 'mode', 'dirmode'):
if field in fl and type(fl[field]) is not int:
failures.append('%s must be a number in %s' % (field, fl))
if failures:
raise errors.InvalidCommandParamsError(
"Validation of files failed: %s" % '; '.join(failures))
return files

421
ironic_python_agent/tests/unit/test_inject_files.py Normal file
View File

@ -0,0 +1,421 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import stat
import tempfile
from unittest import mock
from ironic_python_agent import errors
from ironic_python_agent import inject_files
from ironic_python_agent.tests.unit import base
@mock.patch('ironic_lib.utils.mounted', autospec=True)
@mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True)
@mock.patch('ironic_python_agent.hardware.dispatch_to_managers',
lambda _call: '/dev/fake')
class TestFindPartitionWithPath(base.IronicAgentTest):
def setUp(self):
super().setUp()
self.tempdir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(self.tempdir))
def test_found(self, mock_list_parts, mock_mount):
mock_list_parts.return_value = [
{'number': 1, 'flags': 'lvm'},
{'number': 2, 'flags': 'boot'},
]
mock_mount.return_value.__enter__.return_value = self.tempdir
expected = os.path.join(self.tempdir, "some/path")
os.makedirs(expected)
with inject_files.find_partition_with_path("/some/path") as path:
self.assertEqual(expected, path)
mock_mount.assert_called_once_with('/dev/fake2')
def test_found_with_dev(self, mock_list_parts, mock_mount):
mock_list_parts.return_value = [
{'number': 1, 'flags': 'lvm'},
{'number': 2, 'flags': 'boot'},
]
mock_mount.return_value.__enter__.return_value = self.tempdir
expected = os.path.join(self.tempdir, "some/path")
os.makedirs(expected)
with inject_files.find_partition_with_path("/some/path",
"/dev/nvme0n1") as path:
self.assertEqual(expected, path)
mock_mount.assert_called_once_with('/dev/nvme0n1p2')
def test_not_found(self, mock_list_parts, mock_mount):
mock_list_parts.return_value = [
{'number': 1, 'flags': 'lvm'},
{'number': 2, 'flags': 'boot'},
{'number': 3, 'flags': ''},
]
mock_mount.return_value.__enter__.return_value = self.tempdir
self.assertRaises(
errors.DeviceNotFound,
inject_files.find_partition_with_path("/some/path").__enter__)
mock_mount.assert_has_calls([
mock.call('/dev/fake2'),
mock.call('/dev/fake3'),
], any_order=True)
class TestFindAndMountPath(base.IronicAgentTest):
@mock.patch.object(inject_files, 'find_partition_with_path', autospec=True)
def test_without_on(self, mock_find_part):
mock_find_part.return_value.__enter__.return_value = '/mount/path'
with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
None, '/dev/fake') as result:
# "etc" is included in a real result of find_partition_with_path
self.assertEqual('/mount/path/sysctl.d/my.conf', result)
mock_find_part.assert_called_once_with('etc', '/dev/fake')
def test_without_on_wrong_path(self):
self.assertRaises(
errors.InvalidCommandParamsError,
inject_files._find_and_mount_path('/etc', None,
'/dev/fake').__enter__)
@mock.patch('ironic_lib.utils.mounted', autospec=True)
def test_with_on_as_path(self, mock_mount):
mock_mount.return_value.__enter__.return_value = '/mount/path'
with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
'/dev/on',
'/dev/fake') as result:
self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
mock_mount.assert_called_once_with('/dev/on')
@mock.patch('ironic_lib.utils.mounted', autospec=True)
def test_with_on_as_number(self, mock_mount):
mock_mount.return_value.__enter__.return_value = '/mount/path'
with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
2, '/dev/fake') as result:
self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
mock_mount.assert_called_once_with('/dev/fake2')
@mock.patch('ironic_lib.utils.mounted', autospec=True)
def test_with_on_as_number_nvme(self, mock_mount):
mock_mount.return_value.__enter__.return_value = '/mount/path'
with inject_files._find_and_mount_path('/etc/sysctl.d/my.conf',
2, '/dev/nvme0n1') as result:
self.assertEqual('/mount/path/etc/sysctl.d/my.conf', result)
mock_mount.assert_called_once_with('/dev/nvme0n1p2')
@mock.patch.object(inject_files, '_find_and_mount_path', autospec=True)
class TestInjectOne(base.IronicAgentTest):
def setUp(self):
super().setUp()
self.tempdir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(self.tempdir))
self.dirpath = os.path.join(self.tempdir, 'dir1', 'dir2')
self.path = os.path.join(self.dirpath, 'file.name')
self.http_get = mock.MagicMock()
self.http_get.return_value.__enter__.return_value = iter(
[b'con', b'tent', b''])
self.node = {'uuid': '1234'}
self.ports = [{'address': 'aabb'}]
def test_delete(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True}
os.makedirs(self.dirpath)
with open(self.path, 'wb') as fp:
fp.write(b'content')
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
self.assertFalse(os.path.exists(self.path))
self.assertTrue(os.path.isdir(self.dirpath))
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_not_called()
def test_delete_not_exists(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'deleted': True}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
self.assertFalse(os.path.exists(self.path))
self.assertFalse(os.path.isdir(self.dirpath))
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_not_called()
def test_plain_content(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA=='}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_not_called()
def test_plain_content_with_on(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
'partition': '/dev/sda1'}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], '/dev/sda1',
'/dev/root')
self.http_get.assert_not_called()
def test_plain_content_with_modes(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
'mode': 0o602, 'dirmode': 0o703}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode))
self.assertEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode))
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_not_called()
def test_plain_content_with_modes_exists(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
'mode': 0o602, 'dirmode': 0o703}
os.makedirs(self.dirpath)
with open(self.path, 'wb') as fp:
fp.write(b"I'm not a cat, I'm a lawyer")
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
self.assertEqual(0o602, stat.S_IMODE(os.stat(self.path).st_mode))
# Exising directories do not change their permissions
self.assertNotEqual(0o703, stat.S_IMODE(os.stat(self.dirpath).st_mode))
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_not_called()
@mock.patch.object(os, 'chown', autospec=True)
def test_plain_content_with_owner(self, mock_chown, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
'owner': 42}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
mock_chown.assert_called_once_with(self.path, 42, -1)
self.http_get.assert_not_called()
@mock.patch.object(os, 'chown', autospec=True)
def test_plain_content_with_owner_and_group(self, mock_chown,
mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name', 'content': 'Y29udGVudA==',
'owner': 0, 'group': 0}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
mock_chown.assert_called_once_with(self.path, 0, 0)
self.http_get.assert_not_called()
def test_url(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name',
'content': 'http://example.com/path'}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_called_once_with('http://example.com/path')
def test_url_formatting(self, mock_find_and_mount):
mock_find_and_mount.return_value.__enter__.return_value = self.path
fl = {'path': '/etc/dir1/dir2/file.name',
'content': 'http://example.com/{node[uuid]}/{ports[0][address]}'}
inject_files._inject_one(self.node, self.ports, fl,
'/dev/root', self.http_get)
with open(self.path, 'rb') as fp:
self.assertEqual(b'content', fp.read())
mock_find_and_mount.assert_called_once_with(fl['path'], None,
'/dev/root')
self.http_get.assert_called_once_with('http://example.com/1234/aabb')
@mock.patch('ironic_python_agent.hardware.dispatch_to_managers',
lambda _call: '/dev/root')
@mock.patch.object(inject_files, '_inject_one', autospec=True)
class TestInjectFiles(base.IronicAgentTest):
def test_empty(self, mock_inject):
node = {
'properties': {}
}
inject_files.inject_files(node, [mock.sentinel.port], [])
mock_inject.assert_not_called()
def test_ok(self, mock_inject):
node = {
'properties': {
'inject_files': [
{'path': '/etc/default/grub', 'content': 'abcdef'},
{'path': '/etc/default/bluetooth', 'deleted': True},
]
}
}
files = [
{'path': '/boot/special.conf',
'content': 'http://example.com/data',
'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0},
{'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'},
]
inject_files.inject_files(node, [mock.sentinel.port], files)
mock_inject.assert_has_calls([
mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY)
for fl in node['properties']['inject_files'] + files
])
http_get = mock_inject.call_args_list[0][0][4]
self.assertTrue(http_get.verify)
self.assertIsNone(http_get.cert)
def test_verify_false(self, mock_inject):
node = {
'properties': {
'inject_files': [
{'path': '/etc/default/grub', 'content': 'abcdef'},
{'path': '/etc/default/bluetooth', 'deleted': True},
]
}
}
files = [
{'path': '/boot/special.conf',
'content': 'http://example.com/data',
'mode': 0o600, 'dirmode': 0o750, 'owner': 0, 'group': 0},
{'path': 'service.conf', 'partition': '/dev/disk/by-label/OPT'},
]
inject_files.inject_files(node, [mock.sentinel.port], files, False)
mock_inject.assert_has_calls([
mock.call(node, [mock.sentinel.port], fl, '/dev/root', mock.ANY)
for fl in node['properties']['inject_files'] + files
])
http_get = mock_inject.call_args_list[0][0][4]
self.assertFalse(http_get.verify)
self.assertIsNone(http_get.cert)
def test_invalid_type_on_node(self, mock_inject):
node = {
'properties': {
'inject_files': 42
}
}
self.assertRaises(errors.InvalidCommandParamsError,
inject_files.inject_files, node, [], [])
mock_inject.assert_not_called()
def test_invalid_type_in_param(self, mock_inject):
node = {
'properties': {}
}
self.assertRaises(errors.InvalidCommandParamsError,
inject_files.inject_files, node, [], 42)
mock_inject.assert_not_called()
class TestValidateFiles(base.IronicAgentTest):
def test_missing_path(self):
fl = {'deleted': True}
self.assertRaisesRegex(errors.InvalidCommandParamsError, 'path',
inject_files._validate_files, [fl], [])
def test_unknown_fields(self):
fl = {'path': '/etc/passwd', 'cat': 'meow'}
self.assertRaisesRegex(errors.InvalidCommandParamsError, 'cat',
inject_files._validate_files, [fl], [])
def test_root_without_on(self):
fl = {'path': '/something', 'content': 'abcd'}
self.assertRaisesRegex(errors.InvalidCommandParamsError, 'partition',
inject_files._validate_files, [fl], [])
def test_no_directories(self):
fl = {'path': '/something/else/', 'content': 'abcd'}
self.assertRaisesRegex(errors.InvalidCommandParamsError, 'directories',
inject_files._validate_files, [fl], [])
def test_content_and_deleted(self):
fl = {'path': '/etc/password', 'content': 'abcd', 'deleted': True}
self.assertRaisesRegex(errors.InvalidCommandParamsError,
'content .* with deleted',
inject_files._validate_files, [fl], [])
def test_numeric_fields(self):
for field in ('owner', 'group', 'mode', 'dirmode'):
fl = {'path': '/etc/password', 'content': 'abcd', field: 'name'}
self.assertRaisesRegex(errors.InvalidCommandParamsError,
'must be a number',
inject_files._validate_files, [fl], [])

31
ironic_python_agent/tests/unit/test_utils.py
View File

@ -27,6 +27,7 @@ from ironic_lib import disk_utils
from ironic_lib import utils as ironic_utils
from oslo_concurrency import processutils
from oslo_serialization import base64
import requests
import testtools
from ironic_python_agent import errors
@ -1132,3 +1133,33 @@ class TestCopyConfigFromVmedia(testtools.TestCase):
mock.call(mock.ANY, '/etc/ironic-python-agent/ironic.crt'),
mock.call(mock.ANY, '/etc/ironic-python-agent.d/ironic.conf'),
], any_order=True)
@mock.patch.object(requests, 'get', autospec=True)
class TestStreamingClient(ironic_agent_base.IronicAgentTest):
def test_ok(self, mock_get):
client = utils.StreamingClient()
self.assertTrue(client.verify)
self.assertIsNone(client.cert)
with client("http://url") as result:
response = mock_get.return_value.__enter__.return_value
self.assertIs(result, response.iter_content.return_value)
mock_get.assert_called_once_with("http://url", verify=True, cert=None,
stream=True, timeout=60)
response.iter_content.assert_called_once_with(1024 * 1024)
def test_retries(self, mock_get):
self.config(image_download_connection_retries=1,
image_download_connection_retry_interval=1)
mock_get.side_effect = requests.ConnectionError
client = utils.StreamingClient()
self.assertRaises(errors.CommandExecutionError,
client("http://url").__enter__)
mock_get.assert_called_with("http://url", verify=True, cert=None,
stream=True, timeout=60)
self.assertEqual(2, mock_get.call_count)

41
ironic_python_agent/utils.py
View File

@ -35,6 +35,8 @@ from oslo_log import log as logging
from oslo_serialization import base64
from oslo_serialization import jsonutils
from oslo_utils import units
import requests
import tenacity
from ironic_python_agent import errors
@ -811,3 +813,42 @@ def create_partition_table(dev_name, partition_table_type):
msg = "Failed to create partition table on {}: {}".format(
dev_name, e)
raise errors.CommandExecutionError(msg)
class StreamingClient:
"""A wrapper around HTTP client with TLS, streaming and error handling."""
_CHUNK_SIZE = 1 * units.Mi
def __init__(self, verify_ca=True):
if verify_ca:
self.verify, self.cert = get_ssl_client_options(CONF)
else:
self.verify, self.cert = False, None
@contextlib.contextmanager
def __call__(self, url):
"""Execute a GET request and start streaming.
:param url: Target URL.
:return: A generator yielding chunks of data.
"""
@tenacity.retry(
retry=tenacity.retry_if_exception_type(requests.ConnectionError),
stop=tenacity.stop_after_attempt(
CONF.image_download_connection_retries + 1),
wait=tenacity.wait_fixed(
CONF.image_download_connection_retry_interval),
reraise=True)
def _get_with_retries():
return requests.get(url, verify=self.verify, cert=self.cert,
stream=True,
timeout=CONF.image_download_connection_timeout)
try:
with _get_with_retries() as resp:
resp.raise_for_status()
yield resp.iter_content(self._CHUNK_SIZE)
except requests.RequestException as exc:
raise errors.CommandExecutionError(
"Unable to read data from %s: %s" % (url, exc))

7
releasenotes/notes/inject-files-b411369ce6856dac.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Adds a new deploy step ``deploy.inject_files`` to inject arbitrary files
into the instance. See `the hardware managers documentation
<https://docs.openstack.org/ironic-python-agent/latest/admin/hardware_managers.html>`_
for details.

2
requirements.txt
View File

@ -17,6 +17,6 @@ requests>=2.14.2 # Apache-2.0
rtslib-fb>=2.1.65 # Apache-2.0
stevedore>=1.20.0 # Apache-2.0
tenacity>=6.2.0 # Apache-2.0
ironic-lib>=4.1.0 # Apache-2.0
ironic-lib>=4.5.0 # Apache-2.0
Werkzeug>=1.0.1 # BSD License
cryptography>=2.3 # BSD/Apache-2.0