From 4b70223bf2382f314e61618bb76b3d51ac25c5f0 Mon Sep 17 00:00:00 2001 From: Ethan Gafford Date: Wed, 29 Jun 2016 16:47:10 -0400 Subject: [PATCH] CLI for Plugin-Declared Image Declaration Introduces a new libfuestfs-driven CLI for packing of Sahara images, using the same recipe definition scheme introduced in the validate-image-spi blueprint. Documentation and plugin image packing configurability can and will be provided in separate patches for ease of review. Partially-implements: blueprint image-generation-cli Change-Id: I6788108e3fb6232045fc56937639a6348768a7bc --- sahara/cli/image_pack/__init__.py | 0 sahara/cli/image_pack/api.py | 113 ++++++++++++++++++ sahara/cli/image_pack/cli.py | 93 ++++++++++++++ sahara/plugins/images.py | 21 ++-- sahara/tests/unit/cli/image_pack/__init__.py | 0 .../cli/image_pack/test_image_pack_api.py | 56 +++++++++ sahara/utils/remote.py | 27 +++-- setup.cfg | 1 + tox.ini | 4 + 9 files changed, 299 insertions(+), 16 deletions(-) create mode 100644 sahara/cli/image_pack/__init__.py create mode 100644 sahara/cli/image_pack/api.py create mode 100644 sahara/cli/image_pack/cli.py create mode 100644 sahara/tests/unit/cli/image_pack/__init__.py create mode 100644 sahara/tests/unit/cli/image_pack/test_image_pack_api.py diff --git a/sahara/cli/image_pack/__init__.py b/sahara/cli/image_pack/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sahara/cli/image_pack/api.py b/sahara/cli/image_pack/api.py new file mode 100644 index 0000000000..b90c1f07af --- /dev/null +++ b/sahara/cli/image_pack/api.py @@ -0,0 +1,113 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara import conductor # noqa +from sahara.i18n import _LE +from sahara.i18n import _LI +from sahara.plugins import base as plugins_base +from sahara.utils import remote + +try: + import guestfs +except ImportError: + raise Exception(_LE("The image packing API depends on the system package " + "python-libguestfs (and libguestfs itself.) Please " + "install these packages to proceed.")) + + +LOG = None +CONF = None + + +# This is broken out to support testability +def set_logger(log): + global LOG + LOG = log + + +# This is broken out to support testability +def set_conf(conf): + global CONF + CONF = conf + + +# This is a local exception class that is used to exit routines +# in cases where error information has already been logged. +# It is caught and suppressed everywhere it is used. +class Handled(Exception): + pass + + +class Context(object): + '''Create a pseudo Context object + + Since this tool does not use the REST interface, we + do not have a request from which to build a Context. + ''' + def __init__(self, is_admin=False, tenant_id=None): + self.is_admin = is_admin + self.tenant_id = tenant_id + + +class ImageRemote(remote.TerminalOnlyRemote): + + def __init__(self, image_path, root_drive): + guest = guestfs.GuestFS(python_return_dict=True) + guest.add_drive_opts(image_path, format="qcow2") + guest.set_network(True) + self.guest = guest + self.root_drive = root_drive + + def __enter__(self): + self.guest.launch() + if not self.root_drive: + self.root_drive = self.guest.inspect_os()[0] + self.guest.mount(self.root_drive, '/') + try: + cmd = "echo Testing sudo without tty..." + self.execute_command(cmd, run_as_root=True) + except RuntimeError: + cmd = "sed -i 's/requiretty/!requiretty/' /etc/sudoers" + self.guest.execute_command(cmd) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.guest.sync() + self.guest.umount_all() + self.guest.close() + + def execute_command(self, cmd, run_as_root=False, get_stderr=False, + raise_when_error=True, timeout=300): + try: + LOG.info(_LI("Issuing command: {cmd}").format(cmd=cmd)) + stdout = self.guest.sh(cmd) + LOG.info(_LI("Received response: {stdout}").format(stdout=stdout)) + return 0, stdout + except RuntimeError as ex: + if raise_when_error: + raise ex + else: + return 1, ex.message + + def get_os_distrib(self): + return self.guest.inspect_get_distro() + + +def pack_image(plugin_name, plugin_version, image_path, root_drive=None, + test_only=False, **kwargs): + plugins_base.setup_plugins() + with ImageRemote(image_path, root_drive) as image_remote: + plugin = plugins_base.PLUGINS.get_plugin(plugin_name) + reconcile = not test_only + plugin.pack_image(image_remote, reconcile=reconcile, env_map=kwargs) diff --git a/sahara/cli/image_pack/cli.py b/sahara/cli/image_pack/cli.py new file mode 100644 index 0000000000..ba95039a89 --- /dev/null +++ b/sahara/cli/image_pack/cli.py @@ -0,0 +1,93 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from oslo_config import cfg +from oslo_log import log + +from sahara.cli.image_pack import api +from sahara.i18n import _LI + +LOG = log.getLogger(__name__) + +CONF = cfg.CONF + + +CONF.register_cli_opts([ + cfg.StrOpt( + 'plugin', + required=True, + help="The name of the Sahara plugin for which you would like to " + "generate an image. Use sahara-image-create -p PLUGIN -h to " + "see a set of versions for a specific plugin."), + cfg.StrOpt( + 'plugin-version', + dest='plugin_version', + required=True, + help="The version of the Sahara plugin for which you would like to " + "generate an image. Use sahara-image-create -p PLUGIN -v " + "VERSION -h to see a full set of arguments for a specific plugin " + "and version."), + cfg.StrOpt( + 'image', + required=True, + help="The path to an image to modify. This image will be modified " + "in-place: be sure to target a copy if you wish to maintain a " + "clean master image."), + cfg.StrOpt( + 'root-filesystem', + dest='root_fs', + required=False, + help="The filesystem to mount as the root volume on the image. No" + "value is required if only one filesystem is detected."), + cfg.BoolOpt( + 'test-only', + dest='test_only', + default=False, + help="If this flag is set, no changes will be made to the image; " + "instead, the script will fail if discrepancies are found " + "between the image and the intended state."), +]) + + +def unregister_extra_cli_opt(name): + try: + for cli in CONF._cli_opts: + if cli['opt'].name == name: + CONF.unregister_opt(cli['opt']) + except Exception: + pass + + +for extra_opt in ["log-exchange", "host", "port"]: + unregister_extra_cli_opt(extra_opt) + + +def main(): + CONF(project='sahara') + + CONF.reload_config_files() + log.setup(CONF, "sahara") + + LOG.info(_LI("Command: {command}").format(command=' '.join(sys.argv))) + + api.set_logger(LOG) + api.set_conf(CONF) + + api.pack_image(CONF.plugin, CONF.plugin_version, CONF.image, + CONF.root_fs, CONF.test_only) + + LOG.info(_LI("Finished packing image for {plugin} at version {version}" + ).format(plugin=CONF.plugin, version=CONF.plugin_version)) diff --git a/sahara/plugins/images.py b/sahara/plugins/images.py index b8dc393921..49b1129eaf 100644 --- a/sahara/plugins/images.py +++ b/sahara/plugins/images.py @@ -414,7 +414,8 @@ class SaharaPackageValidator(SaharaImageValidatorBase): _("Unknown distro: cannot verify or install packages.")) try: check(self, remote) - except (ex.SubprocessException, ex.RemoteCommandException): + except (ex.SubprocessException, ex.RemoteCommandException, + RuntimeError): if reconcile: install(self, remote) check(self, remote) @@ -474,6 +475,10 @@ class SaharaScriptValidator(SaharaImageValidatorBase): "output": { "type": "string", "minLength": 1 + }, + "inline": { + "type": "string", + "minLength": 1 } }, } @@ -502,6 +507,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase): """ jsonschema.validate(spec, cls.SPEC_SCHEMA) + script_contents = None if isinstance(spec, six.string_types): script_path = spec env_vars, output_var = cls._DEFAULT_ENV_VARS, None @@ -509,13 +515,14 @@ class SaharaScriptValidator(SaharaImageValidatorBase): script_path, properties = list(six.iteritems(spec))[0] env_vars = cls._DEFAULT_ENV_VARS + properties.get('env_vars', []) output_var = properties.get('output', None) + script_contents = properties.get('inline') - script_contents = None - for root in resource_roots: - file_path = path.join(root, script_path) - script_contents = files.try_get_file_text(file_path) - if script_contents: - break + if not script_contents: + for root in resource_roots: + file_path = path.join(root, script_path) + script_contents = files.try_get_file_text(file_path) + if script_contents: + break if not script_contents: raise p_ex.ImageValidationSpecificationError( diff --git a/sahara/tests/unit/cli/image_pack/__init__.py b/sahara/tests/unit/cli/image_pack/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sahara/tests/unit/cli/image_pack/test_image_pack_api.py b/sahara/tests/unit/cli/image_pack/test_image_pack_api.py new file mode 100644 index 0000000000..1317c59cca --- /dev/null +++ b/sahara/tests/unit/cli/image_pack/test_image_pack_api.py @@ -0,0 +1,56 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import sys +guestfs = mock.Mock() +sys.modules['guestfs'] = guestfs + +from sahara.cli.image_pack import api +from sahara.tests.unit import base + + +class TestSaharaImagePackAPI(base.SaharaTestCase): + + def setUp(self): + super(TestSaharaImagePackAPI, self).setUp() + + def tearDown(self): + super(TestSaharaImagePackAPI, self).tearDown() + + @mock.patch('sahara.cli.image_pack.api.guestfs') + @mock.patch('sahara.cli.image_pack.api.plugins_base') + @mock.patch('sahara.cli.image_pack.api.LOG') + def test_pack_image_call(self, mock_log, mock_plugins_base, mock_guestfs): + guest = mock.Mock() + mock_guestfs.GuestFS = mock.Mock(return_value=guest) + guest.inspect_os = mock.Mock(return_value=['/dev/something1']) + plugin = mock.Mock() + mock_plugins_base.PLUGINS = mock.Mock( + get_plugin=mock.Mock(return_value=plugin)) + + api.pack_image( + "plugin_name", "plugin_version", "image_path", + root_drive=None, test_only=False) + + guest.add_drive_opts.assert_called_with("image_path", format="qcow2") + guest.set_network.assert_called_with(True) + guest.launch.assert_called_once() + guest.mount.assert_called_with('/dev/something1', '/') + guest.sh.assert_called_with("echo Testing sudo without tty...") + guest.sync.assert_called_once() + guest.umount_all.assert_called_once() + guest.close.assert_called_once() diff --git a/sahara/utils/remote.py b/sahara/utils/remote.py index b6ad18d35d..47c3e8df37 100644 --- a/sahara/utils/remote.py +++ b/sahara/utils/remote.py @@ -71,7 +71,24 @@ class RemoteDriver(object): @six.add_metaclass(abc.ABCMeta) -class Remote(object): +class TerminalOnlyRemote(object): + + @abc.abstractmethod + def execute_command(self, cmd, run_as_root=False, get_stderr=False, + raise_when_error=True, timeout=300): + """Execute specified command remotely using existing ssh connection. + + Return exit code, stdout data and stderr data of the executed command. + """ + + @abc.abstractmethod + def get_os_distrib(self): + """Returns the OS distribution running on the target machine.""" + + +@six.add_metaclass(abc.ABCMeta) +class Remote(TerminalOnlyRemote): + @abc.abstractmethod def get_neutron_info(self): """Returns dict which later could be passed to get_http_client.""" @@ -84,14 +101,6 @@ class Remote(object): def close_http_session(self, port): """Closes cached HTTP session for a given instance's port.""" - @abc.abstractmethod - def execute_command(self, cmd, run_as_root=False, get_stderr=False, - raise_when_error=True, timeout=300): - """Execute specified command remotely using existing ssh connection. - - Return exit code, stdout data and stderr data of the executed command. - """ - @abc.abstractmethod def write_file_to(self, remote_file, data, run_as_root=False, timeout=120): """Create remote file and write the given data to it. diff --git a/setup.cfg b/setup.cfg index f999b5464c..6cc0fc178f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,7 @@ console_scripts = sahara-rootwrap = oslo_rootwrap.cmd:main _sahara-subprocess = sahara.cli.sahara_subprocess:main sahara-templates = sahara.db.templates.cli:main + sahara-image-pack = sahara.cli.image_pack.cli:main wsgi_scripts = sahara-wsgi-api = sahara.cli.sahara_api:setup_api diff --git a/tox.ini b/tox.ini index 96face0806..5cfc064238 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,10 @@ commands = [testenv:venv] commands = {posargs} +[testenv:images] +sitepackages = True +commands = {posargs} + [testenv:docs] commands = rm -rf doc/html doc/build