CLI for Plugin-Declared Image Declaration

Introduces a new libfuestfs-driven CLI for packing of
Sahara images, using the same recipe definition scheme
introduced in the validate-image-spi blueprint.

Documentation and plugin image packing configurability
can and will be provided in separate patches for ease
of review.

Partially-implements: blueprint image-generation-cli
Change-Id: I6788108e3fb6232045fc56937639a6348768a7bc

sahara/cli/image_pack/api.py

@@ -0,0 +1,113 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara import conductor  # noqa
+from sahara.i18n import _LE
+from sahara.i18n import _LI
+from sahara.plugins import base as plugins_base
+from sahara.utils import remote
+
+try:
+    import guestfs
+except ImportError:
+    raise Exception(_LE("The image packing API depends on the system package "
+                        "python-libguestfs (and libguestfs itself.) Please "
+                        "install these packages to proceed."))
+
+
+LOG = None
+CONF = None
+
+
+# This is broken out to support testability
+def set_logger(log):
+    global LOG
+    LOG = log
+
+
+# This is broken out to support testability
+def set_conf(conf):
+    global CONF
+    CONF = conf
+
+
+# This is a local exception class that is used to exit routines
+# in cases where error information has already been logged.
+# It is caught and suppressed everywhere it is used.
+class Handled(Exception):
+    pass
+
+
+class Context(object):
+    '''Create a pseudo Context object
+
+    Since this tool does not use the REST interface, we
+    do not have a request from which to build a Context.
+    '''
+    def __init__(self, is_admin=False, tenant_id=None):
+        self.is_admin = is_admin
+        self.tenant_id = tenant_id
+
+
+class ImageRemote(remote.TerminalOnlyRemote):
+
+    def __init__(self, image_path, root_drive):
+        guest = guestfs.GuestFS(python_return_dict=True)
+        guest.add_drive_opts(image_path, format="qcow2")
+        guest.set_network(True)
+        self.guest = guest
+        self.root_drive = root_drive
+
+    def __enter__(self):
+        self.guest.launch()
+        if not self.root_drive:
+            self.root_drive = self.guest.inspect_os()[0]
+        self.guest.mount(self.root_drive, '/')
+        try:
+            cmd = "echo Testing sudo without tty..."
+            self.execute_command(cmd, run_as_root=True)
+        except RuntimeError:
+            cmd = "sed -i 's/requiretty/!requiretty/' /etc/sudoers"
+            self.guest.execute_command(cmd)
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.guest.sync()
+        self.guest.umount_all()
+        self.guest.close()
+
+    def execute_command(self, cmd, run_as_root=False, get_stderr=False,
+                        raise_when_error=True, timeout=300):
+        try:
+            LOG.info(_LI("Issuing command: {cmd}").format(cmd=cmd))
+            stdout = self.guest.sh(cmd)
+            LOG.info(_LI("Received response: {stdout}").format(stdout=stdout))
+            return 0, stdout
+        except RuntimeError as ex:
+            if raise_when_error:
+                raise ex
+            else:
+                return 1, ex.message
+
+    def get_os_distrib(self):
+        return self.guest.inspect_get_distro()
+
+
+def pack_image(plugin_name, plugin_version, image_path, root_drive=None,
+               test_only=False, **kwargs):
+    plugins_base.setup_plugins()
+    with ImageRemote(image_path, root_drive) as image_remote:
+        plugin = plugins_base.PLUGINS.get_plugin(plugin_name)
+        reconcile = not test_only
+        plugin.pack_image(image_remote, reconcile=reconcile, env_map=kwargs)

sahara/cli/image_pack/cli.py

@@ -0,0 +1,93 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from oslo_config import cfg
+from oslo_log import log
+
+from sahara.cli.image_pack import api
+from sahara.i18n import _LI
+
+LOG = log.getLogger(__name__)
+
+CONF = cfg.CONF
+
+
+CONF.register_cli_opts([
+    cfg.StrOpt(
+        'plugin',
+        required=True,
+        help="The name of the Sahara plugin for which you would like to "
+             "generate an image. Use sahara-image-create -p PLUGIN -h to "
+             "see a set of versions for a specific plugin."),
+    cfg.StrOpt(
+        'plugin-version',
+        dest='plugin_version',
+        required=True,
+        help="The version of the Sahara plugin for which you would like to "
+             "generate an image. Use sahara-image-create -p PLUGIN -v "
+             "VERSION -h to see a full set of arguments for a specific plugin "
+             "and version."),
+    cfg.StrOpt(
+        'image',
+        required=True,
+        help="The path to an image to modify. This image will be modified "
+             "in-place: be sure to target a copy if you wish to maintain a "
+             "clean master image."),
+    cfg.StrOpt(
+        'root-filesystem',
+        dest='root_fs',
+        required=False,
+        help="The filesystem to mount as the root volume on the image. No"
+             "value is required if only one filesystem is detected."),
+    cfg.BoolOpt(
+        'test-only',
+        dest='test_only',
+        default=False,
+        help="If this flag is set, no changes will be made to the image; "
+             "instead, the script will fail if discrepancies are found "
+             "between the image and the intended state."),
+])
+
+
+def unregister_extra_cli_opt(name):
+    try:
+        for cli in CONF._cli_opts:
+            if cli['opt'].name == name:
+                CONF.unregister_opt(cli['opt'])
+    except Exception:
+        pass
+
+
+for extra_opt in ["log-exchange", "host", "port"]:
+    unregister_extra_cli_opt(extra_opt)
+
+
+def main():
+    CONF(project='sahara')
+
+    CONF.reload_config_files()
+    log.setup(CONF, "sahara")
+
+    LOG.info(_LI("Command: {command}").format(command=' '.join(sys.argv)))
+
+    api.set_logger(LOG)
+    api.set_conf(CONF)
+
+    api.pack_image(CONF.plugin, CONF.plugin_version, CONF.image,
+                   CONF.root_fs, CONF.test_only)
+
+    LOG.info(_LI("Finished packing image for {plugin} at version {version}"
+                 ).format(plugin=CONF.plugin, version=CONF.plugin_version))

sahara/plugins/images.py

@@ -414,7 +414,8 @@ class SaharaPackageValidator(SaharaImageValidatorBase):
                 _("Unknown distro: cannot verify or install packages."))
         try:
             check(self, remote)
-        except (ex.SubprocessException, ex.RemoteCommandException):
+        except (ex.SubprocessException, ex.RemoteCommandException,
+                RuntimeError):
             if reconcile:
                 install(self, remote)
                 check(self, remote)
@@ -474,6 +475,10 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
                         "output": {
                             "type": "string",
                             "minLength": 1
+                        },
+                        "inline": {
+                            "type": "string",
+                            "minLength": 1
                         }
                     },
                 }
@@ -502,6 +507,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
         """
         jsonschema.validate(spec, cls.SPEC_SCHEMA)
 
+        script_contents = None
         if isinstance(spec, six.string_types):
             script_path = spec
             env_vars, output_var = cls._DEFAULT_ENV_VARS, None
@@ -509,13 +515,14 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
             script_path, properties = list(six.iteritems(spec))[0]
             env_vars = cls._DEFAULT_ENV_VARS + properties.get('env_vars', [])
             output_var = properties.get('output', None)
+            script_contents = properties.get('inline')
 
-        script_contents = None
-        for root in resource_roots:
-            file_path = path.join(root, script_path)
-            script_contents = files.try_get_file_text(file_path)
-            if script_contents:
-                break
+        if not script_contents:
+            for root in resource_roots:
+                file_path = path.join(root, script_path)
+                script_contents = files.try_get_file_text(file_path)
+                if script_contents:
+                    break
 
         if not script_contents:
             raise p_ex.ImageValidationSpecificationError(

sahara/tests/unit/cli/image_pack/test_image_pack_api.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import sys
+guestfs = mock.Mock()
+sys.modules['guestfs'] = guestfs
+
+from sahara.cli.image_pack import api
+from sahara.tests.unit import base
+
+
+class TestSaharaImagePackAPI(base.SaharaTestCase):
+
+    def setUp(self):
+        super(TestSaharaImagePackAPI, self).setUp()
+
+    def tearDown(self):
+        super(TestSaharaImagePackAPI, self).tearDown()
+
+    @mock.patch('sahara.cli.image_pack.api.guestfs')
+    @mock.patch('sahara.cli.image_pack.api.plugins_base')
+    @mock.patch('sahara.cli.image_pack.api.LOG')
+    def test_pack_image_call(self, mock_log, mock_plugins_base, mock_guestfs):
+        guest = mock.Mock()
+        mock_guestfs.GuestFS = mock.Mock(return_value=guest)
+        guest.inspect_os = mock.Mock(return_value=['/dev/something1'])
+        plugin = mock.Mock()
+        mock_plugins_base.PLUGINS = mock.Mock(
+            get_plugin=mock.Mock(return_value=plugin))
+
+        api.pack_image(
+            "plugin_name", "plugin_version", "image_path",
+            root_drive=None, test_only=False)
+
+        guest.add_drive_opts.assert_called_with("image_path", format="qcow2")
+        guest.set_network.assert_called_with(True)
+        guest.launch.assert_called_once()
+        guest.mount.assert_called_with('/dev/something1', '/')
+        guest.sh.assert_called_with("echo Testing sudo without tty...")
+        guest.sync.assert_called_once()
+        guest.umount_all.assert_called_once()
+        guest.close.assert_called_once()

sahara/utils/remote.py

@@ -71,7 +71,24 @@ class RemoteDriver(object):
 
 
 @six.add_metaclass(abc.ABCMeta)
-class Remote(object):
+class TerminalOnlyRemote(object):
+
+    @abc.abstractmethod
+    def execute_command(self, cmd, run_as_root=False, get_stderr=False,
+                        raise_when_error=True, timeout=300):
+        """Execute specified command remotely using existing ssh connection.
+
+        Return exit code, stdout data and stderr data of the executed command.
+        """
+
+    @abc.abstractmethod
+    def get_os_distrib(self):
+        """Returns the OS distribution running on the target machine."""
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Remote(TerminalOnlyRemote):
+
     @abc.abstractmethod
     def get_neutron_info(self):
         """Returns dict which later could be passed to get_http_client."""
@@ -84,14 +101,6 @@ class Remote(object):
     def close_http_session(self, port):
         """Closes cached HTTP session for a given instance's port."""
 
-    @abc.abstractmethod
-    def execute_command(self, cmd, run_as_root=False, get_stderr=False,
-                        raise_when_error=True, timeout=300):
-        """Execute specified command remotely using existing ssh connection.
-
-        Return exit code, stdout data and stderr data of the executed command.
-        """
-
     @abc.abstractmethod
     def write_file_to(self, remote_file, data, run_as_root=False, timeout=120):
         """Create remote file and write the given data to it.

setup.cfg

@@ -35,6 +35,7 @@ console_scripts =
     sahara-rootwrap = oslo_rootwrap.cmd:main
     _sahara-subprocess = sahara.cli.sahara_subprocess:main
     sahara-templates = sahara.db.templates.cli:main
+    sahara-image-pack = sahara.cli.image_pack.cli:main
 
 wsgi_scripts =
     sahara-wsgi-api = sahara.cli.sahara_api:setup_api

tox.ini

@@ -39,6 +39,10 @@ commands =
 [testenv:venv]
 commands = {posargs}
 
+[testenv:images]
+sitepackages = True
+commands = {posargs}
+
 [testenv:docs]
 commands =
     rm -rf doc/html doc/build