Move console scripts to entrypoints.

As part of the move of plugins to entrypoints, take advantage of the
entrypoints based console scripts, which will make our command line scripts
available for unittesting.

Part of blueprint entrypoints-plugins

Co-authored-by: Michael Still <mikal@stillhq.com>

Change-Id: I5f17348b7b3cc896c92263dd518abb128757d81f

nova/cmd/baremetal_deploy_helper.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2012 NTT DOCOMO, INC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for Bare-Metal Deployment Service."""
+
+
+import os
+import sys
+import threading
+import time
+
+import cgi
+import Queue
+import re
+import socket
+import stat
+from wsgiref import simple_server
+
+from nova import config
+from nova import context as nova_context
+from nova import exception
+from nova.openstack.common import log as logging
+from nova import utils
+from nova.virt.baremetal import baremetal_states
+from nova.virt.baremetal import db
+
+
+LOG = logging.getLogger('nova.virt.baremetal.deploy_helper')
+
+QUEUE = Queue.Queue()
+
+
+# All functions are called from deploy() directly or indirectly.
+# They are split for stub-out.
+
+def discovery(portal_address, portal_port):
+    """Do iSCSI discovery on portal."""
+    utils.execute('iscsiadm',
+                  '-m', 'discovery',
+                  '-t', 'st',
+                  '-p', '%s:%s' % (portal_address, portal_port),
+                  run_as_root=True,
+                  check_exit_code=[0])
+
+
+def login_iscsi(portal_address, portal_port, target_iqn):
+    """Login to an iSCSI target."""
+    utils.execute('iscsiadm',
+                  '-m', 'node',
+                  '-p', '%s:%s' % (portal_address, portal_port),
+                  '-T', target_iqn,
+                  '--login',
+                  run_as_root=True,
+                  check_exit_code=[0])
+    # Ensure the login complete
+    time.sleep(3)
+
+
+def logout_iscsi(portal_address, portal_port, target_iqn):
+    """Logout from an iSCSI target."""
+    utils.execute('iscsiadm',
+                  '-m', 'node',
+                  '-p', '%s:%s' % (portal_address, portal_port),
+                  '-T', target_iqn,
+                  '--logout',
+                  run_as_root=True,
+                  check_exit_code=[0])
+
+
+def make_partitions(dev, root_mb, swap_mb):
+    """Create partitions for root and swap on a disk device."""
+    # Lead in with 1MB to allow room for the partition table itself, otherwise
+    # the way sfdisk adjusts doesn't shift the partition up to compensate, and
+    # we lose the space.
+    # http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/raring/util-linux/
+    # raring/view/head:/fdisk/sfdisk.c#L1940
+    stdin_command = ('1 %d 83;\n- %d 82;\n0 0;\n0 0;\n' % (root_mb, swap_mb))
+    utils.execute('sfdisk', '-uM', dev, process_input=stdin_command,
+            run_as_root=True,
+            check_exit_code=[0])
+    # avoid "device is busy"
+    time.sleep(3)
+
+
+def is_block_device(dev):
+    """Check whether a device is block or not."""
+    s = os.stat(dev)
+    return stat.S_ISBLK(s.st_mode)
+
+
+def dd(src, dst):
+    """Execute dd from src to dst."""
+    utils.execute('dd',
+                  'if=%s' % src,
+                  'of=%s' % dst,
+                  'bs=1M',
+                  'oflag=direct',
+                  run_as_root=True,
+                  check_exit_code=[0])
+
+
+def mkswap(dev, label='swap1'):
+    """Execute mkswap on a device."""
+    utils.execute('mkswap',
+                  '-L', label,
+                  dev,
+                  run_as_root=True,
+                  check_exit_code=[0])
+
+
+def block_uuid(dev):
+    """Get UUID of a block device."""
+    out, _ = utils.execute('blkid', '-s', 'UUID', '-o', 'value', dev,
+                           run_as_root=True,
+                           check_exit_code=[0])
+    return out.strip()
+
+
+def switch_pxe_config(path, root_uuid):
+    """Switch a pxe config from deployment mode to service mode."""
+    with open(path) as f:
+        lines = f.readlines()
+    root = 'UUID=%s' % root_uuid
+    rre = re.compile(r'\$\{ROOT\}')
+    dre = re.compile('^default .*$')
+    with open(path, 'w') as f:
+        for line in lines:
+            line = rre.sub(root, line)
+            line = dre.sub('default boot', line)
+            f.write(line)
+
+
+def notify(address, port):
+    """Notify a node that it becomes ready to reboot."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.connect((address, port))
+        s.send('done')
+    finally:
+        s.close()
+
+
+def get_dev(address, port, iqn, lun):
+    """Returns a device path for given parameters."""
+    dev = "/dev/disk/by-path/ip-%s:%s-iscsi-%s-lun-%s" \
+            % (address, port, iqn, lun)
+    return dev
+
+
+def get_image_mb(image_path):
+    """Get size of an image in Megabyte."""
+    mb = 1024 * 1024
+    image_byte = os.path.getsize(image_path)
+    # round up size to MB
+    image_mb = int((image_byte + mb - 1) / mb)
+    return image_mb
+
+
+def work_on_disk(dev, root_mb, swap_mb, image_path):
+    """Creates partitions and write an image to the root partition."""
+    root_part = "%s-part1" % dev
+    swap_part = "%s-part2" % dev
+
+    if not is_block_device(dev):
+        LOG.warn("parent device '%s' not found", dev)
+        return
+    make_partitions(dev, root_mb, swap_mb)
+    if not is_block_device(root_part):
+        LOG.warn("root device '%s' not found", root_part)
+        return
+    if not is_block_device(swap_part):
+        LOG.warn("swap device '%s' not found", swap_part)
+        return
+    dd(image_path, root_part)
+    mkswap(swap_part)
+    root_uuid = block_uuid(root_part)
+    return root_uuid
+
+
+def deploy(address, port, iqn, lun, image_path, pxe_config_path,
+           root_mb, swap_mb):
+    """All-in-one function to deploy a node."""
+    dev = get_dev(address, port, iqn, lun)
+    image_mb = get_image_mb(image_path)
+    if image_mb > root_mb:
+        root_mb = image_mb
+    discovery(address, port)
+    login_iscsi(address, port, iqn)
+    try:
+        root_uuid = work_on_disk(dev, root_mb, swap_mb, image_path)
+    except exception.ProcessExecutionError, err:
+        # Log output if there was a error
+        LOG.error("Cmd     : %s" % err.cmd)
+        LOG.error("StdOut  : %s" % err.stdout)
+        LOG.error("StdErr  : %s" % err.stderr)
+    finally:
+        logout_iscsi(address, port, iqn)
+    switch_pxe_config(pxe_config_path, root_uuid)
+    # Ensure the node started netcat on the port after POST the request.
+    time.sleep(3)
+    notify(address, 10000)
+
+
+class Worker(threading.Thread):
+    """Thread that handles requests in queue."""
+
+    def __init__(self):
+        super(Worker, self).__init__()
+        self.setDaemon(True)
+        self.stop = False
+        self.queue_timeout = 1
+
+    def run(self):
+        while not self.stop:
+            try:
+                # Set timeout to check self.stop periodically
+                (node_id, params) = QUEUE.get(block=True,
+                                                    timeout=self.queue_timeout)
+            except Queue.Empty:
+                pass
+            else:
+                # Requests comes here from BareMetalDeploy.post()
+                LOG.info(_('start deployment for node %(node_id)s, '
+                           'params %(params)s') % locals())
+                context = nova_context.get_admin_context()
+                try:
+                    db.bm_node_update(context, node_id,
+                          {'task_state': baremetal_states.DEPLOYING})
+                    deploy(**params)
+                except Exception:
+                    LOG.error(_('deployment to node %s failed') % node_id)
+                    db.bm_node_update(context, node_id,
+                          {'task_state': baremetal_states.DEPLOYFAIL})
+                else:
+                    LOG.info(_('deployment to node %s done') % node_id)
+                    db.bm_node_update(context, node_id,
+                          {'task_state': baremetal_states.DEPLOYDONE})
+
+
+class BareMetalDeploy(object):
+    """WSGI server for bare-metal deployment."""
+
+    def __init__(self):
+        self.worker = Worker()
+        self.worker.start()
+
+    def __call__(self, environ, start_response):
+        method = environ['REQUEST_METHOD']
+        if method == 'POST':
+            return self.post(environ, start_response)
+        else:
+            start_response('501 Not Implemented',
+                           [('Content-type', 'text/plain')])
+            return 'Not Implemented'
+
+    def post(self, environ, start_response):
+        LOG.info("post: environ=%s", environ)
+        inpt = environ['wsgi.input']
+        length = int(environ.get('CONTENT_LENGTH', 0))
+
+        x = inpt.read(length)
+        q = dict(cgi.parse_qsl(x))
+        try:
+            node_id = q['i']
+            deploy_key = q['k']
+            address = q['a']
+            port = q.get('p', '3260')
+            iqn = q['n']
+            lun = q.get('l', '1')
+        except KeyError as e:
+            start_response('400 Bad Request', [('Content-type', 'text/plain')])
+            return "parameter '%s' is not defined" % e
+
+        context = nova_context.get_admin_context()
+        d = db.bm_node_get(context, node_id)
+
+        if d['deploy_key'] != deploy_key:
+            start_response('400 Bad Request', [('Content-type', 'text/plain')])
+            return 'key is not match'
+
+        params = {'address': address,
+                  'port': port,
+                  'iqn': iqn,
+                  'lun': lun,
+                  'image_path': d['image_path'],
+                  'pxe_config_path': d['pxe_config_path'],
+                  'root_mb': int(d['root_mb']),
+                  'swap_mb': int(d['swap_mb']),
+                 }
+        # Restart worker, if needed
+        if not self.worker.isAlive():
+            self.worker = Worker()
+            self.worker.start()
+        LOG.info("request is queued: node %s, params %s", node_id, params)
+        QUEUE.put((node_id, params))
+        # Requests go to Worker.run()
+        start_response('200 OK', [('Content-type', 'text/plain')])
+        return ''
+
+
+def main():
+    config.parse_args(sys.argv)
+    logging.setup("nova")
+    app = BareMetalDeploy()
+    srv = simple_server.make_server('', 10000, app)
+    srv.serve_forever()

nova/cmd/baremetal_manage.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Interactive shell based on Django:
+#
+# Copyright (c) 2005, the Lawrence Journal-World
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     1. Redistributions of source code must retain the above copyright notice,
+#        this list of conditions and the following disclaimer.
+#
+#     2. Redistributions in binary form must reproduce the above copyright
+#        notice, this list of conditions and the following disclaimer in the
+#        documentation and/or other materials provided with the distribution.
+#
+#     3. Neither the name of Django nor the names of its contributors may be
+#        used to endorse or promote products derived from this software without
+#        specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+"""
+  CLI interface for nova bare-metal management.
+"""
+
+import gettext
+import os
+import sys
+
+from oslo.config import cfg
+
+gettext.install('nova', unicode=1)
+
+from nova import config
+from nova.openstack.common import cliutils
+from nova.openstack.common import log as logging
+from nova import version
+from nova.virt.baremetal.db import migration as bmdb_migration
+
+CONF = cfg.CONF
+
+
+# Decorators for actions
+def args(*args, **kwargs):
+    def _decorator(func):
+        func.__dict__.setdefault('args', []).insert(0, (args, kwargs))
+        return func
+    return _decorator
+
+
+class BareMetalDbCommands(object):
+    """Class for managing the bare-metal database."""
+
+    def __init__(self):
+        pass
+
+    @args('--version', dest='version', metavar='<version>',
+            help='Bare-metal Database version')
+    def sync(self, version=None):
+        """Sync the database up to the most recent version."""
+        bmdb_migration.db_sync(version)
+
+    def version(self):
+        """Print the current database version."""
+        v = bmdb_migration.db_version()
+        print(v)
+        # return for unittest
+        return v
+
+
+CATEGORIES = {
+    'db': BareMetalDbCommands,
+}
+
+
+def methods_of(obj):
+    """Get all callable methods of an object that don't start with underscore
+    returns a list of tuples of the form (method_name, method)"""
+    result = []
+    for i in dir(obj):
+        if callable(getattr(obj, i)) and not i.startswith('_'):
+            result.append((i, getattr(obj, i)))
+    return result
+
+
+def add_command_parsers(subparsers):
+    parser = subparsers.add_parser('bash-completion')
+    parser.add_argument('query_category', nargs='?')
+
+    for category in CATEGORIES:
+        command_object = CATEGORIES[category]()
+
+        parser = subparsers.add_parser(category)
+        parser.set_defaults(command_object=command_object)
+
+        category_subparsers = parser.add_subparsers(dest='action')
+
+        for (action, action_fn) in methods_of(command_object):
+            parser = category_subparsers.add_parser(action)
+
+            action_kwargs = []
+            for args, kwargs in getattr(action_fn, 'args', []):
+                action_kwargs.append(kwargs['dest'])
+                kwargs['dest'] = 'action_kwarg_' + kwargs['dest']
+                parser.add_argument(*args, **kwargs)
+
+            parser.set_defaults(action_fn=action_fn)
+            parser.set_defaults(action_kwargs=action_kwargs)
+
+            parser.add_argument('action_args', nargs='*')
+
+
+category_opt = cfg.SubCommandOpt('category',
+                                 title='Command categories',
+                                 help='Available categories',
+                                 handler=add_command_parsers)
+
+
+def main():
+    """Parse options and call the appropriate class/method."""
+    CONF.register_cli_opt(category_opt)
+    try:
+        config.parse_args(sys.argv)
+        logging.setup("nova")
+    except cfg.ConfigFilesNotFoundError:
+        cfgfile = CONF.config_file[-1] if CONF.config_file else None
+        if cfgfile and not os.access(cfgfile, os.R_OK):
+            st = os.stat(cfgfile)
+            print(_("Could not read %s. Re-running with sudo") % cfgfile)
+            try:
+                os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv)
+            except Exception:
+                print(_('sudo failed, continuing as if nothing happened'))
+
+        print(_('Please re-run nova-manage as root.'))
+        return(2)
+
+    if CONF.category.name == "version":
+        print(version.version_string_with_package())
+        return(0)
+
+    if CONF.category.name == "bash-completion":
+        if not CONF.category.query_category:
+            print(" ".join(CATEGORIES.keys()))
+        elif CONF.category.query_category in CATEGORIES:
+            fn = CATEGORIES[CONF.category.query_category]
+            command_object = fn()
+            actions = methods_of(command_object)
+            print(" ".join([k for (k, v) in actions]))
+        return(0)
+
+    fn = CONF.category.action_fn
+    fn_args = [arg.decode('utf-8') for arg in CONF.category.action_args]
+    fn_kwargs = {}
+    for k in CONF.category.action_kwargs:
+        v = getattr(CONF.category, 'action_kwarg_' + k)
+        if v is None:
+            continue
+        if isinstance(v, basestring):
+            v = v.decode('utf-8')
+        fn_kwargs[k] = v
+
+    # call the action with the remaining arguments
+    # check arguments
+    try:
+        cliutils.validate_args(fn, *fn_args, **fn_kwargs)
+    except cliutils.MissingArgs as e:
+        print(fn.__doc__)
+        parser.print_help()
+        print(e)
+        return(1)
+    try:
+        fn(*fn_args, **fn_kwargs)
+        return(0)
+    except Exception:
+        print(_("Command failed, please check log for more info"))
+        raise

nova/tests/baremetal/test_nova_baremetal_deploy_helper.py

@@ -16,29 +16,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import imp
 import os
-import sys
 import tempfile
 import time
 
 import mox
 
+from nova.cmd import baremetal_deploy_helper as bmdh
 from nova import test
 from nova.tests.baremetal.db import base as bm_db_base
 from nova.virt.baremetal import db as bm_db
 
-TOPDIR = os.path.normpath(os.path.join(
-                            os.path.dirname(os.path.abspath(__file__)),
-                            os.pardir,
-                            os.pardir,
-                            os.pardir))
-BMDH_PATH = os.path.join(TOPDIR, 'bin', 'nova-baremetal-deploy-helper')
-
-sys.dont_write_bytecode = True
-bmdh = imp.load_source('bmdh', BMDH_PATH)
-sys.dont_write_bytecode = False
-
 _PXECONF_DEPLOY = """
 default deploy
 

nova/tests/baremetal/test_nova_baremetal_manage.py

@@ -16,23 +16,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import imp
+from nova.cmd import baremetal_manage as bm_man
-import os
-import sys
-
 from nova.tests.baremetal.db import base as bm_db_base
 
-TOPDIR = os.path.normpath(os.path.join(
-                            os.path.dirname(os.path.abspath(__file__)),
-                            os.pardir,
-                            os.pardir,
-                            os.pardir))
-BM_MAN_PATH = os.path.join(TOPDIR, 'bin', 'nova-baremetal-manage')
-
-sys.dont_write_bytecode = True
-bm_man = imp.load_source('bm_man', BM_MAN_PATH)
-sys.dont_write_bytecode = False
-
 
 class BareMetalDbCommandsTestCase(bm_db_base.BMDBTestCase):
     def setUp(self):