add SmartOS datasource supporting Joyent cloud

This adds a datasource designed to work on Joyent cloud (SmartOS).

cloudinit/settings.py

@@ -37,6 +37,7 @@ CFG_BUILTIN = {
         'MAAS',
         'Ec2',
         'CloudStack',
+        'SmartOS',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],

cloudinit/sources/DataSourceSmartOS.py

@@ -0,0 +1,190 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2013 Canonical Ltd.
+#
+#    Author: Ben Howard <ben.howard@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+#    Datasource for provisioning on SmartOS. This works on Joyent
+#        and public/private Clouds using SmartOS.
+#
+#    SmartOS hosts use a serial console (/dev/ttyS1) on Linux Guests.
+#        The meta-data is transmitted via key/value pairs made by
+#        requests on the console. For example, to get the hostname, you
+#        would send "GET hostname" on /dev/ttyS1.
+#
+
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+import os
+import os.path
+import serial
+
+
+DEF_TTY_LOC = '/dev/ttyS1'
+DEF_TTY_TIMEOUT = 60
+LOG = logging.getLogger(__name__)
+
+SMARTOS_ATTRIB_MAP = {
+    #Cloud-init Key : (SmartOS Key, Strip line endings)
+    'local-hostname': ('hostname', True),
+    'public-keys': ('root_authorized_keys', True),
+    'user-script': ('user-script', False),
+    'user-data': ('user-data', False),
+    'iptables_disable': ('iptables_disable', True),
+    'motd_sys_info': ('motd_sys_info', True),
+}
+
+
+class DataSourceSmartOS(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
+        self.is_smartdc = None
+        self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
+        self.seed_timeout = self.sys_cfg.get("serial_timeout",
+                                             DEF_TTY_TIMEOUT)
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
+
+    def get_data(self):
+        md = {}
+        ud = ""
+
+        if not os.path.exists(self.seed):
+            LOG.debug("Host does not appear to be on SmartOS")
+            return False
+        self.seed = self.seed
+
+        system_uuid, system_type = dmi_data()
+        if 'smartdc' not in system_type.lower():
+            LOG.debug("Host is not on SmartOS")
+            return False
+        self.is_smartdc = True
+        md['instance-id'] = system_uuid
+
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
+            smartos_noun, strip = attribute
+            md[ci_noun] = query_data(smartos_noun, self.seed,
+                                     self.seed_timeout, strip=strip)
+
+        if not md['local-hostname']:
+            md['local-hostname'] = system_uuid
+
+        if md['user-data']:
+            ud = md['user-data']
+        else:
+            ud = md['user-script']
+
+        self.metadata = md
+        self.userdata_raw = ud
+        return True
+
+    def get_instance_id(self):
+        return self.metadata['instance-id']
+
+
+def get_serial(seed_device, seed_timeout):
+    """This is replaced in unit testing, allowing us to replace
+        serial.Serial with a mocked class
+
+        The timeout value of 60 seconds should never be hit. The value
+        is taken from SmartOS own provisioning tools. Since we are reading
+        each line individually up until the single ".", the transfer is
+        usually very fast (i.e. microseconds) to get the response.
+    """
+    if not seed_device:
+        raise AttributeError("seed_device value is not set")
+
+    ser = serial.Serial(seed_device, timeout=seed_timeout)
+    if not ser.isOpen():
+        raise SystemError("Unable to open %s" % seed_device)
+
+    return ser
+
+
+def query_data(noun, seed_device, seed_timeout, strip=False):
+    """Makes a request to via the serial console via "GET <NOUN>"
+
+        In the response, the first line is the status, while subsequent lines
+        are is the value. A blank line with a "." is used to indicate end of
+        response.
+    """
+
+    if not noun:
+        return False
+
+    ser = get_serial(seed_device, seed_timeout)
+    ser.write("GET %s\n" % noun.rstrip())
+    status = str(ser.readline()).rstrip()
+    response = []
+    eom_found = False
+
+    if 'SUCCESS' not in status:
+        ser.close()
+        return None
+
+    while not eom_found:
+        m = ser.readline()
+        if m.rstrip() == ".":
+            eom_found = True
+        else:
+            response.append(m)
+
+    ser.close()
+    if not strip:
+        return "".join(response)
+    else:
+        return "".join(response).rstrip()
+
+    return None
+
+
+def dmi_data():
+    sys_uuid, sys_type = None, None
+    dmidecode_path = util.which('dmidecode')
+    if not dmidecode_path:
+        return False
+
+    sys_uuid_cmd = [dmidecode_path, "-s", "system-uuid"]
+    try:
+        LOG.debug("Getting hostname from dmidecode")
+        (sys_uuid, _err) = util.subp(sys_uuid_cmd)
+    except Exception as e:
+        util.logexc(LOG, "Failed to get system UUID", e)
+
+    sys_type_cmd = [dmidecode_path, "-s", "system-product-name"]
+    try:
+        LOG.debug("Determining hypervisor product name via dmidecode")
+        (sys_type, _err) = util.subp(sys_type_cmd)
+    except Exception as e:
+        util.logexc(LOG, "Failed to get system UUID", e)
+
+    return sys_uuid.lower(), sys_type
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)

cloudinit/util.py

@@ -1751,3 +1751,22 @@ def get_mount_info(path, log=LOG):
     mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
     lines = load_file(mountinfo_path).splitlines()
     return parse_mount_info(path, lines, log)
+
+
+def which(program):
+    # Return path of program for execution if found in path
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    _fpath, _ = os.path.split(program)
+    if _fpath:
+        if is_exe(program):
+            return program
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            exe_file = os.path.join(path, program)
+            if is_exe(exe_file):
+                return exe_file
+
+    return None

doc/examples/cloud-config-datasources.txt

@@ -46,3 +46,9 @@ datasource:
   Azure:
     agent_command: [service, walinuxagent, start]
 
+  SmartOS:
+    # Smart OS datasource works over a serial console interacting with
+    # a server on the other end. By default, the second serial console is the
+    # device. SmartOS also uses a serial timeout of 60 seconds.
+    serial device: /dev/ttyS1
+    serial timeout: 60

tests/unittests/test_datasource/test_smartos.py

@@ -0,0 +1,191 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2013 Canonical Ltd.
+#
+#    Author: Ben Howard <ben.howard@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+#   This is a testcase for the SmartOS datasource. It replicates a serial
+#   console and acts like the SmartOS console does in order to validate
+#   return responses.
+#
+
+from cloudinit import helpers
+from cloudinit.sources import DataSourceSmartOS
+
+from mocker import MockerTestCase
+import uuid
+
+mock_returns = {
+    'hostname': 'test-host',
+    'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
+    'disable_iptables_flag': None,
+    'enable_motd_sys_info': None,
+    'system_uuid': str(uuid.uuid4()),
+    'smartdc': 'smartdc',
+    'userdata': """
+#!/bin/sh
+/bin/true
+""",
+}
+
+
+class MockSerial(object):
+    """Fake a serial terminal for testing the code that
+        interfaces with the serial"""
+
+    port = None
+
+    def __init__(self):
+        self.last = None
+        self.last = None
+        self.new = True
+        self.count = 0
+        self.mocked_out = []
+
+    def open(self):
+        return True
+
+    def close(self):
+        return True
+
+    def isOpen(self):
+        return True
+
+    def write(self, line):
+        line = line.replace('GET ', '')
+        self.last = line.rstrip()
+
+    def readline(self):
+        if self.new:
+            self.new = False
+            if self.last in mock_returns:
+                return 'SUCCESS\n'
+            else:
+                return 'NOTFOUND %s\n' % self.last
+
+        if self.last in mock_returns:
+            if not self.mocked_out:
+                self.mocked_out = [x for x in self._format_out()]
+                print self.mocked_out
+
+            if len(self.mocked_out) > self.count:
+                self.count += 1
+                return self.mocked_out[self.count - 1]
+
+    def _format_out(self):
+        if self.last in mock_returns:
+            try:
+                for l in mock_returns[self.last].splitlines():
+                    yield "%s\n" % l
+            except:
+                yield "%s\n" % mock_returns[self.last]
+
+            yield '\n'
+            yield '.'
+
+
+class TestSmartOSDataSource(MockerTestCase):
+    def setUp(self):
+        # makeDir comes from MockerTestCase
+        self.tmp = self.makeDir()
+
+        # patch cloud_dir, so our 'seed_dir' is guaranteed empty
+        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+
+        self.unapply = []
+        super(TestSmartOSDataSource, self).setUp()
+
+    def tearDown(self):
+        apply_patches([i for i in reversed(self.unapply)])
+        super(TestSmartOSDataSource, self).tearDown()
+
+    def apply_patches(self, patches):
+        ret = apply_patches(patches)
+        self.unapply += ret
+
+    def _get_ds(self):
+
+        def _get_serial(*_):
+            return MockSerial()
+
+        def _dmi_data():
+            return mock_returns['system_uuid'], 'smartdc'
+
+        data = {'sys_cfg': {}}
+        mod = DataSourceSmartOS
+        self.apply_patches([(mod, 'get_serial', _get_serial)])
+        self.apply_patches([(mod, 'dmi_data', _dmi_data)])
+        dsrc = mod.DataSourceSmartOS(
+            data.get('sys_cfg', {}), distro=None, paths=self.paths)
+        return dsrc
+
+    def test_seed(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals('/dev/ttyS1', dsrc.seed)
+
+    def test_issmartdc(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertTrue(dsrc.is_smartdc)
+
+    def test_uuid(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(mock_returns['system_uuid'],
+                          dsrc.metadata['instance-id'])
+
+    def test_root_keys(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(mock_returns['root_authorized_keys'],
+                          dsrc.metadata['public-keys'])
+
+    def test_hostname(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(mock_returns['hostname'],
+                          dsrc.metadata['local-hostname'])
+
+    def test_disable_iptables_flag(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(mock_returns['disable_iptables_flag'],
+                          dsrc.metadata['iptables_disable'])
+
+    def test_motd_sys_info(self):
+        dsrc = self._get_ds()
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(mock_returns['enable_motd_sys_info'],
+                          dsrc.metadata['motd_sys_info'])
+
+
+def apply_patches(patches):
+    ret = []
+    for (ref, name, replace) in patches:
+        if replace is None:
+            continue
+        orig = getattr(ref, name)
+        setattr(ref, name, replace)
+        ret.append((ref, name, orig))
+    return ret