new: FreeBSD module to support cloud-init on the FBSD10 platform. In its

current form its still missing some modules though.

Supported:
-SSH-keys
-growpart
-growfs
-adduser
-powerstate

cloudinit/config/cc_growpart.py

@@ -22,6 +22,7 @@ import os
 import os.path
 import re
 import stat
+import sys
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_ALWAYS
@@ -137,6 +138,35 @@ class ResizeGrowPart(object):
 
         return (before, get_size(partdev))
 
+class ResizeGpart(object):
+    def available(self):
+        if not os.path.exists('/usr/local/sbin/gpart'):
+            return False
+        return True
+
+    def resize(self, diskdev, partnum, partdev):
+        """
+        GPT disks store metadata at the beginning (primary) and at the
+        end (secondary) of the disk. When launching an image with a
+        larger disk compared to the original image, the secondary copy
+        is lost. Thus, the metadata will be marked CORRUPT, and need to
+        be recovered.
+        """
+        try:
+            util.subp(["gpart", "recover", diskdev])
+        except util.ProcessExecutionError as e:
+            if e.exit_code != 0:
+                util.logexc(LOG, "Failed: gpart recover %s", diskdev)
+                raise ResizeFailedException(e)
+
+        before = get_size(partdev)
+        try:
+            util.subp(["gpart", "resize", "-i", partnum, diskdev])
+        except util.ProcessExecutionError as e:
+	    util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
+            raise ResizeFailedException(e)
+
+        return (before, get_size(partdev))
 
 def get_size(filename):
     fd = os.open(filename, os.O_RDONLY)
@@ -156,6 +186,12 @@ def device_part_info(devpath):
     bname = os.path.basename(rpath)
     syspath = "/sys/class/block/%s" % bname
 
+    # FreeBSD doesn't know of sysfs so just get everything we need from
+    # the device, like /dev/vtbd0p2.
+    if sys.platform.startswith('freebsd'):
+        m = re.search('^(/dev/.+)p([0-9])$', devpath)
+        return (m.group(1), m.group(2))
+
     if not os.path.exists(syspath):
         raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
 
@@ -206,7 +242,7 @@ def resize_devices(resizer, devices):
                          "stat of '%s' failed: %s" % (blockdev, e),))
             continue
 
-        if not stat.S_ISBLK(statret.st_mode):
+        if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
             info.append((devent, RESIZE.SKIPPED,
                          "device '%s' not a block device" % blockdev,))
             continue
@@ -281,4 +317,4 @@ def handle(_name, cfg, _cloud, log, _args):
 
 # LP: 1212444 FIXME re-order and favor ResizeParted
 #RESIZERS = (('growpart', ResizeGrowPart),)
-RESIZERS = (('growpart', ResizeGrowPart), ('parted', ResizeParted))
+RESIZERS = (('growpart', ResizeGrowPart), ('parted', ResizeParted), ('gpart', ResizeGpart))

cloudinit/config/cc_power_state_change.py

@@ -23,12 +23,34 @@ import errno
 import os
 import re
 import subprocess
+import sys
 import time
 
 frequency = PER_INSTANCE
 
 EXIT_FAIL = 254
 
+#
+# Returns the cmdline for the given process id.
+#
+
+def givecmdline(pid):
+    # Check if this pid still exists by sending it the harmless 0 signal.
+    try:
+	os.kill(pid, 0)
+    except OSError:
+        return None
+    else:
+        # Example output from procstat -c 16357
+        #   PID COMM             ARGS
+        #     1 init             /bin/init --
+        if sys.platform.startswith('freebsd'):
+            (output, _err) = util.subp(['procstat', '-c', str(pid)])
+            line = output.splitlines()[1]
+            m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line)
+            return m.group(2)
+        else:
+            return util.load_file("/proc/%s/cmdline" % pid)
 
 def handle(_name, cfg, _cloud, log, _args):
 
@@ -42,8 +64,8 @@ def handle(_name, cfg, _cloud, log, _args):
         return
 
     mypid = os.getpid()
-    cmdline = util.load_file("/proc/%s/cmdline" % mypid)
 
+    cmdline = givecmdline(mypid)
     if not cmdline:
         log.warn("power_state: failed to get cmdline of current process")
         return
@@ -119,8 +141,6 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
     msg = None
     end_time = time.time() + timeout
 
-    cmdline_f = "/proc/%s/cmdline" % pid
-
     def fatal(msg):
         if log:
             log.warn(msg)
@@ -134,16 +154,14 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
             break
 
         try:
-            cmdline = ""
+            cmdline = givecmdline(pid)
-            with open(cmdline_f) as fp:
-                cmdline = fp.read()
             if cmdline != pidcmdline:
                 msg = "cmdline changed for %s [now: %s]" % (pid, cmdline)
                 break
 
         except IOError as ioerr:
             if ioerr.errno in known_errnos:
-                msg = "pidfile '%s' gone [%d]" % (cmdline_f, ioerr.errno)
+                msg = "pidfile gone [%d]" % ioerr.errno
             else:
                 fatal("IOError during wait: %s" % ioerr)
             break

cloudinit/config/cc_resizefs.py

@@ -39,6 +39,10 @@ def _resize_ext(mount_point, devpth):  # pylint: disable=W0613
 def _resize_xfs(mount_point, devpth):  # pylint: disable=W0613
     return ('xfs_growfs', devpth)
 
+
+def _resize_ufs(mount_point, devpth):  # pylint: disable=W0613
+    return ('growfs', devpth)
+
 # Do not use a dictionary as these commands should be able to be used
 # for multiple filesystem types if possible, e.g. one command for
 # ext2, ext3 and ext4.
@@ -46,6 +50,7 @@ RESIZE_FS_PREFIXES_CMDS = [
     ('btrfs', _resize_btrfs),
     ('ext', _resize_ext),
     ('xfs', _resize_xfs),
+    ('ufs', _resize_ufs),
 ]
 
 NOBLOCK = "noblock"
@@ -91,7 +96,7 @@ def handle(name, cfg, _cloud, log, args):
             raise exc
         return
 
-    if not stat.S_ISBLK(statret.st_mode):
+    if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
         if util.is_container():
             log.debug("device '%s' not a block device in container."
                       " cannot resize: %s" % (devpth, info))

cloudinit/distros/__init__.py

@@ -39,6 +39,7 @@ from cloudinit.distros.parsers import hosts
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['fedora', 'rhel'],
+    'freebsd': ['freebsd'],
     'suse': ['sles']
 }
 

cloudinit/distros/freebsd.py

@@ -0,0 +1,208 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import netinfo
+from cloudinit import ssh_util
+from cloudinit import util
+
+from cloudinit.settings import PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+class Distro(distros.Distro):
+    def __init__(self, name, cfg, paths):
+        distros.Distro.__init__(self, name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        self.osfamily = 'freebsd'
+
+    def updatercconf(self, key, value):
+	LOG.debug("updatercconf: %s => %s" % (key, value))
+        conf = {}
+	configchanged = False
+	with open("/etc/rc.conf") as file:
+            for line in file:
+                tok = line.split('=')
+		# TODO: Handle keys with spaces, make this a bit more robust.
+		if tok[0] == key:
+		    if tok[1] != value:
+		        conf[tok[0]] = value 
+			LOG.debug("[rc.conf]: Value %s for key %s needs to be changed" % (value, key))
+			configchanged = True
+		else:
+		    conf[tok[0]] = tok[1].rstrip()
+
+        if configchanged:
+            LOG.debug("Writing new /etc/rc.conf file")
+            with open ('/etc/rc.conf', 'w') as file:
+               for keyval in conf.items():
+                   file.write("%s=%s\n" % keyval)
+
+    def _read_hostname():
+	return
+
+    def _read_system_hostname():
+	return
+
+    def _select_hostname(self, hostname, fqdn):
+        if not hostname:
+            return fqdn
+        return hostname
+
+    def _write_hostname(self, your_hostname, out_fn):
+       self.updatercconf('hostname', your_hostname)
+
+    def create_group(self, name, members):
+        group_add_cmd = ['pw', '-n', name]
+        if util.is_group(name):
+            LOG.warn("Skipping creation of existing group '%s'" % name)
+        else:
+            try:
+                util.subp(group_add_cmd)
+                LOG.info("Created new group %s" % name)
+            except Exception:
+                util.logexc("Failed to create group %s", name)
+
+        if len(members) > 0:
+            for member in members:
+                if not util.is_user(member):
+                    LOG.warn("Unable to add group member '%s' to group '%s'"
+                                     "; user does not exist.", member, name)
+                    continue
+                util.subp(['pw', 'usermod', '-n', name, '-G', member])
+                LOG.info("Added user '%s' to group '%s'" % (member, name))
+
+    def add_user(self, name, **kwargs):
+        if util.is_user(name):
+            LOG.info("User %s already exists, skipping." % name)
+            return False
+
+        adduser_cmd = ['pw', 'useradd', '-n', name]
+        log_adduser_cmd = ['pw', 'useradd', '-n', name]
+
+	adduser_opts = {
+		"homedir": '-d',
+		"gecos": '-c',
+		"primary_group": '-g',
+		"groups": '-G',
+		"passwd": '-h',
+		"shell": '-s',
+		"inactive": '-E',
+	}
+	adduser_flags = {
+		"no_user_group": '--no-user-group',
+		"system": '--system',
+		"no_log_init": '--no-log-init',
+	}
+
+	redact_opts = ['passwd']
+
+	for key, val in kwargs.iteritems():
+	    if key in adduser_opts and val and isinstance(val, str):
+               	adduser_cmd.extend([adduser_opts[key], val])
+
+                # Redact certain fields from the logs
+                if key in redact_opts:
+                    log_adduser_cmd.extend([adduser_opts[key], 'REDACTED'])
+                else:
+                    log_adduser_cmd.extend([adduser_opts[key], val])
+
+            elif key in adduser_flags and val:
+                adduser_cmd.append(adduser_flags[key])
+                log_adduser_cmd.append(adduser_flags[key])
+
+        if 'no_create_home' in kwargs or 'system' in kwargs:
+            adduser_cmd.append('-d/nonexistent')
+            log_adduser_cmd.append('-d/nonexistent')
+        else:
+            adduser_cmd.append('-d/usr/home/%s' % name)
+	    adduser_cmd.append('-m')
+            log_adduser_cmd.append('-d/usr/home/%s' % name)
+	    log_adduser_cmd.append('-m')
+
+        # Run the command
+        LOG.info("Adding user %s", name)
+        try:
+            util.subp(adduser_cmd, logstring=log_adduser_cmd)
+        except Exception as e:
+            util.logexc(LOG, "Failed to create user %s", name)
+            raise e
+
+    # TODO:
+    def set_passwd(self, name, **kwargs):
+	return False
+
+    def lock_passwd(self, name):
+        try:
+            util.subp(['pw', 'usermod', name, '-h', '-'])
+        except Exception as e:
+            util.logexc(LOG, "Failed to lock user %s", name)
+            raise e
+
+    # TODO:
+    def write_sudo_rules(self, name, rules, sudo_file=None):
+	LOG.debug("[write_sudo_rules] Name: %s" % name)
+
+    def create_user(self, name, **kwargs):
+        self.add_user(name, **kwargs)
+
+        # Set password if plain-text password provided and non-empty
+        if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']:
+            self.set_passwd(name, kwargs['plain_text_passwd'])
+
+        # Default locking down the account. 'lock_passwd' defaults to True.
+        # lock account unless lock_password is False.
+        if kwargs.get('lock_passwd', True):
+            self.lock_passwd(name)
+
+        # Configure sudo access
+        if 'sudo' in kwargs:
+            self.write_sudo_rules(name, kwargs['sudo'])
+
+        # Import SSH keys
+        if 'ssh_authorized_keys' in kwargs:
+            keys = set(kwargs['ssh_authorized_keys']) or []
+            ssh_util.setup_user_keys(keys, name, options=None)
+
+    def _write_network(self, settings):
+	return
+	
+    def apply_locale():
+	return
+	
+    def install_packages():
+	return
+
+    def package_command():
+	return
+
+    def set_timezone():
+	return
+
+    def update_package_sources():
+	return
+

cloudinit/netinfo.py

@@ -34,6 +34,7 @@ def netdev_info(empty=""):
             continue
         if line[0] not in ("\t", " "):
             curdev = line.split()[0]
+            # TODO: up/down detection fails on FreeBSD
             devs[curdev] = {"up": False}
             for field in fields:
                 devs[curdev][field] = ""
@@ -46,21 +47,32 @@ def netdev_info(empty=""):
             fieldpost = "6"
 
         for i in range(len(toks)):
-            if toks[i] == "hwaddr":
+            if toks[i] == "hwaddr" or toks[i] == "ether":
                 try:
                     devs[curdev]["hwaddr"] = toks[i + 1]
                 except IndexError:
                     pass
-            for field in ("addr", "bcast", "mask"):
+
+            """
+            Couple the different items we're interested in with the correct field
+            since FreeBSD/CentOS/Fedora differ in the output.
+            """
+
+            ifconfigfields = {
+                "addr:":"addr", "inet":"addr",
+                "bcast:":"bcast", "broadcast":"bcast",
+                "mask:":"mask", "netmask":"mask"
+            }
+	    for origfield, field in ifconfigfields.items():
                 target = "%s%s" % (field, fieldpost)
                 if devs[curdev].get(target, ""):
                     continue
-                if toks[i] == "%s:" % field:
+                if toks[i] == "%s" % origfield:
                     try:
                         devs[curdev][target] = toks[i + 1]
                     except IndexError:
                         pass
-                elif toks[i].startswith("%s:" % field):
+                elif toks[i].startswith("%s" % origfield):
                     devs[curdev][target] = toks[i][len(field) + 1:]
 
     if empty != "":
@@ -71,17 +83,38 @@ def netdev_info(empty=""):
 
     return devs
 
+#
+# Use netstat instead of route since that produces more portable output.
+#
 
 def route_info():
-    (route_out, _err) = util.subp(["route", "-n"])
+    (route_out, _err) = util.subp(["netstat", "-rn"])
     routes = []
     entries = route_out.splitlines()[1:]
     for line in entries:
         if not line:
             continue
         toks = line.split()
-        if len(toks) < 8 or toks[0] == "Kernel" or toks[0] == "Destination":
+
+        """
+        FreeBSD shows 6 items in the routing table:
+          Destination        Gateway            Flags    Refs      Use  Netif Expire
+          default            10.65.0.1          UGS         0    34920 vtnet0
+	
+        Linux netstat shows 2 more:
+          Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
+          0.0.0.0         10.65.0.1       0.0.0.0         UG        0 0          0 eth0
+	"""
+
+        if len(toks) < 6 or toks[0] == "Kernel" or toks[0] == "Destination" or toks[0] == "Internet" or toks[0] == "Internet6" or toks[0] == "Routing":
             continue
+
+        if len(toks) < 8:
+            toks.append("-")
+            toks.append("-")
+            toks[7] = toks[5]
+            toks[5] = "-"
+
         entry = {
             'destination': toks[0],
             'gateway': toks[1],
@@ -92,6 +125,7 @@ def route_info():
             'use': toks[6],
             'iface': toks[7],
         }
+
         routes.append(entry)
     return routes
 

cloudinit/sources/__init__.py

@@ -119,7 +119,7 @@ class DataSource(object):
         # when the kernel named them 'vda' or 'xvda'
         # we want to return the correct value for what will actually
         # exist in this instance
-        mappings = {"sd": ("vd", "xvd")}
+        mappings = {"sd": ("vd", "xvd", "vtb")}
         for (nfrom, tlist) in mappings.iteritems():
             if not short_name.startswith(nfrom):
                 continue

cloudinit/util.py

@@ -26,6 +26,7 @@ from StringIO import StringIO
 
 import contextlib
 import copy as obj_copy
+import ctypes
 import errno
 import glob
 import grp
@@ -36,6 +37,7 @@ import os.path
 import platform
 import pwd
 import random
+import re
 import shutil
 import socket
 import stat
@@ -1300,11 +1302,25 @@ def mounts():
     mounted = {}
     try:
         # Go through mounts to see what is already mounted
-        mount_locs = load_file("/proc/mounts").splitlines()
+	if os.path.exists("/proc/mounts"):
+            mount_locs = load_file("/proc/mounts").splitlines()
+            method = 'proc'
+	else:
+            (mountoutput, _err) = subp("mount")
+            mount_locs = mountoutput.splitlines()
+            method = 'mount'
         for mpline in mount_locs:
-            # Format at: man fstab
+            # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered)
+            # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates)
             try:
-                (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+		if method == 'proc' and len(mpline) == 6:
+                    (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+                elif method == 'mount':
+                    m = re.search('^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', mpline)
+                    dev = m.group(1)
+                    mp = m.group(2)
+                    fstype = m.group(3)
+                    opts = m.group(4)
             except:
                 continue
             # If the name of the mount point contains spaces these
@@ -1315,9 +1331,9 @@ def mounts():
                 'mountpoint': mp,
                 'opts': opts,
             }
-        LOG.debug("Fetched %s mounts from %s", mounted, "/proc/mounts")
+        LOG.debug("Fetched %s mounts from %s", mounted, method)
     except (IOError, OSError):
-        logexc(LOG, "Failed fetching mount points from /proc/mounts")
+        logexc(LOG, "Failed fetching mount points")
     return mounted
 
 
@@ -1403,11 +1419,22 @@ def time_rfc2822():
 def uptime():
     uptime_str = '??'
     try:
-        contents = load_file("/proc/uptime").strip()
+        if os.path.exists("/proc/uptime"):
-        if contents:
+            contents = load_file("/proc/uptime").strip()
-            uptime_str = contents.split()[0]
+            if contents:
+                uptime_str = contents.split()[0]
+        else:
+            libc = ctypes.CDLL('/lib/libc.so.7')
+            size = ctypes.c_size_t()
+            buf = ctypes.c_int()
+            size.value = ctypes.sizeof(buf)
+            libc.sysctlbyname("kern.boottime", ctypes.byref(buf), ctypes.byref(size), None, 0)
+            now = time.time()
+            bootup = buf.value
+            uptime_str = now - bootup
+
     except:
-        logexc(LOG, "Unable to read uptime from /proc/uptime")
+        logexc(LOG, "Unable to read uptime")
     return uptime_str
 
 
@@ -1746,6 +1773,18 @@ def parse_mtab(path):
     return None
 
 
+def parse_mount(path):
+    (mountoutput, _err) = subp("mount")
+    mount_locs = mountoutput.splitlines()
+    for line in mount_locs:
+        m = re.search('^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line)
+        devpth = m.group(1)
+        mount_point = m.group(2)
+        fs_type = m.group(3)
+        if mount_point == path:
+            return devpth, fs_type, mount_point
+    return None
+
 def get_mount_info(path, log=LOG):
     # Use /proc/$$/mountinfo to find the device where path is mounted.
     # This is done because with a btrfs filesystem using os.stat(path)
@@ -1779,8 +1818,10 @@ def get_mount_info(path, log=LOG):
     if os.path.exists(mountinfo_path):
         lines = load_file(mountinfo_path).splitlines()
         return parse_mount_info(path, lines, log)
-    else:
+    elif os.path.exists("/etc/mtab"):
         return parse_mtab(path)
+    else:
+        return parse_mount(path)
 
 
 def which(program):

tools/read-version

@@ -25,7 +25,7 @@ if [ ! -e "$CHNG_LOG" ]; then
     fail "Unable to find 'ChangeLog' file located at '$CHNG_LOG'"
 fi
 
-VERSION=$(sed -n '/^[0-9]\+[.][0-9]\+[.][0-9]\+:/ {s/://; p; :a;n; ba; }' \
+VERSION=$(grep -m1 -o -E '^[0-9]+(\.[0-9]+)+' \
           "$CHNG_LOG") &&
    [ -n "$VERSION" ] ||
    fail "failed to get version from '$CHNG_LOG'"