add write-files module for "injecting" files (LP: #1012854)

This implements file writing via cloud-config.  It also 
 * adjusts other code to have user/group parsing in util instead
   of in stages.py, 
 * renames decomp_str to decomp_gzip since it is more meaningful when named
   that (as thats all it can decompress).

ChangeLog

@@ -1,4 +1,5 @@
 0.7.0:
+ - add write-files module (LP: #1012854)
  - Add setuptools + cheetah to debian package build dependencies (LP: #1022101)
  - Adjust the sysvinit local script to provide 'cloud-init-local' and have
    the cloud-config script depend on that as well.

cloudinit/config/cc_write_files.py

@@ -0,0 +1,102 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import os
+
+from cloudinit import util
+from cloudinit.settings import PER_INSTANCE
+
+frequency = PER_INSTANCE
+
+DEFAULT_OWNER = "root:root"
+DEFAULT_PERMS = 0644
+UNKNOWN_ENC = 'text/plain'
+
+
+def handle(name, cfg, _cloud, log, _args):
+    files = cfg.get('write_files')
+    if not files:
+        log.debug(("Skipping module named %s,"
+                   " no/empty 'write_files' key in configuration"), name)
+        return
+    write_files(name, files, log)
+
+
+def canonicalize_extraction(encoding_type, log):
+    if not encoding_type:
+        encoding_type = ''
+    encoding_type = encoding_type.lower().strip()
+    if encoding_type in ['gz', 'gzip']:
+        return ['application/x-gzip']
+    if encoding_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']:
+        return ['application/base64', 'application/x-gzip']
+    # Yaml already encodes binary data as base64 if it is given to the 
+    # yaml file as binary, so those will be automatically decoded for you.
+    # But the above b64 is just for people that are more 'comfortable'
+    # specifing it manually (which might be a possiblity)
+    if encoding_type in ['b64', 'base64']:
+        return ['application/base64']
+    if encoding_type:
+        log.warn("Unknown encoding type %s, assuming %s",
+                 encoding_type, UNKNOWN_ENC)
+    return [UNKNOWN_ENC]
+
+
+def write_files(name, files, log):
+    if not files:
+        return
+
+    for (i, f_info) in enumerate(files):
+        path = f_info.get('path')
+        if not path:
+            log.warn("No path provided to write for entry %s in module %s",
+                     i + 1, name)
+            continue
+        path = os.path.abspath(path)
+        extractions = canonicalize_extraction(f_info.get('encoding'), log)
+        contents = extract_contents(f_info.get('content', ''), extractions)
+        (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER))
+        perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS, log)
+        util.write_file(path, contents, mode=perms)
+        util.chownbyname(path, u, g)
+
+
+def decode_perms(perm, default, log):
+    try:
+        if isinstance(perm, (int, long, float)):
+            # Just 'downcast' it (if a float)
+            return int(perm)
+        else:
+            # Force to string and try octal conversion
+            return int(str(perm), 8)
+    except (TypeError, ValueError):
+        log.warn("Undecodable permissions %s, assuming %s", perm, default)
+        return default
+
+
+def extract_contents(contents, extraction_types):
+    result = str(contents)
+    for t in extraction_types:
+        if t == 'application/x-gzip':
+            result = util.decomp_gzip(result, quiet=False)
+        elif t == 'application/base64':
+            result = base64.b64decode(result)
+        elif t == UNKNOWN_ENC:
+            pass
+    return result

cloudinit/stages.py

@@ -133,18 +133,7 @@ class Init(object):
         if log_file:
             util.ensure_file(log_file)
             if perms:
-                perms_parted = perms.split(':', 1)
+                u, g = util.extract_usergroup(perms)
-                u = perms_parted[0]
-                if len(perms_parted) == 2:
-                    g = perms_parted[1]
-                else:
-                    g = ''
-                u = u.strip()
-                g = g.strip()
-                if u == "-1" or u.lower() == "none":
-                    u = None
-                if g == "-1" or g.lower() == "none":
-                    g = None
                 try:
                     util.chownbyname(log_file, u, g)
                 except OSError:

cloudinit/user_data.py

@@ -227,7 +227,7 @@ def convert_string(raw_data, headers=None):
         raw_data = ''
     if not headers:
         headers = {}
-    data = util.decomp_str(raw_data)
+    data = util.decomp_gzip(raw_data)
     if "mime-version:" in data[0:4096].lower():
         msg = email.message_from_string(data)
         for (key, val) in headers.iteritems():

cloudinit/util.py

@@ -159,6 +159,10 @@ class MountFailedError(Exception):
     pass
 
 
+class DecompressionError(Exception):
+    pass
+
+
 def ExtendedTemporaryFile(**kwargs):
     fh = tempfile.NamedTemporaryFile(**kwargs)
     # Replace its unlink with a quiet version
@@ -256,13 +260,32 @@ def clean_filename(fn):
     return fn
 
 
-def decomp_str(data):
+def decomp_gzip(data, quiet=True):
     try:
         buf = StringIO(str(data))
         with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh:
             return gh.read()
-    except:
+    except Exception as e:
-        return data
+        if quiet:
+            return data
+        else:
+            raise DecompressionError(str(e))
+
+
+def extract_usergroup(ug_pair):
+    if not ug_pair:
+        return (None, None)
+    ug_parted = ug_pair.split(':', 1)
+    u = ug_parted[0].strip()
+    if len(ug_parted) == 2:
+        g = ug_parted[1].strip()
+    else:
+        g = None
+    if not u or u == "-1" or u.lower() == "none":
+        u = None
+    if not g or g == "-1" or g.lower() == "none":
+        g = None
+    return (u, g)
 
 
 def find_modules(root_dir):

config/cloud.cfg

@@ -21,6 +21,7 @@ preserve_hostname: false
 # The modules that run in the 'init' stage
 cloud_init_modules:
  - bootcmd
+ - write-files
  - resizefs
  - set_hostname
  - update_hostname

doc/examples/cloud-config-write-files.txt

@@ -0,0 +1,33 @@
+#cloud-config
+# vim: syntax=yaml
+#
+# This is the configuration syntax that the write_files module
+# will know how to understand. encoding can be given b64 or gzip or (gz+b64).
+# The content will be decoded accordingly and then written to the path that is
+# provided. 
+#
+# Note: Content strings here are truncated for example purposes.
+write_files:
+-   encoding: b64
+    content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4...
+    owner: root:root
+    path: /etc/sysconfig/selinux
+    perms: '0644'
+-   content: |
+        # My new /etc/sysconfig/samba file
+
+        SMBDOPTIONS="-D"
+    path: /etc/sysconfig/samba
+-   content: !!binary |
+        f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI
+        AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA
+        AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA
+        ....
+    path: /bin/arch
+    perms: '0555'
+-   encoding: gzip
+    content: !!binary |
+        H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA=
+    path: /usr/bin/hello
+    perms: '0755'
+