Helpful cleanups.

1. Ensure the changelog is in ascii
2. Allow post-download patches to come from a directory instead of from a single set of files.
3. Allow rpm patches to come from a directory instead of a single list of files
4. Update smithy so that the epel rpm location can come from somewhere not epel (if desired)
5. Ensure we cleanup the 'negative' phase files correctly at the various phases.

anvil/action.py

@@ -261,6 +261,8 @@ class Action(object):
                 LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
                 change_activate(instance, True)
                 component_results[c] = None
+                for n in neg_phase_recs:
+                    n.unmark(c)
             else:
                 try:
                     result = None

anvil/actions/install.py

@@ -56,6 +56,7 @@ class InstallAction(action.Action):
                                logger=LOG)
 
     def _run(self, persona, component_order, instances):
+        removals = []
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.name)),
@@ -64,8 +65,22 @@ class InstallAction(action.Action):
             ),
             component_order,
             instances,
-            "download"
+            "download",
+            *removals
             )
+        self._run_phase(
+            PhaseFunctors(
+                start=lambda i: LOG.info('Post-download patching %s.', colorizer.quote(i.name)),
+                run=lambda i: i.patch("download"),
+                end=None,
+            ),
+            component_order,
+            instances,
+            "download-patch",
+            *removals
+            )
+
+        removals += ['uninstall', 'unconfigure']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.name)),
@@ -75,7 +90,7 @@ class InstallAction(action.Action):
             component_order,
             instances,
             "configure",
-            'unconfigure'
+            *removals
             )
 
         if self.only_configure:
@@ -85,6 +100,7 @@ class InstallAction(action.Action):
             LOG.info("Exiting early, only asked to download and configure!")
             return
 
+        removals += ['pre-uninstall', 'post-uninstall']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Preinstalling %s.', colorizer.quote(i.name)),
@@ -93,7 +109,8 @@ class InstallAction(action.Action):
             ),
             component_order,
             instances,
-            "pre-install"
+            "pre-install",
+            *removals
             )
 
         def install_start(instance):
@@ -120,7 +137,7 @@ class InstallAction(action.Action):
             component_order,
             instances,
             "install",
-            'uninstall'
+            *removals
             )
         self._run_phase(
             PhaseFunctors(
@@ -131,5 +148,5 @@ class InstallAction(action.Action):
             component_order,
             instances,
             "post-install",
-            'uninstall', 'unconfigure', 'pre-uninstall', 'post-uninstall'
+            *removals
             )

anvil/actions/start.py

@@ -29,6 +29,7 @@ class StartAction(action.Action):
         return 'running'
 
     def _run(self, persona, component_order, instances):
+        removals = []
         self._run_phase(
             PhaseFunctors(
                 start=None,
@@ -38,7 +39,9 @@ class StartAction(action.Action):
             component_order,
             instances,
             "pre-start",
+            *removals
             )
+        removals += ['stopped']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Starting %s.', i.name),
@@ -48,7 +51,7 @@ class StartAction(action.Action):
             component_order,
             instances,
             "start",
-            'stopped'
+            *removals
             )
         self._run_phase(
             PhaseFunctors(
@@ -59,5 +62,5 @@ class StartAction(action.Action):
             component_order,
             instances,
             "post-start",
-            'stopped'
+            *removals
             )

anvil/actions/stop.py

@@ -34,6 +34,7 @@ class StopAction(action.Action):
         return components
 
     def _run(self, persona, component_order, instances):
+        removals = ['pre-start', 'start', 'post-start']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.name)),
@@ -43,5 +44,5 @@ class StopAction(action.Action):
             component_order,
             instances,
             "stopped",
-            'pre-start', 'start', 'post-start'
+            *removals
             )

anvil/actions/uninstall.py

@@ -34,6 +34,7 @@ class UninstallAction(action.Action):
         return components
 
     def _run(self, persona, component_order, instances):
+        removals = ['configure']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.name)),
@@ -43,8 +44,9 @@ class UninstallAction(action.Action):
             component_order,
             instances,
             'unconfigure',
-            'configure'
+            *removals
             )
+        removals += ['post-install']
         self._run_phase(
             PhaseFunctors(
                 start=None,
@@ -54,8 +56,9 @@ class UninstallAction(action.Action):
             component_order,
             instances,
             'pre-uninstall',
-            'post-install'
+            *removals
             )
+        removals += ['install']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.name)),
@@ -65,8 +68,9 @@ class UninstallAction(action.Action):
             component_order,
             instances,
             'uninstall',
-            'install'
+            *removals
             )
+        removals += ['download', 'configure', "download-patch", 'pre-install', 'post-install']
         self._run_phase(
             PhaseFunctors(
                 start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.name)),
@@ -76,5 +80,5 @@ class UninstallAction(action.Action):
             component_order,
             instances,
             'post-uninstall',
-            'download', 'configure', 'pre-install', 'install', 'post-install'
+            *removals
             )

anvil/components/__init__.py

@@ -45,6 +45,7 @@ from anvil import exceptions as excp
 from anvil import importer
 from anvil import log as logging
 from anvil import packager
+from anvil import patcher
 from anvil import shell as sh
 from anvil import trace as tr
 from anvil import utils
@@ -121,6 +122,20 @@ class PkgInstallComponent(component.Component):
             down.download(self.distro, from_uri, target_dir)
             return uris
 
+    def patch(self, section):
+        what_patches = self.get_option('patches', section)
+        (_from_uri, target_dir) = self._get_download_location()
+        if not what_patches:
+            what_patches = []
+        canon_what_patches = []
+        for path in what_patches:
+            if sh.isdir(path):
+                canon_what_patches.extend(sh.listdir(path, files_only=True))
+            elif sh.isfile(path):
+                canon_what_patches.append(path)
+        if canon_what_patches:
+            patcher.apply_patches(canon_what_patches, target_dir)
+
     def config_params(self, config_fn):
         mp = dict(self.params)
         if config_fn:

anvil/packaging/helpers/changelog.py

@@ -28,12 +28,19 @@ import textwrap
 
 import iso8601
 
+from anvil import log as logging
 from anvil import shell as sh
 from anvil import utils
 
+LOG = logging.getLogger(__name__)
+
 PER_CALL_AM = 50
 
 
+def translate_utf8(text):
+    return text.decode('utf8').encode('ascii', 'replace')
+
+
 class GitChangeLog(object):
     __meta__ = abc.ABCMeta
 
@@ -71,6 +78,12 @@ class GitChangeLog(object):
         return self.date_buckets
 
     def _skip_entry(self, summary, date, email, name):
+        for f in [summary, name, email]:
+            try:
+                translate_utf8(f)
+            except UnicodeError:
+                LOG.warn("Non-utf8 field %s found", f)
+                return True
         email = email.lower().strip()
         if email in ['jenkins@review.openstack.org']:
             return True
@@ -118,10 +131,10 @@ class GitChangeLog(object):
                     if self._skip_entry(summary, date, author_email, author_name):
                         continue
                     log.append({
-                        'summary': summary,
+                        'summary': translate_utf8(summary),
                         'when': date,
-                        'author_email': author_email,
+                        'author_email': translate_utf8(author_email),
-                        'author_name': author_name,
+                        'author_name': translate_utf8(author_name),
                     })
 
         # Bucketize the dates by day
@@ -158,7 +171,4 @@ class RpmChangeLog(GitChangeLog):
                     if len(sublines) > 1:
                         for subline in sublines[1:]:
                             lines.append("  %s" % subline)
-        # Replace utf8 oddities with ? just incase
+        return "\n".join(lines)
-        contents = "\n".join(lines)
-        contents = contents.decode('utf8').encode('ascii', 'replace')
-        return contents

anvil/packaging/rpm.py

@@ -54,6 +54,23 @@ class DependencyPackager(comp.Component):
             self._build_paths = bpaths
         return dict(self._build_paths)
 
+    def _patches(self):
+        your_patches = []
+        in_patches = self.get_option('patches', 'package')
+        if in_patches:
+            for path in in_patches:
+                path = sh.abspth(path)
+                if sh.isdir(path):
+                    for c_path in sh.listdir(path, files_only=True):
+                        tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(c_path))
+                        sh.copy(c_path, tgt_fn)
+                        your_patches.append(sh.basename(tgt_fn))
+                else:
+                    tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(path))
+                    sh.copy(path, tgt_fn)
+                    your_patches.append(sh.basename(tgt_fn))
+        return your_patches
+
     def _requirements(self):
         return {
             'install': self._install_requirements(),
@@ -142,6 +159,7 @@ class DependencyPackager(comp.Component):
             'build': self._build_details(),
             'who': sh.getuser(),
             'date': utils.iso8601(),
+            'patches': self._patches(),
             'details': self.details,
         }
         (_fn, content) = utils.load_template('packaging', 'spec.tmpl')

anvil/patcher.py

@@ -0,0 +1,48 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from anvil import log as logging
+from anvil import shell as sh
+from anvil import utils
+
+
+LOG = logging.getLogger(__name__)
+
+# TODO(harlowja): use git patching vs. raw patching??
+PATCH_CMD = ['patch', '-p1']
+
+
+def apply_patches(patch_files, working_dir):
+    if not patch_files:
+        return
+    apply_files = []
+    for p in patch_files:
+        p = sh.abspth(p)
+        if not sh.isfile(p):
+            LOG.warn("Can not apply non-file patch %s", p)
+        apply_files.append(p)
+    if not apply_files:
+        return
+    if not sh.isdir(working_dir):
+        LOG.warn("Can only apply %s patches 'inside' a directory and not '%s'",
+                 len(apply_files), working_dir)
+        return
+    with utils.chdir(working_dir):
+        for p in apply_files:
+            LOG.debug("Applying patch %s in directory %s", p, working_dir)
+            patch_contents = sh.load_file(p)
+            if patch_contents:
+                sh.execute(*PATCH_CMD, process_input=patch_contents)

anvil/shell.py

@@ -254,8 +254,23 @@ def fileperms(path):
     return (os.stat(path).st_mode & 0777)
 
 
-def listdir(path):
+def listdir(path, recursive=False, dirs_only=False, files_only=False):
-    return os.listdir(path)
+    path = abspth(path)
+    all_contents = []
+    if not recursive:
+        all_contents = os.listdir(path)
+        all_contents = [os.path.join(path, f) for f in all_contents]
+    else:
+        for (root, dirs, files) in os.walk(path):
+            for d in dirs:
+                all_contents.append(joinpths(root, d))
+            for f in files:
+                all_contents.append(joinpths(root, f))
+    if dirs_only:
+        all_contents = [f for f in all_contents if isdir(f)]
+    if files_only:
+        all_contents = [f for f in all_contents if isfile(f)]
+    return all_contents
 
 
 def isfile(fn):

conf/templates/packaging/spec.tmpl

@@ -85,6 +85,13 @@ Requires: ${i}
 #end for
 #end if
 
+# Custom patches
+#set $size = 0
+#for $p in $patches
+Patch${size}: $p
+#set $size += 1
+#end for
+
 %description
 #if $details.description
 $details.description
@@ -98,6 +105,13 @@ $details.summary
 %setup $build.setup
 #end if
 
+# Custom patches activation
+#set $size = 0
+#for $p in $patches
+%patch${size} -p1
+#set $size += 1
+#end for
+
 #if $build.has_key('action')
 %build
 $build.action

smithy

@@ -7,6 +7,11 @@ fi
 
 shopt -s nocasematch
 
+# Possible locations of the epel rpm/list url
+RHEL_VERSION=$(lsb_release  -r  | awk '{ print $2 }' | cut -d"." -f1)
+EPEL_RPM_LIST="http://mirrors.kernel.org/fedora-epel/$RHEL_VERSION/i386"
+NODE_RPM_URL="http://nodejs.tchol.org/repocfg/el/nodejs-stable-release.noarch.rpm"
+
 ARGS="$@"
 VER=$(python -c "from anvil import version; print version.version_string()")
 PWD=`pwd`
@@ -41,39 +46,41 @@ bootstrap_rh()
     echo "Please wait..."
 
     echo "Installing node.js yum repository configuration."
-    JS_REPO_RPM_FN="nodejs-stable-release.noarch.rpm"
+    JS_REPO_RPM_FN=$(basename $NODE_RPM_URL)
     if [ ! -f "/tmp/$JS_REPO_RPM_FN" ]; then
-        echo "Downloading $JS_REPO_RPM_FN"
+        echo "Downloading $JS_REPO_RPM_FN to /tmp/$JS_REPO_RPM_FN..."
-        wget -q -O "/tmp/$JS_REPO_RPM_FN" "http://nodejs.tchol.org/repocfg/el/$JS_REPO_RPM_FN"
+        wget -q -O "/tmp/$JS_REPO_RPM_FN" "$NODE_RPM_URL"
         if [ $? -ne 0 ]; then
             return 1
         fi
     fi
-    echo "Installing /tmp/$JS_REPO_RPM_FN."
+    echo "Installing /tmp/$JS_REPO_RPM_FN..."
     yum install --assumeyes --nogpgcheck -t "/tmp/$JS_REPO_RPM_FN" 2>&1
 
-    echo "Locating the EPEL rpm."
+    echo "Locating the EPEL rpm..."
-    EPEL_RPM=$(curl -s "http://mirrors.kernel.org/fedora-epel/6/i386/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm")
+    if [ -z "$EPEL_RPM" ]; then
+        EPEL_RPM=$(curl -s "$EPEL_RPM_LIST/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm")
         if [ $? -ne 0 ]; then
             return 1
         fi
+    fi
     if [ ! -f "/tmp/$EPEL_RPM" ]; then
-        echo "Downloading $EPEL_RPM."
+        echo "Downloading $EPEL_RPM to /tmp/$EPEL_RPM"
-        wget -q -O "/tmp/$EPEL_RPM" "http://mirrors.kernel.org/fedora-epel/6/i386/$EPEL_RPM"
+        wget -q -O "/tmp/$EPEL_RPM" "$EPEL_RPM_LIST/$EPEL_RPM"
         if [ $? -ne 0 ]; then
             return 1
         fi
     fi
-    echo "Installing /tmp/$EPEL_RPM."
+    echo "Installing /tmp/$EPEL_RPM..."
     yum install --assumeyes --nogpgcheck -t "/tmp/$EPEL_RPM" 2>&1
 
-    echo "Installing needed distribution dependencies:"
+    echo "Installing distribution dependencies..."
     pkgs="gcc git pylint python python-netifaces python-pep8 python-cheetah"
     pkgs="$pkgs python-pip python-progressbar PyYAML python-ordereddict python-iso8601"
     yum install -y $pkgs 2>&1
 
-    echo "Installing needed pypi dependencies:"
+    echo "Installing pypi dependencies..."
-    pip-python install -U -I termcolor iniparse "keyring==0.9.2"
+    pip-python install -U -I termcolor iniparse "keyring>=0.9.2"
     return 0
 }