From 9697e7c2f990938838b9a2034b417a3230bafece Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 26 Oct 2012 17:07:19 -0700 Subject: [PATCH] Helpful cleanups. 1. Ensure the changelog is in ascii 2. Allow post-download patches to come from a directory instead of from a single set of files. 3. Allow rpm patches to come from a directory instead of a single list of files 4. Update smithy so that the epel rpm location can come from somewhere not epel (if desired) 5. Ensure we cleanup the 'negative' phase files correctly at the various phases. --- anvil/action.py | 2 ++ anvil/actions/install.py | 27 +++++++++++++--- anvil/actions/start.py | 7 ++-- anvil/actions/stop.py | 3 +- anvil/actions/uninstall.py | 12 ++++--- anvil/components/__init__.py | 15 +++++++++ anvil/packaging/helpers/changelog.py | 24 ++++++++++---- anvil/packaging/rpm.py | 18 +++++++++++ anvil/patcher.py | 48 ++++++++++++++++++++++++++++ anvil/shell.py | 19 +++++++++-- conf/templates/packaging/spec.tmpl | 14 ++++++++ smithy | 39 ++++++++++++---------- 12 files changed, 191 insertions(+), 37 deletions(-) create mode 100644 anvil/patcher.py diff --git a/anvil/action.py b/anvil/action.py index 85ab0ea7..a6ed69cc 100644 --- a/anvil/action.py +++ b/anvil/action.py @@ -261,6 +261,8 @@ class Action(object): LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c) change_activate(instance, True) component_results[c] = None + for n in neg_phase_recs: + n.unmark(c) else: try: result = None diff --git a/anvil/actions/install.py b/anvil/actions/install.py index 5154b1e6..f737808c 100644 --- a/anvil/actions/install.py +++ b/anvil/actions/install.py @@ -56,6 +56,7 @@ class InstallAction(action.Action): logger=LOG) def _run(self, persona, component_order, instances): + removals = [] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.name)), @@ -64,8 +65,22 @@ class InstallAction(action.Action): ), component_order, instances, - "download" + "download", + *removals ) + self._run_phase( + PhaseFunctors( + start=lambda i: LOG.info('Post-download patching %s.', colorizer.quote(i.name)), + run=lambda i: i.patch("download"), + end=None, + ), + component_order, + instances, + "download-patch", + *removals + ) + + removals += ['uninstall', 'unconfigure'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.name)), @@ -75,7 +90,7 @@ class InstallAction(action.Action): component_order, instances, "configure", - 'unconfigure' + *removals ) if self.only_configure: @@ -85,6 +100,7 @@ class InstallAction(action.Action): LOG.info("Exiting early, only asked to download and configure!") return + removals += ['pre-uninstall', 'post-uninstall'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Preinstalling %s.', colorizer.quote(i.name)), @@ -93,7 +109,8 @@ class InstallAction(action.Action): ), component_order, instances, - "pre-install" + "pre-install", + *removals ) def install_start(instance): @@ -120,7 +137,7 @@ class InstallAction(action.Action): component_order, instances, "install", - 'uninstall' + *removals ) self._run_phase( PhaseFunctors( @@ -131,5 +148,5 @@ class InstallAction(action.Action): component_order, instances, "post-install", - 'uninstall', 'unconfigure', 'pre-uninstall', 'post-uninstall' + *removals ) diff --git a/anvil/actions/start.py b/anvil/actions/start.py index 070c9848..34b1b8d3 100644 --- a/anvil/actions/start.py +++ b/anvil/actions/start.py @@ -29,6 +29,7 @@ class StartAction(action.Action): return 'running' def _run(self, persona, component_order, instances): + removals = [] self._run_phase( PhaseFunctors( start=None, @@ -38,7 +39,9 @@ class StartAction(action.Action): component_order, instances, "pre-start", + *removals ) + removals += ['stopped'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Starting %s.', i.name), @@ -48,7 +51,7 @@ class StartAction(action.Action): component_order, instances, "start", - 'stopped' + *removals ) self._run_phase( PhaseFunctors( @@ -59,5 +62,5 @@ class StartAction(action.Action): component_order, instances, "post-start", - 'stopped' + *removals ) diff --git a/anvil/actions/stop.py b/anvil/actions/stop.py index 3835d78e..e4a2beff 100644 --- a/anvil/actions/stop.py +++ b/anvil/actions/stop.py @@ -34,6 +34,7 @@ class StopAction(action.Action): return components def _run(self, persona, component_order, instances): + removals = ['pre-start', 'start', 'post-start'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.name)), @@ -43,5 +44,5 @@ class StopAction(action.Action): component_order, instances, "stopped", - 'pre-start', 'start', 'post-start' + *removals ) diff --git a/anvil/actions/uninstall.py b/anvil/actions/uninstall.py index 1ba7d2b0..2fbfd73a 100644 --- a/anvil/actions/uninstall.py +++ b/anvil/actions/uninstall.py @@ -34,6 +34,7 @@ class UninstallAction(action.Action): return components def _run(self, persona, component_order, instances): + removals = ['configure'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.name)), @@ -43,8 +44,9 @@ class UninstallAction(action.Action): component_order, instances, 'unconfigure', - 'configure' + *removals ) + removals += ['post-install'] self._run_phase( PhaseFunctors( start=None, @@ -54,8 +56,9 @@ class UninstallAction(action.Action): component_order, instances, 'pre-uninstall', - 'post-install' + *removals ) + removals += ['install'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.name)), @@ -65,8 +68,9 @@ class UninstallAction(action.Action): component_order, instances, 'uninstall', - 'install' + *removals ) + removals += ['download', 'configure', "download-patch", 'pre-install', 'post-install'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.name)), @@ -76,5 +80,5 @@ class UninstallAction(action.Action): component_order, instances, 'post-uninstall', - 'download', 'configure', 'pre-install', 'install', 'post-install' + *removals ) diff --git a/anvil/components/__init__.py b/anvil/components/__init__.py index 15b282f6..673fd3d3 100644 --- a/anvil/components/__init__.py +++ b/anvil/components/__init__.py @@ -45,6 +45,7 @@ from anvil import exceptions as excp from anvil import importer from anvil import log as logging from anvil import packager +from anvil import patcher from anvil import shell as sh from anvil import trace as tr from anvil import utils @@ -121,6 +122,20 @@ class PkgInstallComponent(component.Component): down.download(self.distro, from_uri, target_dir) return uris + def patch(self, section): + what_patches = self.get_option('patches', section) + (_from_uri, target_dir) = self._get_download_location() + if not what_patches: + what_patches = [] + canon_what_patches = [] + for path in what_patches: + if sh.isdir(path): + canon_what_patches.extend(sh.listdir(path, files_only=True)) + elif sh.isfile(path): + canon_what_patches.append(path) + if canon_what_patches: + patcher.apply_patches(canon_what_patches, target_dir) + def config_params(self, config_fn): mp = dict(self.params) if config_fn: diff --git a/anvil/packaging/helpers/changelog.py b/anvil/packaging/helpers/changelog.py index 1694e33a..5ade1ec8 100644 --- a/anvil/packaging/helpers/changelog.py +++ b/anvil/packaging/helpers/changelog.py @@ -28,12 +28,19 @@ import textwrap import iso8601 +from anvil import log as logging from anvil import shell as sh from anvil import utils +LOG = logging.getLogger(__name__) + PER_CALL_AM = 50 +def translate_utf8(text): + return text.decode('utf8').encode('ascii', 'replace') + + class GitChangeLog(object): __meta__ = abc.ABCMeta @@ -71,6 +78,12 @@ class GitChangeLog(object): return self.date_buckets def _skip_entry(self, summary, date, email, name): + for f in [summary, name, email]: + try: + translate_utf8(f) + except UnicodeError: + LOG.warn("Non-utf8 field %s found", f) + return True email = email.lower().strip() if email in ['jenkins@review.openstack.org']: return True @@ -118,10 +131,10 @@ class GitChangeLog(object): if self._skip_entry(summary, date, author_email, author_name): continue log.append({ - 'summary': summary, + 'summary': translate_utf8(summary), 'when': date, - 'author_email': author_email, - 'author_name': author_name, + 'author_email': translate_utf8(author_email), + 'author_name': translate_utf8(author_name), }) # Bucketize the dates by day @@ -158,7 +171,4 @@ class RpmChangeLog(GitChangeLog): if len(sublines) > 1: for subline in sublines[1:]: lines.append(" %s" % subline) - # Replace utf8 oddities with ? just incase - contents = "\n".join(lines) - contents = contents.decode('utf8').encode('ascii', 'replace') - return contents + return "\n".join(lines) diff --git a/anvil/packaging/rpm.py b/anvil/packaging/rpm.py index afb5e745..b39b35cc 100644 --- a/anvil/packaging/rpm.py +++ b/anvil/packaging/rpm.py @@ -54,6 +54,23 @@ class DependencyPackager(comp.Component): self._build_paths = bpaths return dict(self._build_paths) + def _patches(self): + your_patches = [] + in_patches = self.get_option('patches', 'package') + if in_patches: + for path in in_patches: + path = sh.abspth(path) + if sh.isdir(path): + for c_path in sh.listdir(path, files_only=True): + tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(c_path)) + sh.copy(c_path, tgt_fn) + your_patches.append(sh.basename(tgt_fn)) + else: + tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(path)) + sh.copy(path, tgt_fn) + your_patches.append(sh.basename(tgt_fn)) + return your_patches + def _requirements(self): return { 'install': self._install_requirements(), @@ -142,6 +159,7 @@ class DependencyPackager(comp.Component): 'build': self._build_details(), 'who': sh.getuser(), 'date': utils.iso8601(), + 'patches': self._patches(), 'details': self.details, } (_fn, content) = utils.load_template('packaging', 'spec.tmpl') diff --git a/anvil/patcher.py b/anvil/patcher.py new file mode 100644 index 00000000..ca144144 --- /dev/null +++ b/anvil/patcher.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from anvil import log as logging +from anvil import shell as sh +from anvil import utils + + +LOG = logging.getLogger(__name__) + +# TODO(harlowja): use git patching vs. raw patching?? +PATCH_CMD = ['patch', '-p1'] + + +def apply_patches(patch_files, working_dir): + if not patch_files: + return + apply_files = [] + for p in patch_files: + p = sh.abspth(p) + if not sh.isfile(p): + LOG.warn("Can not apply non-file patch %s", p) + apply_files.append(p) + if not apply_files: + return + if not sh.isdir(working_dir): + LOG.warn("Can only apply %s patches 'inside' a directory and not '%s'", + len(apply_files), working_dir) + return + with utils.chdir(working_dir): + for p in apply_files: + LOG.debug("Applying patch %s in directory %s", p, working_dir) + patch_contents = sh.load_file(p) + if patch_contents: + sh.execute(*PATCH_CMD, process_input=patch_contents) diff --git a/anvil/shell.py b/anvil/shell.py index c26da89d..71387e48 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -254,8 +254,23 @@ def fileperms(path): return (os.stat(path).st_mode & 0777) -def listdir(path): - return os.listdir(path) +def listdir(path, recursive=False, dirs_only=False, files_only=False): + path = abspth(path) + all_contents = [] + if not recursive: + all_contents = os.listdir(path) + all_contents = [os.path.join(path, f) for f in all_contents] + else: + for (root, dirs, files) in os.walk(path): + for d in dirs: + all_contents.append(joinpths(root, d)) + for f in files: + all_contents.append(joinpths(root, f)) + if dirs_only: + all_contents = [f for f in all_contents if isdir(f)] + if files_only: + all_contents = [f for f in all_contents if isfile(f)] + return all_contents def isfile(fn): diff --git a/conf/templates/packaging/spec.tmpl b/conf/templates/packaging/spec.tmpl index 553115c9..84842c98 100644 --- a/conf/templates/packaging/spec.tmpl +++ b/conf/templates/packaging/spec.tmpl @@ -85,6 +85,13 @@ Requires: ${i} #end for #end if +# Custom patches +#set $size = 0 +#for $p in $patches +Patch${size}: $p +#set $size += 1 +#end for + %description #if $details.description $details.description @@ -98,6 +105,13 @@ $details.summary %setup $build.setup #end if +# Custom patches activation +#set $size = 0 +#for $p in $patches +%patch${size} -p1 +#set $size += 1 +#end for + #if $build.has_key('action') %build $build.action diff --git a/smithy b/smithy index 611d74e5..a25ba1ac 100755 --- a/smithy +++ b/smithy @@ -7,6 +7,11 @@ fi shopt -s nocasematch +# Possible locations of the epel rpm/list url +RHEL_VERSION=$(lsb_release -r | awk '{ print $2 }' | cut -d"." -f1) +EPEL_RPM_LIST="http://mirrors.kernel.org/fedora-epel/$RHEL_VERSION/i386" +NODE_RPM_URL="http://nodejs.tchol.org/repocfg/el/nodejs-stable-release.noarch.rpm" + ARGS="$@" VER=$(python -c "from anvil import version; print version.version_string()") PWD=`pwd` @@ -41,39 +46,41 @@ bootstrap_rh() echo "Please wait..." echo "Installing node.js yum repository configuration." - JS_REPO_RPM_FN="nodejs-stable-release.noarch.rpm" + JS_REPO_RPM_FN=$(basename $NODE_RPM_URL) if [ ! -f "/tmp/$JS_REPO_RPM_FN" ]; then - echo "Downloading $JS_REPO_RPM_FN" - wget -q -O "/tmp/$JS_REPO_RPM_FN" "http://nodejs.tchol.org/repocfg/el/$JS_REPO_RPM_FN" + echo "Downloading $JS_REPO_RPM_FN to /tmp/$JS_REPO_RPM_FN..." + wget -q -O "/tmp/$JS_REPO_RPM_FN" "$NODE_RPM_URL" if [ $? -ne 0 ]; then return 1 fi fi - echo "Installing /tmp/$JS_REPO_RPM_FN." + echo "Installing /tmp/$JS_REPO_RPM_FN..." yum install --assumeyes --nogpgcheck -t "/tmp/$JS_REPO_RPM_FN" 2>&1 - echo "Locating the EPEL rpm." - EPEL_RPM=$(curl -s "http://mirrors.kernel.org/fedora-epel/6/i386/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm") - if [ $? -ne 0 ]; then - return 1 - fi - if [ ! -f "/tmp/$EPEL_RPM" ]; then - echo "Downloading $EPEL_RPM." - wget -q -O "/tmp/$EPEL_RPM" "http://mirrors.kernel.org/fedora-epel/6/i386/$EPEL_RPM" + echo "Locating the EPEL rpm..." + if [ -z "$EPEL_RPM" ]; then + EPEL_RPM=$(curl -s "$EPEL_RPM_LIST/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm") if [ $? -ne 0 ]; then return 1 fi fi - echo "Installing /tmp/$EPEL_RPM." + if [ ! -f "/tmp/$EPEL_RPM" ]; then + echo "Downloading $EPEL_RPM to /tmp/$EPEL_RPM" + wget -q -O "/tmp/$EPEL_RPM" "$EPEL_RPM_LIST/$EPEL_RPM" + if [ $? -ne 0 ]; then + return 1 + fi + fi + echo "Installing /tmp/$EPEL_RPM..." yum install --assumeyes --nogpgcheck -t "/tmp/$EPEL_RPM" 2>&1 - echo "Installing needed distribution dependencies:" + echo "Installing distribution dependencies..." pkgs="gcc git pylint python python-netifaces python-pep8 python-cheetah" pkgs="$pkgs python-pip python-progressbar PyYAML python-ordereddict python-iso8601" yum install -y $pkgs 2>&1 - echo "Installing needed pypi dependencies:" - pip-python install -U -I termcolor iniparse "keyring==0.9.2" + echo "Installing pypi dependencies..." + pip-python install -U -I termcolor iniparse "keyring>=0.9.2" return 0 }