diff --git a/anvil/action.py b/anvil/action.py index 85ab0ea7..a6ed69cc 100644 --- a/anvil/action.py +++ b/anvil/action.py @@ -261,6 +261,8 @@ class Action(object): LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c) change_activate(instance, True) component_results[c] = None + for n in neg_phase_recs: + n.unmark(c) else: try: result = None diff --git a/anvil/actions/install.py b/anvil/actions/install.py index 5154b1e6..f737808c 100644 --- a/anvil/actions/install.py +++ b/anvil/actions/install.py @@ -56,6 +56,7 @@ class InstallAction(action.Action): logger=LOG) def _run(self, persona, component_order, instances): + removals = [] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.name)), @@ -64,8 +65,22 @@ class InstallAction(action.Action): ), component_order, instances, - "download" + "download", + *removals ) + self._run_phase( + PhaseFunctors( + start=lambda i: LOG.info('Post-download patching %s.', colorizer.quote(i.name)), + run=lambda i: i.patch("download"), + end=None, + ), + component_order, + instances, + "download-patch", + *removals + ) + + removals += ['uninstall', 'unconfigure'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.name)), @@ -75,7 +90,7 @@ class InstallAction(action.Action): component_order, instances, "configure", - 'unconfigure' + *removals ) if self.only_configure: @@ -85,6 +100,7 @@ class InstallAction(action.Action): LOG.info("Exiting early, only asked to download and configure!") return + removals += ['pre-uninstall', 'post-uninstall'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Preinstalling %s.', colorizer.quote(i.name)), @@ -93,7 +109,8 @@ class InstallAction(action.Action): ), component_order, instances, - "pre-install" + "pre-install", + *removals ) def install_start(instance): @@ -120,7 +137,7 @@ class InstallAction(action.Action): component_order, instances, "install", - 'uninstall' + *removals ) self._run_phase( PhaseFunctors( @@ -131,5 +148,5 @@ class InstallAction(action.Action): component_order, instances, "post-install", - 'uninstall', 'unconfigure', 'pre-uninstall', 'post-uninstall' + *removals ) diff --git a/anvil/actions/start.py b/anvil/actions/start.py index 070c9848..34b1b8d3 100644 --- a/anvil/actions/start.py +++ b/anvil/actions/start.py @@ -29,6 +29,7 @@ class StartAction(action.Action): return 'running' def _run(self, persona, component_order, instances): + removals = [] self._run_phase( PhaseFunctors( start=None, @@ -38,7 +39,9 @@ class StartAction(action.Action): component_order, instances, "pre-start", + *removals ) + removals += ['stopped'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Starting %s.', i.name), @@ -48,7 +51,7 @@ class StartAction(action.Action): component_order, instances, "start", - 'stopped' + *removals ) self._run_phase( PhaseFunctors( @@ -59,5 +62,5 @@ class StartAction(action.Action): component_order, instances, "post-start", - 'stopped' + *removals ) diff --git a/anvil/actions/stop.py b/anvil/actions/stop.py index 3835d78e..e4a2beff 100644 --- a/anvil/actions/stop.py +++ b/anvil/actions/stop.py @@ -34,6 +34,7 @@ class StopAction(action.Action): return components def _run(self, persona, component_order, instances): + removals = ['pre-start', 'start', 'post-start'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.name)), @@ -43,5 +44,5 @@ class StopAction(action.Action): component_order, instances, "stopped", - 'pre-start', 'start', 'post-start' + *removals ) diff --git a/anvil/actions/uninstall.py b/anvil/actions/uninstall.py index 1ba7d2b0..2fbfd73a 100644 --- a/anvil/actions/uninstall.py +++ b/anvil/actions/uninstall.py @@ -34,6 +34,7 @@ class UninstallAction(action.Action): return components def _run(self, persona, component_order, instances): + removals = ['configure'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.name)), @@ -43,8 +44,9 @@ class UninstallAction(action.Action): component_order, instances, 'unconfigure', - 'configure' + *removals ) + removals += ['post-install'] self._run_phase( PhaseFunctors( start=None, @@ -54,8 +56,9 @@ class UninstallAction(action.Action): component_order, instances, 'pre-uninstall', - 'post-install' + *removals ) + removals += ['install'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.name)), @@ -65,8 +68,9 @@ class UninstallAction(action.Action): component_order, instances, 'uninstall', - 'install' + *removals ) + removals += ['download', 'configure', "download-patch", 'pre-install', 'post-install'] self._run_phase( PhaseFunctors( start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.name)), @@ -76,5 +80,5 @@ class UninstallAction(action.Action): component_order, instances, 'post-uninstall', - 'download', 'configure', 'pre-install', 'install', 'post-install' + *removals ) diff --git a/anvil/components/__init__.py b/anvil/components/__init__.py index 15b282f6..673fd3d3 100644 --- a/anvil/components/__init__.py +++ b/anvil/components/__init__.py @@ -45,6 +45,7 @@ from anvil import exceptions as excp from anvil import importer from anvil import log as logging from anvil import packager +from anvil import patcher from anvil import shell as sh from anvil import trace as tr from anvil import utils @@ -121,6 +122,20 @@ class PkgInstallComponent(component.Component): down.download(self.distro, from_uri, target_dir) return uris + def patch(self, section): + what_patches = self.get_option('patches', section) + (_from_uri, target_dir) = self._get_download_location() + if not what_patches: + what_patches = [] + canon_what_patches = [] + for path in what_patches: + if sh.isdir(path): + canon_what_patches.extend(sh.listdir(path, files_only=True)) + elif sh.isfile(path): + canon_what_patches.append(path) + if canon_what_patches: + patcher.apply_patches(canon_what_patches, target_dir) + def config_params(self, config_fn): mp = dict(self.params) if config_fn: diff --git a/anvil/packaging/helpers/changelog.py b/anvil/packaging/helpers/changelog.py index 1694e33a..5ade1ec8 100644 --- a/anvil/packaging/helpers/changelog.py +++ b/anvil/packaging/helpers/changelog.py @@ -28,12 +28,19 @@ import textwrap import iso8601 +from anvil import log as logging from anvil import shell as sh from anvil import utils +LOG = logging.getLogger(__name__) + PER_CALL_AM = 50 +def translate_utf8(text): + return text.decode('utf8').encode('ascii', 'replace') + + class GitChangeLog(object): __meta__ = abc.ABCMeta @@ -71,6 +78,12 @@ class GitChangeLog(object): return self.date_buckets def _skip_entry(self, summary, date, email, name): + for f in [summary, name, email]: + try: + translate_utf8(f) + except UnicodeError: + LOG.warn("Non-utf8 field %s found", f) + return True email = email.lower().strip() if email in ['jenkins@review.openstack.org']: return True @@ -118,10 +131,10 @@ class GitChangeLog(object): if self._skip_entry(summary, date, author_email, author_name): continue log.append({ - 'summary': summary, + 'summary': translate_utf8(summary), 'when': date, - 'author_email': author_email, - 'author_name': author_name, + 'author_email': translate_utf8(author_email), + 'author_name': translate_utf8(author_name), }) # Bucketize the dates by day @@ -158,7 +171,4 @@ class RpmChangeLog(GitChangeLog): if len(sublines) > 1: for subline in sublines[1:]: lines.append(" %s" % subline) - # Replace utf8 oddities with ? just incase - contents = "\n".join(lines) - contents = contents.decode('utf8').encode('ascii', 'replace') - return contents + return "\n".join(lines) diff --git a/anvil/packaging/rpm.py b/anvil/packaging/rpm.py index afb5e745..b39b35cc 100644 --- a/anvil/packaging/rpm.py +++ b/anvil/packaging/rpm.py @@ -54,6 +54,23 @@ class DependencyPackager(comp.Component): self._build_paths = bpaths return dict(self._build_paths) + def _patches(self): + your_patches = [] + in_patches = self.get_option('patches', 'package') + if in_patches: + for path in in_patches: + path = sh.abspth(path) + if sh.isdir(path): + for c_path in sh.listdir(path, files_only=True): + tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(c_path)) + sh.copy(c_path, tgt_fn) + your_patches.append(sh.basename(tgt_fn)) + else: + tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(path)) + sh.copy(path, tgt_fn) + your_patches.append(sh.basename(tgt_fn)) + return your_patches + def _requirements(self): return { 'install': self._install_requirements(), @@ -142,6 +159,7 @@ class DependencyPackager(comp.Component): 'build': self._build_details(), 'who': sh.getuser(), 'date': utils.iso8601(), + 'patches': self._patches(), 'details': self.details, } (_fn, content) = utils.load_template('packaging', 'spec.tmpl') diff --git a/anvil/patcher.py b/anvil/patcher.py new file mode 100644 index 00000000..ca144144 --- /dev/null +++ b/anvil/patcher.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from anvil import log as logging +from anvil import shell as sh +from anvil import utils + + +LOG = logging.getLogger(__name__) + +# TODO(harlowja): use git patching vs. raw patching?? +PATCH_CMD = ['patch', '-p1'] + + +def apply_patches(patch_files, working_dir): + if not patch_files: + return + apply_files = [] + for p in patch_files: + p = sh.abspth(p) + if not sh.isfile(p): + LOG.warn("Can not apply non-file patch %s", p) + apply_files.append(p) + if not apply_files: + return + if not sh.isdir(working_dir): + LOG.warn("Can only apply %s patches 'inside' a directory and not '%s'", + len(apply_files), working_dir) + return + with utils.chdir(working_dir): + for p in apply_files: + LOG.debug("Applying patch %s in directory %s", p, working_dir) + patch_contents = sh.load_file(p) + if patch_contents: + sh.execute(*PATCH_CMD, process_input=patch_contents) diff --git a/anvil/shell.py b/anvil/shell.py index c26da89d..71387e48 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -254,8 +254,23 @@ def fileperms(path): return (os.stat(path).st_mode & 0777) -def listdir(path): - return os.listdir(path) +def listdir(path, recursive=False, dirs_only=False, files_only=False): + path = abspth(path) + all_contents = [] + if not recursive: + all_contents = os.listdir(path) + all_contents = [os.path.join(path, f) for f in all_contents] + else: + for (root, dirs, files) in os.walk(path): + for d in dirs: + all_contents.append(joinpths(root, d)) + for f in files: + all_contents.append(joinpths(root, f)) + if dirs_only: + all_contents = [f for f in all_contents if isdir(f)] + if files_only: + all_contents = [f for f in all_contents if isfile(f)] + return all_contents def isfile(fn): diff --git a/conf/templates/packaging/spec.tmpl b/conf/templates/packaging/spec.tmpl index 553115c9..84842c98 100644 --- a/conf/templates/packaging/spec.tmpl +++ b/conf/templates/packaging/spec.tmpl @@ -85,6 +85,13 @@ Requires: ${i} #end for #end if +# Custom patches +#set $size = 0 +#for $p in $patches +Patch${size}: $p +#set $size += 1 +#end for + %description #if $details.description $details.description @@ -98,6 +105,13 @@ $details.summary %setup $build.setup #end if +# Custom patches activation +#set $size = 0 +#for $p in $patches +%patch${size} -p1 +#set $size += 1 +#end for + #if $build.has_key('action') %build $build.action diff --git a/smithy b/smithy index 611d74e5..a25ba1ac 100755 --- a/smithy +++ b/smithy @@ -7,6 +7,11 @@ fi shopt -s nocasematch +# Possible locations of the epel rpm/list url +RHEL_VERSION=$(lsb_release -r | awk '{ print $2 }' | cut -d"." -f1) +EPEL_RPM_LIST="http://mirrors.kernel.org/fedora-epel/$RHEL_VERSION/i386" +NODE_RPM_URL="http://nodejs.tchol.org/repocfg/el/nodejs-stable-release.noarch.rpm" + ARGS="$@" VER=$(python -c "from anvil import version; print version.version_string()") PWD=`pwd` @@ -41,39 +46,41 @@ bootstrap_rh() echo "Please wait..." echo "Installing node.js yum repository configuration." - JS_REPO_RPM_FN="nodejs-stable-release.noarch.rpm" + JS_REPO_RPM_FN=$(basename $NODE_RPM_URL) if [ ! -f "/tmp/$JS_REPO_RPM_FN" ]; then - echo "Downloading $JS_REPO_RPM_FN" - wget -q -O "/tmp/$JS_REPO_RPM_FN" "http://nodejs.tchol.org/repocfg/el/$JS_REPO_RPM_FN" + echo "Downloading $JS_REPO_RPM_FN to /tmp/$JS_REPO_RPM_FN..." + wget -q -O "/tmp/$JS_REPO_RPM_FN" "$NODE_RPM_URL" if [ $? -ne 0 ]; then return 1 fi fi - echo "Installing /tmp/$JS_REPO_RPM_FN." + echo "Installing /tmp/$JS_REPO_RPM_FN..." yum install --assumeyes --nogpgcheck -t "/tmp/$JS_REPO_RPM_FN" 2>&1 - echo "Locating the EPEL rpm." - EPEL_RPM=$(curl -s "http://mirrors.kernel.org/fedora-epel/6/i386/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm") - if [ $? -ne 0 ]; then - return 1 - fi - if [ ! -f "/tmp/$EPEL_RPM" ]; then - echo "Downloading $EPEL_RPM." - wget -q -O "/tmp/$EPEL_RPM" "http://mirrors.kernel.org/fedora-epel/6/i386/$EPEL_RPM" + echo "Locating the EPEL rpm..." + if [ -z "$EPEL_RPM" ]; then + EPEL_RPM=$(curl -s "$EPEL_RPM_LIST/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm") if [ $? -ne 0 ]; then return 1 fi fi - echo "Installing /tmp/$EPEL_RPM." + if [ ! -f "/tmp/$EPEL_RPM" ]; then + echo "Downloading $EPEL_RPM to /tmp/$EPEL_RPM" + wget -q -O "/tmp/$EPEL_RPM" "$EPEL_RPM_LIST/$EPEL_RPM" + if [ $? -ne 0 ]; then + return 1 + fi + fi + echo "Installing /tmp/$EPEL_RPM..." yum install --assumeyes --nogpgcheck -t "/tmp/$EPEL_RPM" 2>&1 - echo "Installing needed distribution dependencies:" + echo "Installing distribution dependencies..." pkgs="gcc git pylint python python-netifaces python-pep8 python-cheetah" pkgs="$pkgs python-pip python-progressbar PyYAML python-ordereddict python-iso8601" yum install -y $pkgs 2>&1 - echo "Installing needed pypi dependencies:" - pip-python install -U -I termcolor iniparse "keyring==0.9.2" + echo "Installing pypi dependencies..." + pip-python install -U -I termcolor iniparse "keyring>=0.9.2" return 0 }