x/anvil
Code Issues Proposed changes

Helpful cleanups.

Browse Source 
1. Ensure the changelog is in ascii
2. Allow post-download patches to come from a directory instead of from a single set of files.
3. Allow rpm patches to come from a directory instead of a single list of files
4. Update smithy so that the epel rpm location can come from somewhere not epel (if desired)
5. Ensure we cleanup the 'negative' phase files correctly at the various phases.
This commit is contained in:
Joshua Harlow 2012-10-26 17:07:19 -07:00
parent 857bc0c9fd
commit 9697e7c2f9
12 changed files with 191 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
anvil/action.py
View File

@ -261,6 +261,8 @@ class Action(object):
LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
change_activate(instance, True)
component_results[c] = None
for n in neg_phase_recs:
n.unmark(c)
else:
try:
result = None

27
anvil/actions/install.py
View File

@ -56,6 +56,7 @@ class InstallAction(action.Action):
logger=LOG)
def _run(self, persona, component_order, instances):
removals = []
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.name)),
@ -64,8 +65,22 @@ class InstallAction(action.Action):
),
component_order,
instances,
"download"
"download",
*removals
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Post-download patching %s.', colorizer.quote(i.name)),
run=lambda i: i.patch("download"),
end=None,
),
component_order,
instances,
"download-patch",
*removals
)
removals += ['uninstall', 'unconfigure']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.name)),
@ -75,7 +90,7 @@ class InstallAction(action.Action):
component_order,
instances,
"configure",
'unconfigure'
*removals
)
if self.only_configure:
@ -85,6 +100,7 @@ class InstallAction(action.Action):
LOG.info("Exiting early, only asked to download and configure!")
return
removals += ['pre-uninstall', 'post-uninstall']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Preinstalling %s.', colorizer.quote(i.name)),
@ -93,7 +109,8 @@ class InstallAction(action.Action):
),
component_order,
instances,
"pre-install"
"pre-install",
*removals
)
def install_start(instance):
@ -120,7 +137,7 @@ class InstallAction(action.Action):
component_order,
instances,
"install",
'uninstall'
*removals
)
self._run_phase(
PhaseFunctors(
@ -131,5 +148,5 @@ class InstallAction(action.Action):
component_order,
instances,
"post-install",
'uninstall', 'unconfigure', 'pre-uninstall', 'post-uninstall'
*removals
)

7
anvil/actions/start.py
View File

@ -29,6 +29,7 @@ class StartAction(action.Action):
return 'running'
def _run(self, persona, component_order, instances):
removals = []
self._run_phase(
PhaseFunctors(
start=None,
@ -38,7 +39,9 @@ class StartAction(action.Action):
component_order,
instances,
"pre-start",
*removals
)
removals += ['stopped']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Starting %s.', i.name),
@ -48,7 +51,7 @@ class StartAction(action.Action):
component_order,
instances,
"start",
'stopped'
*removals
)
self._run_phase(
PhaseFunctors(
@ -59,5 +62,5 @@ class StartAction(action.Action):
component_order,
instances,
"post-start",
'stopped'
*removals
)

3
anvil/actions/stop.py
View File

@ -34,6 +34,7 @@ class StopAction(action.Action):
return components
def _run(self, persona, component_order, instances):
removals = ['pre-start', 'start', 'post-start']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.name)),
@ -43,5 +44,5 @@ class StopAction(action.Action):
component_order,
instances,
"stopped",
'pre-start', 'start', 'post-start'
*removals
)

12
anvil/actions/uninstall.py
View File

@ -34,6 +34,7 @@ class UninstallAction(action.Action):
return components
def _run(self, persona, component_order, instances):
removals = ['configure']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.name)),
@ -43,8 +44,9 @@ class UninstallAction(action.Action):
component_order,
instances,
'unconfigure',
'configure'
*removals
)
removals += ['post-install']
self._run_phase(
PhaseFunctors(
start=None,
@ -54,8 +56,9 @@ class UninstallAction(action.Action):
component_order,
instances,
'pre-uninstall',
'post-install'
*removals
)
removals += ['install']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.name)),
@ -65,8 +68,9 @@ class UninstallAction(action.Action):
component_order,
instances,
'uninstall',
'install'
*removals
)
removals += ['download', 'configure', "download-patch", 'pre-install', 'post-install']
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.name)),
@ -76,5 +80,5 @@ class UninstallAction(action.Action):
component_order,
instances,
'post-uninstall',
'download', 'configure', 'pre-install', 'install', 'post-install'
*removals
)

15
anvil/components/__init__.py
View File

@ -45,6 +45,7 @@ from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import packager
from anvil import patcher
from anvil import shell as sh
from anvil import trace as tr
from anvil import utils
@ -121,6 +122,20 @@ class PkgInstallComponent(component.Component):
down.download(self.distro, from_uri, target_dir)
return uris
def patch(self, section):
what_patches = self.get_option('patches', section)
(_from_uri, target_dir) = self._get_download_location()
if not what_patches:
what_patches = []
canon_what_patches = []
for path in what_patches:
if sh.isdir(path):
canon_what_patches.extend(sh.listdir(path, files_only=True))
elif sh.isfile(path):
canon_what_patches.append(path)
if canon_what_patches:
patcher.apply_patches(canon_what_patches, target_dir)
def config_params(self, config_fn):
mp = dict(self.params)
if config_fn:

24
anvil/packaging/helpers/changelog.py
View File

@ -28,12 +28,19 @@ import textwrap
import iso8601
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
LOG = logging.getLogger(__name__)
PER_CALL_AM = 50
def translate_utf8(text):
return text.decode('utf8').encode('ascii', 'replace')
class GitChangeLog(object):
__meta__ = abc.ABCMeta
@ -71,6 +78,12 @@ class GitChangeLog(object):
return self.date_buckets
def _skip_entry(self, summary, date, email, name):
for f in [summary, name, email]:
try:
translate_utf8(f)
except UnicodeError:
LOG.warn("Non-utf8 field %s found", f)
return True
email = email.lower().strip()
if email in ['jenkins@review.openstack.org']:
return True
@ -118,10 +131,10 @@ class GitChangeLog(object):
if self._skip_entry(summary, date, author_email, author_name):
continue
log.append({
'summary': summary,
'summary': translate_utf8(summary),
'when': date,
'author_email': author_email,
'author_name': author_name,
'author_email': translate_utf8(author_email),
'author_name': translate_utf8(author_name),
})
# Bucketize the dates by day
@ -158,7 +171,4 @@ class RpmChangeLog(GitChangeLog):
if len(sublines) > 1:
for subline in sublines[1:]:
lines.append(" %s" % subline)
# Replace utf8 oddities with ? just incase
contents = "\n".join(lines)
contents = contents.decode('utf8').encode('ascii', 'replace')
return contents
return "\n".join(lines)

18
anvil/packaging/rpm.py
View File

@ -54,6 +54,23 @@ class DependencyPackager(comp.Component):
self._build_paths = bpaths
return dict(self._build_paths)
def _patches(self):
your_patches = []
in_patches = self.get_option('patches', 'package')
if in_patches:
for path in in_patches:
path = sh.abspth(path)
if sh.isdir(path):
for c_path in sh.listdir(path, files_only=True):
tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(c_path))
sh.copy(c_path, tgt_fn)
your_patches.append(sh.basename(tgt_fn))
else:
tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(path))
sh.copy(path, tgt_fn)
your_patches.append(sh.basename(tgt_fn))
return your_patches
def _requirements(self):
return {
'install': self._install_requirements(),
@ -142,6 +159,7 @@ class DependencyPackager(comp.Component):
'build': self._build_details(),
'who': sh.getuser(),
'date': utils.iso8601(),
'patches': self._patches(),
'details': self.details,
}
(_fn, content) = utils.load_template('packaging', 'spec.tmpl')

48
anvil/patcher.py Normal file
View File

@ -0,0 +1,48 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
LOG = logging.getLogger(__name__)
# TODO(harlowja): use git patching vs. raw patching??
PATCH_CMD = ['patch', '-p1']
def apply_patches(patch_files, working_dir):
if not patch_files:
return
apply_files = []
for p in patch_files:
p = sh.abspth(p)
if not sh.isfile(p):
LOG.warn("Can not apply non-file patch %s", p)
apply_files.append(p)
if not apply_files:
return
if not sh.isdir(working_dir):
LOG.warn("Can only apply %s patches 'inside' a directory and not '%s'",
len(apply_files), working_dir)
return
with utils.chdir(working_dir):
for p in apply_files:
LOG.debug("Applying patch %s in directory %s", p, working_dir)
patch_contents = sh.load_file(p)
if patch_contents:
sh.execute(*PATCH_CMD, process_input=patch_contents)

19
anvil/shell.py
View File

@ -254,8 +254,23 @@ def fileperms(path):
return (os.stat(path).st_mode & 0777)
def listdir(path):
return os.listdir(path)
def listdir(path, recursive=False, dirs_only=False, files_only=False):
path = abspth(path)
all_contents = []
if not recursive:
all_contents = os.listdir(path)
all_contents = [os.path.join(path, f) for f in all_contents]
else:
for (root, dirs, files) in os.walk(path):
for d in dirs:
all_contents.append(joinpths(root, d))
for f in files:
all_contents.append(joinpths(root, f))
if dirs_only:
all_contents = [f for f in all_contents if isdir(f)]
if files_only:
all_contents = [f for f in all_contents if isfile(f)]
return all_contents
def isfile(fn):

14
conf/templates/packaging/spec.tmpl
View File

@ -85,6 +85,13 @@ Requires: ${i}
#end for
#end if
# Custom patches
#set $size = 0
#for $p in $patches
Patch${size}: $p
#set $size += 1
#end for
%description
#if $details.description
$details.description
@ -98,6 +105,13 @@ $details.summary
%setup $build.setup
#end if
# Custom patches activation
#set $size = 0
#for $p in $patches
%patch${size} -p1
#set $size += 1
#end for
#if $build.has_key('action')
%build
$build.action

39
smithy
View File

@ -7,6 +7,11 @@ fi
shopt -s nocasematch
# Possible locations of the epel rpm/list url
RHEL_VERSION=$(lsb_release -r | awk '{ print $2 }' | cut -d"." -f1)
EPEL_RPM_LIST="http://mirrors.kernel.org/fedora-epel/$RHEL_VERSION/i386"
NODE_RPM_URL="http://nodejs.tchol.org/repocfg/el/nodejs-stable-release.noarch.rpm"
ARGS="$@"
VER=$(python -c "from anvil import version; print version.version_string()")
PWD=`pwd`
@ -41,39 +46,41 @@ bootstrap_rh()
echo "Please wait..."
echo "Installing node.js yum repository configuration."
JS_REPO_RPM_FN="nodejs-stable-release.noarch.rpm"
JS_REPO_RPM_FN=$(basename $NODE_RPM_URL)
if [ ! -f "/tmp/$JS_REPO_RPM_FN" ]; then
echo "Downloading $JS_REPO_RPM_FN"
wget -q -O "/tmp/$JS_REPO_RPM_FN" "http://nodejs.tchol.org/repocfg/el/$JS_REPO_RPM_FN"
echo "Downloading $JS_REPO_RPM_FN to /tmp/$JS_REPO_RPM_FN..."
wget -q -O "/tmp/$JS_REPO_RPM_FN" "$NODE_RPM_URL"
if [ $? -ne 0 ]; then
return 1
fi
fi
echo "Installing /tmp/$JS_REPO_RPM_FN."
echo "Installing /tmp/$JS_REPO_RPM_FN..."
yum install --assumeyes --nogpgcheck -t "/tmp/$JS_REPO_RPM_FN" 2>&1
echo "Locating the EPEL rpm."
EPEL_RPM=$(curl -s "http://mirrors.kernel.org/fedora-epel/6/i386/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm")
if [ $? -ne 0 ]; then
return 1
fi
if [ ! -f "/tmp/$EPEL_RPM" ]; then
echo "Downloading $EPEL_RPM."
wget -q -O "/tmp/$EPEL_RPM" "http://mirrors.kernel.org/fedora-epel/6/i386/$EPEL_RPM"
echo "Locating the EPEL rpm..."
if [ -z "$EPEL_RPM" ]; then
EPEL_RPM=$(curl -s "$EPEL_RPM_LIST/" | grep -io ">\s*epel.*.rpm\s*<" | grep -io "epel.*.rpm")
if [ $? -ne 0 ]; then
return 1
fi
fi
echo "Installing /tmp/$EPEL_RPM."
if [ ! -f "/tmp/$EPEL_RPM" ]; then
echo "Downloading $EPEL_RPM to /tmp/$EPEL_RPM"
wget -q -O "/tmp/$EPEL_RPM" "$EPEL_RPM_LIST/$EPEL_RPM"
if [ $? -ne 0 ]; then
return 1
fi
fi
echo "Installing /tmp/$EPEL_RPM..."
yum install --assumeyes --nogpgcheck -t "/tmp/$EPEL_RPM" 2>&1
echo "Installing needed distribution dependencies:"
echo "Installing distribution dependencies..."
pkgs="gcc git pylint python python-netifaces python-pep8 python-cheetah"
pkgs="$pkgs python-pip python-progressbar PyYAML python-ordereddict python-iso8601"
yum install -y $pkgs 2>&1
echo "Installing needed pypi dependencies:"
pip-python install -U -I termcolor iniparse "keyring==0.9.2"
echo "Installing pypi dependencies..."
pip-python install -U -I termcolor iniparse "keyring>=0.9.2"
return 0
}