x/anvil
Code Issues Proposed changes

Do not use yum to guess RPM name in py2rpm

Browse Source 
This can lead to unpredictable results: nobody knows
what repos and what package names will be available.

Also, removing `yum list` makes py2rpm faster.

Change-Id: I4892d46b9b13ea5f94a9805293381284c388cebe
This commit is contained in:
Alessio Ababilov 2013-06-06 08:00:56 +04:00 committed by Joshua Harlow
parent f016749dd0
commit 10d54505fc
4 changed files with 131 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
anvil/distro.py
View File

@ -97,7 +97,7 @@ class Distro(object):
@property
def dependency_handler_class(self):
"""Return a dependency handler that will work for this distro."""
return importer.import_entry_point(self._dependency_handler)
return importer.import_entry_point(self._dependency_handler["name"])
def extract_component(self, name, action):
"""Return the class + component info to use for doing the action w/the component."""

42
anvil/packaging/yum.py
View File

@ -64,10 +64,30 @@ class YumDependencyHandler(base.DependencyHandler):
self.tracereader = tr.TraceReader(trace_fn)
self.helper = yum_helper.Helper()
def _epoch_list(self):
return [
"--epoch-list",
] + ["%s==%s" % (name, self.OPENSTACK_EPOCH) for name in self.python_names]
def py2rpm_start_cmdline(self):
cmdline = [
self.py2rpm_executable,
"--rpm-base",
self.rpmbuild_dir,
]
if self.python_names:
cmdline += [
"--epoch-map",
] + ["%s==%s" % (name, self.OPENSTACK_EPOCH)
for name in self.python_names]
package_map = self.distro._dependency_handler.get("package_map", {})
if package_map:
cmdline += [
"--package-map",
] + ["%s==%s" % (key, value)
for key, value in package_map.iteritems()]
arch_dependent = self.distro._dependency_handler.get(
"arch_dependent", [])
if arch_dependent:
cmdline += [
"--arch-dependent",
] + arch_dependent
return cmdline
def package(self):
super(YumDependencyHandler, self).package()
@ -229,11 +249,7 @@ BuildArch: noarch
return
utils.log_iterable(sorted(package_files), logger=LOG,
header="Building RPM packages from files")
cmdline = [
self.py2rpm_executable,
"--rpm-base",
self.rpmbuild_dir,
] + self._epoch_list() + ["--"] + package_files
cmdline = self.py2rpm_start_cmdline() + ["--"] + package_files
out_filename = sh.joinpths(self.deps_dir, "py2rpm.deps.out")
LOG.info("You can watch progress in another terminal with:")
LOG.info(" tail -f %s" % out_filename)
@ -248,11 +264,7 @@ BuildArch: noarch
def _build_openstack(self):
utils.log_iterable(sorted(self.package_dirs), logger=LOG,
header="Building RPM packages for directories")
cmdline = [
self.py2rpm_executable,
"--rpm-base",
self.rpmbuild_dir,
] + self._epoch_list() + ["--"] + self.package_dirs
cmdline = self.py2rpm_start_cmdline() + ["--"] + self.package_dirs
out_filename = sh.joinpths(self.deps_dir, "py2rpm.openstack.out")
LOG.info("You can watch progress in another terminal with:")
LOG.info(" tail -f %s" % out_filename)
@ -282,7 +294,7 @@ BuildArch: noarch
if not self.python_names:
return []
cmdline = [self.py2rpm_executable, "--convert"] + python_names
cmdline = self.py2rpm_start_cmdline() + ["--convert"] + python_names
rpm_names = []
for name in sh.execute(cmdline)[0].splitlines():
# name is "Requires: rpm-name"

18
conf/distros/rhel.yaml
View File

@ -3,7 +3,23 @@
name: rhel
platform_pattern: redhat(.*)|centos(.*)
install_helper: anvil.packaging.yum:YumInstallHelper
dependency_handler: anvil.packaging.yum:YumDependencyHandler
dependency_handler:
name: anvil.packaging.yum:YumDependencyHandler
package_map:
django: Django
distribute: python-setuptools
mysql-python: MySQL-python
pam: python-pam
pastedeploy: python-paste-deploy
pycrypto: python-crypto
pyflakes: pyflakes
pylint: pylint
pyopenssl: pyOpenSSL
pyparsing: pyparsing
pysendfile: pysendfile
pytz: pytz
arch_dependent:
- selenium
commands:
apache:
name: httpd

201
tools/py2rpm
View File

@ -31,19 +31,8 @@ headers = [
'Group',
]
package_map = {
"django": "Django",
"distribute": "python-setuptools",
"pam": "python-pam",
"pycrypto": "python-crypto",
}
package_names = {}
arch_dependent = [
"selenium",
]
package_map = {}
arch_dependent = set()
epoch_map = {}
requirements_section_re = re.compile(r'\[(.*?)\]')
@ -55,7 +44,11 @@ class InstallationError(Exception):
pass
def package_name_python2rpm(python_name):
def python_name_to_key(name):
return pkg_resources.Requirement.parse(name).key
def python_key_to_rpm(python_name):
python_name = python_name.lower()
try:
return package_map[python_name]
@ -66,14 +59,6 @@ def package_name_python2rpm(python_name):
prefixed_name = python_name
else:
prefixed_name = "python-%s" % python_name
try:
return package_names[prefixed_name]
except:
pass
try:
return package_names[python_name]
except:
pass
return prefixed_name
@ -179,7 +164,7 @@ def create_parser():
"--arch-dependent", "-a",
metavar="<Python package name>",
nargs="+",
default=arch_dependent,
default=[],
help="Known architecture dependent packages")
parser.add_argument(
"--epoch", "-e",
@ -188,11 +173,17 @@ def create_parser():
default=None,
help="RPM epoch for generated packages")
parser.add_argument(
"--epoch-list", "-l",
"--epoch-map", "-x",
metavar="<Python package name == epoch number>",
nargs="+",
default=[],
help="Forced RPM epochs for packages")
parser.add_argument(
"--package-map", "-p",
metavar="<Python package name == RPM name>",
nargs="+",
default=[],
help="Correspondence between Python and RPM package names")
return parser
@ -224,20 +215,7 @@ def setup_logging(options):
logger.setLevel(level)
def build_name_map():
cmdline = ["yum", "list", "-q"]
try:
yum_list = call_subprocess(cmdline, show_stdout=False)[0]
except Exception as ex:
logging.warning(str(ex))
return
for line in yum_list.split("\n")[1:]:
if line:
line = line.split(None, 1)[0].split(".", 1)[0]
package_names[line.lower()] = line
def truncate(text, max_len):
def truncate(text, max_len=77):
if max_len <= 0:
return ''
if len(text) < max_len:
@ -246,17 +224,19 @@ def truncate(text, max_len):
return text
def build_epoch_map(options):
for epoch_spec in options.epoch_list:
def build_map(arguments):
result = {}
for arg in arguments:
try:
(name, epoch) = epoch_spec.split("==")
name = name.strip().lower()
epoch = epoch.strip()
assert(name and epoch)
except (IndexError, AssertionError):
raise InstallationError("Bad epoch specifier: `%s'" % epoch_spec)
(key, value) = arg.split("==")
key = python_name_to_key(key)
value = value.strip()
assert value
except (IndexError, ValueError, AssertionError):
raise InstallationError("Bad specifier: `%s'" % value_spec)
else:
epoch_map[name] = epoch
result[key] = value
return result
def run_egg_info(source_dir, options):
@ -306,7 +286,7 @@ def requires_and_conflicts(req_list, multiline):
req = pkg_resources.Requirement.parse(line)
except:
continue
rpm_name = package_name_python2rpm(req.key)
rpm_name = python_key_to_rpm(req.key)
if not req.specs:
if multiline:
rpm_requires += "\nRequires:"
@ -335,66 +315,6 @@ def requires_and_conflicts(req_list, multiline):
return rpm_requires, rpm_conflicts
def adjust_spec(options, spec_file_name, pkg_name, archive_name, build_dir, source_dir):
global epoch_map
rpm_name = package_name_python2rpm(pkg_name)
cmdline = [
"sed", "-i",
"-e", "s/^Name:.*$/Name: %s/" % rpm_name,
"-e", "s/%{name}/%{pkg_name}/g",
"-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name,
]
epoch = epoch_map.get(pkg_name.lower(), options.epoch)
if epoch is not None:
cmdline += [
"-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch,
]
if pkg_name.lower() in options.arch_dependent:
cmdline += [
"-e", "/^BuildArch/d",
]
if archive_name:
cmdline += [
"-e",
"s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name)
]
shutil.copy(archive_name, os.path.join(build_dir, "SOURCES"))
call_subprocess(cmdline + [spec_file_name])
cmdline = [
"sed", "-i", "-r",
"-e", "/%doc/s/ man[^ ]+//",
]
call_subprocess(cmdline + [spec_file_name])
# Do any adjustments inside the file.
with open(spec_file_name, "rb") as fh:
contents = fh.read()
# Hook up a changelog if we can.
if not re.search(r"%changelog", contents):
changelog = build_changelog(source_dir, options)
contents += changelog
# Fix how summary is not supposed to be more than 1 line but sometimes
# seems to be (distribute has this problem).
def summary_cleaner(mtch):
summary = mtch.group(1)
summary = summary.strip()
summary = summary.replace("\n", " ")
summary = truncate(summary, 77)
summary = summary.strip()
return summary + "\n" + mtch.group(2)
spec_headers = [h + ":" for h in headers]
spec_headers = "|".join(spec_headers)
contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S),
summary_cleaner, contents)
with open(spec_file_name, "wb") as fh:
fh.write(contents)
def build_rpm(options, filename):
if os.path.isfile(filename):
temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
@ -414,6 +334,7 @@ def build_rpm(options, filename):
egg_info_requirements(source_dir), multiline=False)
pkg_name = setup_py_one_line(source_dir, "--name")
pkg_key = python_name_to_key(pkg_name)
build_dir = options.rpm_base
cmdline = [
sys.executable, setup_py, "bdist_rpm",
@ -427,7 +348,7 @@ def build_rpm(options, filename):
cmdline += ["--conflicts", rpm_conflicts]
call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False)
rpm_name = package_name_python2rpm(pkg_name)
rpm_name = python_key_to_rpm(pkg_key)
spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % pkg_name)
if not os.path.exists(spec_name):
raise InstallationError("`%s' does not exist" % spec_name)
@ -435,10 +356,56 @@ def build_rpm(options, filename):
old_name = spec_name
spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)
os.rename(old_name, spec_name)
cmdline = [
"sed", "-i",
"-e", "s/^Name:.*$/Name: %s/" % rpm_name,
"-e", "s/%{name}/%{pkg_name}/g",
"-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name,
]
epoch = epoch_map.get(pkg_key, options.epoch)
if epoch is not None:
cmdline += [
"-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch,
]
if pkg_key in arch_dependent:
cmdline += [
"-e", "/^BuildArch/d",
]
if archive_name:
cmdline += [
"-e",
"s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name)
]
shutil.copy(archive_name,
os.path.join(build_dir, "SOURCES"))
call_subprocess(cmdline + [spec_name])
cmdline = [
"sed", "-i", "-r",
"-e", "/%doc/s/ man[^ ]+//",
]
call_subprocess(cmdline + [spec_name])
# Clean up (or add new things) to the spec file.
adjust_spec(options, spec_name, pkg_name, archive_name, build_dir,
source_dir)
# Do any adjustments inside the file.
with open(spec_file_name, "rb") as fh:
contents = fh.read()
# Fix how summary is not supposed to be more than 1 line but sometimes
# seems to be.
def summary_cleaner(mtch):
summary = mtch.group(1)
summary = summary.strip()
summary = summary.replace("\n", " ")
summary = truncate(summary)
summary = summary.strip()
return summary + "\n" + mtch.group(2)
spec_headers = [h + ":" for h in headers]
spec_headers = "|".join(spec_headers)
contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S),
summary_cleaner, contents)
with open(spec_file_name, "wb") as fh:
fh.write(contents)
if options.source_only:
rpmbuild_what = "-bs"
@ -457,8 +424,13 @@ def main():
parser = create_parser()
options = parser.parse_args()
setup_logging(options)
build_name_map()
build_epoch_map(options)
global arch_dependent
global package_map
global epoch_map
arch_dependent = set(python_name_to_key(pkg)
for pkg in options.arch_dependent)
package_map = build_map(options.package_map)
epoch_map = build_map(options.epoch_map)
if options.convert:
rpm_requires, rpm_conflicts = requires_and_conflicts(
@ -499,7 +471,6 @@ mv -f INSTALLED_FILES{.tmp,}
os.close(tmp_install_script[0])
else:
tmp_install_script = None
options.arch_dependent = set(pkg.lower() for pkg in options.arch_dependent)
failed_pkgs = []
for src in (os.path.abspath(sdir) for sdir in options.sources):
try: