From 10d54505fc23ac1fbafab855af59f81d0701f4e7 Mon Sep 17 00:00:00 2001 From: Alessio Ababilov Date: Thu, 6 Jun 2013 08:00:56 +0400 Subject: [PATCH] Do not use yum to guess RPM name in py2rpm This can lead to unpredictable results: nobody knows what repos and what package names will be available. Also, removing `yum list` makes py2rpm faster. Change-Id: I4892d46b9b13ea5f94a9805293381284c388cebe --- anvil/distro.py | 2 +- anvil/packaging/yum.py | 42 ++++++--- conf/distros/rhel.yaml | 18 +++- tools/py2rpm | 201 ++++++++++++++++++----------------------- 4 files changed, 131 insertions(+), 132 deletions(-) diff --git a/anvil/distro.py b/anvil/distro.py index 40903fbb..ea58ec54 100644 --- a/anvil/distro.py +++ b/anvil/distro.py @@ -97,7 +97,7 @@ class Distro(object): @property def dependency_handler_class(self): """Return a dependency handler that will work for this distro.""" - return importer.import_entry_point(self._dependency_handler) + return importer.import_entry_point(self._dependency_handler["name"]) def extract_component(self, name, action): """Return the class + component info to use for doing the action w/the component.""" diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 6b7af13f..a2602b4e 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -64,10 +64,30 @@ class YumDependencyHandler(base.DependencyHandler): self.tracereader = tr.TraceReader(trace_fn) self.helper = yum_helper.Helper() - def _epoch_list(self): - return [ - "--epoch-list", - ] + ["%s==%s" % (name, self.OPENSTACK_EPOCH) for name in self.python_names] + def py2rpm_start_cmdline(self): + cmdline = [ + self.py2rpm_executable, + "--rpm-base", + self.rpmbuild_dir, + ] + if self.python_names: + cmdline += [ + "--epoch-map", + ] + ["%s==%s" % (name, self.OPENSTACK_EPOCH) + for name in self.python_names] + package_map = self.distro._dependency_handler.get("package_map", {}) + if package_map: + cmdline += [ + "--package-map", + ] + ["%s==%s" % (key, value) + for key, value in package_map.iteritems()] + arch_dependent = self.distro._dependency_handler.get( + "arch_dependent", []) + if arch_dependent: + cmdline += [ + "--arch-dependent", + ] + arch_dependent + return cmdline def package(self): super(YumDependencyHandler, self).package() @@ -229,11 +249,7 @@ BuildArch: noarch return utils.log_iterable(sorted(package_files), logger=LOG, header="Building RPM packages from files") - cmdline = [ - self.py2rpm_executable, - "--rpm-base", - self.rpmbuild_dir, - ] + self._epoch_list() + ["--"] + package_files + cmdline = self.py2rpm_start_cmdline() + ["--"] + package_files out_filename = sh.joinpths(self.deps_dir, "py2rpm.deps.out") LOG.info("You can watch progress in another terminal with:") LOG.info(" tail -f %s" % out_filename) @@ -248,11 +264,7 @@ BuildArch: noarch def _build_openstack(self): utils.log_iterable(sorted(self.package_dirs), logger=LOG, header="Building RPM packages for directories") - cmdline = [ - self.py2rpm_executable, - "--rpm-base", - self.rpmbuild_dir, - ] + self._epoch_list() + ["--"] + self.package_dirs + cmdline = self.py2rpm_start_cmdline() + ["--"] + self.package_dirs out_filename = sh.joinpths(self.deps_dir, "py2rpm.openstack.out") LOG.info("You can watch progress in another terminal with:") LOG.info(" tail -f %s" % out_filename) @@ -282,7 +294,7 @@ BuildArch: noarch if not self.python_names: return [] - cmdline = [self.py2rpm_executable, "--convert"] + python_names + cmdline = self.py2rpm_start_cmdline() + ["--convert"] + python_names rpm_names = [] for name in sh.execute(cmdline)[0].splitlines(): # name is "Requires: rpm-name" diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index 4b351d38..6a8a4116 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -3,7 +3,23 @@ name: rhel platform_pattern: redhat(.*)|centos(.*) install_helper: anvil.packaging.yum:YumInstallHelper -dependency_handler: anvil.packaging.yum:YumDependencyHandler +dependency_handler: + name: anvil.packaging.yum:YumDependencyHandler + package_map: + django: Django + distribute: python-setuptools + mysql-python: MySQL-python + pam: python-pam + pastedeploy: python-paste-deploy + pycrypto: python-crypto + pyflakes: pyflakes + pylint: pylint + pyopenssl: pyOpenSSL + pyparsing: pyparsing + pysendfile: pysendfile + pytz: pytz + arch_dependent: + - selenium commands: apache: name: httpd diff --git a/tools/py2rpm b/tools/py2rpm index 08290a88..7bede1e6 100755 --- a/tools/py2rpm +++ b/tools/py2rpm @@ -31,19 +31,8 @@ headers = [ 'Group', ] -package_map = { - "django": "Django", - "distribute": "python-setuptools", - "pam": "python-pam", - "pycrypto": "python-crypto", -} - -package_names = {} - -arch_dependent = [ - "selenium", -] - +package_map = {} +arch_dependent = set() epoch_map = {} requirements_section_re = re.compile(r'\[(.*?)\]') @@ -55,7 +44,11 @@ class InstallationError(Exception): pass -def package_name_python2rpm(python_name): +def python_name_to_key(name): + return pkg_resources.Requirement.parse(name).key + + +def python_key_to_rpm(python_name): python_name = python_name.lower() try: return package_map[python_name] @@ -66,14 +59,6 @@ def package_name_python2rpm(python_name): prefixed_name = python_name else: prefixed_name = "python-%s" % python_name - try: - return package_names[prefixed_name] - except: - pass - try: - return package_names[python_name] - except: - pass return prefixed_name @@ -179,7 +164,7 @@ def create_parser(): "--arch-dependent", "-a", metavar="", nargs="+", - default=arch_dependent, + default=[], help="Known architecture dependent packages") parser.add_argument( "--epoch", "-e", @@ -188,11 +173,17 @@ def create_parser(): default=None, help="RPM epoch for generated packages") parser.add_argument( - "--epoch-list", "-l", + "--epoch-map", "-x", metavar="", nargs="+", default=[], help="Forced RPM epochs for packages") + parser.add_argument( + "--package-map", "-p", + metavar="", + nargs="+", + default=[], + help="Correspondence between Python and RPM package names") return parser @@ -224,20 +215,7 @@ def setup_logging(options): logger.setLevel(level) -def build_name_map(): - cmdline = ["yum", "list", "-q"] - try: - yum_list = call_subprocess(cmdline, show_stdout=False)[0] - except Exception as ex: - logging.warning(str(ex)) - return - for line in yum_list.split("\n")[1:]: - if line: - line = line.split(None, 1)[0].split(".", 1)[0] - package_names[line.lower()] = line - - -def truncate(text, max_len): +def truncate(text, max_len=77): if max_len <= 0: return '' if len(text) < max_len: @@ -246,17 +224,19 @@ def truncate(text, max_len): return text -def build_epoch_map(options): - for epoch_spec in options.epoch_list: +def build_map(arguments): + result = {} + for arg in arguments: try: - (name, epoch) = epoch_spec.split("==") - name = name.strip().lower() - epoch = epoch.strip() - assert(name and epoch) - except (IndexError, AssertionError): - raise InstallationError("Bad epoch specifier: `%s'" % epoch_spec) + (key, value) = arg.split("==") + key = python_name_to_key(key) + value = value.strip() + assert value + except (IndexError, ValueError, AssertionError): + raise InstallationError("Bad specifier: `%s'" % value_spec) else: - epoch_map[name] = epoch + result[key] = value + return result def run_egg_info(source_dir, options): @@ -306,7 +286,7 @@ def requires_and_conflicts(req_list, multiline): req = pkg_resources.Requirement.parse(line) except: continue - rpm_name = package_name_python2rpm(req.key) + rpm_name = python_key_to_rpm(req.key) if not req.specs: if multiline: rpm_requires += "\nRequires:" @@ -335,66 +315,6 @@ def requires_and_conflicts(req_list, multiline): return rpm_requires, rpm_conflicts -def adjust_spec(options, spec_file_name, pkg_name, archive_name, build_dir, source_dir): - global epoch_map - - rpm_name = package_name_python2rpm(pkg_name) - cmdline = [ - "sed", "-i", - "-e", "s/^Name:.*$/Name: %s/" % rpm_name, - "-e", "s/%{name}/%{pkg_name}/g", - "-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name, - ] - epoch = epoch_map.get(pkg_name.lower(), options.epoch) - if epoch is not None: - cmdline += [ - "-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch, - ] - if pkg_name.lower() in options.arch_dependent: - cmdline += [ - "-e", "/^BuildArch/d", - ] - if archive_name: - cmdline += [ - "-e", - "s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name) - ] - shutil.copy(archive_name, os.path.join(build_dir, "SOURCES")) - call_subprocess(cmdline + [spec_file_name]) - cmdline = [ - "sed", "-i", "-r", - "-e", "/%doc/s/ man[^ ]+//", - ] - call_subprocess(cmdline + [spec_file_name]) - - # Do any adjustments inside the file. - with open(spec_file_name, "rb") as fh: - contents = fh.read() - - # Hook up a changelog if we can. - if not re.search(r"%changelog", contents): - changelog = build_changelog(source_dir, options) - contents += changelog - - # Fix how summary is not supposed to be more than 1 line but sometimes - # seems to be (distribute has this problem). - def summary_cleaner(mtch): - summary = mtch.group(1) - summary = summary.strip() - summary = summary.replace("\n", " ") - summary = truncate(summary, 77) - summary = summary.strip() - return summary + "\n" + mtch.group(2) - - spec_headers = [h + ":" for h in headers] - spec_headers = "|".join(spec_headers) - contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S), - summary_cleaner, contents) - - with open(spec_file_name, "wb") as fh: - fh.write(contents) - - def build_rpm(options, filename): if os.path.isfile(filename): temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-') @@ -414,6 +334,7 @@ def build_rpm(options, filename): egg_info_requirements(source_dir), multiline=False) pkg_name = setup_py_one_line(source_dir, "--name") + pkg_key = python_name_to_key(pkg_name) build_dir = options.rpm_base cmdline = [ sys.executable, setup_py, "bdist_rpm", @@ -427,7 +348,7 @@ def build_rpm(options, filename): cmdline += ["--conflicts", rpm_conflicts] call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False) - rpm_name = package_name_python2rpm(pkg_name) + rpm_name = python_key_to_rpm(pkg_key) spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % pkg_name) if not os.path.exists(spec_name): raise InstallationError("`%s' does not exist" % spec_name) @@ -435,10 +356,56 @@ def build_rpm(options, filename): old_name = spec_name spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name) os.rename(old_name, spec_name) + cmdline = [ + "sed", "-i", + "-e", "s/^Name:.*$/Name: %s/" % rpm_name, + "-e", "s/%{name}/%{pkg_name}/g", + "-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name, + ] + epoch = epoch_map.get(pkg_key, options.epoch) + if epoch is not None: + cmdline += [ + "-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch, + ] + if pkg_key in arch_dependent: + cmdline += [ + "-e", "/^BuildArch/d", + ] + if archive_name: + cmdline += [ + "-e", + "s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name) + ] + shutil.copy(archive_name, + os.path.join(build_dir, "SOURCES")) + call_subprocess(cmdline + [spec_name]) + cmdline = [ + "sed", "-i", "-r", + "-e", "/%doc/s/ man[^ ]+//", + ] + call_subprocess(cmdline + [spec_name]) - # Clean up (or add new things) to the spec file. - adjust_spec(options, spec_name, pkg_name, archive_name, build_dir, - source_dir) + # Do any adjustments inside the file. + with open(spec_file_name, "rb") as fh: + contents = fh.read() + + # Fix how summary is not supposed to be more than 1 line but sometimes + # seems to be. + def summary_cleaner(mtch): + summary = mtch.group(1) + summary = summary.strip() + summary = summary.replace("\n", " ") + summary = truncate(summary) + summary = summary.strip() + return summary + "\n" + mtch.group(2) + + spec_headers = [h + ":" for h in headers] + spec_headers = "|".join(spec_headers) + contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S), + summary_cleaner, contents) + + with open(spec_file_name, "wb") as fh: + fh.write(contents) if options.source_only: rpmbuild_what = "-bs" @@ -457,8 +424,13 @@ def main(): parser = create_parser() options = parser.parse_args() setup_logging(options) - build_name_map() - build_epoch_map(options) + global arch_dependent + global package_map + global epoch_map + arch_dependent = set(python_name_to_key(pkg) + for pkg in options.arch_dependent) + package_map = build_map(options.package_map) + epoch_map = build_map(options.epoch_map) if options.convert: rpm_requires, rpm_conflicts = requires_and_conflicts( @@ -499,7 +471,6 @@ mv -f INSTALLED_FILES{.tmp,} os.close(tmp_install_script[0]) else: tmp_install_script = None - options.arch_dependent = set(pkg.lower() for pkg in options.arch_dependent) failed_pkgs = [] for src in (os.path.abspath(sdir) for sdir in options.sources): try: