Do not use yum to guess RPM name in py2rpm

This can lead to unpredictable results: nobody knows
what repos and what package names will be available.

Also, removing `yum list` makes py2rpm faster.

Change-Id: I4892d46b9b13ea5f94a9805293381284c388cebe

anvil/distro.py

@@ -97,7 +97,7 @@ class Distro(object):
     @property
     def dependency_handler_class(self):
         """Return a dependency handler that will work for this distro."""
-        return importer.import_entry_point(self._dependency_handler)
+        return importer.import_entry_point(self._dependency_handler["name"])
 
     def extract_component(self, name, action):
         """Return the class + component info to use for doing the action w/the component."""

anvil/packaging/yum.py

@@ -64,10 +64,30 @@ class YumDependencyHandler(base.DependencyHandler):
         self.tracereader = tr.TraceReader(trace_fn)
         self.helper = yum_helper.Helper()
 
-    def _epoch_list(self):
+    def py2rpm_start_cmdline(self):
-        return [
+        cmdline = [
-            "--epoch-list",
+            self.py2rpm_executable,
-        ] + ["%s==%s" % (name, self.OPENSTACK_EPOCH) for name in self.python_names]
+            "--rpm-base",
+            self.rpmbuild_dir,
+        ]
+        if self.python_names:
+            cmdline += [
+               "--epoch-map",
+            ] + ["%s==%s" % (name, self.OPENSTACK_EPOCH)
+                 for name in self.python_names]
+        package_map = self.distro._dependency_handler.get("package_map", {})
+        if package_map:
+            cmdline += [
+                "--package-map",
+            ] + ["%s==%s" % (key, value)
+                 for key, value in package_map.iteritems()]
+        arch_dependent = self.distro._dependency_handler.get(
+            "arch_dependent", [])
+        if arch_dependent:
+            cmdline += [
+                "--arch-dependent",
+            ] + arch_dependent
+        return cmdline
 
     def package(self):
         super(YumDependencyHandler, self).package()
@@ -229,11 +249,7 @@ BuildArch: noarch
             return
         utils.log_iterable(sorted(package_files), logger=LOG,
                            header="Building RPM packages from files")
-        cmdline = [
+        cmdline = self.py2rpm_start_cmdline() + ["--"] + package_files
-            self.py2rpm_executable,
-            "--rpm-base",
-            self.rpmbuild_dir,
-        ] + self._epoch_list() + ["--"] + package_files
         out_filename = sh.joinpths(self.deps_dir, "py2rpm.deps.out")
         LOG.info("You can watch progress in another terminal with:")
         LOG.info("    tail -f %s" % out_filename)
@@ -248,11 +264,7 @@ BuildArch: noarch
     def _build_openstack(self):
         utils.log_iterable(sorted(self.package_dirs), logger=LOG,
                            header="Building RPM packages for directories")
-        cmdline = [
+        cmdline = self.py2rpm_start_cmdline() + ["--"] + self.package_dirs
-            self.py2rpm_executable,
-            "--rpm-base",
-            self.rpmbuild_dir,
-        ] + self._epoch_list() + ["--"] + self.package_dirs
         out_filename = sh.joinpths(self.deps_dir, "py2rpm.openstack.out")
         LOG.info("You can watch progress in another terminal with:")
         LOG.info("    tail -f %s" % out_filename)
@@ -282,7 +294,7 @@ BuildArch: noarch
         if not self.python_names:
             return []
 
-        cmdline = [self.py2rpm_executable, "--convert"] + python_names
+        cmdline = self.py2rpm_start_cmdline() + ["--convert"] + python_names
         rpm_names = []
         for name in sh.execute(cmdline)[0].splitlines():
             # name is "Requires: rpm-name"

conf/distros/rhel.yaml

@@ -3,7 +3,23 @@
 name: rhel
 platform_pattern: redhat(.*)|centos(.*)
 install_helper: anvil.packaging.yum:YumInstallHelper
-dependency_handler: anvil.packaging.yum:YumDependencyHandler
+dependency_handler:
+    name: anvil.packaging.yum:YumDependencyHandler
+    package_map:
+        django: Django
+        distribute: python-setuptools
+        mysql-python: MySQL-python
+        pam: python-pam
+        pastedeploy: python-paste-deploy
+        pycrypto: python-crypto
+        pyflakes: pyflakes
+        pylint: pylint
+        pyopenssl: pyOpenSSL
+        pyparsing: pyparsing
+        pysendfile: pysendfile
+        pytz: pytz
+    arch_dependent:
+    -   selenium
 commands:
     apache:
         name: httpd

tools/py2rpm

@@ -31,19 +31,8 @@ headers = [
     'Group',
 ]
 
-package_map = {
+package_map = {}
-    "django": "Django",
+arch_dependent = set()
-    "distribute": "python-setuptools",
-    "pam": "python-pam",
-    "pycrypto": "python-crypto",
-}
-
-package_names = {}
-
-arch_dependent = [
-    "selenium",
-]
-
 epoch_map = {}
 
 requirements_section_re = re.compile(r'\[(.*?)\]')
@@ -55,7 +44,11 @@ class InstallationError(Exception):
     pass
 
 
-def package_name_python2rpm(python_name):
+def python_name_to_key(name):
+    return pkg_resources.Requirement.parse(name).key
+
+
+def python_key_to_rpm(python_name):
     python_name = python_name.lower()
     try:
         return package_map[python_name]
@@ -66,14 +59,6 @@ def package_name_python2rpm(python_name):
         prefixed_name = python_name
     else:
         prefixed_name = "python-%s" % python_name
-    try:
-        return package_names[prefixed_name]
-    except:
-        pass
-    try:
-        return package_names[python_name]
-    except:
-        pass
     return prefixed_name
 
 
@@ -179,7 +164,7 @@ def create_parser():
         "--arch-dependent", "-a",
         metavar="<Python package name>",
         nargs="+",
-        default=arch_dependent,
+        default=[],
         help="Known architecture dependent packages")
     parser.add_argument(
         "--epoch", "-e",
@@ -188,11 +173,17 @@ def create_parser():
         default=None,
         help="RPM epoch for generated packages")
     parser.add_argument(
-        "--epoch-list", "-l",
+        "--epoch-map", "-x",
         metavar="<Python package name == epoch number>",
         nargs="+",
         default=[],
         help="Forced RPM epochs for packages")
+    parser.add_argument(
+        "--package-map", "-p",
+        metavar="<Python package name == RPM name>",
+        nargs="+",
+        default=[],
+        help="Correspondence between Python and RPM package names")
     return parser
 
 
@@ -224,20 +215,7 @@ def setup_logging(options):
     logger.setLevel(level)
 
 
-def build_name_map():
+def truncate(text, max_len=77):
-    cmdline = ["yum", "list", "-q"]
-    try:
-        yum_list = call_subprocess(cmdline, show_stdout=False)[0]
-    except Exception as ex:
-        logging.warning(str(ex))
-        return
-    for line in yum_list.split("\n")[1:]:
-        if line:
-            line = line.split(None, 1)[0].split(".", 1)[0]
-            package_names[line.lower()] = line
-
-
-def truncate(text, max_len):
     if max_len <= 0:
         return ''
     if len(text) < max_len:
@@ -246,17 +224,19 @@ def truncate(text, max_len):
     return text
 
 
-def build_epoch_map(options):
+def build_map(arguments):
-    for epoch_spec in options.epoch_list:
+    result = {}
+    for arg in arguments:
         try:
-            (name, epoch) = epoch_spec.split("==")
+            (key, value) = arg.split("==")
-            name = name.strip().lower()
+            key = python_name_to_key(key)
-            epoch = epoch.strip()
+            value = value.strip()
-            assert(name and epoch)
+            assert value
-        except (IndexError, AssertionError):
+        except (IndexError, ValueError, AssertionError):
-            raise InstallationError("Bad epoch specifier: `%s'" % epoch_spec)
+            raise InstallationError("Bad specifier: `%s'" % value_spec)
         else:
-            epoch_map[name] = epoch
+            result[key] = value
+    return result
 
 
 def run_egg_info(source_dir, options):
@@ -306,7 +286,7 @@ def requires_and_conflicts(req_list, multiline):
             req = pkg_resources.Requirement.parse(line)
         except:
             continue
-        rpm_name = package_name_python2rpm(req.key)
+        rpm_name = python_key_to_rpm(req.key)
         if not req.specs:
             if multiline:
                 rpm_requires += "\nRequires:"
@@ -335,66 +315,6 @@ def requires_and_conflicts(req_list, multiline):
     return rpm_requires, rpm_conflicts
 
 
-def adjust_spec(options, spec_file_name, pkg_name, archive_name, build_dir, source_dir):
-    global epoch_map
-
-    rpm_name = package_name_python2rpm(pkg_name)
-    cmdline = [
-        "sed", "-i",
-        "-e", "s/^Name:.*$/Name: %s/" % rpm_name,
-        "-e", "s/%{name}/%{pkg_name}/g",
-        "-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name,
-    ]
-    epoch = epoch_map.get(pkg_name.lower(), options.epoch)
-    if epoch is not None:
-        cmdline += [
-            "-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch,
-        ]
-    if pkg_name.lower() in options.arch_dependent:
-        cmdline += [
-            "-e", "/^BuildArch/d",
-        ]
-    if archive_name:
-        cmdline += [
-            "-e",
-            "s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name)
-        ]
-        shutil.copy(archive_name, os.path.join(build_dir, "SOURCES"))
-    call_subprocess(cmdline + [spec_file_name])
-    cmdline = [
-        "sed", "-i", "-r",
-        "-e", "/%doc/s/ man[^ ]+//",
-    ]
-    call_subprocess(cmdline + [spec_file_name])
-
-    # Do any adjustments inside the file.
-    with open(spec_file_name, "rb") as fh:
-        contents = fh.read()
-
-    # Hook up a changelog if we can.
-    if not re.search(r"%changelog", contents):
-        changelog = build_changelog(source_dir, options)
-        contents += changelog
-
-    # Fix how summary is not supposed to be more than 1 line but sometimes
-    # seems to be (distribute has this problem).
-    def summary_cleaner(mtch):
-        summary = mtch.group(1)
-        summary = summary.strip()
-        summary = summary.replace("\n", " ")
-        summary = truncate(summary, 77)
-        summary = summary.strip()
-        return summary + "\n" + mtch.group(2)
-
-    spec_headers = [h + ":" for h in headers]
-    spec_headers = "|".join(spec_headers)
-    contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S),
-                      summary_cleaner, contents)
-
-    with open(spec_file_name, "wb") as fh:
-        fh.write(contents)
-
-
 def build_rpm(options, filename):
     if os.path.isfile(filename):
         temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
@@ -414,6 +334,7 @@ def build_rpm(options, filename):
         egg_info_requirements(source_dir), multiline=False)
 
     pkg_name = setup_py_one_line(source_dir, "--name")
+    pkg_key = python_name_to_key(pkg_name)
     build_dir = options.rpm_base
     cmdline = [
         sys.executable, setup_py, "bdist_rpm",
@@ -427,7 +348,7 @@ def build_rpm(options, filename):
         cmdline += ["--conflicts",  rpm_conflicts]
     call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False)
 
-    rpm_name = package_name_python2rpm(pkg_name)
+    rpm_name = python_key_to_rpm(pkg_key)
     spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % pkg_name)
     if not os.path.exists(spec_name):
         raise InstallationError("`%s' does not exist" % spec_name)
@@ -435,10 +356,56 @@ def build_rpm(options, filename):
         old_name = spec_name
         spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)
         os.rename(old_name, spec_name)
+    cmdline = [
+        "sed", "-i",
+        "-e", "s/^Name:.*$/Name: %s/" % rpm_name,
+        "-e", "s/%{name}/%{pkg_name}/g",
+        "-e", "s/^%%define name.*$/%%define pkg_name %s/" % pkg_name,
+    ]
+    epoch = epoch_map.get(pkg_key, options.epoch)
+    if epoch is not None:
+        cmdline += [
+            "-e", "s/^Version:/Epoch: %s\\nVersion:/" % epoch,
+        ]
+    if pkg_key in arch_dependent:
+        cmdline += [
+            "-e", "/^BuildArch/d",
+        ]
+    if archive_name:
+        cmdline += [
+            "-e",
+            "s/^Source0: .*$/Source0: %s/" % os.path.basename(archive_name)
+        ]
+        shutil.copy(archive_name,
+                    os.path.join(build_dir, "SOURCES"))
+    call_subprocess(cmdline + [spec_name])
+    cmdline = [
+        "sed", "-i", "-r",
+        "-e", "/%doc/s/ man[^ ]+//",
+    ]
+    call_subprocess(cmdline + [spec_name])
 
-    # Clean up (or add new things) to the spec file.
+    # Do any adjustments inside the file.
-    adjust_spec(options, spec_name, pkg_name, archive_name, build_dir,
+    with open(spec_file_name, "rb") as fh:
-                source_dir)
+        contents = fh.read()
+
+    # Fix how summary is not supposed to be more than 1 line but sometimes
+    # seems to be.
+    def summary_cleaner(mtch):
+        summary = mtch.group(1)
+        summary = summary.strip()
+        summary = summary.replace("\n", " ")
+        summary = truncate(summary)
+        summary = summary.strip()
+        return summary + "\n" + mtch.group(2)
+
+    spec_headers = [h + ":" for h in headers]
+    spec_headers = "|".join(spec_headers)
+    contents = re.sub(re.compile(r"(^Summary:.*?)(" + spec_headers+ ")", re.M|re.S),
+                      summary_cleaner, contents)
+
+    with open(spec_file_name, "wb") as fh:
+        fh.write(contents)
 
     if options.source_only:
         rpmbuild_what = "-bs"
@@ -457,8 +424,13 @@ def main():
     parser = create_parser()
     options = parser.parse_args()
     setup_logging(options)
-    build_name_map()
+    global arch_dependent
-    build_epoch_map(options)
+    global package_map
+    global epoch_map
+    arch_dependent = set(python_name_to_key(pkg)
+                         for pkg in options.arch_dependent)
+    package_map = build_map(options.package_map)
+    epoch_map = build_map(options.epoch_map)
 
     if options.convert:
         rpm_requires, rpm_conflicts = requires_and_conflicts(
@@ -499,7 +471,6 @@ mv -f INSTALLED_FILES{.tmp,}
         os.close(tmp_install_script[0])
     else:
         tmp_install_script = None
-    options.arch_dependent = set(pkg.lower() for pkg in options.arch_dependent)
     failed_pkgs = []
     for src in (os.path.abspath(sdir) for sdir in options.sources):
         try: