#!/usr/bin/python import argparse import distutils.spawn import email.parser import logging import os import os.path import re import shutil import subprocess import sys import tempfile import pip.util import pkg_resources logger = logging.getLogger() package_map = {} arch_dependent = set() epoch_map = {} requirements_section_re = re.compile(r'\[(.*?)\]') version_re = re.compile(r"^(.*[^.0])(\.0+)*$") setup_py = "setup.py" DEFAULT_SCRIPTS = { "prep": """%setup -n %{pkg_name}-%{unmangled_version} -n %{pkg_name}-%{unmangled_version}""", "build": """python setup.py build""", "install": """python setup.py install -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES abspath_installed_files=$(readlink -f INSTALLED_FILES) ( cd $RPM_BUILD_ROOT for i in usr/*/python*/site-packages/* usr/bin/*; do if [ -e "$i" ]; then sed -i "s@/$i/@DELETE_ME@" "$abspath_installed_files" echo "/$i" fi done if [ -d usr/man ]; then rm -rf usr/share/man mkdir -p usr/share mv usr/man usr/share/ sed -i "s@/usr/man/@DELETE_ME@" "$abspath_installed_files" for i in usr/share/man/*; do echo "/$i/*" done fi ) >> GATHERED_FILES { sed '/^DELETE_ME/d' INSTALLED_FILES; cat GATHERED_FILES; } | sort -u > INSTALLED_FILES.tmp mv -f INSTALLED_FILES{.tmp,}""", "clean": """rm -rf $RPM_BUILD_ROOT""", } class InstallationError(Exception): pass def python_name_to_key(name): return pkg_resources.Requirement.parse(name).key def python_key_to_rpm(python_name): python_name = python_name.lower() try: return package_map[python_name] except KeyError: pass python_name = python_name.replace("_", "-").replace(".", "-") if python_name.startswith("python-"): prefixed_name = python_name else: prefixed_name = "python-%s" % python_name return prefixed_name def egg_info_path(source_dir, filename): base = os.path.join(source_dir, "pip-egg-info") filenames = os.listdir(base) if not filenames: raise InstallationError("No files/directories in %s (from %s)" % (base, filename)) # if we have more than one match, we pick the toplevel one. if len(filenames) > 1: filenames.sort(key=lambda x: x.count(os.path.sep) + (os.path.altsep and x.count(os.path.altsep) or 0)) return os.path.join(base, filenames[0], filename) def egg_info_lines(source_dir, filename): filename = egg_info_path(source_dir, filename) if not os.path.exists(filename): return [] with open(filename, "r") as f: return f.readlines() def egg_info_requirements(source_dir, extras=()): in_extra = None for line in egg_info_lines(source_dir, 'requires.txt'): match = requirements_section_re.match(line.lower()) if match: in_extra = match.group(1) continue if in_extra and in_extra not in extras: # Skip requirement for an extra we aren't requiring continue yield line def pkg_info(source_dir): p = email.parser.FeedParser() filename = egg_info_path(source_dir, "PKG-INFO") if not os.path.exists(filename): logger.warn('No PKG-INFO file found in %s' % source_dir) else: with open(filename, "r") as f: for line in f.readlines(): # NOTE(aababilov): d2to1 has bad PKG-INFO # that is fixed this way: if line and not line[0].isspace() and not ":" in line: line = " " + line p.feed(line) return p.close() def setup_py_one_line(source_dir, command): """Run `python setup.py $command` and return the last line. python ldap is so clever that is prints extra stuff before package name or version. Lets return the last line """ return call_subprocess( [sys.executable, setup_py, command], cwd=source_dir, show_stdout=False)[0].splitlines()[-1].strip() def create_parser(): parser = argparse.ArgumentParser() rpm_base = os.path.expanduser("~/rpmbuild") source_dir = os.getcwd() rpmbuild_executable = (distutils.spawn.find_executable("rpmbuild") or distutils.spawn.find_executable("rpm")) parser.add_argument( "--pip-verbose", "-f", action="store_true", default=False, help="Show pip stdout") parser.add_argument( "--debug", "-d", action="store_true", default=False, help="Print debug information") parser.add_argument( "--source-only", "-s", action="store_true", default=False, help="Only generate source RPM") parser.add_argument( "--binary-only", "-b", action="store_true", default=False, help="Only generate binary RPM") parser.add_argument( "--rpm-base", metavar="", default=rpm_base, help="rpmbuild directory (default: %s)" % rpm_base) parser.add_argument( "--rpmbuild", metavar="", default=rpmbuild_executable, help="rpmbuild executable (default: %s)" % rpmbuild_executable) parser.add_argument( "--convert", "-c", dest="convert", metavar="", nargs="+", default=[], help="Python requirement name to be converted to RPM package names") parser.add_argument( dest="sources", metavar="", nargs="*", default=[source_dir], help="Source directories of packages (default: current directory)") parser.add_argument( "--scripts-dir", metavar="", default=None, help="Specify a directory with scripts for packages") parser.add_argument( "--arch-dependent", "-a", metavar="", nargs="+", default=[], help="Known architecture dependent packages") parser.add_argument( "--epoch", "-e", metavar="", type=int, default=None, help="RPM epoch for generated packages") parser.add_argument( "--epoch-map", "-x", metavar="", nargs="+", default=[], help="Forced RPM epochs for packages") parser.add_argument( "--release", "-r", metavar="", default="0%{?dist}", help="RPM release for generated packages") parser.add_argument( "--package-map", "-p", metavar="", nargs="+", default=[], help="Correspondence between Python and RPM package names") return parser def call_subprocess(cmd, cwd=None, show_stdout=True, raise_on_returncode=True): if show_stdout: stdout = None else: stdout = subprocess.PIPE proc = subprocess.Popen(cmd, cwd=cwd, stderr=None, stdin=None, stdout=stdout) ret = proc.communicate() if proc.returncode: cwd = cwd or os.getcwd() command_desc = " ".join(cmd) if raise_on_returncode: raise InstallationError( "Command %s failed with error code %s in %s" % (command_desc, proc.returncode, cwd)) else: logger.warn( "Command %s had error code %s in %s" % (command_desc, proc.returncode, cwd)) return ret def setup_logging(options): level = logging.DEBUG if options.debug else logging.WARNING handler = logging.StreamHandler(sys.stderr) logger.addHandler(handler) logger.setLevel(level) def truncate(text, max_len=77): if max_len <= 0: return '' if len(text) < max_len: return text text = text[0:max_len] + "..." return text def build_map(arguments): result = {} for arg in arguments: try: (key, value) = arg.split("==") key = python_name_to_key(key) value = value.strip() assert value except (IndexError, ValueError, AssertionError): raise InstallationError("Bad specifier: `%s'" % arg) else: result[key] = value return result def run_egg_info(source_dir, options): script = """ __file__ = __SETUP_PY__ from setuptools.command import egg_info import pkg_resources import os def replacement_run(self): self.mkpath(self.egg_info) installer = self.distribution.fetch_build_egg if self.distribution.has_ext_modules(): with open("%s/ext_modules.txt" % self.egg_info, "w") as f: pass for ep in pkg_resources.iter_entry_points('egg_info.writers'): # require=False is the change we're making: writer = ep.load(require=False) if writer: writer(self, ep.name, os.path.join(self.egg_info,ep.name)) self.find_sources() egg_info.egg_info.run = replacement_run exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec')) """ script = script.replace('__SETUP_PY__', "'setup.py'") egg_info_dir = os.path.join(source_dir, 'pip-egg-info') if not os.path.exists(egg_info_dir): os.makedirs(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] call_subprocess( [sys.executable, '-c', script, 'egg_info'] + egg_base_option, cwd=source_dir, show_stdout=options.pip_verbose) def trim_zeroes(version): """RPM mishandles versions like "0.8.0". Make it happy.""" match = version_re.match(version) if match: return match.group(1) return version def requires_and_conflicts(req_list): rpm_requires = "" rpm_conflicts = "" for line in req_list: try: req = pkg_resources.Requirement.parse(line) except Exception: continue rpm_name = python_key_to_rpm(req.key) if not req.specs: rpm_requires += "\nRequires:" rpm_requires = "%s %s" % ( rpm_requires, rpm_name) for spec in req.specs: # kind in ("==", "<=", ">=", "!=") kind = spec[0] version = trim_zeroes(spec[1]) try: version = "%s:%s" % (epoch_map[req.key], version) except KeyError: pass if kind == "!=": rpm_conflicts += "\nConflicts:" rpm_conflicts = "%s %s = %s" % ( rpm_conflicts, rpm_name, version) continue if kind == "==": kind = "=" rpm_requires += "\nRequires:" rpm_requires = "%s %s %s %s" % ( rpm_requires, rpm_name, kind, version) return rpm_requires, rpm_conflicts def one_line(line, max_len=80): line = line.replace("\n", " ") if max_len > 0: return line[:max_len] return line def build_rpm(options, filename): if os.path.isfile(filename): temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-') pip.util.unpack_file(filename, temp_dir, None, None) source_dir = temp_dir archive_name = filename elif os.path.isdir(filename): temp_dir = None archive_name = None source_dir = filename else: raise InstallationError( "`%s' is not a regular file nor a directory" % filename) run_egg_info(source_dir, options) info = pkg_info(source_dir) rpm_requires, rpm_conflicts = requires_and_conflicts( egg_info_requirements(source_dir)) # NOTE(aababilov): do not use info["name"] to get the name - it is # the key (e.g., "nose-plugin"), not the name ("nose_plugin") pkg_name = setup_py_one_line(source_dir, "--name") pkg_key = python_name_to_key(pkg_name) build_dir = options.rpm_base rpm_name = python_key_to_rpm(pkg_key) version = one_line(info["version"]) for path in (os.path.join(build_dir, "SPECS"), os.path.join(build_dir, "SOURCES")): if not os.path.isdir(path): os.makedirs(path) spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name) if not archive_name: cmdline = [ sys.executable, setup_py, "sdist", ] call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False) archive_name = "%s/dist/%s-%s.tar.gz" % (source_dir, pkg_name, version) shutil.copy(archive_name, os.path.join(build_dir, "SOURCES")) # We need to do this so that when a package such as hacking depends on # flake8 v2 that we don't go ahead and build a v2.0 version. # # Note(harlowja): Not sure why rpm seems to not understand these are the same... cleaned_version = trim_zeroes(version.replace('-', '_')) with open(spec_name, "w") as spec_file: print >> spec_file, "%define pkg_name", pkg_name print >> spec_file, "%define rpm_name", rpm_name print >> spec_file, "%define version", cleaned_version print >> spec_file, "%define release", options.release print >> spec_file, "%define unmangled_version", version print >> spec_file, "" tags = [] tags.append(("Name", "%{rpm_name}")) epoch = epoch_map.get(pkg_key, options.epoch) if epoch is not None: tags.append(("Epoch", epoch)) tags.append(("Version", "%{version}")) tags.append(("Release", "%{release}")) tags.append(("Summary", info["summary"])) archive_name = os.path.basename(archive_name) if archive_name == ("%s-%s.tar.gz" % (pkg_name, version)): tags.append(("Source0", "%{pkg_name}-%{unmangled_version}.tar.gz")) else: tags.append(("Source0", archive_name)) tags.append(("License", info["license"])) tags.append(("Group", "Development/Libraries")) tags.append(("BuildRoot", "%{_tmppath}/%{pkg_name}-%{unmangled_version}-%{release}-buildroot")) tags.append(("Prefix", "%{_prefix}")) if pkg_key not in arch_dependent: if not os.path.exists(egg_info_path(source_dir, "ext_modules.txt")): tags.append(("BuildArch", "noarch")) tags.append(("Vendor", "%s <%s>" % (info["author"], info["author-email"]))) tags.append(("Url", info["home-page"])) max_name_len = max(len(tag[0]) for tag in tags) for tag in tags: print >> spec_file, "%s:" % tag[0], " " * ( max_name_len - len(tag[0]) + 1), one_line(tag[1], max_len=-1) if rpm_requires: print >> spec_file, rpm_requires if rpm_conflicts: print >> spec_file, rpm_conflicts print >> spec_file, "\n%description\n", info["description"] for script in "prep", "build", "install", "clean": print >> spec_file, "\n\n%%%s" % script if options.scripts_dir: script_filename = "%s/%s-%s.sh" % (options.scripts_dir, pkg_key, script) if os.path.isfile(script_filename): with open(script_filename) as f_in: print >> spec_file, f_in.read() continue print >> spec_file, DEFAULT_SCRIPTS[script] print >> spec_file, """ %files -f INSTALLED_FILES %defattr(-,root,root) """ if options.source_only: rpmbuild_what = "-bs" elif options.binary_only: rpmbuild_what = "-bb" else: rpmbuild_what = "-ba" call_subprocess( [options.rpmbuild, rpmbuild_what, "--define", "_topdir %s" % build_dir, spec_name]) if temp_dir: shutil.rmtree(temp_dir) def main(): parser = create_parser() options = parser.parse_args() setup_logging(options) global arch_dependent global package_map global epoch_map arch_dependent = set(python_name_to_key(pkg) for pkg in options.arch_dependent) package_map = build_map(options.package_map) epoch_map = build_map(options.epoch_map) if options.convert: rpm_requires, rpm_conflicts = requires_and_conflicts(options.convert) if rpm_requires: print rpm_requires.strip() if rpm_conflicts: print rpm_conflicts.strip() return failed_pkgs = [] for src in (os.path.abspath(sdir) for sdir in options.sources): try: build_rpm(options, src) except Exception as ex: failed_pkgs.append((src, ex)) print >> sys.stderr, ex if failed_pkgs: print >> sys.stderr, "These packages failed to build:" for descr in failed_pkgs: print >> sys.stderr, "%s:\n\t%s" % descr sys.exit(1) if __name__ == "__main__": try: main() except Exception as exp: print >> sys.stderr, exp