#!/usr/bin/python
import argparse
import distutils.spawn
import email.parser
import logging
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
import pip.util
import pkg_resources
logger = logging.getLogger()
package_map = {}
arch_dependent = set()
epoch_map = {}
requirements_section_re = re.compile(r'\[(.*?)\]')
version_re = re.compile(r"^(.*[^.0])(\.0+)*$")
setup_py = "setup.py"
DEFAULT_SCRIPTS = {
"prep":
"""%setup -n %{pkg_name}-%{unmangled_version} -n %{pkg_name}-%{unmangled_version}""",
"build":
"""python setup.py build""",
"install":
"""python setup.py install -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
abspath_installed_files=$(readlink -f INSTALLED_FILES)
(
cd $RPM_BUILD_ROOT
for i in usr/*/python*/site-packages/* usr/bin/*; do
if [ -e "$i" ]; then
sed -i "s@/$i/@DELETE_ME@" "$abspath_installed_files"
echo "/$i"
fi
done
if [ -d usr/man ]; then
rm -rf usr/share/man
mkdir -p usr/share
mv usr/man usr/share/
sed -i "s@/usr/man/@DELETE_ME@" "$abspath_installed_files"
for i in usr/share/man/*; do
echo "/$i/*"
done
fi
) >> GATHERED_FILES
{ sed '/^DELETE_ME/d' INSTALLED_FILES; cat GATHERED_FILES; } | sort -u > INSTALLED_FILES.tmp
mv -f INSTALLED_FILES{.tmp,}""",
"clean":
"""rm -rf $RPM_BUILD_ROOT""",
}
class InstallationError(Exception):
pass
def python_name_to_key(name):
return pkg_resources.Requirement.parse(name).key
def python_key_to_rpm(python_name):
python_name = python_name.lower()
try:
return package_map[python_name]
except KeyError:
pass
python_name = python_name.replace("_", "-").replace(".", "-")
if python_name.startswith("python-"):
prefixed_name = python_name
else:
prefixed_name = "python-%s" % python_name
return prefixed_name
def egg_info_path(source_dir, filename):
base = os.path.join(source_dir, "pip-egg-info")
filenames = os.listdir(base)
if not filenames:
raise InstallationError("No files/directories in %s (from %s)"
% (base, filename))
# if we have more than one match, we pick the toplevel one.
if len(filenames) > 1:
filenames.sort(key=lambda x: x.count(os.path.sep) +
(os.path.altsep and
x.count(os.path.altsep) or 0))
return os.path.join(base, filenames[0], filename)
def egg_info_lines(source_dir, filename):
filename = egg_info_path(source_dir, filename)
if not os.path.exists(filename):
return []
with open(filename, "r") as f:
return f.readlines()
def egg_info_requirements(source_dir, extras=()):
in_extra = None
for line in egg_info_lines(source_dir, 'requires.txt'):
match = requirements_section_re.match(line.lower())
if match:
in_extra = match.group(1)
continue
if in_extra and in_extra not in extras:
# Skip requirement for an extra we aren't requiring
continue
yield line
def pkg_info(source_dir):
p = email.parser.FeedParser()
filename = egg_info_path(source_dir, "PKG-INFO")
if not os.path.exists(filename):
logger.warn('No PKG-INFO file found in %s' % source_dir)
else:
with open(filename, "r") as f:
for line in f.readlines():
# NOTE(aababilov): d2to1 has bad PKG-INFO
# that is fixed this way:
if line and not line[0].isspace() and not ":" in line:
line = " " + line
p.feed(line)
return p.close()
def setup_py_one_line(source_dir, command):
"""Run `python setup.py $command` and return the last line.
python ldap is so clever that is prints extra stuff
before package name or version. Lets return the last line
"""
return call_subprocess(
[sys.executable, setup_py, command],
cwd=source_dir, show_stdout=False)[0].splitlines()[-1].strip()
def create_parser():
parser = argparse.ArgumentParser()
rpm_base = os.path.expanduser("~/rpmbuild")
source_dir = os.getcwd()
rpmbuild_executable = (distutils.spawn.find_executable("rpmbuild") or
distutils.spawn.find_executable("rpm"))
parser.add_argument(
"--pip-verbose", "-f",
action="store_true",
default=False,
help="Show pip stdout")
parser.add_argument(
"--debug", "-d",
action="store_true",
default=False,
help="Print debug information")
parser.add_argument(
"--source-only", "-s",
action="store_true",
default=False,
help="Only generate source RPM")
parser.add_argument(
"--binary-only", "-b",
action="store_true",
default=False,
help="Only generate binary RPM")
parser.add_argument(
"--rpm-base",
metavar="
",
default=rpm_base,
help="rpmbuild directory (default: %s)" % rpm_base)
parser.add_argument(
"--rpmbuild",
metavar="",
default=rpmbuild_executable,
help="rpmbuild executable (default: %s)" % rpmbuild_executable)
parser.add_argument(
"--convert", "-c",
dest="convert",
metavar="",
nargs="+",
default=[],
help="Python requirement name to be converted to RPM package names")
parser.add_argument(
dest="sources",
metavar="",
nargs="*",
default=[source_dir],
help="Source directories of packages (default: current directory)")
parser.add_argument(
"--scripts-dir",
metavar="",
default=None,
help="Specify a directory with scripts for packages")
parser.add_argument(
"--arch-dependent", "-a",
metavar="",
nargs="+",
default=[],
help="Known architecture dependent packages")
parser.add_argument(
"--epoch", "-e",
metavar="",
type=int,
default=None,
help="RPM epoch for generated packages")
parser.add_argument(
"--epoch-map", "-x",
metavar="",
nargs="+",
default=[],
help="Forced RPM epochs for packages")
parser.add_argument(
"--release", "-r",
metavar="",
default="0%{?dist}",
help="RPM release for generated packages")
parser.add_argument(
"--package-map", "-p",
metavar="",
nargs="+",
default=[],
help="Correspondence between Python and RPM package names")
return parser
def call_subprocess(cmd, cwd=None, show_stdout=True, raise_on_returncode=True):
if show_stdout:
stdout = None
else:
stdout = subprocess.PIPE
proc = subprocess.Popen(cmd, cwd=cwd, stderr=None, stdin=None, stdout=stdout)
ret = proc.communicate()
if proc.returncode:
cwd = cwd or os.getcwd()
command_desc = " ".join(cmd)
if raise_on_returncode:
raise InstallationError(
"Command %s failed with error code %s in %s"
% (command_desc, proc.returncode, cwd))
else:
logger.warn(
"Command %s had error code %s in %s"
% (command_desc, proc.returncode, cwd))
return ret
def setup_logging(options):
level = logging.DEBUG if options.debug else logging.WARNING
handler = logging.StreamHandler(sys.stderr)
logger.addHandler(handler)
logger.setLevel(level)
def truncate(text, max_len=77):
if max_len <= 0:
return ''
if len(text) < max_len:
return text
text = text[0:max_len] + "..."
return text
def build_map(arguments):
result = {}
for arg in arguments:
try:
(key, value) = arg.split("==")
key = python_name_to_key(key)
value = value.strip()
assert value
except (IndexError, ValueError, AssertionError):
raise InstallationError("Bad specifier: `%s'" % arg)
else:
result[key] = value
return result
def run_egg_info(source_dir, options):
script = """
__file__ = __SETUP_PY__
from setuptools.command import egg_info
import pkg_resources
import os
def replacement_run(self):
self.mkpath(self.egg_info)
installer = self.distribution.fetch_build_egg
if self.distribution.has_ext_modules():
with open("%s/ext_modules.txt" % self.egg_info, "w") as f:
pass
for ep in pkg_resources.iter_entry_points('egg_info.writers'):
# require=False is the change we're making:
writer = ep.load(require=False)
if writer:
writer(self, ep.name, os.path.join(self.egg_info,ep.name))
self.find_sources()
egg_info.egg_info.run = replacement_run
exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
"""
script = script.replace('__SETUP_PY__', "'setup.py'")
egg_info_dir = os.path.join(source_dir, 'pip-egg-info')
if not os.path.exists(egg_info_dir):
os.makedirs(egg_info_dir)
egg_base_option = ['--egg-base', 'pip-egg-info']
call_subprocess(
[sys.executable, '-c', script, 'egg_info'] + egg_base_option,
cwd=source_dir,
show_stdout=options.pip_verbose)
def trim_zeroes(version):
"""RPM mishandles versions like "0.8.0". Make it happy."""
match = version_re.match(version)
if match:
return match.group(1)
return version
def requires_and_conflicts(req_list):
rpm_requires = ""
rpm_conflicts = ""
for line in req_list:
try:
req = pkg_resources.Requirement.parse(line)
except Exception:
continue
rpm_name = python_key_to_rpm(req.key)
if not req.specs:
rpm_requires += "\nRequires:"
rpm_requires = "%s %s" % (
rpm_requires, rpm_name)
for spec in req.specs:
# kind in ("==", "<=", ">=", "!=")
kind = spec[0]
version = trim_zeroes(spec[1])
try:
version = "%s:%s" % (epoch_map[req.key], version)
except KeyError:
pass
if kind == "!=":
rpm_conflicts += "\nConflicts:"
rpm_conflicts = "%s %s = %s" % (
rpm_conflicts, rpm_name, version)
continue
if kind == "==":
kind = "="
rpm_requires += "\nRequires:"
rpm_requires = "%s %s %s %s" % (
rpm_requires, rpm_name, kind, version)
return rpm_requires, rpm_conflicts
def one_line(line, max_len=80):
line = line.replace("\n", " ")
if max_len > 0:
return line[:max_len]
return line
def build_rpm(options, filename):
if os.path.isfile(filename):
temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
pip.util.unpack_file(filename, temp_dir, None, None)
source_dir = temp_dir
archive_name = filename
elif os.path.isdir(filename):
temp_dir = None
archive_name = None
source_dir = filename
else:
raise InstallationError(
"`%s' is not a regular file nor a directory" % filename)
run_egg_info(source_dir, options)
info = pkg_info(source_dir)
rpm_requires, rpm_conflicts = requires_and_conflicts(
egg_info_requirements(source_dir))
# NOTE(aababilov): do not use info["name"] to get the name - it is
# the key (e.g., "nose-plugin"), not the name ("nose_plugin")
pkg_name = setup_py_one_line(source_dir, "--name")
pkg_key = python_name_to_key(pkg_name)
build_dir = options.rpm_base
rpm_name = python_key_to_rpm(pkg_key)
version = one_line(info["version"])
for path in (os.path.join(build_dir, "SPECS"),
os.path.join(build_dir, "SOURCES")):
if not os.path.isdir(path):
os.makedirs(path)
spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)
if not archive_name:
cmdline = [
sys.executable, setup_py, "sdist",
]
call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False)
archive_name = "%s/dist/%s-%s.tar.gz" % (source_dir, pkg_name, version)
shutil.copy(archive_name, os.path.join(build_dir, "SOURCES"))
# We need to do this so that when a package such as hacking depends on
# flake8 v2 that we don't go ahead and build a v2.0 version.
#
# Note(harlowja): Not sure why rpm seems to not understand these are the same...
cleaned_version = trim_zeroes(version.replace('-', '_'))
with open(spec_name, "w") as spec_file:
print >> spec_file, "%define pkg_name", pkg_name
print >> spec_file, "%define rpm_name", rpm_name
print >> spec_file, "%define version", cleaned_version
print >> spec_file, "%define release", options.release
print >> spec_file, "%define unmangled_version", version
print >> spec_file, ""
tags = []
tags.append(("Name", "%{rpm_name}"))
epoch = epoch_map.get(pkg_key, options.epoch)
if epoch is not None:
tags.append(("Epoch", epoch))
tags.append(("Version", "%{version}"))
tags.append(("Release", "%{release}"))
tags.append(("Summary", info["summary"]))
archive_name = os.path.basename(archive_name)
if archive_name == ("%s-%s.tar.gz" % (pkg_name, version)):
tags.append(("Source0", "%{pkg_name}-%{unmangled_version}.tar.gz"))
else:
tags.append(("Source0", archive_name))
tags.append(("License", info["license"]))
tags.append(("Group", "Development/Libraries"))
tags.append(("BuildRoot", "%{_tmppath}/%{pkg_name}-%{unmangled_version}-%{release}-buildroot"))
tags.append(("Prefix", "%{_prefix}"))
if pkg_key not in arch_dependent:
if not os.path.exists(egg_info_path(source_dir, "ext_modules.txt")):
tags.append(("BuildArch", "noarch"))
tags.append(("Vendor", "%s <%s>" % (info["author"], info["author-email"])))
tags.append(("Url", info["home-page"]))
max_name_len = max(len(tag[0]) for tag in tags)
for tag in tags:
print >> spec_file, "%s:" % tag[0], " " * (
max_name_len - len(tag[0]) + 1), one_line(tag[1], max_len=-1)
if rpm_requires:
print >> spec_file, rpm_requires
if rpm_conflicts:
print >> spec_file, rpm_conflicts
print >> spec_file, "\n%description\n", info["description"]
for script in "prep", "build", "install", "clean":
print >> spec_file, "\n\n%%%s" % script
if options.scripts_dir:
script_filename = "%s/%s-%s.sh" % (options.scripts_dir, pkg_key, script)
if os.path.isfile(script_filename):
with open(script_filename) as f_in:
print >> spec_file, f_in.read()
continue
print >> spec_file, DEFAULT_SCRIPTS[script]
print >> spec_file, """
%files -f INSTALLED_FILES
%defattr(-,root,root)
"""
if options.source_only:
rpmbuild_what = "-bs"
elif options.binary_only:
rpmbuild_what = "-bb"
else:
rpmbuild_what = "-ba"
call_subprocess(
[options.rpmbuild, rpmbuild_what,
"--define", "_topdir %s" % build_dir,
spec_name])
if temp_dir:
shutil.rmtree(temp_dir)
def main():
parser = create_parser()
options = parser.parse_args()
setup_logging(options)
global arch_dependent
global package_map
global epoch_map
arch_dependent = set(python_name_to_key(pkg)
for pkg in options.arch_dependent)
package_map = build_map(options.package_map)
epoch_map = build_map(options.epoch_map)
if options.convert:
rpm_requires, rpm_conflicts = requires_and_conflicts(options.convert)
if rpm_requires:
print rpm_requires.strip()
if rpm_conflicts:
print rpm_conflicts.strip()
return
failed_pkgs = []
for src in (os.path.abspath(sdir) for sdir in options.sources):
try:
build_rpm(options, src)
except Exception as ex:
failed_pkgs.append((src, ex))
print >> sys.stderr, ex
if failed_pkgs:
print >> sys.stderr, "These packages failed to build:"
for descr in failed_pkgs:
print >> sys.stderr, "%s:\n\t%s" % descr
sys.exit(1)
if __name__ == "__main__":
try:
main()
except Exception as exp:
print >> sys.stderr, exp