anvil/tools/py2rpm

#!/usr/bin/env python

import argparse
import collections
import distutils.spawn
import email.parser
import logging
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile

import six

import pip.util
import pkg_resources


logger = logging.getLogger()

package_map = {}
arch_dependent = set()
epoch_map = {}

requirements_section_re = re.compile(r'\[(.*?)\]')
version_re = re.compile(r"^(.*[^.0])(\.0+)*$")
setup_py = "setup.py"

DEFAULT_SCRIPTS = {
    "prep":
    """%setup -n %{pkg_name}-%{unmangled_version} -n %{pkg_name}-%{unmangled_version}""",
    "build":
    """%{__python} setup.py build""",
    "install":
    """%{__python} setup.py install -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
abspath_installed_files=$(readlink -f INSTALLED_FILES)
(
    cd $RPM_BUILD_ROOT
    for i in usr/*/python*/site-packages/* usr/bin/*; do
        if [ -e "$i" ]; then
            sed -i "s@/$i/@DELETE_ME@" "$abspath_installed_files"
            echo "/$i"
        fi
    done
    if [ -d usr/man ]; then
        rm -rf usr/share/man
        mkdir -p usr/share
        mv usr/man usr/share/
    fi
    sed -i "s@/usr/man/@DELETE_ME@" "$abspath_installed_files"
    sed -i "s@/usr/share/man/@DELETE_ME@" "$abspath_installed_files"
    for i in usr/share/man/*; do
        if [ -d "$i" ]; then
            echo "%%doc /$i/*"
        fi
    done
) >> GATHERED_FILES
sed '/^DELETE_ME/d' INSTALLED_FILES >> GATHERED_FILES
sort -u GATHERED_FILES > INSTALLED_FILES
""",
    "clean":
    """rm -rf $RPM_BUILD_ROOT""",
}

DEFAULT_TESTS_SCRIPTS = {
    "install": """
install -d -m 755 %{buildroot}%{tests_data_dir}
tar -cf "%{buildroot}%{tests_data_dir}/test_env.tar" \
    --exclude-vcs  --exclude ./%{pkg_path} \
    --exclude './build*' --exclude './bin' --exclude './smoketest*' \
    .
if [ -d "./%{pkg_path}/tests" ]; then
    tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \
        ./%{pkg_path}/tests
fi
if [ -r "./%{pkg_path}/test.py" ]; then
    tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \
        ./%{pkg_path}/test.py
fi
gzip -9 "%{buildroot}%{tests_data_dir}/test_env.tar"


# Make simple test runner
install -d -m 755 %{buildroot}%{_bindir}
cat > %{buildroot}%{_bindir}/%{pkg_name}-make-test-env <<"EOF"
#!/bin/bash

set -e

if [ -z "$1" ] || [ "$1" == "--help" ] ; then
    echo "Usage: $0 [dir]"
    echo "       $0 --tmpdir"
    exit 1
fi

if [ "$1" == "--tmpdir" ]; then
    target_dir=$(mktemp -dt "${0##*/}.XXXXXXXX")
    echo "Created temporary directory: $target_dir"
else
    target_dir="$1"
fi

cd "$target_dir"
tar -xzf "%{tests_data_dir}/test_env.tar.gz"
cp -a %{python_sitelib}/%{pkg_path} `dirname %{pkg_path}`
ln -s /usr/bin ./bin

EOF
chmod 0755 %{buildroot}%{_bindir}/%{pkg_name}-make-test-env
"""
}

class InstallationError(Exception):
    pass


def python_name_to_key(name):
    return pkg_resources.Requirement.parse(name).key


def python_key_to_rpm(python_name):
    python_name = python_name.lower()
    try:
        return package_map[python_name]
    except KeyError:
        pass
    python_name = python_name.replace("_", "-").replace(".", "-")
    if python_name.startswith("python-"):
        prefixed_name = python_name
    else:
        prefixed_name = "python-%s" % python_name
    return prefixed_name


def egg_info_path(source_dir, filename):
    base = os.path.join(source_dir, "pip-egg-info")
    filenames = os.listdir(base)
    if not filenames:
        raise InstallationError("No files/directories in %s (from %s)"
                                % (base, filename))

    # if we have more than one match, we pick the toplevel one.
    if len(filenames) > 1:
        filenames.sort(key=lambda x: x.count(os.path.sep) +
                       (os.path.altsep and
                        x.count(os.path.altsep) or 0))
    return os.path.join(base, filenames[0], filename)


def egg_info_lines(source_dir, filename):
    filename = egg_info_path(source_dir, filename)
    if not os.path.exists(filename):
        return []
    with open(filename, "r") as f:
        return f.readlines()


def egg_info_requirements(source_dir, extras=(), filename='requires.txt'):
    in_extra = None
    for line in egg_info_lines(source_dir, filename):
        match = requirements_section_re.match(line.lower())
        if match:
            in_extra = match.group(1)
            continue
        if in_extra and in_extra not in extras:
            # Skip requirement for an extra we aren't requiring
            continue
        yield line


def pkg_info(source_dir):
    p = email.parser.FeedParser()
    filename = egg_info_path(source_dir, "PKG-INFO")
    if not os.path.exists(filename):
        logger.warn('No PKG-INFO file found in %s' % source_dir)
    else:
        with open(filename, "r") as f:
            for line in f.readlines():
                # NOTE(aababilov): d2to1 has bad PKG-INFO
                # that is fixed this way:
                if line and not line[0].isspace() and not ":" in line:
                    line = " " + line
                p.feed(line)
    return p.close()


def setup_py_one_line(source_dir, command):
    """Run `python setup.py $command` and return the last line.

    python ldap is so clever that is prints extra stuff
    before package name or version. Lets return the last line
    """
    return call_subprocess(
        [sys.executable, setup_py, command],
        cwd=source_dir, show_stdout=False)[0].splitlines()[-1].strip()


def create_parser():
    parser = argparse.ArgumentParser()

    rpm_base = os.path.expanduser("~/rpmbuild")
    source_dir = os.getcwd()

    rpmbuild_executable = (distutils.spawn.find_executable("rpmbuild") or
                           distutils.spawn.find_executable("rpm"))
    parser.add_argument(
        "--pip-verbose", "-f",
        action="store_true",
        default=False,
        help="Show pip stdout")
    parser.add_argument(
        "--debug", "-d",
        action="store_true",
        default=False,
        help="Print debug information")
    parser.add_argument(
        "--source-only", "-s",
        action="store_true",
        default=False,
        help="Only generate source RPM")
    parser.add_argument(
        "--binary-only", "-b",
        action="store_true",
        default=False,
        help="Only generate binary RPM")
    parser.add_argument(
        "--rpm-base",
        metavar="<dir>",
        default=rpm_base,
        help="rpmbuild directory (default: %s)" % rpm_base)
    parser.add_argument(
        "--rpmbuild",
        metavar="<dir>",
        default=rpmbuild_executable,
        help="rpmbuild executable (default: %s)" % rpmbuild_executable)
    parser.add_argument(
        "--convert", "-c",
        dest="convert",
        metavar="<name>",
        nargs="+",
        default=[],
        help="Python requirement name to be converted to RPM package names")
    parser.add_argument(
        dest="sources",
        metavar="<dir or archive>",
        nargs="*",
        default=[source_dir],
        help="Source directories of packages (default: current directory)")
    parser.add_argument(
        "--scripts-dir",
        metavar="<dir>",
        default=None,
        help="Specify a directory with scripts for packages")
    parser.add_argument(
        "--arch-dependent", "-a",
        metavar="<Python package name>",
        nargs="+",
        default=[],
        help="Known architecture dependent packages")
    parser.add_argument(
        "--epoch", "-e",
        metavar="<number>",
        type=int,
        default=None,
        help="RPM epoch for generated packages")
    parser.add_argument(
        "--epoch-map", "-x",
        metavar="<Python package name == epoch number>",
        nargs="+",
        default=[],
        help="Forced RPM epochs for packages")
    parser.add_argument(
        "--release", "-r",
        metavar="<number>",
        default="0%{?dist}",
        help="RPM release for generated packages")
    parser.add_argument(
        "--package-map", "-p",
        metavar="<Python package name == RPM name>",
        nargs="+",
        default=[],
        help="Correspondence between Python and RPM package names")
    parser.add_argument(
        "--with-tests",
        action="store_true",
        default=False,
        help="Add subpackage with tests")
    return parser


def call_subprocess(cmd, cwd=None, show_stdout=True, raise_on_returncode=True):
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    proc = subprocess.Popen(cmd, cwd=cwd, stderr=None, stdin=None, stdout=stdout)
    ret = proc.communicate()
    if proc.returncode:
        cwd = cwd or os.getcwd()
        command_desc = " ".join(cmd)
        if raise_on_returncode:
            raise InstallationError(
                "Command %s failed with error code %s in %s"
                % (command_desc, proc.returncode, cwd))
        else:
            logger.warn(
                "Command %s had error code %s in %s"
                % (command_desc, proc.returncode, cwd))
    return ret


def setup_logging(options):
    level = logging.DEBUG if options.debug else logging.WARNING
    handler = logging.StreamHandler(sys.stderr)
    logger.addHandler(handler)
    logger.setLevel(level)


def truncate(text, max_len=77):
    if max_len <= 0:
        return ''
    if len(text) < max_len:
        return text
    text = text[0:max_len] + "..."
    return text


def build_map(arguments):
    result = {}
    for arg in arguments:
        try:
            (key, value) = arg.split("==")
            key = python_name_to_key(key)
            value = value.strip()
            assert value
        except (IndexError, ValueError, AssertionError):
            raise InstallationError("Bad specifier: `%s'" % arg)
        else:
            result[key] = value
    return result


def run_egg_info(source_dir, options):
    script = """
__file__ = __SETUP_PY__
from setuptools.command import egg_info
import pkg_resources
import os
def replacement_run(self):
    self.mkpath(self.egg_info)
    installer = self.distribution.fetch_build_egg
    if self.distribution.has_ext_modules():
        with open("%s/ext_modules.txt" % self.egg_info, "w") as f:
            pass
    for ep in pkg_resources.iter_entry_points('egg_info.writers'):
        # require=False is the change we're making:
        writer = ep.load(require=False)
        if writer:
            writer(self, ep.name, os.path.join(self.egg_info,ep.name))
    self.find_sources()
    if self.distribution.tests_require:
        with open(os.path.join(self.egg_info, 'test-requires.txt'), 'w') as f:
            f.write('\\n'.join(self.distribution.tests_require))
egg_info.egg_info.run = replacement_run
exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
"""
    script = script.replace('__SETUP_PY__', "'setup.py'")
    egg_info_dir = os.path.join(source_dir, 'pip-egg-info')
    if not os.path.exists(egg_info_dir):
        os.makedirs(egg_info_dir)
    egg_base_option = ['--egg-base', 'pip-egg-info']
    call_subprocess(
        [sys.executable, '-c', script, 'egg_info'] + egg_base_option,
        cwd=source_dir,
        show_stdout=options.pip_verbose)


def trim_zeroes(version):
    """Trim zeroes from the and of a version"""
    match = version_re.match(version)
    if match:
        return match.group(1)
    return version


_DOT_ZEROES_TAIL = '.0' * 10
_CAND_MAP = {
    'final-': 'f',
    '@': 'd',
}


def version_release(version):

    # Unformats the parsed versions zero fill.
    def undo_zfill(piece):
        piece = piece.lstrip("0")
        if not piece:
            piece = '0'
        return piece

    # Translate usage of pre-release versions into
    # version and release since rpm will have conflicts
    # when trying to compare against pre-release versions.
    parsed_version = pkg_resources.parse_version(version)
    cand_start = -1
    for i, piece in enumerate(parsed_version):
        if piece.startswith("*"):
            cand_start = i
            break
    if cand_start == -1:
        return (version, None)
    version = []
    for v in list(parsed_version)[0:cand_start]:
        version.append(undo_zfill(v))
    version = ".".join(version)
    if not version:
        version = "0"
    release = []
    candidates = collections.deque(parsed_version[cand_start:])
    while len(candidates):
        v = candidates.popleft()
        if v == '*final':
            break
        # TODO(harlowja): this will likely require some more work as the
        # way python and rpm compare release versions is not the same, but the
        # usage of these types is limited so should not be a major problem.
        piece = []
        if v.startswith("*"):
            v = v[1:]
            v = _CAND_MAP.get(v, v)
            piece.append(v)
        while len(candidates):
            v = candidates.popleft()
            if not v.isdigit():
                candidates.appendleft(v)
                break
            else:
                piece.append(undo_zfill(v))
        release.append("".join(piece))
    release = ".".join(release)
    return (version, release)


def format_version(req, version):
    version, release = version_release(version)
    try:
        version = "%s:%s" % (epoch_map[req.key], version)
    except KeyError:
        pass
    # NOTE(imelnikov): rpm and pip compare versions differently, and
    # this used to lead to lots problems, pain and sorrows. The most
    # visible outcome of the difference is that from rpm's point of
    # view version '2' != '2.0', as well as '2.0' != '2.0.0', but for
    # pip it's same version.
    #
    # Current workaround for this works as follows: if python module
    # requires module of some version, (like 2.0.0), the actual rpm
    # version of the module will have the same non-zero beginning and
    # some '.0's at the end ('2', '2.0', '2.0.0', '2.0.0.0' ...). Thus,
    # we can calculate lower bound for requirement by trimming '.0'
    # from the version (we get '2'), and then set upper bound to lower
    # bound + tail of '.0' repeated several times.  Luckily, '2.0' and
    # '2.00' is the same version for rpm.
    lower_version = trim_zeroes(version)
    upper_version = '%s%s' % (lower_version, _DOT_ZEROES_TAIL)
    # Attach on the release (if any)
    if release:
        version = version + "-" + release
        lower_version = lower_version + "-" + release
        upper_version = upper_version + "-" + release
    return (version, lower_version, upper_version)


def requires_and_conflicts(req_list, skip_req_names=()):
    rpm_requires = []
    rpm_conflicts = []
    rpm_mapping = {}
    for line in req_list:
        try:
            req = pkg_resources.Requirement.parse(line)
        except Exception:
            continue
        if req.key in skip_req_names:
            continue
        rpm_name = python_key_to_rpm(req.key)
        if not req.specs:
            rpm_requires.append(rpm_name)
            rpm_mapping[rpm_name] = req
            continue
        for kind, version in req.specs:
            version, lower_version, upper_version = format_version(req, version)
            if kind == "!=":
                # NOTE(imelnikov): we can't conflict with ranges, so we
                # put version as is and with trimmed zeroes just in case
                rpm_conflicts.append('%s == %s' % (rpm_name, version))
                rpm_mapping[rpm_conflicts[-1]] = req
                if version != lower_version:
                    rpm_conflicts.append('%s == %s' % (rpm_name, lower_version))
                    rpm_mapping[rpm_conflicts[-1]] = req
            elif kind == '==':
                rpm_requires.extend((
                    '%s >= %s' % (rpm_name, lower_version),
                    '%s <= %s' % (rpm_name, upper_version)
                ))
                rpm_mapping[rpm_requires[-1]] = req
                rpm_mapping[rpm_requires[-2]] = req
            elif kind in ('<=', '<'):
                rpm_requires.append('%s <= %s' % (rpm_name, upper_version))
                rpm_mapping[rpm_requires[-1]] = req
            elif kind in ('>=', '>'):
                rpm_requires.append('%s >= %s' % (rpm_name, lower_version))
                rpm_mapping[rpm_requires[-1]] = req
            else:
                raise ValueError('Invalid requirement kind: %r' % kind)

    rpm_requires_str = six.StringIO()
    for req in rpm_requires:
        rpm_requires_str.write("# Source: %s\n" % (rpm_mapping[req]))
        rpm_requires_str.write("Requires: %s\n\n" % (req))

    rpm_conflicts_str = six.StringIO()
    for req in rpm_conflicts:
        rpm_conflicts_str.write("# Source: %s\n" % (rpm_mapping[req]))
        rpm_conflicts_str.write("Conflicts: %s\n\n" % (req))
        rpm_conflicts_str.write("\n\n")

    return rpm_requires_str.getvalue(), rpm_conflicts_str.getvalue()


def one_line(line, max_len=80):
    line = line.replace("\n", " ")
    if max_len > 0:
        return line[:max_len]
    return line


def build_rpm(options, filename):
    if os.path.isfile(filename):
        temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
        pip.util.unpack_file(filename, temp_dir, None, None)
        source_dir = temp_dir
        archive_name = filename
    elif os.path.isdir(filename):
        temp_dir = None
        archive_name = None
        source_dir = filename
    else:
        raise InstallationError(
            "`%s' is not a regular file nor a directory" % filename)

    run_egg_info(source_dir, options)
    info = pkg_info(source_dir)
    rpm_requires, rpm_conflicts = requires_and_conflicts(
        egg_info_requirements(source_dir))
    test_rpm_requires, test_rpm_conflicts = requires_and_conflicts(
        egg_info_requirements(source_dir, filename="test-requires.txt"),
        skip_req_names=('sphinx', 'setuptools', 'setuptools-git', 'docutils'))
    # NOTE(aababilov): do not use info["name"] to get the name - it is
    # the key (e.g., "nose-plugin"), not the name ("nose_plugin")
    pkg_name = setup_py_one_line(source_dir, "--name")
    pkg_key = python_name_to_key(pkg_name)
    build_dir = options.rpm_base
    rpm_name = python_key_to_rpm(pkg_key)
    version = one_line(info["version"])
    for path in (os.path.join(build_dir, "SPECS"),
                 os.path.join(build_dir, "SOURCES")):
        if not os.path.isdir(path):
            os.makedirs(path)
    spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)

    if not archive_name:
        cmdline = [
            sys.executable, setup_py, "sdist",
        ]
        call_subprocess(cmdline, cwd=source_dir, raise_on_returncode=False)
        archive_name = "%s/dist/%s-%s.tar.gz" % (source_dir, pkg_name, version)
    shutil.copy(archive_name, os.path.join(build_dir, "SOURCES"))

    cleaned_version = version.replace('-', '_')
    with open(spec_name, "w") as spec_file:
        print >> spec_file, "%define pkg_name", pkg_name
        print >> spec_file, "%define pkg_path", os.path.join(*pkg_name.split('.'))
        print >> spec_file, "%define rpm_name", rpm_name
        print >> spec_file, "%define version", cleaned_version
        print >> spec_file, "%define release", options.release
        print >> spec_file, "%define unmangled_version", version
        if options.with_tests:
            print >> spec_file, ("%define tests_data_dir "
                                 "%{_datarootdir}/%{pkg_name}-tests")
        print >> spec_file, ""
        tags = []
        tags.append(("Name", "%{rpm_name}"))
        epoch = epoch_map.get(pkg_key, options.epoch)
        if epoch is not None:
            tags.append(("Epoch", epoch))
        tags.append(("Version", "%{version}"))
        tags.append(("Release", "%{release}"))
        tags.append(("Summary", info["summary"]))
        archive_name = os.path.basename(archive_name)
        if archive_name == ("%s-%s.tar.gz" % (pkg_name, version)):
            tags.append(("Source0", "%{pkg_name}-%{unmangled_version}.tar.gz"))
        else:
            tags.append(("Source0", archive_name))
        tags.append(("License", info["license"]))
        tags.append(("Group", "Development/Libraries"))
        tags.append(("BuildRoot", "%{_tmppath}/%{pkg_name}-%{unmangled_version}-%{release}-buildroot"))
        tags.append(("Prefix", "%{_prefix}"))
        if pkg_key not in arch_dependent:
            if not os.path.exists(egg_info_path(source_dir, "ext_modules.txt")):
                tags.append(("BuildArch", "noarch"))
        tags.append(("Vendor", "%s <%s>" % (info["author"], info["author-email"])))
        tags.append(("Url", info["home-page"]))
        max_name_len = max(len(tag[0]) for tag in tags)
        for tag_name, tag_value in tags:
            if not tag_value:
                tag_value = 'Unknown'
            print >> spec_file, "%s:" % tag_name, " " * (
                max_name_len - len(tag_name) + 1), one_line(tag_value, max_len=-1)
        if rpm_requires:
            print >> spec_file, rpm_requires
        if rpm_conflicts:
            print >> spec_file, rpm_conflicts
        print >> spec_file, "\n%description\n", info["description"]

        if options.with_tests:
            print >> spec_file, "\n%package tests"
            print >> spec_file, "Group: Development/Libraries"
            print >> spec_file, "Summary: tests for %{name}"
            for req in ("%{name} = %{epoch}:%{version}-%{release}",
                        "python-nose",
                        "python-openstack-nose-plugin",
                        "python-nose-exclude"):
                print >> spec_file, "Requires:", req
            print >> spec_file, test_rpm_requires
            print >> spec_file, test_rpm_conflicts
            print >> spec_file, "\n%description tests"
            print >> spec_file, "Tests for %{name}"


        for script in "prep", "build", "install", "clean":
            print >> spec_file, "\n\n%%%s" % script
            if options.scripts_dir:
                script_filename = "%s/%s-%s.sh" % (options.scripts_dir, pkg_key, script)
                if os.path.isfile(script_filename):
                    with open(script_filename) as f_in:
                        print >> spec_file, f_in.read()
                    continue
            print >> spec_file, DEFAULT_SCRIPTS[script]
            if options.with_tests and script in DEFAULT_TESTS_SCRIPTS:
                print >> spec_file, DEFAULT_TESTS_SCRIPTS[script]
        print >> spec_file, """

%files -f INSTALLED_FILES
%defattr(-,root,root)
"""
        if options.with_tests:
            print >> spec_file, "\n%files tests"
            print >> spec_file, "%{_bindir}/%{pkg_name}-make-test-env"
            print >> spec_file, "%{tests_data_dir}"

    if options.source_only:
        rpmbuild_what = "-bs"
    elif options.binary_only:
        rpmbuild_what = "-bb"
    else:
        rpmbuild_what = "-ba"
    call_subprocess(
        [options.rpmbuild, rpmbuild_what,
         "--define", "_topdir %s" % build_dir,
         spec_name])
    if temp_dir:
        shutil.rmtree(temp_dir)


def main():
    parser = create_parser()
    options = parser.parse_args()
    setup_logging(options)
    global arch_dependent
    global package_map
    global epoch_map
    arch_dependent = set(python_name_to_key(pkg)
                         for pkg in options.arch_dependent)
    package_map = build_map(options.package_map)
    epoch_map = build_map(options.epoch_map)

    if options.convert:
        rpm_requires, rpm_conflicts = requires_and_conflicts(options.convert)
        if rpm_requires:
            print rpm_requires.strip()
        if rpm_conflicts:
            print rpm_conflicts.strip()
        return

    failed_pkgs = []
    for src in (os.path.abspath(sdir) for sdir in options.sources):
        try:
            build_rpm(options, src)
        except Exception as ex:
            failed_pkgs.append((src, ex))
            print >> sys.stderr, ex
    if failed_pkgs:
        print >> sys.stderr, "These packages failed to build:"
        for descr in failed_pkgs:
            print >> sys.stderr, "%s:\n\t%s" % descr
        sys.exit(1)


if __name__ == "__main__":
    try:
        main()
    except Exception as exp:
        print >> sys.stderr, exp