anvil/tools/pip-download

#!/usr/bin/python

# vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from distutils.version import LooseVersion
from distutils.version import StrictVersion

import collections
import distutils
import glob
import optparse
import os
import pkg_resources
import shutil
import subprocess

from pip import req as pip_req
from pip import util as pip_util

PIP_CMDS = ['pip-python', 'pip']
ARCHIVE_EXTS = ['.zip', '.tgz', '.tbz', '.tar.gz', '.tar', '.gz', '.bz2']

# The support for wiping existing downloads was added in 1.1
if LooseVersion(pkg_resources.get_distribution("pip").version) < LooseVersion('1.1'):
    SUPPORTS_EXISTS = False
else:
    SUPPORTS_EXISTS = True


def call(cmd):
    proc = subprocess.Popen(cmd, stderr=None, stdin=None, stdout=None)
    ret = proc.communicate()
    if proc.returncode != 0:
        raise RuntimeError("Failed running %s" % (" ".join(cmd)))
    return ret


def find_pip():
    for pp in PIP_CMDS:
        bin_name = distutils.spawn.find_executable(pp)
        if bin_name:
            return bin_name
    raise RuntimeError("Unable to find pip via any of %s commands" % (PIP_CMDS))


def execute_download(options, deps, download_dir, cache_dir, build_dir):
    cmd = [find_pip()]
    if options.verbose:
        cmd.extend(['-v'])
    else:
        cmd.extend(['-q'])
    cmd.extend(['install', '-I', '-U',
                '--download', download_dir,
                '--build', build_dir,
                '--download-cache', cache_dir])
    if SUPPORTS_EXISTS:
        cmd.extend(['--exists-action', 'w'])
    cmd.extend([str(d) for d in deps])
    call(cmd)


def remove_archive_extensions(path):
    for i in ARCHIVE_EXTS:
        if path.endswith(i):
            path = path[0:-len(i)]
    return path


def extract_requirement(source_dir):
    # Remove old egg-infos since it appears that pip will not work if there
    # are previous egg-info directories existing in the source directory.
    source_dir = os.path.abspath(source_dir)
    for egg_d in glob.glob(os.path.join(source_dir, "*.egg-info")):
        shutil.rmtree(egg_d)
    req = pip_req.InstallRequirement.from_line(source_dir)
    req.source_dir = source_dir
    req.run_egg_info()
    return req


def perform_download(options, deps, download_dir, extract_dir, cache_dir, build_dir):
    execute_download(options, deps, download_dir, cache_dir, build_dir)
    files_examined = {}
    for basename in os.listdir(download_dir):
        if basename.startswith("."):
            continue
        filename = os.path.join(download_dir, basename)
        if not os.path.isfile(filename):
            continue
        untar_dir = os.path.join(extract_dir, remove_archive_extensions(basename))
        if not os.path.isdir(untar_dir):
            if options.verbose:
                print("Extracting %s -> %s" % (filename, untar_dir))
            pip_util.unpack_file(filename, untar_dir, content_type='', link='')
        if options.verbose:
            print("Examining %s" % (untar_dir))
        files_examined[filename] = extract_requirement(untar_dir)
    return files_examined


def evict_equivalent(options, downloaded):

    def ver_comp(name_req1, name_req2):
        file1, req1 = name_req1
        file2, req2 = name_req2
        if file1 == file2:
            return 0
        try:
            return cmp(StrictVersion(req1.installed_version),
                       StrictVersion(req2.installed_version))
        except ValueError:
            return cmp(LooseVersion(req1.installed_version),
                       LooseVersion(req2.installed_version))

    duplicates = collections.defaultdict(list)
    for (filename, req) in downloaded.items():
        duplicates[req.name].append((filename, req))
    dups_found = 0
    for (name, matches) in duplicates.items():
        if len(matches) > 1:
            dups_found += 1
    if not dups_found:
        return
    if options.verbose:
        print("%s duplicate found..." % (dups_found))
    for (name, matches) in duplicates.items():
        if len(matches) <= 1:
            continue
        versions = []
        for (filename, req) in matches:
            if options.verbose:
                print("Duplicate %s at %s with version %s" % (name, filename, req.installed_version))
            versions.append((filename, req))
        selected_filename, selected_req = list(sorted(versions, cmp=ver_comp))[-1]
        if options.verbose:
            print('Keeping %s with version %s' % (selected_filename, selected_req.installed_version))
        for (filename, req) in matches:
            if filename != selected_filename:
                if options.verbose:
                    print("Deleting %s" % (filename))
                os.unlink(filename)
                downloaded.pop(filename)


if __name__ == '__main__':
    usage = "usage: %prog [options] req req2 ..."
    parser = optparse.OptionParser(usage=usage)
    parser.add_option("-d", action="store", dest="download_dir",
                      help='directory to download dependencies too', metavar="DIR")
    parser.add_option("-v", '--verbose', action="store_true", help='enable verbose output',
                     dest="verbose", default=False)
    (options, packages) = parser.parse_args()
    download_dir = options.download_dir
    if not options.download_dir:
        raise IOError("Download directory required")
    if not packages:
        raise IOError("Download requirement/s expected")
    extract_dir = os.path.join(download_dir, '.extract')
    cache_dir = os.path.join(download_dir, '.cache')
    build_dir = os.path.join(download_dir, '.build')
    # Clear out the build & extraction directory if it exists so we don't
    # conflict with previous build or extraction attempts.
    for d in [extract_dir, build_dir]:
        if os.path.isdir(d):
            shutil.rmtree(d)
    for d in [download_dir, extract_dir, cache_dir, build_dir]:
        if not os.path.isdir(d):
            os.makedirs(d)
    downloaded = perform_download(options, list(packages),
                                  download_dir, extract_dir, cache_dir, build_dir)
    evict_equivalent(options, downloaded)
    for filename in sorted(downloaded.keys()):
        print("Saved %s" % (filename))