#!/usr/bin/python # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from distutils.version import LooseVersion from distutils.version import StrictVersion import collections import distutils import glob import optparse import os import pkg_resources import shutil import subprocess from pip import req as pip_req from pip import util as pip_util PIP_CMDS = ['pip-python', 'pip'] ARCHIVE_EXTS = ['.zip', '.tgz', '.tbz', '.tar.gz', '.tar', '.gz', '.bz2'] # The support for wiping existing downloads was added in 1.1 if LooseVersion(pkg_resources.get_distribution("pip").version) < LooseVersion('1.1'): SUPPORTS_EXISTS = False else: SUPPORTS_EXISTS = True def call(cmd): proc = subprocess.Popen(cmd, stderr=None, stdin=None, stdout=None) ret = proc.communicate() if proc.returncode != 0: raise RuntimeError("Failed running %s" % (" ".join(cmd))) return ret def find_pip(): for pp in PIP_CMDS: bin_name = distutils.spawn.find_executable(pp) if bin_name: return bin_name raise RuntimeError("Unable to find pip via any of %s commands" % (PIP_CMDS)) def execute_download(options, deps, download_dir, cache_dir, build_dir): cmd = [find_pip()] if options.verbose: cmd.extend(['-v']) else: cmd.extend(['-q']) cmd.extend(['install', '-I', '-U', '--download', download_dir, '--build', build_dir, '--download-cache', cache_dir]) if SUPPORTS_EXISTS: cmd.extend(['--exists-action', 'w']) cmd.extend([str(d) for d in deps]) call(cmd) def remove_archive_extensions(path): for i in ARCHIVE_EXTS: if path.endswith(i): path = path[0:-len(i)] return path def extract_requirement(source_dir): # Remove old egg-infos since it appears that pip will not work if there # are previous egg-info directories existing in the source directory. source_dir = os.path.abspath(source_dir) for egg_d in glob.glob(os.path.join(source_dir, "*.egg-info")): shutil.rmtree(egg_d) req = pip_req.InstallRequirement.from_line(source_dir) req.source_dir = source_dir req.run_egg_info() return req def perform_download(options, deps, download_dir, extract_dir, cache_dir, build_dir): execute_download(options, deps, download_dir, cache_dir, build_dir) files_examined = {} for basename in os.listdir(download_dir): if basename.startswith("."): continue filename = os.path.join(download_dir, basename) if not os.path.isfile(filename): continue untar_dir = os.path.join(extract_dir, remove_archive_extensions(basename)) if not os.path.isdir(untar_dir): if options.verbose: print("Extracting %s -> %s" % (filename, untar_dir)) pip_util.unpack_file(filename, untar_dir, content_type='', link='') if options.verbose: print("Examining %s" % (untar_dir)) files_examined[filename] = extract_requirement(untar_dir) return files_examined def evict_equivalent(options, downloaded): def ver_comp(name_req1, name_req2): file1, req1 = name_req1 file2, req2 = name_req2 if file1 == file2: return 0 try: return cmp(StrictVersion(req1.installed_version), StrictVersion(req2.installed_version)) except ValueError: return cmp(LooseVersion(req1.installed_version), LooseVersion(req2.installed_version)) duplicates = collections.defaultdict(list) for (filename, req) in downloaded.items(): duplicates[req.name].append((filename, req)) dups_found = 0 for (name, matches) in duplicates.items(): if len(matches) > 1: dups_found += 1 if not dups_found: return if options.verbose: print("%s duplicate found..." % (dups_found)) for (name, matches) in duplicates.items(): if len(matches) <= 1: continue versions = [] for (filename, req) in matches: if options.verbose: print("Duplicate %s at %s with version %s" % (name, filename, req.installed_version)) versions.append((filename, req)) selected_filename, selected_req = list(sorted(versions, cmp=ver_comp))[-1] if options.verbose: print('Keeping %s with version %s' % (selected_filename, selected_req.installed_version)) for (filename, req) in matches: if filename != selected_filename: if options.verbose: print("Deleting %s" % (filename)) os.unlink(filename) downloaded.pop(filename) if __name__ == '__main__': usage = "usage: %prog [options] req req2 ..." parser = optparse.OptionParser(usage=usage) parser.add_option("-d", action="store", dest="download_dir", help='directory to download dependencies too', metavar="DIR") parser.add_option("-v", '--verbose', action="store_true", help='enable verbose output', dest="verbose", default=False) (options, packages) = parser.parse_args() download_dir = options.download_dir if not options.download_dir: raise IOError("Download directory required") if not packages: raise IOError("Download requirement/s expected") extract_dir = os.path.join(download_dir, '.extract') cache_dir = os.path.join(download_dir, '.cache') build_dir = os.path.join(download_dir, '.build') # Clear out the build & extraction directory if it exists so we don't # conflict with previous build or extraction attempts. for d in [extract_dir, build_dir]: if os.path.isdir(d): shutil.rmtree(d) for d in [download_dir, extract_dir, cache_dir, build_dir]: if not os.path.isdir(d): os.makedirs(d) downloaded = perform_download(options, list(packages), download_dir, extract_dir, cache_dir, build_dir) evict_equivalent(options, downloaded) for filename in sorted(downloaded.keys()): print("Saved %s" % (filename))