cb0542dea3
Have the output of pip-download be more intituive and informative, also clear the download dir target before running by default to avoid unwanted dependencies from getting included. Add a nice little spinner that is shown when downloading. Change-Id: I23acd5021fe95e7ce48f60182daa332b76191ccc
229 lines
7.3 KiB
Python
Executable File
229 lines
7.3 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import distutils
|
|
import glob
|
|
import optparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import thread
|
|
import threading
|
|
import time
|
|
|
|
from pip import download as pip_down
|
|
from pip import req as pip_req
|
|
from pip import util as pip_util
|
|
|
|
PIP_CMDS = (
|
|
"pip-%s" % (".".join([str(s) for s in sys.version_info[0:2]])),
|
|
'pip-python',
|
|
'pip',
|
|
)
|
|
ARCHIVE_EXTS = (
|
|
'.zip', '.tgz', '.tbz',
|
|
'.tar.gz', '.tar', '.gz', '.bz2',
|
|
'.pybundle', '.whl',
|
|
)
|
|
|
|
|
|
class Spinner(threading.Thread):
|
|
def __init__(self, wait_time=0.15, stream=sys.stdout):
|
|
super(Spinner, self).__init__()
|
|
self.daemon = True
|
|
self.death_event = threading.Event()
|
|
self.wait_time = float(wait_time)
|
|
self.stream = stream
|
|
|
|
def run(self):
|
|
if not self.stream.isatty():
|
|
tpl = "%s"
|
|
selectors = (".",)
|
|
else:
|
|
tpl = "\b%s"
|
|
selectors = ('-', '\\', '|', '/')
|
|
try:
|
|
index = 0
|
|
has_output_something = False
|
|
selector_count = len(selectors)
|
|
while not self.death_event.is_set():
|
|
self.stream.write(tpl % selectors[index])
|
|
self.stream.flush()
|
|
has_output_something = True
|
|
if selector_count > 1:
|
|
index = (index + 1) % selector_count
|
|
time.sleep(self.wait_time)
|
|
if has_output_something:
|
|
self.stream.write("\n")
|
|
self.stream.flush()
|
|
except KeyboardInterrupt:
|
|
thread.interrupt_main()
|
|
|
|
|
|
def call(cmd):
|
|
proc = subprocess.Popen(cmd, stderr=None, stdin=None, stdout=None)
|
|
ret = proc.communicate()
|
|
if proc.returncode != 0:
|
|
raise RuntimeError("Failed running command: %s\n"
|
|
"Exit code: %s" % (" ".join(cmd), proc.returncode))
|
|
return ret
|
|
|
|
|
|
def find_pip():
|
|
for pp in PIP_CMDS:
|
|
bin_name = distutils.spawn.find_executable(pp)
|
|
if bin_name:
|
|
return bin_name
|
|
raise RuntimeError("Unable to find pip via any of %s commands"
|
|
% list(PIP_CMDS))
|
|
|
|
|
|
def execute_download(options, deps, download_dir, cache_dir, build_dir):
|
|
cmd = [find_pip()]
|
|
if options.verbose:
|
|
cmd.extend(['-v'])
|
|
else:
|
|
cmd.extend(['-q'])
|
|
cmd.extend(['install', '-I', '-U',
|
|
'--download', download_dir,
|
|
'--build', build_dir,
|
|
'--download-cache', cache_dir])
|
|
cmd.extend([str(d) for d in deps])
|
|
call(cmd)
|
|
|
|
|
|
def remove_archive_extensions(path):
|
|
for i in ARCHIVE_EXTS:
|
|
if path.endswith(i):
|
|
path = path[0:-len(i)]
|
|
return path
|
|
|
|
|
|
def extract_requirement(source_dir):
|
|
# Remove old egg-infos since it appears that pip will not work if there
|
|
# are previous egg-info directories existing in the source directory.
|
|
source_dir = os.path.abspath(source_dir)
|
|
for egg_d in glob.glob(os.path.join(source_dir, "*.egg-info")):
|
|
pip_util.rmtree(egg_d)
|
|
req = pip_req.InstallRequirement.from_line(source_dir)
|
|
req.source_dir = source_dir
|
|
req.run_egg_info()
|
|
return req
|
|
|
|
|
|
def examine_file(filename, extract_dir):
|
|
if not os.path.isfile(filename):
|
|
raise IOError("Can not extract missing file: %s" % (filename))
|
|
if not os.path.isdir(extract_dir):
|
|
raise IOError("Can not extract to missing dir: %s" % (extract_dir))
|
|
basename = os.path.basename(filename)
|
|
untar_dir = os.path.join(extract_dir, remove_archive_extensions(basename))
|
|
if os.path.isdir(untar_dir):
|
|
pip_util.rmtree(untar_dir)
|
|
pip_util.unpack_file(filename, untar_dir, content_type='', link='')
|
|
return extract_requirement(untar_dir)
|
|
|
|
|
|
def iter_archives_in(base_dir):
|
|
for basename in os.listdir(base_dir):
|
|
if basename.startswith("."):
|
|
continue
|
|
if not pip_down.is_archive_file(basename):
|
|
continue
|
|
filename = os.path.join(base_dir, basename)
|
|
if not os.path.isfile(filename):
|
|
continue
|
|
yield filename
|
|
|
|
|
|
def perform_download(options, deps,
|
|
download_dir, extract_dir, cache_dir, build_dir):
|
|
execute_download(options, deps, download_dir, cache_dir, build_dir)
|
|
files_examined = {}
|
|
for filename in iter_archives_in(download_dir):
|
|
files_examined[filename] = examine_file(filename, extract_dir)
|
|
return files_examined
|
|
|
|
|
|
def print_header(text):
|
|
if not text:
|
|
return
|
|
print("-" * len(text))
|
|
print(text)
|
|
print("-" * len(text))
|
|
|
|
|
|
def main():
|
|
usage = "usage: %prog [options] req req2 ..."
|
|
parser = optparse.OptionParser(usage=usage)
|
|
parser.add_option("-d", action="store", dest="download_dir",
|
|
help='directory to download dependencies too',
|
|
metavar="DIR")
|
|
parser.add_option("-n", "--no-wipe", action="store_false", dest="wipe",
|
|
help='do not clear old downloaded items', default=True)
|
|
parser.add_option("-v", '--verbose', action="store_true",
|
|
help='enable verbose output',
|
|
dest="verbose", default=False)
|
|
(options, packages) = parser.parse_args()
|
|
|
|
download_dir = options.download_dir
|
|
if not options.download_dir:
|
|
parser.error("Download directory required")
|
|
if not packages:
|
|
parser.error("Download requirement/s expected")
|
|
|
|
# Clear out the build & extraction directory if it exists so we don't
|
|
# conflict with previous build or extraction attempts.
|
|
extract_dir = os.path.join(download_dir, '.extract')
|
|
cache_dir = os.path.join(download_dir, '.cache')
|
|
build_dir = os.path.join(download_dir, '.build')
|
|
for d in (extract_dir, build_dir):
|
|
if os.path.isdir(d):
|
|
pip_util.rmtree(d)
|
|
for d in (download_dir, extract_dir, cache_dir, build_dir):
|
|
if not os.path.isdir(d):
|
|
os.makedirs(d)
|
|
|
|
# Clear out the previously existing files so that we don't have conflicts
|
|
# with the files we are about to download.
|
|
if options.wipe:
|
|
existing_files = list(iter_archives_in(download_dir))
|
|
if existing_files:
|
|
print_header("Cleaning")
|
|
for path in sorted(existing_files):
|
|
print(" - %s (X)" % (os.path.basename(path)))
|
|
os.unlink(path)
|
|
|
|
print_header("Downloading")
|
|
s = Spinner()
|
|
s.start()
|
|
downloaded = perform_download(options, list(packages),
|
|
download_dir, extract_dir, cache_dir,
|
|
build_dir)
|
|
s.death_event.set()
|
|
s.join()
|
|
|
|
print_header("Saved")
|
|
for filename in sorted(downloaded.keys()):
|
|
print(" - %s (%s)" % (os.path.basename(filename),
|
|
downloaded[filename].req))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|