diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 01789af3..1d5da417 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -18,6 +18,8 @@ # R0921: Abstract class not referenced #pylint: disable=R0902,R0921 +import functools + from anvil import colorizer from anvil import exceptions as exc from anvil import log as logging @@ -180,36 +182,9 @@ class DependencyHandler(object): sh.unlink(self.tracereader.filename()) def _scan_pip_requires(self, requires_files): + own_eggs = self._python_eggs(False) - def validate_requirement(filename, source_req): - install_egg = None - for egg_info in self._python_eggs(False): - if egg_info['name'] == source_req.key: - install_egg = egg_info - break - if not install_egg: - return - # Ensure what we are about to install/create will actually work - # with the desired version. If it is not compatible then we should - # abort and someone should update the tag/branch in the origin - # file (or fix it via some other mechanism). - if install_egg['version'] not in source_req: - msg = ("Can not satisfy '%s' with '%s', version" - " conflict found in %s") - raise exc.DependencyException(msg % (source_req, - install_egg['req'], - filename)) - - if not requires_files: - return - utils.log_iterable(sorted(requires_files), - logger=LOG, - header="Scanning %s pip 'requires' files" % (len(requires_files))) - forced_by_key = {} - for pkg in self.forced_pips: - forced_by_key[pkg.key] = pkg - mutations = 0 - for fn in sorted(requires_files): + def replace_forced_requirements(fn, forced_by_key): old_lines = sh.load_file(fn).splitlines() new_lines = [] alterations = [] @@ -237,12 +212,52 @@ class DependencyHandler(object): if alterations: contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines)) sh.write_file_and_backup(fn, contents) - mutations += len(alterations) utils.log_iterable(alterations, logger=LOG, header="Replaced %s requirements in %s" % (len(alterations), fn), color=None) + return len(alterations) + + def on_replace_done(fn, time_taken): + LOG.debug("Replacing potential forced requirements in %s" + " took %s seconds", colorizer.quote(fn), time_taken) + + def validate_requirement(filename, source_req): + install_egg = None + for egg_info in own_eggs: + if egg_info['name'] == source_req.key: + install_egg = egg_info + break + if not install_egg: + return + # Ensure what we are about to install/create will actually work + # with the desired version. If it is not compatible then we should + # abort and someone should update the tag/branch in the origin + # file (or fix it via some other mechanism). + if install_egg['version'] not in source_req: + msg = ("Can not satisfy '%s' with '%s', version" + " conflict found in %s") + raise exc.DependencyException(msg % (source_req, + install_egg['req'], + filename)) + + if not requires_files: + return + requires_files = sorted(requires_files) + utils.log_iterable(requires_files, + logger=LOG, + header="Scanning %s pip 'requires' files" % (len(requires_files))) + forced_by_key = {} + for pkg in self.forced_pips: + forced_by_key[pkg.key] = pkg + mutations = 0 + for fn in requires_files: + LOG.debug("Replacing any potential forced requirements in %s", + colorizer.quote(fn)) + mutations += utils.time_it(functools.partial(on_replace_done, fn), + replace_forced_requirements, + fn, forced_by_key) # NOTE(imelnikov): after updating requirement lists we should re-fetch # data from them again, so we drop pip helper caches here. if mutations > 0: diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 58ffdde9..e1b6e334 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from distutils import version as dist_version import glob import pkg_resources import re @@ -22,16 +21,13 @@ import sys import tempfile import threading +from pip import download as pip_download from pip import req as pip_req +from pip import utils as pip_util + import pkginfo import six -try: - from pip import util as pip_util -except ImportError: - # pip >=6 changed this location for some reason... - from pip import utils as pip_util - from anvil import log as logging from anvil import shell as sh from anvil import utils @@ -45,7 +41,6 @@ EGGS_DETAILED = {} EGGS_DETAILED_LOCK = threading.RLock() PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$") -PIP_VERSION = pkg_resources.get_distribution('pip').version PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python']) @@ -208,6 +203,7 @@ def parse_requirements(contents): def read_requirement_files(files): pip_requirements = [] + session = pip_download.PipSession() for filename in files: if sh.isfile(filename): cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename)) @@ -215,7 +211,8 @@ def read_requirement_files(files): try: reqs = REQUIREMENT_FILE_CACHE[cache_key] except KeyError: - reqs = tuple(pip_req.parse_requirements(filename)) + reqs = tuple(pip_req.parse_requirements(filename, + session=session)) REQUIREMENT_FILE_CACHE[cache_key] = reqs pip_requirements.extend(reqs) return (pip_requirements, @@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename): if sh.isdir(build_path): sh.deldir(build_path) sh.mkdir(build_path) - # Ensure certain directories exist that we want to exist (but we don't - # want to delete them run after run). - cache_path = sh.joinpths(download_dir, ".cache") - if not sh.isdir(cache_path): - sh.mkdir(cache_path) cmdline = [ PIP_EXECUTABLE, '-v', 'install', '-I', '-U', '--download', download_dir, '--build', build_path, - '--download-cache', cache_path, + # Don't download wheels since we lack the ability to create + # rpms from them (until future when we will have it, if ever)... + "--no-use-wheel", ] - # Don't download wheels... - # - # See: https://github.com/pypa/pip/issues/1439 - if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'): - cmdline.append("--no-use-wheel") for p in pips_to_download: for p_seg in _split(p): if p_seg: diff --git a/anvil/packaging/venv.py b/anvil/packaging/venv.py index 5a307d04..f386dc68 100644 --- a/anvil/packaging/venv.py +++ b/anvil/packaging/venv.py @@ -58,7 +58,6 @@ class VenvDependencyHandler(base.DependencyHandler): super(VenvDependencyHandler, self).__init__(distro, root_dir, instances, opts, group, prior_groups) - self.cache_dir = sh.joinpths(self.root_dir, "pip-cache") self.jobs = max(0, int(opts.get('jobs', 0))) self.install_counters = {} @@ -76,14 +75,9 @@ class VenvDependencyHandler(base.DependencyHandler): } if extra_env_overrides: env_overrides.update(extra_env_overrides) - sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter) cmd = list(base_pip) + ['install'] if upgrade: cmd.append("--upgrade") - cmd.extend([ - '--download-cache', - self.cache_dir, - ]) if isinstance(requirements, six.string_types): cmd.extend([ '--requirement', diff --git a/requirements.txt b/requirements.txt index 13e08191..563174ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process (and later running processes...). -pip<6 +pip>=6 pkginfo cheetah>=2.4.4 iniparse diff --git a/tools/multipip b/tools/multipip index f1b6e678..ed357b1a 100755 --- a/tools/multipip +++ b/tools/multipip @@ -5,16 +5,15 @@ from __future__ import print_function import collections import itertools import logging +import pkg_resources import re import sys import argparse import six -import pip.index -import pip.req -import pkg_resources - +from pip import download as pip_download +from pip import req as pip_req # Use this for the sorting order of operators OP_ORDER = ('!=', '==', '<', '<=', '>', '>=') @@ -84,21 +83,6 @@ def setup_logging(options): LOGGER.setLevel(level) -def install_requirement_ensure_req_field(req): - if not hasattr(req, 'req') or not req.req: - # pip 0.8 or so - link = pip.index.Link(req.url) - name = link.egg_fragment - if not name: - raise Exception("Cannot find package name from `%s'" % req.url) - req.req = pkg_resources.Requirement.parse(name) - return req - - -def install_requirement_str(req): - return req.url or str(req.req) - - def install_requirement_parse(line, comes_from): line = line.strip() if line.startswith('-e') or line.startswith('--editable'): @@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from): line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - req = pip.req.InstallRequirement.from_editable( + req = pip_req.InstallRequirement.from_editable( line, comes_from=comes_from) else: - req = pip.req.InstallRequirement.from_line(line, comes_from) - return install_requirement_ensure_req_field(req) + req = pip_req.InstallRequirement.from_line(line, comes_from) + return req def iter_combinations(elements, include_empty=False): @@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None): for (op, version) in specs: spec_pieces.append("%s%s" % (op, version)) spec = "%s%s" % (req_key, ",".join(spec_pieces)) - sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled')) + sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled')) return sources def reform_incompatibles(incompatible_specs, versions): @@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None): if not matches: spec_pieces = "%s%s" % (op, version) spec = "%s%s" % (req_key, spec_pieces) - causes.append(pip.req.InstallRequirement.from_line(spec, + causes.append(pip_req.InstallRequirement.from_line(spec, "compiled conflict")) return causes @@ -370,10 +354,10 @@ def parse_requirements(options): all_requirements.setdefault(req_key(req), []).append(req) except Exception as ex: raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex)) + session = pip_download.PipSession() for filename in options.requirements: try: - for req in pip.req.parse_requirements(filename): - req = install_requirement_ensure_req_field(req) + for req in pip_req.parse_requirements(filename, session=session): if skip_match and skip_match.search(req_key(req)): continue all_requirements.setdefault(req_key(req), []).append(req) @@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements): return (joined_requirements, incompatibles) +def reform_req(req): + if req.editable: + return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req) + else: + return str(req.req) + + def print_requirements(joined_requirements): formatted_requirements = [] for req_key in sorted(six.iterkeys(joined_requirements)): req = joined_requirements[req_key][0] - req_prefix = "" - if req.editable: - req_prefix = "-e " - if req.url: - req = "%s#egg=%s" % (req.url, req.req) - else: - req = str(req.req) - formatted_requirements.append("%s%s" % (req_prefix, req)) + formatted_requirements.append(reform_req(req)) for req in formatted_requirements: print(req) @@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements): print("Choosing:", file=sys.stderr) for chosen in chosen_reqs: print("\t%s: %s" % (chosen.comes_from, - install_requirement_str(chosen)), + reform_req(chosen)), file=sys.stderr) print("Conflicting:", file=sys.stderr) for conflicting in req_incompatibles: print("\t%s: %s" % (conflicting.comes_from, - install_requirement_str(conflicting)), + reform_req(conflicting)), file=sys.stderr) diff --git a/tools/py2rpm b/tools/py2rpm index 272fdf3f..2ffa5f3a 100755 --- a/tools/py2rpm +++ b/tools/py2rpm @@ -15,9 +15,9 @@ import sys import tempfile import argparse +from pip import utils as pip_util import six -import pip.util import pkg_resources @@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs): def build_rpm(options, filename, build_options): if os.path.isfile(filename): temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-') - pip.util.unpack_file(filename, temp_dir, None, None) + pip_util.unpack_file(filename, temp_dir, None, None) source_dir = temp_dir archive_name = filename elif os.path.isdir(filename): @@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options): pkg_key = python_name_to_key(pkg_name) build_dir = options.rpm_base rpm_name = python_key_to_rpm(pkg_key) - version = one_line(info["version"]) + + # NOTE(harlowja): try not to use info["version"] to get the version, since + # currently that is getting normalized by setuptools to be a normalized + # version which can be different from the actual version... + try: + version = setup_py_one_line(source_dir, "--version") + except Exception: + version = info['version'] + logger.warning("Failed extracting version, falling back to '%s'", + version, exc_info=True) cleaned_version = version.replace('-', '_') spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)