x/anvil
Code Issues Proposed changes

Add new multipip helper module and adjust multipip help

Browse Source 
Create a helper module that can be used to interact with
the multipip program without having to worry about how that
interaction occurs internally.

Adjust the multipip utilities argument help to reflect that
it does not install anything but only analyzes requirements.

Change-Id: Ibc3d888928f251f76d4de8792f4a65cdab377b1d
This commit is contained in:
Joshua Harlow 2014-04-12 19:38:13 -07:00
parent daa5baa98b
commit c12e141c58
3 changed files with 74 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
anvil/packaging/base.py
View File

@ -18,12 +18,11 @@
# R0921: Abstract class not referenced
#pylint: disable=R0902,R0921
import collections
from anvil import colorizer
from anvil import decorators
from anvil import exceptions as exc
from anvil import log as logging
from anvil.packaging.helpers import multipip_helper
from anvil.packaging.helpers import pip_helper
from anvil import shell as sh
from anvil import trace as tr
@ -69,8 +68,7 @@ class DependencyHandler(object):
self.gathered_requires_filename = sh.joinpths(self.deps_dir, "pip-requires")
self.forced_requires_filename = sh.joinpths(self.deps_dir, "forced-requires")
self.download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
# Executables we require to operate
self.multipip_executable = sh.which("multipip", ["tools/"])
self.multipip = multipip_helper.Helper()
# List of requirements
self.pips_to_install = []
self.forced_packages = []
@ -235,36 +233,6 @@ class DependencyHandler(object):
if mutations > 0:
pip_helper.drop_caches()
def _call_multipip(self, requires_pips, requires_files, ignore_pips):
cmdline = [self.multipip_executable]
if requires_files:
cmdline.append("-r")
cmdline.extend(requires_files)
if ignore_pips:
cmdline.append("--ignore-package")
cmdline.extend(ignore_pips)
if requires_pips:
cmdline.append("--")
cmdline.extend(requires_pips)
(stdout, stderr) = sh.execute(cmdline, check_exit_code=False)
compatibles = list(utils.splitlines_not_empty(stdout))
incompatibles = collections.defaultdict(list)
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
cleaned_incompatibles = dict()
for (req_name, lines) in incompatibles.iteritems():
req_name = req_name.strip()
if not req_name:
continue
cleaned_incompatibles[req_name] = lines
return (compatibles, cleaned_incompatibles)
def _gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
@ -277,9 +245,9 @@ class DependencyHandler(object):
ignore_pips = set(self.python_names)
ignore_pips.update(self.ignore_pips)
compatibles, incompatibles = self._call_multipip(extra_pips,
requires_files,
ignore_pips)
compatibles, incompatibles = self.multipip.resolve(extra_pips,
requires_files,
ignore_pips)
self.pips_to_install = compatibles
sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install))
pips_to_install = pip_helper.read_requirement_files([self.gathered_requires_filename])

67
anvil/packaging/helpers/multipip_helper.py Normal file
View File

@ -0,0 +1,67 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import six
from anvil import shell as sh
from anvil import utils
class Helper(object):
def __init__(self):
self._multipip_executable = sh.which("multipip", ["tools/"])
def _call_multipip(self, requirements,
requires_files=None, ignore_requirements=None):
cmdline = [self._multipip_executable]
if requires_files:
cmdline.append("-r")
cmdline.extend(requires_files)
if ignore_requirements:
cmdline.append("--ignore-package")
cmdline.extend(ignore_requirements)
if requirements:
cmdline.append("--")
cmdline.extend(requirements)
(stdout, stderr) = sh.execute(cmdline, check_exit_code=False)
compatibles = list(utils.splitlines_not_empty(stdout))
incompatibles = collections.defaultdict(list)
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
cleaned_incompatibles = dict()
for (requirement, lines) in six.iteritems(incompatibles):
requirement = requirement.strip()
if not requirement:
continue
if not lines:
continue
cleaned_incompatibles[requirement] = lines
incompatibles = cleaned_incompatibles
return (compatibles, incompatibles)
def resolve(self, requirements,
requires_files=None, ignore_requirements=None):
return self._call_multipip(requirements,
requires_files=requires_files,
ignore_requirements=ignore_requirements)

4
tools/multipip
View File

@ -39,13 +39,13 @@ def create_parser():
nargs="*",
default=[],
metavar="<file>",
help="Install all the packages listed in the given requirements file")
help="Analyze all the packages listed in the given requirements file")
parser.add_argument(
"requirement_specs",
nargs="*",
default=[],
metavar="<requirement specifier>",
help="Install specified package")
help="Analyze specified package")
parser.add_argument(
# A regex to be used to skip requirements
"--skip-requirements-regex",