openstack
/
openstack-ansible-plugins
Code Issues Proposed changes

Remove py_pkgs and packages_file lookups 

Neither of these lookups are used in OSA any more, so we can
remove them.

Change-Id: Ib1c19eaf5a16d59e91d5c6c5ced9b30781b1f69b
This commit is contained in:
Jesse Pretorius 2019-04-24 14:13:53 +01:00
parent 5cdb163ffb
commit 00c22491d0
8 changed files with 6 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/source/index.rst
View File

@ -7,7 +7,6 @@ OpenStack-Ansible plugins
actions.rst
filters.rst
lookups.rst
Example ansible.cfg file
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

167
doc/source/lookups.rst
View File

@ -1,167 +0,0 @@
=======
Lookups
=======
py_pkgs
~~~~~~~
The ``py_pkgs`` lookup crawls a given list of directories to parse variables
and generate lists of Python packages, git repo information and Ansible group
memberships which is used within OpenStack-Ansible's repo_build role to build
wheels and virtual environments.
Files and paths containing the following strings are evaluated:
- test-requirements.txt
- dev-requirements.txt
- requirements.txt
- global-requirements.txt
- global-requirement-pins.txt
- /defaults/
- /vars/
- /user_*
Variables parsed within any evaluated files include:
- service_pip_dependencies
- pip_common_packages
- pip_container_packages
- pip_packages
Example
-------
.. code-block:: yaml
- name: Load local packages
debug:
msg: "Loading Packages"
with_py_pkgs: "{{ pkg_locations }}"
register: local_packages
vars:
pkg_locations:
- "/etc/ansible/roles/os_nova"
# => {
# "packages": [
# "httplib2",
# "keystonemiddleware",
# "libvirt-python",
# "nova",
# "nova-lxd",
# "nova-powervm",
# "pyasn1-modules",
# "pycrypto",
# "pylxd",
# "pymysql",
# "python-ironicclient",
# "python-keystoneclient",
# "python-memcached",
# "python-novaclient",
# "virtualenv",
# "websockify"
# ],
# "remote_package_parts": [
# {
# "egg_name": "nova",
# "fragment": null,
# "name": "nova",
# "original":
# "git+https://git.openstack.org/openstack/nova@stable/newton#egg=nova&gitname=nova&projectgroup=all",
# "project_group": "all",
# "url": "https://git.openstack.org/openstack/nova",
# "version": "stable/newton"
# },
# {
# "egg_name": "nova_lxd",
# "fragment": null,
# "name": "nova-lxd",
# "original":
# "git+https://git.openstack.org/openstack/nova-lxd@stable/newton#egg=nova_lxd&gitname=nova-lxd&projectgroup=all",
# "project_group": "all",
# "url": "https://git.openstack.org/openstack/nova-lxd",
# "version": "stable/newton"
# },
# {
# "egg_name": "novnc",
# "fragment": null,
# "name": "novnc",
# "original":
# "git+https://github.com/kanaka/novnc@master#egg=novnc&gitname=novnc&projectgroup=all",
# "project_group": "all",
# "url": "https://github.com/kanaka/novnc",
# "version": "master"
# },
# {
# "egg_name": "spice_html5",
# "fragment": null,
# "name": "spice-html5",
# "original":
# "git+https://gitlab.freedesktop.org/spice/spice-html5.git@master#egg=spice_html5&gitname=spice-html5&projectgroup=all",
# "project_group": "all",
# "url": "https://gitlab.freedesktop.org/spice/spice-html5.git",
# "version": "master"
# }
# ],
# "remote_packages": [
# "git+https://git.openstack.org/openstack/nova-lxd@stable/newton#egg=nova_lxd&gitname=nova-lxd&projectgroup=all",
# "git+https://git.openstack.org/openstack/nova@stable/newton#egg=nova&gitname=nova&projectgroup=all",
# "git+https://gitlab.freedesktop.org/spice/spice-html5.git@master#egg=spice_html5&gitname=spice-html5&projectgroup=all",
# "git+https://github.com/kanaka/novnc@master#egg=novnc&gitname=novnc&projectgroup=all"
# ],
# "role_packages": {
# "os_nova": [
# "httplib2",
# "keystonemiddleware",
# "libvirt-python",
# "nova",
# "nova-lxd",
# "nova-powervm",
# "pyasn1-modules",
# "pycrypto",
# "pylxd",
# "pymysql",
# "python-ironicclient",
# "python-keystoneclient",
# "python-memcached",
# "python-novaclient",
# "virtualenv",
# "websockify"
# ]
# },
# "role_project_groups": {
# "os_nova": "nova_all"
# },
# "role_requirement_files": {},
# "role_requirements": {
# "os_nova": {
# "nova_compute_ironic_pip_packages": [
# "python-ironicclient"
# ],
# "nova_compute_lxd_pip_packages": [
# "nova-lxd",
# "pylxd"
# ],
# "nova_compute_pip_packages": [
# "libvirt-python"
# ],
# "nova_compute_powervm_pip_packages": [
# "nova-powervm",
# "pyasn1-modules"
# ],
# "nova_novnc_pip_packages": [
# "websockify"
# ],
# "nova_pip_packages": [
# "keystonemiddleware",
# "nova",
# "pycrypto",
# "pymysql",
# "python-keystoneclient",
# "python-memcached",
# "python-novaclient"
# ],
# "nova_requires_pip_packages": [
# "httplib2",
# "python-keystoneclient",
# "virtualenv",
# ],
# "project_group": "nova_all"
# }
# }

1
examples/example.ini
View File

@ -1,5 +1,4 @@
[defaults]
lookup_plugins = /etc/ansible/plugins/lookups
filter_plugins = /etc/ansible/plugins/filters
action_plugins = /etc/ansible/plugins/actions
library = /etc/ansible/plugins/library

79
lookup/packages_file.py
View File

@ -1,79 +0,0 @@
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2017, Jean-Philippe Evrard <jean-philippe.evrard@rackspace.co.uk>
# Take a path to a debian mirror release file, and outputs a dict with
# package names, each of the packages holding the following:
# - package version
# - checksums
# - Relative location to pool folder
#
# example:
# get_url:
# url: http://rpc-repo.rackspace.com/apt-mirror/integrated/dists/r14.0.0rc1-trusty/main/binary-amd64/Packages
# dest: /tmp/trusty-amd64-Packages
# debug:
# var: item
# with_packages_file:
# - /tmp/trusty-amd64-Packages
import os
from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleLookupError
IMPORTANT_FIELDS = ['Version', 'Filename', 'MD5sum', 'SHA1', 'SHA256']
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
def parse_fields(line):
for field in IMPORTANT_FIELDS:
if line.startswith(field + ":"):
return (field, line.split(":")[1].strip())
class LookupModule(LookupBase):
def run(self, terms, variables=None, **kwargs):
ret = []
for term in terms:
pkg_details = {}
with open(term, 'r') as f:
for line in f:
#non empty line means pkg data
if line.strip():
if line.startswith('Package:'):
currentpkg = line.split(":")[1].strip()
pkg_details[currentpkg] = {}
elif line.startswith('Provides:'):
pkg_details[line.split(":")[1].strip()] = pkg_details[currentpkg]
else:
# Now doing package data
parsed = parse_fields(line)
if parsed:
pkg_details[currentpkg][parsed[0]] = parsed[1]
else:
currentpkg=""
ret.append(pkg_details)
return ret
# For debug purposes
if __name__ == '__main__':
import sys
import json
print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))

736
lookup/py_pkgs.py
View File

@ -1,736 +0,0 @@
# Copyright 2014, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
import os
import re
import traceback
import yaml
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
try:
basestring
except NameError:
basestring = str
# Used to keep track of git package parts as various files are processed
GIT_PACKAGE_DEFAULT_PARTS = dict()
# Role based package indexes
ROLE_BREAKOUT_REQUIREMENTS = dict()
ROLE_PACKAGES = dict()
ROLE_REQUIREMENTS = dict()
REQUIREMENTS_FILE_TYPES = [
'requirements.txt',
'global-requirement-pins.txt'
]
# List of variable names that could be used within the yaml files that
# represent lists of python packages.
BUILT_IN_PIP_PACKAGE_VARS = [
'service_pip_dependencies',
'pip_common_packages',
'pip_container_packages',
'pip_packages'
]
PACKAGE_MAPPING = {
'packages': set(),
'remote_packages': set(),
'remote_package_parts': list(),
'role_packages': dict(),
'role_project_groups': dict(),
}
def map_base_and_remote_packages(package, package_map):
"""Determine whether a package is a base package or a remote package
and add to the appropriate set.
:type package: ``str``
:type package_map: ``dict``
"""
def check_for_ignore(p):
p_parts = GIT_PACKAGE_DEFAULT_PARTS.get(p)
if p_parts:
fragments = p_parts.get('fragments', '') or ''
if 'ignorerequirements=True' not in fragments:
package_map['packages'].add(p)
else:
package_map['packages'].add(p)
if package.startswith(('http:', 'https:', 'git+')):
if '@' not in package:
check_for_ignore(p=package)
else:
git_parts = git_pip_link_parse(package)
package_name = git_parts[-2]
if not package_name:
package_name = git_pip_link_parse(package)[0]
for rpkg in list(package_map['remote_packages']):
rpkg_name = git_pip_link_parse(rpkg)[-2]
if not rpkg_name:
rpkg_name = git_pip_link_parse(package)[0]
if rpkg_name == package_name:
package_map['remote_packages'].remove(rpkg)
package_map['remote_packages'].add(package)
break
else:
package_map['remote_packages'].add(package)
else:
check_for_ignore(p=package)
def parse_remote_package_parts(package_map):
"""Parse parts of each remote package and add them to
the remote_package_parts list.
:type package_map: ``dict``
"""
keys = [
'name',
'version',
'fragment',
'url',
'original',
'egg_name',
'project_group'
]
remote_pkg_parts = [
dict(
zip(
keys, git_pip_link_parse(i)
)
) for i in package_map['remote_packages']
]
package_map['remote_package_parts'].extend(remote_pkg_parts)
package_map['remote_package_parts'] = list(
dict(
(i['name'], i)
for i in package_map['remote_package_parts']
).values()
)
def map_role_packages(package_map):
"""Add and sort packages belonging to a role to the role_packages dict.
:type package_map: ``dict``
"""
for k, v in ROLE_PACKAGES.items():
role_pkgs = package_map['role_packages'][k] = list()
package_map['role_project_groups'][k] = v.pop('project_group', 'all')
for pkg_list in v.values():
role_pkgs.extend(pkg_list)
else:
package_map['role_packages'][k] = sorted(set(role_pkgs))
def map_base_package_details(package_map):
"""Parse package version and marker requirements and add to the
base packages set.
:type package_map: ``dict``
"""
check_pkgs = dict()
base_packages = sorted(list(package_map['packages']))
for pkg in base_packages:
name, versions, markers = _pip_requirement_split(pkg)
if versions and markers:
versions = '%s;%s' % (versions, markers)
elif not versions and markers:
versions = ';%s' % markers
if name in check_pkgs:
if versions and not check_pkgs[name]:
check_pkgs[name] = versions
else:
check_pkgs[name] = versions
else:
return_pkgs = list()
for k, v in check_pkgs.items():
if v:
return_pkgs.append('%s%s' % (k, v))
else:
return_pkgs.append(k)
package_map['packages'] = set(return_pkgs)
def git_pip_link_parse(repo):
"""Return a tuple containing the parts of a git repository.
Example parsing a standard git repo:
>>> git_pip_link_parse('git+https://github.com/username/repo-name@tag')
('repo-name',
'tag',
None,
'https://github.com/username/repo',
'git+https://github.com/username/repo@tag',
'repo_name')
Example parsing a git repo that uses an installable from a subdirectory:
>>> git_pip_link_parse(
... 'git+https://github.com/username/repo@tag#egg=plugin.name'
... '&subdirectory=remote_path/plugin.name'
... )
('plugin.name',
'tag',
'remote_path/plugin.name',
'https://github.com/username/repo',
'git+https://github.com/username/repo@tag#egg=plugin.name&'
'subdirectory=remote_path/plugin.name',
'plugin.name')
:param repo: git repo string to parse.
:type repo: ``str``
:returns: ``tuple``
"""'meta'
def _meta_return(meta_data, item):
"""Return the value of an item in meta data."""
return meta_data.lstrip('#').split('%s=' % item)[-1].split('&')[0]
_git_url = repo.split('+')
if len(_git_url) >= 2:
_git_url = _git_url[1]
else:
_git_url = _git_url[0]
git_branch_sha = _git_url.split('@')
if len(git_branch_sha) > 2:
branch = git_branch_sha.pop()
url = '@'.join(git_branch_sha)
elif len(git_branch_sha) > 1:
url, branch = git_branch_sha
else:
url = git_branch_sha[0]
branch = 'master'
egg_name = name = os.path.basename(url.rstrip('/'))
egg_name = egg_name.replace('-', '_')
_branch = branch.split('#')
branch = _branch[0]
plugin_path = None
# Determine if the package is a plugin type
if len(_branch) > 1:
if 'subdirectory=' in _branch[-1]:
plugin_path = _meta_return(_branch[-1], 'subdirectory')
name = os.path.basename(plugin_path)
if 'egg=' in _branch[-1]:
egg_name = _meta_return(_branch[-1], 'egg')
egg_name = egg_name.replace('-', '_')
if 'gitname=' in _branch[-1]:
name = _meta_return(_branch[-1], 'gitname')
project_group = 'all'
if 'projectgroup=' in _branch[-1]:
project_group = _meta_return(_branch[-1], 'projectgroup')
return name.lower(), branch, plugin_path, url, repo, egg_name, project_group
def _pip_requirement_split(requirement):
"""Split pip versions from a given requirement.
The method will return the package name, versions, and any markers.
:type requirement: ``str``
:returns: ``tuple``
"""
version_descriptors = "(>=|<=|>|<|==|~=|!=)"
requirement = requirement.split(';')
requirement_info = re.split(r'%s\s*' % version_descriptors, requirement[0])
name = requirement_info[0]
marker = None
if len(requirement) > 1:
marker = requirement[-1]
versions = None
if len(requirement_info) > 1:
versions = ''.join(requirement_info[1:])
return name, versions, marker
class DependencyFileProcessor(object):
def __init__(self, local_path):
"""Find required files.
:type local_path: ``str``
:return:
"""
self.pip = dict()
self.pip['git_package'] = list()
self.pip['py_package'] = list()
self.pip['git_data'] = list()
self.git_pip_install = 'git+%s@%s'
self.file_names = self._get_files(path=local_path)
# Process everything simply by calling the method
self._process_files()
def _py_pkg_extend(self, packages, py_package=None):
if py_package is None:
py_package = self.pip['py_package']
for pkg in packages:
pkg_name = _pip_requirement_split(pkg)[0]
for py_pkg in py_package:
py_pkg_name = _pip_requirement_split(py_pkg)[0]
if pkg_name == py_pkg_name:
py_package.remove(py_pkg)
else:
norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')]
py_package.extend(norm_pkgs)
return py_package
@staticmethod
def _filter_files(file_names, ext):
"""Filter the files and return a sorted list.
:type file_names:
:type ext: ``str`` or ``tuple``
:returns: ``list``
"""
_file_names = list()
file_name_words = ['/defaults/', '/vars/', '/user_']
file_name_words.extend(REQUIREMENTS_FILE_TYPES)
for file_name in file_names:
if file_name.endswith(ext):
if any(i in file_name for i in file_name_words):
_file_names.append(file_name)
else:
return _file_names
@staticmethod
def _get_files(path):
"""Return a list of all files in the defaults/repo_packages directory.
:type path: ``str``
:returns: ``list``
"""
paths = os.walk(os.path.abspath(path), followlinks=True)
files = list()
for fpath, _, afiles in paths:
for afile in afiles:
files.append(os.path.join(fpath, afile))
else:
return files
def _check_plugins(self, git_repo_plugins, git_data):
"""Check if the git url is a plugin type.
:type git_repo_plugins: ``dict``
:type git_data: ``dict``
"""
for repo_plugin in git_repo_plugins:
strip_plugin_path = repo_plugin['package'].lstrip('/')
plugin = '%s/%s' % (
repo_plugin['path'].strip('/'),
strip_plugin_path
)
name = git_data['name'] = os.path.basename(strip_plugin_path)
git_data['egg_name'] = name.replace('-', '_')
package = self.git_pip_install % (
git_data['repo'], git_data['branch']
)
package += '#egg=%s' % git_data['egg_name']
package += '&subdirectory=%s' % plugin
package += '&gitname=%s' % name
if git_data['fragments']:
package += '&%s' % git_data['fragments']
self.pip['git_data'].append(git_data)
self.pip['git_package'].append(package)
if name not in GIT_PACKAGE_DEFAULT_PARTS:
GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
else:
GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data.copy())
@staticmethod
def _check_defaults(git_data, name, item):
"""Check if a default exists and use it if an item is undefined.
:type git_data: ``dict``
:type name: ``str``
:type item: ``str``
"""
if not git_data[item] and name in GIT_PACKAGE_DEFAULT_PARTS:
check_item = GIT_PACKAGE_DEFAULT_PARTS[name].get(item)
if check_item:
git_data[item] = check_item
def _process_git(self, loaded_yaml, git_item, yaml_file_name):
"""Process git repos.
:type loaded_yaml: ``dict``
:type git_item: ``str``
"""
git_data = dict()
if git_item.split('_')[0] == 'git':
prefix = ''
else:
prefix = '%s_' % git_item.split('_git_repo')[0].replace('.', '_')
# Set the various variable definitions
repo_var = prefix + 'git_repo'
name_var = prefix + 'git_package_name'
branch_var = prefix + 'git_install_branch'
fragment_var = prefix + 'git_install_fragments'
plugins_var = prefix + 'repo_plugins'
group_var = prefix + 'git_project_group'
# get the repo definition
git_data['repo'] = loaded_yaml.get(repo_var)
group = git_data['project_group'] = loaded_yaml.get(group_var, 'all')
# get the repo name definition
name = git_data['name'] = loaded_yaml.get(name_var)
if not name:
# NOTE: strip off trailing /, .git, or .git/
name = git_data['name'] = os.path.basename(
re.sub(r'(\/$|\.git(\/)?$)', '', git_data['repo'])
)
git_data['egg_name'] = name.replace('-', '_')
# This conditional is set to ensure we're only processing git
# repos from the defaults file when those same repos are not
# being set in the repo_packages files.
if '/defaults/main' in yaml_file_name:
if name in GIT_PACKAGE_DEFAULT_PARTS:
return
# get the repo branch definition
git_data['branch'] = loaded_yaml.get(branch_var)
self._check_defaults(git_data, name, 'branch')
if not git_data['branch']:
git_data['branch'] = 'master'
package = self.git_pip_install % (git_data['repo'], git_data['branch'])
# get the repo fragment definitions, if any
git_data['fragments'] = loaded_yaml.get(fragment_var)
self._check_defaults(git_data, name, 'fragments')
package += '#egg=%s' % git_data['egg_name']
package += '&gitname=%s' % name
package += '&projectgroup=%s' % group
if git_data['fragments']:
package += '&%s' % git_data['fragments']
self.pip['git_package'].append(package)
self.pip['git_data'].append(git_data.copy())
# Set the default package parts to track data during the run
if name not in GIT_PACKAGE_DEFAULT_PARTS:
GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
else:
GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data)
# get the repo plugin definitions, if any
git_data['plugins'] = loaded_yaml.get(plugins_var)
self._check_defaults(git_data, name, 'plugins')
if git_data['plugins']:
self._check_plugins(
git_repo_plugins=git_data['plugins'],
git_data=git_data
)
def _package_build_index(self, packages, role_name, var_name, pkg_index,
project_group='all', var_file_name=None,
pip_packages=True):
if pip_packages:
self._py_pkg_extend(packages)
if role_name:
if role_name in pkg_index:
role_pkgs = pkg_index[role_name]
else:
role_pkgs = pkg_index[role_name] = dict()
role_pkgs['project_group'] = project_group
if var_file_name:
_name = os.path.splitext(os.path.basename(var_file_name))[0]
if _name in pkg_index[role_name]:
file_name_index = pkg_index[role_name][_name]
else:
file_name_index = pkg_index[role_name][_name] = dict()
pkgs = file_name_index.get(var_name, list())
pkgs = self._py_pkg_extend(packages, pkgs)
file_name_index[var_name] = sorted(set(pkgs))
else:
pkgs = role_pkgs.get(var_name, list())
pkgs.extend(packages)
if 'pip' in var_name:
pkgs = [i.lower() for i in pkgs if not i.startswith('{{')]
else:
pkgs = [i for i in pkgs if not i.startswith('{{')]
if pkgs:
pkg_index[role_name][var_name] = sorted(set(pkgs))
else:
for k, v in pkg_index.items():
for item_name in v.keys():
if var_name == item_name:
pkg_index[k][item_name] = self._py_pkg_extend(
packages,
pkg_index[k][item_name]
)
def _process_files(self):
"""Process all of the requirement files."""
self._process_files_defaults()
self._process_files_requirements()
def _process_files_defaults(self):
"""Process files."""
for file_name in self._filter_files(self.file_names, ('yaml', 'yml')):
with open(file_name, 'r') as f:
# If there is an exception loading the file continue
# and if the loaded_config is None continue. This makes
# no bad config gets passed to the rest of the process.
try:
loaded_config = yaml.safe_load(f.read())
except Exception: # Broad exception so everything is caught
continue
else:
if not loaded_config or not isinstance(loaded_config, dict):
continue
if 'roles' in file_name:
_role_name = file_name.split('roles%s' % os.sep)[-1]
role_name = _role_name.split(os.sep)[0]
else:
role_name = None
for key, value in loaded_config.items():
if key.endswith('role_project_group'):
project_group = value
break
else:
project_group = 'all'
if role_name is not None:
PACKAGE_MAPPING['role_project_groups'][role_name] = project_group
for key, values in loaded_config.items():
key = key.lower()
if key.endswith('git_repo'):
self._process_git(
loaded_yaml=loaded_config,
git_item=key,
yaml_file_name=file_name
)
# Process pip packages
self._process_packages(
pkg_constant=BUILT_IN_PIP_PACKAGE_VARS,
pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS,
pkg_role_index=ROLE_PACKAGES,
pkg_var_name=key,
packages=values,
role_name=role_name,
project_group=project_group
)
def _process_packages(self, pkg_constant, pkg_breakout_index,
pkg_role_index, pkg_var_name, packages, role_name,
project_group, role_index=True, var_file_name=None,
pip_packages=True):
"""Process variables to build the package data structures.
:param pkg_constant: CONSTANT used to validate package names
:type pkg_constant: ``list``
:param pkg_breakout_index: CONSTANT used to store indexed packages
:type pkg_breakout_index: ``dict``
:param pkg_role_index: CONSTANT used to store role indexed packages
:type pkg_role_index: ``dict``
:param pkg_var_name: package variable name
:type pkg_var_name: ``str``
:param packages: list of packages to index
:type packages: ``list``
:param role_name: Name of the role where the packages came from
:type role_name: ``str``
:param project_group: Name of the group being indexed
:type project_group: ``str``
:param role_index: Enable or disable the use of the role index
:type role_index: ``bool``
:param var_file_name: Variable file name used to index packages
:type var_file_name: ``str``
:param pip_packages: Enable or disable pip index types
:type pip_packages: ``bool``
"""
if [i for i in pkg_constant if i in pkg_var_name]:
if 'proprietary' in pkg_var_name:
return
self._package_build_index(
packages=packages,
role_name=role_name,
var_name=pkg_var_name,
pkg_index=pkg_breakout_index,
project_group=project_group,
var_file_name=var_file_name,
pip_packages=pip_packages
)
if not role_index:
return
elif 'optional' in pkg_var_name:
return
else:
self._package_build_index(
packages=packages,
role_name=role_name,
var_name=pkg_var_name,
pkg_index=pkg_role_index,
project_group=project_group,
var_file_name=var_file_name,
pip_packages=pip_packages
)
def _process_files_requirements(self):
"""Process requirements files."""
return_list = self._filter_files(self.file_names, 'txt')
for file_name in return_list:
base_name = os.path.basename(file_name)
if base_name in REQUIREMENTS_FILE_TYPES:
index = REQUIREMENTS_FILE_TYPES.index(base_name)
return_list.remove(file_name)
return_list.insert(index, file_name)
else:
for file_name in return_list:
if file_name.endswith('doc/requirements.txt'):
continue
if file_name.endswith('other-requirements.txt'):
continue
elif file_name.endswith('bindep.txt'):
continue
elif 'roles' in file_name:
continue
else:
role_name = 'default'
with open(file_name, 'r') as f:
packages = [
i.split()[0].lower() for i in f.read().splitlines()
if i
if not i.startswith('#')
]
base_file_name = os.path.basename(file_name)
if base_file_name.endswith('test-requirements.txt'):
continue
if base_file_name.endswith('global-requirement-pins.txt'):
self._package_build_index(
packages=packages,
role_name='global_pins',
var_name='pinned_packages',
pkg_index=ROLE_REQUIREMENTS,
project_group='all'
)
self._package_build_index(
packages=packages,
role_name=role_name,
var_name='txt_file_packages',
pkg_index=ROLE_REQUIREMENTS,
project_group='all'
)
def _abs_path(path):
return os.path.abspath(
os.path.expanduser(
path
)
)
class LookupModule(LookupBase):
def __init__(self, basedir=None, **kwargs):
"""Run the lookup module.
:type basedir:
:type kwargs:
"""
def run(self, terms, variables=None, **kwargs):
"""Run the main application.
:type terms: ``str``
:type variables: ``str``
:type kwargs: ``dict``
:returns: ``list``
"""
if isinstance(terms, str):
terms = [terms]
return_data = PACKAGE_MAPPING
for term in terms:
return_list = list()
try:
dfp = DependencyFileProcessor(
local_path=_abs_path(str(term))
)
return_list.extend(dfp.pip['py_package'])
return_list.extend(dfp.pip['git_package'])
except Exception as exp:
raise AnsibleError(
'lookup_plugin.py_pkgs(%s) returned "%s" error "%s"' % (
term,
str(exp),
traceback.format_exc()
)
)
for item in return_list:
map_base_and_remote_packages(item, return_data)
else:
parse_remote_package_parts(return_data)
else:
map_role_packages(return_data)
map_base_package_details(return_data)
# Sort everything within the returned data
for key, value in return_data.items():
if isinstance(value, (list, set)):
try:
if all(isinstance(item, dict) for item in value):
return_data[key] = sorted(value, key = lambda k: k['name'])
else:
return_data[key] = sorted(value)
except TypeError:
return_data[key] = value
return_data['role_requirement_files'] = ROLE_REQUIREMENTS
return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS
return [return_data]
# Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`
if __name__ == '__main__':
import sys
import json
print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))

6
releasenotes/notes/remove-py-pkgs-db29425b9e1855a6.yaml Normal file
View File

@ -0,0 +1,6 @@
---
deprecations:
- |
The ``py_pkgs`` and ``packages_file`` Ansible lookups are no
longer used in OSA and have been removed from the plugins
repository.

40
tests/test-lookups.yml
View File

@ -1,40 +0,0 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2017, Jean-Philippe Evrard <jean-philippe.evrard@rackspace.co.uk>
- name: Test lookups
hosts: localhost
connection: local
gather_facts: no
tasks:
- name: Ensure Package list contains the files listed in Package
set_fact:
packages_dict: "{{ lookup('packages_file', playbook_dir ~ '/files/Packages') }}"
- name: Validate that Filename, MD5sum, SHA1, SHA256 and Version are properly generated
assert:
that:
- packages_dict['0ad']['Filename'] == "pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb"
- packages_dict['0ad']['MD5sum'] == "7e5f2ba5e1a95e47753eeb962af32e26"
- packages_dict['0ad']['SHA1'] == "3cf898d4595092daa274e6cd8d9afd0332b0afbe"
- packages_dict['0ad']['SHA256'] == "f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378"
- packages_dict['0ad']['Version'] == "0.0.16-2~ubuntu14.04.1"
- name: Validate that a provides package is listed as a package too
assert:
that:
- packages_dict['aide'] is defined
- packages_dict['aide-binary'] is defined
- packages_dict['aide'] == packages_dict['aide-binary']

2
tests/test.yml
View File

@ -21,8 +21,6 @@
- import_playbook: test-filters.yml
- import_playbook: test-lookups.yml
- import_playbook: test-strategy-tagfilter.yml
- import_playbook: test-connection-plugin.yml