Update roles requiring python

Change-Id: Iae5cd4affc3e7d148221e55768fbdf6d28907e54
This commit is contained in:
Federico Ressi 2020-04-02 15:02:50 +02:00
parent 427ec04806
commit 86441d7aac
27 changed files with 332 additions and 301 deletions

View File

@ -6,17 +6,13 @@
test_collect_dir: '{{ zuul_output_dir }}/logs'
roles:
- role: tobiko-ensure-python
- role: tobiko-ensure-python3
- name: "run Tox InfraRed plugin"
role: tox
vars:
tox_envlist: infrared
tox_extra_args:
'-- --collect-dir {{ test_collect_dir | quote }}'
tox_environment:
PYTHON: '{{ unversioned_python_executable }}'
ANSIBLE_PYTHON_INTERPRETER: '{{ unversioned_python_executable }}'
tox_extra_args: '-- --collect-dir {{ test_collect_dir | quote }}'
tasks:
- name: "list collected files"

View File

@ -20,17 +20,11 @@ subparsers:
default: localhost
help: Target host where test cases are deployed and executed
ansible_variable: test_host
ansible-python-interpreter:
type: Value
default: python3
help: Python interpreter executed by ansible on target hosts
ansible_variable: ansible_python_interpreter
- title: Control flow
options:
stage:
type: Value
default: all
ansible_variable: test_stage
- title: Deploy stage

View File

@ -1,24 +1,29 @@
---
# --- Tobiko deploy options ---------------------------------------------------
# --- Tobiko workflow stages
user_home: '{{ ansible_env.get("HOME", "~") }}'
tobiko_dir: '{{ user_home }}/src/tobiko'
tobiko_src_dir: '{{ role_path | dirname | dirname }}'
tobiko_git_repo: 'https://opendev.org/x/tobiko.git'
tobiko_git_refspec: ''
tobiko_git_version: ''
test_stage: all
# --- Test deploy options -----------------------------------------------------
test_home: '{{ (ansible_user_dir + "/src") | realpath }}'
test_dir: '{{ tobiko_dir | realpath }}'
test_src_dir: '{{ tobiko_src_dir | realpath }}'
test_git_repo: '{{ tobiko_git_repo | realpath }}'
test_src_dir: ''
test_git_repo: ''
test_git_refspec: ''
test_git_version: ''
# --- Tobiko deploy options ---------------------------------------------------
tobiko_dir: '{{ test_home | realpath }}/tobiko'
tobiko_src_dir: '{{ role_path | realpath | dirname | dirname }}'
tobiko_git_repo: 'https://opendev.org/x/tobiko.git'
tobiko_git_refspec: ''
tobiko_git_version: ''
# --- Test configuration options ----------------------------------------------
# Configuration options

View File

@ -1,14 +0,0 @@
---
- name: "expand actual user home directory"
command: 'echo "{{ user_home }}"'
register: expand_user_home
- name: "update user home directory"
set_fact:
user_home: '{{ expand_user_home.stdout.strip() | realpath }}'
- name: "show actual user home directory"
debug: var=user_home

View File

@ -2,4 +2,4 @@
dependencies:
- role: tobiko-common
- role: tobiko-ensure-python
- role: tobiko-ensure-python3

View File

@ -0,0 +1,23 @@
---
- name: "ensure Bindep is installed"
command: |
'{{ python_executable }}' -m pip install --user 'bindep>={{ bindep_min_version }}'
register: result
changed_when: "'Successfully installed' in result.stdout"
- name: "set bindep_executable fact"
set_fact:
bindep_executable: "{{ ansible_user_dir }}/.local/bin/bindep"
cacheable: true
when: result is changed
- name: "get installed Bindep version"
command: "{{ bindep_executable }} --version"
register: get_installed_bindep_version
- name: "show Bindep installed version"
debug: var=get_installed_bindep_version.stdout_lines

View File

@ -1,23 +1,6 @@
---
- name: "ensure Bindep is installed"
command: |
'{{ python_executable }}' -m pip install --user 'bindep>={{ bindep_min_version }}'
register: result
changed_when: "'Successfully installed' in result.stdout"
- name: "set bindep_executable fact"
set_fact:
bindep_executable: "{{ ansible_user_dir }}/.local/bin/bindep"
cacheable: true
when: result is changed
- name: "get installed Bindep version"
command: "{{ bindep_executable }} --version"
register: get_installed_bindep_version
- name: "show Bindep installed version"
debug: var=get_installed_bindep_version.stdout_lines
- name: "ensures '{{ bindep_executable }}' command is available"
include_tasks: bindep.yaml
when:
- test_stage in ['all', 'pre-run', 'bindep']

View File

@ -1,7 +0,0 @@
---
python_version: '3'
python_command: 'python{{ python_version }}'
unversioned_python_command: '/usr/bin/python'
unversioned_python_alternative: '/usr/bin/python3'

View File

@ -1,102 +0,0 @@
---
- name: "validate python_version value: {{ python_version }}"
assert:
that:
- (python_version|string).split(".") | length >= 1
- (python_version|string).split(".") | length <= 2
- (python_version|string).split(".")[0] == '3'
- name: "include OS-specific variables"
include_vars: "{{ item }}"
ignore_errors: yes
with_first_found:
- "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml"
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- block:
- name: "get Python info for '{{ python_command }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python_command }}'
register: get_python_info
rescue:
- name: "install Python {{ python_version }} packages"
become: true
package:
name: "{{ python_packages }}"
- name: "get Python info for '{{ python_command }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python_command }}'
register: get_python_info
- name: "set python_info fact"
set_fact:
python_info: '{{ get_python_info.stdout | from_json }}'
- name: "update Python executable facts"
set_fact:
python_command: '{{ python_info.executables | first | basename }}'
python_executable: '{{ python_info.executables | first }}'
python_version: '{{ python_info.version }}'
- name: "show Python executables facts"
debug:
msg:
python_command: '{{ python_command }}'
python_executable: '{{ python_executable }}'
python_info: '{{ python_info }}'
python_version: '{{ python_version }}'
- block:
- name: "get Python info for '{{ unversioned_python_command }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ unversioned_python_command }}'
register: get_unversioned_python_info
rescue:
- name: "set '{{ python_executable }}' as default alternative for python"
become: true
command: "alternatives --set python '{{ unversioned_python_alternative }}'"
- name: "get Python info for '{{ unversioned_python_command }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ unversioned_python_command }}'
register: get_unversioned_python_info
- name: "set unversioned_python_info fact"
set_fact:
unversioned_python_info: '{{ get_unversioned_python_info.stdout | from_json }}'
- name: "update unversioned Python executable facts"
set_fact:
unversioned_python_command:
'{{ unversioned_python_info.executables | first | basename }}'
unversioned_python_executable:
'{{ unversioned_python_info.executables | first }}'
unversioned_python_version:
'{{ unversioned_python_info.version }}'
- name: "show unversioned Python executables facts"
debug:
msg:
unversioned_python_command: '{{ unversioned_python_command }}'
unversioned_python_executable: '{{ unversioned_python_executable }}'
unversioned_python_info: '{{ unversioned_python_info }}'
unversioned_python_version: '{{ unversioned_python_version }}'

View File

@ -1,18 +0,0 @@
---
python_command: '/usr/bin/python{{ python_version }}'
python_package_name: "python{{ python_version | regex_replace('\\.', '') }}"
python_packages:
- bzip2-devel
- gcc
- make
- openssl-devel
- readline-devel
- sqlite-devel
- zlib-devel
- '{{ python_package_name }}'
- '{{ python_package_name }}-devel'
- '{{ python_package_name }}-setuptools'
- '{{ python_package_name }}-virtualenv'

View File

@ -0,0 +1,5 @@
---
python_executable: '/usr/bin/python'
python_version: '3'
python3_executable: '/usr/bin/python{{ python_version }}'

View File

@ -26,7 +26,7 @@ LOG = logging.getLogger(__name__)
GET_PYTHON_VERSION_SCRIPT = """
import sys
version = '.'.join(str(i) for i in sys.version_info[:3])
version = '.'.join(str(i) for i in sys.version_info)
print(version)
"""
@ -47,7 +47,9 @@ def main():
info['executable']
for info in iter_python_executables_info(match_version=version,
base=args.base)]
info = {'version': version, 'executables': executables}
info = {'version': version,
'executable': sys.executable,
'executables': executables}
output = json.dumps(info, indent=4, sort_keys=True)
print(output)
@ -64,7 +66,7 @@ def setup_logging(quiet=False):
def get_python_version():
return '.'.join(str(i) for i in sys.version_info[:3])
return '.'.join(str(i) for i in sys.version_info)
def iter_python_executables_info(match_version=None, base=None):
@ -84,29 +86,12 @@ def iter_python_executables_info(match_version=None, base=None):
raise last_error
def iter_python_executables(base=None):
iterated = set()
for executable in _iter_python_executables(base=base):
# Iterate every executable only once
if executable not in iterated:
iterated.add(executable)
yield executable
def _iter_python_executables(base):
def iter_python_executables(base):
if base:
base_prefix = getattr(sys, 'base_prefix', None)
if base_prefix:
for executable in iter_prefix_executables(base_prefix):
yield executable
for executable in iter_prefix_executables(sys.prefix):
yield executable
yield sys.executable
def iter_prefix_executables(prefix):
prefix = getattr(sys, 'base_prefix', sys.prefix)
else:
prefix = sys.prefix
if prefix:
if os.path.isdir(prefix):
for python_name in iter_versioned_names():
executable = os.path.join(prefix, 'bin', python_name)

View File

@ -0,0 +1,31 @@
---
- name: "show ansible distro variables"
debug:
msg:
ansible_architecture: '{{ ansible_architecture }}'
ansible_distribution: '{{ ansible_distribution }}'
ansible_distribution_major_version: '{{ ansible_distribution_major_version }}'
ansible_os_family: '{{ ansible_os_family }}'
- name: "include OS-specific variables"
include_vars: "{{ item }}"
ignore_errors: yes
with_first_found:
- "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml"
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- block:
- name: "ensures '{{ python3_executable }}' command is available"
include_tasks: python3.yaml
when:
- python3_info is not defined
- name: "ensures '{{ python_executable }}' command is available"
include_tasks: python.yaml
when:
- python_info is not defined

View File

@ -0,0 +1,41 @@
---
- block:
- name: "get Python info for '{{ python_executable }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python_executable }}'
register: get_python_info
rescue:
- block:
- name: "set '{{ python_alternative }}' as default alternative for python"
become: true
command: "alternatives --set python '{{ python_alternative }}'"
- name: "get Python info for '{{ python_executable }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python_executable }}'
register: get_python_info
when:
- python_alternative is defined
- name: "report '{{ python_executable }}' failure"
fail:
msg: get_python_info
when: get_python_info is failed
- name: "set python_info fact"
set_fact:
python_executable: '{{ (get_python_info.stdout | from_json).executables | first }}'
python_info: '{{ get_python_info.stdout | from_json }}'
- name: "show '{{ python_executable }}' executables facts"
debug:
msg:
python_executable: '{{ python_executable }}'
python_info: '{{ python_info }}'

View File

@ -0,0 +1,50 @@
---
- name: "validate python_version value: {{ python_version }}"
assert:
that:
- (python_version | string).split(".") | length >= 1
- (python_version | string).split(".") | length <= 2
- (python_version | string).split(".")[0] == '3'
- block:
- name: "get Python info for '{{ python3_executable }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python3_executable }}'
register: get_python_info
rescue:
- block:
- name: "install Python '{{ python3_executable }}' packages"
become: true
package:
name: "{{ python3_packages }}"
- name: "get Python info for '{{ python3_executable }}'"
script:
cmd: get_python_info.py --base --quiet
executable: '{{ python3_executable }}'
register: get_python_info
when:
- python3_packages is defined
- name: "report '{{ python3_executable }}' failure"
fail:
msg: get_python_info
when: get_python_info is failed
- name: "set python3_info fact"
set_fact:
python3_executable: '{{ (get_python_info.stdout | from_json).executables | first }}'
python3_info: '{{ get_python_info.stdout | from_json }}'
- name: "show '{{ python3_executable }}' facts"
debug:
msg:
python3_executable: '{{ python3_executable }}'
python3_info: '{{ python3_info }}'

View File

@ -0,0 +1,3 @@
---
python3_executable: python3

View File

@ -0,0 +1,18 @@
---
python_alternative: '/usr/bin/python3'
python3_package_name: "python{{ python_version | regex_replace('\\.', '') }}"
python3_packages:
- bzip2-devel
- gcc
- make
- openssl-devel
- readline-devel
- sqlite-devel
- zlib-devel
- '{{ python3_package_name }}'
- '{{ python3_package_name }}-devel'
- '{{ python3_package_name }}-setuptools'
- '{{ python3_package_name }}-virtualenv'

View File

@ -2,4 +2,4 @@
dependencies:
- role: tobiko-common
- role: tobiko-ensure-python
- role: tobiko-ensure-python3

View File

@ -1,23 +1,6 @@
---
- name: "ensure Tox is installed"
command: |
'{{ python_executable }}' -m pip install --user 'tox>={{ tox_min_version }}'
register: result
changed_when: "'Successfully installed' in result.stdout"
- name: "set tox_executable fact"
set_fact:
tox_executable: "{{ ansible_user_dir }}/.local/bin/tox"
cacheable: true
when: result is changed
- name: "get installed Tox version"
command: "{{ tox_executable }} --version"
register: get_installed_tox_version
- name: "show Tox installed version"
debug: var=get_installed_tox_version.stdout_lines
- name: "ensures '{{ tox_executable }}' command is available"
include_tasks: tox.yaml
when:
- test_stage in ['all', 'pre-run', 'tox']

View File

@ -0,0 +1,23 @@
---
- name: "ensure Tox is installed"
command: |
'{{ python_executable }}' -m pip install --user 'tox>={{ tox_min_version }}'
register: result
changed_when: "'Successfully installed' in result.stdout"
- name: "set tox_executable fact"
set_fact:
tox_executable: "{{ ansible_user_dir }}/.local/bin/tox"
cacheable: true
when: result is changed
- name: "get installed Tox version"
command: "{{ tox_executable }} --version"
register: get_installed_tox_version
- name: "show Tox installed version"
debug: var=get_installed_tox_version.stdout_lines

View File

@ -11,7 +11,7 @@ tox_command_line: >
{% if tox_envlist %} -e {{ tox_envlist | quote }} {% endif %}
{{ tox_extra_args }}
tox_python: '{{ python_executable }}'
tox_python: '{{ python3_executable }}'
tox_report_dir: '{{ test_report_dir | realpath }}'
tox_report_name: '{{ test_report_name }}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}'
tox_report_env:

View File

@ -2,5 +2,5 @@
dependencies:
- role: tobiko-common
- role: tobiko-ensure-python
- role: tobiko-ensure-python3
- role: tobiko-ensure-tox

View File

@ -1,53 +1,6 @@
---
- name: "show tox environment"
debug: var=tox_environment
- name: "normalize white spaces from Tox command line"
set_fact:
tox_command_line: '{{ tox_command_line.split() | join(" ") }}'
- name: "run Tox on direcory '{{ tox_dir }}': '{{ tox_command_line }}'"
command:
chdir: '{{ tox_dir }}'
cmd: '{{ tox_command_line }}'
register:
run_tox
environment: '{{ tox_environment | combine(tox_constrain_env) }}'
ignore_errors: yes
- name: "show test cases results"
debug: var=run_tox.stdout_lines
- name: 'run test cases'
include_tasks: tox.yaml
when:
- (run_tox.stdout_lines | length) > 0
- name: "generate test case report files"
shell:
chdir: "{{ tobiko_dir }}"
cmd: |
{{ tox_command }} -e report
register:
make_report
environment: '{{ tox_report_env | combine(tox_constrain_env) }}'
ignore_errors: yes
- name:
block:
- name: "show test cases errors"
debug: var=run_tox.stderr_lines
when:
- (run_tox.stderr_lines | length) > 0
- name: 'report test cases failure'
debug:
msg: 'test cases have failed'
failed_when: yes
when:
- run_tox is defined
- run_tox is failed
- test_stage in ['all', 'run', 'tox']

View File

@ -0,0 +1,49 @@
---
- name: "normalize white spaces from Tox command line"
set_fact:
tox_command_line: '{{ tox_command_line.split() | join(" ") }}'
- name: "run Tox on direcory '{{ tox_dir }}': '{{ tox_command_line }}'"
command:
chdir: '{{ tox_dir }}'
cmd: '{{ tox_command_line }}'
register:
run_tox
environment: '{{ tox_environment | combine(tox_constrain_env) }}'
ignore_errors: yes
- name: "show test cases results"
debug: var=run_tox.stdout_lines
when:
- (run_tox.stdout_lines | length) > 0
- name: "generate test case report files"
shell:
chdir: "{{ tobiko_dir }}"
cmd: |
{{ tox_command }} -e report
register:
make_report
environment: '{{ tox_report_env | combine(tox_constrain_env) }}'
ignore_errors: yes
- name:
block:
- name: "show test cases errors"
debug: var=run_tox.stderr_lines
when:
- (run_tox.stderr_lines | length) > 0
- name: 'report test cases failure'
debug:
msg: 'test cases have failed'
failed_when: yes
when:
- run_tox is defined
- run_tox is failed

View File

@ -24,23 +24,33 @@ LOG = logging.getLogger(__name__)
def main():
setup_logging()
add_plugin('tobiko', os.environ.get('IR_TOBIKO_PLUGIN'))
import_workspace(os.environ.get('IR_WORKSPACE_FILE'))
add_tobiko_plugin()
import_workspace()
copy_ansible_inventory()
def setup_logging(level=logging.DEBUG):
logging.basicConfig(
level=level, stream=sys.stderr,
level=level,
stream=sys.stderr,
format='%(name)-s: %(levelname)-7s %(asctime)-15s | %(message)s')
def add_tobiko_plugin():
add_plugin('tobiko', os.environ.get('IR_TOBIKO_PLUGIN'))
def add_plugin(name, path):
path = path or os.environ.get('IR_TOBIKO_PLUGIN')
if path:
path = normalize_path(path)
if os.path.isdir(path):
remove_plugin(name)
execute('ir plugin add "{}"', path)
else:
LOG.debug("Plug-in '%s' directory not found: '%s'", name, path)
else:
LOG.debug("Plug-in '%s' path not specified", name)
def remove_plugin(name):
@ -52,7 +62,8 @@ def remove_plugin(name):
return True
def import_workspace(filename):
def import_workspace(filename=None):
filename = filename or os.environ.get('IR_WORKSPACE_FILE')
if filename:
filename = normalize_path(filename)
if os.path.isfile(filename):
@ -63,6 +74,24 @@ def import_workspace(filename):
# workspace
workspace = name_from_path(filename)
execute('ir workspace checkout "{}"', workspace)
else:
LOG.debug("No such workspace file: '%s'", filename)
else:
LOG.debug('Workspace file not specified')
def copy_ansible_inventory(filename=None):
filename = filename or os.environ.get('ANSIBLE_INVENTORY')
if filename:
if os.path.isfile(filename):
destination = execute('ir workspace inventory')
if not os.path.exists(os.path.basename(destination)):
os.makedirs(os.path.basename(destination))
execute('cp {} {}', filename, destination)
else:
LOG.debug('Inventary file not found: %r', filename)
else:
LOG.debug('Ansible inventory file not specified')
def normalize_path(path):
@ -73,7 +102,8 @@ def execute(command, *args, **kwargs):
if args or kwargs:
command = command.format(*args, **kwargs)
LOG.info('%s', command)
return subprocess.check_output(command, shell=True)
return subprocess.check_output(command, shell=True,
universal_newlines=True)
def name_from_path(path):

12
tox.ini
View File

@ -177,11 +177,14 @@ commands = {posargs:bash}
[testenv:infrared]
basepython = {env:PYTHON:python3}
# On RedHat Linux must use the default unversioned python because of dependency on native SELinux
# package available only for /usr/bin/python interpreter
basepython = {env:IR_PYTHON:/usr/bin/python}
usedevelop = false
skipdist = true
skip_install = true
sitepackages = true
deps =
-r infrared-requirements.txt
passenv =
@ -194,19 +197,16 @@ passenv =
setenv =
{[testenv:venv]setenv}
ANSIBLE_CONFIG={env:ANSIBLE_CONFIG:{toxinidir}/ansible.cfg}
ANSIBLE_PYTHON_INTERPRETER={env:ANSIBLE_PYTHON_INTERPRETER:python3}
ANSIBLE_INVENTORY={env:ANSIBLE_INVENTORY:{toxinidir}/ansible_hosts}
IR_HOME={env:IR_HOME:{envdir}/home/infrared}
IR_TOBIKO_PLUGIN={env:IR_TOBIKO_PLUGIN:{toxinidir}/roles}
IR_WORKSPACE_FILE={env:IR_WORKSPACE_FILE:{toxinidir}/workspace.tgz}
PYTHON={env:PYTHON:python3}
commands_pre =
{envpython} {toxinidir}/tools/setup_infrared.py
commands =
ir tobiko \
--ansible-python-interpreter {env:ANSIBLE_PYTHON_INTERPRETER} \
--tobiko-src-dir {toxinidir} {posargs}
ir tobiko --tobiko-src-dir {toxinidir} {posargs}
# --- documentation environments ----------------------------------------------