Replace Ansible 6 with Ansible 9

Ansible 6 is EOL and Ansible 9 is available.  Remove 6 and add 9.

This is usually done in two changes, but this time it's in one
since we can just rotate the 6 around to make it a 9.

command.py has been updated for ansible 9.

Change-Id: I537667f66ba321d057b6637aa4885e48c8b96f04
This commit is contained in:
James E. Blair 2024-02-07 14:07:49 -08:00
parent 773af06b5a
commit 5a8e373c3b
39 changed files with 917 additions and 165 deletions

View File

@ -42,18 +42,18 @@
- playbooks/zuul-stream/.*
- setup.cfg
- job:
name: zuul-stream-functional-6
parent: zuul-stream-functional
vars:
zuul_ansible_version: 6
- job:
name: zuul-stream-functional-8
parent: zuul-stream-functional
vars:
zuul_ansible_version: 8
- job:
name: zuul-stream-functional-9
parent: zuul-stream-functional
vars:
zuul_ansible_version: 9
- job:
name: zuul-nox
description: |
@ -374,8 +374,8 @@
files:
- web/.*
nodeset: ubuntu-jammy
- zuul-stream-functional-6
- zuul-stream-functional-8
- zuul-stream-functional-9
- zuul-nox-remote
- zuul-quick-start:
requires: nodepool-container-image
@ -405,8 +405,8 @@
files:
- web/.*
nodeset: ubuntu-jammy
- zuul-stream-functional-6
- zuul-stream-functional-8
- zuul-stream-functional-9
- zuul-nox-remote
- zuul-quick-start:
requires: nodepool-container-image

View File

@ -33,7 +33,7 @@ services:
- "lib-zuul-executor:/var/lib/zuul:z"
# NOTE(pabelanger): Be sure to update this line each time we change the
# default version of ansible for Zuul.
command: "/usr/local/lib/zuul/ansible/6/bin/ansible-playbook /var/playbooks/setup.yaml"
command: "/usr/local/lib/zuul/ansible/8/bin/ansible-playbook /var/playbooks/setup.yaml"
networks:
- zuul
zk:

View File

@ -71,7 +71,6 @@
git push http://admin:secret@gerrit:8080/All-Projects +HEAD:refs/meta/config
args:
chdir: "{{ all_projects_repo }}"
warn: false
- name: Create zuul-config project
include_role:

View File

@ -0,0 +1,11 @@
---
features:
- |
Ansible version 9 is now available. The default Ansible version
is still 8, but version 9 may be selected by using
:attr:`job.ansible-version`.
upgrade:
- |
Support for Ansible version 6 has been removed. Migrate any
existing jobs which rely on this version to Ansible version 8
before upgrading.

View File

@ -19,8 +19,7 @@ class CallbackModule(CallbackBase):
test callback
"""
CALLBACK_VERSION = 2.0
CALLBACK_NEEDS_WHITELIST = True # 6.0
CALLBACK_NEEDS_ENABLED = True # 8.0
CALLBACK_NEEDS_ENABLED = True
# aggregate means we can be loaded and not be the stdout plugin
CALLBACK_TYPE = 'aggregate'
CALLBACK_NAME = 'test_callback'

View File

@ -1,6 +1,6 @@
- tenant:
name: tenant-one
default-ansible-version: '6'
default-ansible-version: '9'
source:
gerrit:
config-projects:

View File

@ -37,15 +37,7 @@
parent: ansible-version
vars:
test_ansible_version_major: 2
test_ansible_version_minor: 13
- job:
name: ansible-6
parent: ansible-version
ansible-version: 6
vars:
test_ansible_version_major: 2
test_ansible_version_minor: 13
test_ansible_version_minor: 16
- job:
name: ansible-8
@ -55,18 +47,26 @@
test_ansible_version_major: 2
test_ansible_version_minor: 15
- job:
name: ansible-9
parent: ansible-version
ansible-version: 9
vars:
test_ansible_version_major: 2
test_ansible_version_minor: 16
- project:
name: common-config
check:
jobs:
- ansible-default
- ansible-6
- ansible-8
- ansible-9
- project:
name: org/project
check:
jobs:
- ansible-default-zuul-conf
- ansible-6
- ansible-8
- ansible-9

View File

@ -1,6 +1,6 @@
- tenant:
name: tenant-one
default-ansible-version: '6'
default-ansible-version: '9'
source:
gerrit:
config-projects:

View File

@ -1,6 +1,6 @@
- tenant:
name: tenant-one
default-ansible-version: '6'
default-ansible-version: '9'
source:
gerrit:
config-projects:

View File

@ -114,15 +114,6 @@
ansible_network_os: foo
run: playbooks/network.yaml
- job:
name: ansible-version6-inventory
nodeset:
nodes:
- name: ubuntu-xenial
label: ubuntu-xenial
ansible-version: '6'
run: playbooks/ansible-version.yaml
- job:
name: ansible-version8-inventory
nodeset:
@ -131,3 +122,12 @@
label: ubuntu-xenial
ansible-version: '8'
run: playbooks/ansible-version.yaml
- job:
name: ansible-version9-inventory
nodeset:
nodes:
- name: ubuntu-xenial
label: ubuntu-xenial
ansible-version: '9'
run: playbooks/ansible-version.yaml

View File

@ -7,5 +7,5 @@
- executor-only-inventory
- group-inventory
- hostvars-inventory
- ansible-version6-inventory
- ansible-version8-inventory
- ansible-version9-inventory

View File

@ -7,7 +7,7 @@ server=127.0.0.1
tenant_config=main.yaml
relative_priority=true
# Used by ansible-default-zuul-conf job
default_ansible_version=6
default_ansible_version=9
[merger]
git_dir=/tmp/zuul-test/merger-git

View File

@ -21,7 +21,7 @@ from tests.base import AnsibleZuulTestCase
class FunctionalActionModulesMixIn:
tenant_config_file = 'config/remote-action-modules/main.yaml'
# This should be overriden in child classes.
ansible_version = '6'
ansible_version = 'X'
wait_timeout = 120
def _setUp(self):
@ -87,17 +87,17 @@ class FunctionalActionModulesMixIn:
self._run_job('shell-good', 'SUCCESS')
class TestActionModules6(AnsibleZuulTestCase, FunctionalActionModulesMixIn):
ansible_version = '6'
def setUp(self):
super().setUp()
self._setUp()
class TestActionModules8(AnsibleZuulTestCase, FunctionalActionModulesMixIn):
ansible_version = '8'
def setUp(self):
super().setUp()
self._setUp()
class TestActionModules9(AnsibleZuulTestCase, FunctionalActionModulesMixIn):
ansible_version = '9'
def setUp(self):
super().setUp()
self._setUp()

View File

@ -22,7 +22,8 @@ from tests.base import AnsibleZuulTestCase
class FunctionalZuulJSONMixIn:
tenant_config_file = 'config/remote-zuul-json/main.yaml'
ansible_version = '2.6'
# This should be overriden in child classes.
ansible_version = 'X'
def _setUp(self):
self.fake_nodepool.remote_ansible = True
@ -144,17 +145,17 @@ class FunctionalZuulJSONMixIn:
dateutil.parser.parse(play_end_time)
class TestZuulJSON6(AnsibleZuulTestCase, FunctionalZuulJSONMixIn):
ansible_version = '6'
def setUp(self):
super().setUp()
self._setUp()
class TestZuulJSON8(AnsibleZuulTestCase, FunctionalZuulJSONMixIn):
ansible_version = '8'
def setUp(self):
super().setUp()
self._setUp()
class TestZuulJSON9(AnsibleZuulTestCase, FunctionalZuulJSONMixIn):
ansible_version = '9'
def setUp(self):
super().setUp()
self._setUp()

View File

@ -26,7 +26,8 @@ from tests.base import AnsibleZuulTestCase
class FunctionalZuulStreamMixIn:
tenant_config_file = 'config/remote-zuul-stream/main.yaml'
# This should be overriden in child classes.
ansible_version = '6'
ansible_version = 'X'
ansible_core_version = 'X.Y'
def _setUp(self):
self.log_console_port = 19000 + int(
@ -379,15 +380,6 @@ class FunctionalZuulStreamMixIn:
self.assertLogLine(regex, text)
class TestZuulStream6(AnsibleZuulTestCase, FunctionalZuulStreamMixIn):
ansible_version = '6'
ansible_core_version = '2.13'
def setUp(self):
super().setUp()
self._setUp()
class TestZuulStream8(AnsibleZuulTestCase, FunctionalZuulStreamMixIn):
ansible_version = '8'
ansible_core_version = '2.15'
@ -395,3 +387,12 @@ class TestZuulStream8(AnsibleZuulTestCase, FunctionalZuulStreamMixIn):
def setUp(self):
super().setUp()
self._setUp()
class TestZuulStream9(AnsibleZuulTestCase, FunctionalZuulStreamMixIn):
ansible_version = '9'
ansible_core_version = '2.16'
def setUp(self):
super().setUp()
self._setUp()

View File

@ -935,16 +935,16 @@ class ExecutorFactsMixin:
output)
class TestExecutorFacts6(AnsibleZuulTestCase, ExecutorFactsMixin):
tenant_config_file = 'config/executor-facts/main6.yaml'
ansible_major_minor = '2.13'
class TestExecutorFacts8(AnsibleZuulTestCase, ExecutorFactsMixin):
tenant_config_file = 'config/executor-facts/main8.yaml'
ansible_major_minor = '2.15'
class TestExecutorFacts9(AnsibleZuulTestCase, ExecutorFactsMixin):
tenant_config_file = 'config/executor-facts/main9.yaml'
ansible_major_minor = '2.16'
class AnsibleCallbackConfigsMixin:
config_file = 'zuul-executor-ansible-callback.conf'
@ -996,13 +996,6 @@ class AnsibleCallbackConfigsMixin:
output)
class TestAnsibleCallbackConfigs6(AnsibleZuulTestCase,
AnsibleCallbackConfigsMixin):
config_file = 'zuul-executor-ansible-callback.conf'
tenant_config_file = 'config/ansible-callbacks/main6.yaml'
ansible_major_minor = '2.13'
class TestAnsibleCallbackConfigs8(AnsibleZuulTestCase,
AnsibleCallbackConfigsMixin):
config_file = 'zuul-executor-ansible-callback.conf'
@ -1010,6 +1003,13 @@ class TestAnsibleCallbackConfigs8(AnsibleZuulTestCase,
ansible_major_minor = '2.15'
class TestAnsibleCallbackConfigs9(AnsibleZuulTestCase,
AnsibleCallbackConfigsMixin):
config_file = 'zuul-executor-ansible-callback.conf'
tenant_config_file = 'config/ansible-callbacks/main9.yaml'
ansible_major_minor = '2.16'
class TestExecutorEnvironment(AnsibleZuulTestCase):
tenant_config_file = 'config/zuul-environment-filter/main.yaml'

View File

@ -186,7 +186,7 @@ class TestInventoryShellType(TestInventoryBase):
class InventoryAutoPythonMixin:
ansible_version = 'X'
def test_auto_python_ansible6_inventory(self):
def test_auto_python_ansible_inventory(self):
inventory = self._get_build_inventory(
f'ansible-version{self.ansible_version}-inventory')
@ -213,16 +213,16 @@ class InventoryAutoPythonMixin:
self.waitUntilSettled()
class TestInventoryAutoPythonAnsible6(TestInventoryBase,
InventoryAutoPythonMixin):
ansible_version = '6'
class TestInventoryAutoPythonAnsible8(TestInventoryBase,
InventoryAutoPythonMixin):
ansible_version = '8'
class TestInventoryAutoPythonAnsible9(TestInventoryBase,
InventoryAutoPythonMixin):
ansible_version = '9'
class TestInventory(TestInventoryBase):
def test_single_inventory(self):

View File

@ -4269,16 +4269,16 @@ class FunctionalAnsibleMixIn(object):
output)
class TestAnsible6(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
tenant_config_file = 'config/ansible/main6.yaml'
ansible_major_minor = '2.13'
class TestAnsible8(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
tenant_config_file = 'config/ansible/main8.yaml'
ansible_major_minor = '2.15'
class TestAnsible9(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
tenant_config_file = 'config/ansible/main9.yaml'
ansible_major_minor = '2.16'
class TestPrePlaybooks(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
@ -9462,8 +9462,8 @@ class TestAnsibleVersion(AnsibleZuulTestCase):
self.assertHistory([
dict(name='ansible-default', result='SUCCESS', changes='1,1'),
dict(name='ansible-6', result='SUCCESS', changes='1,1'),
dict(name='ansible-8', result='SUCCESS', changes='1,1'),
dict(name='ansible-9', result='SUCCESS', changes='1,1'),
], ordered=False)
@ -9482,8 +9482,8 @@ class TestDefaultAnsibleVersion(AnsibleZuulTestCase):
self.assertHistory([
dict(name='ansible-default-zuul-conf', result='SUCCESS',
changes='1,1'),
dict(name='ansible-6', result='SUCCESS', changes='1,1'),
dict(name='ansible-8', result='SUCCESS', changes='1,1'),
dict(name='ansible-9', result='SUCCESS', changes='1,1'),
], ordered=False)

View File

@ -1359,7 +1359,7 @@ class TestSystemConfigCache(ZooKeeperBaseTestCase):
"use_relative_priority": True,
"max_hold_expiration": 7200,
"default_hold_expiration": 3600,
"default_ansible_version": "6",
"default_ansible_version": "X",
"web_root": "/web/root",
"web_status_url": "/web/status",
"websocket_url": "/web/socket",

View File

@ -1 +0,0 @@
../../base/library/command.py

771
zuul/ansible/8/library/command.py Executable file
View File

@ -0,0 +1,771 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others
# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
# flake8: noqa
# This file shares a significant chunk of code with an upstream ansible
# function, run_command. The goal is to not have to fork quite so much
# of that function, and discussing that design with upstream means we
# should keep the changes to substantive ones only. For that reason, this
# file is purposely not enforcing pep8, as making the function pep8 clean
# would remove our ability to easily have a discussion with our friends
# upstream
DOCUMENTATION = r'''
---
module: command
short_description: Execute commands on targets
version_added: historical
description:
- The C(command) module takes the command name followed by a list of space-delimited arguments.
- The given command will be executed on all selected nodes.
- The command(s) will not be
processed through the shell, so variables like C($HOSTNAME) and operations
like C("*"), C("<"), C(">"), C("|"), C(";") and C("&") will not work.
Use the M(ansible.builtin.shell) module if you need these features.
- To create C(command) tasks that are easier to read than the ones using space-delimited
arguments, pass parameters using the C(args) L(task keyword,../reference_appendices/playbooks_keywords.html#task)
or use C(cmd) parameter.
- Either a free form command or C(cmd) parameter is required, see the examples.
- For Windows targets, use the M(ansible.windows.win_command) module instead.
extends_documentation_fragment:
- action_common_attributes
- action_common_attributes.raw
attributes:
check_mode:
details: while the command itself is arbitrary and cannot be subject to the check mode semantics it adds C(creates)/C(removes) options as a workaround
support: partial
diff_mode:
support: none
platform:
support: full
platforms: posix
raw:
support: full
options:
free_form:
description:
- The command module takes a free form string as a command to run.
- There is no actual parameter named 'free form'.
cmd:
type: str
description:
- The command to run.
argv:
type: list
elements: str
description:
- Passes the command as a list rather than a string.
- Use C(argv) to avoid quoting values that would otherwise be interpreted incorrectly (for example "user name").
- Only the string (free form) or the list (argv) form can be provided, not both. One or the other must be provided.
version_added: "2.6"
creates:
type: path
description:
- A filename or (since 2.0) glob pattern. If a matching file already exists, this step B(will not) be run.
- This is checked before I(removes) is checked.
removes:
type: path
description:
- A filename or (since 2.0) glob pattern. If a matching file exists, this step B(will) be run.
- This is checked after I(creates) is checked.
version_added: "0.8"
chdir:
type: path
description:
- Change into this directory before running the command.
version_added: "0.6"
warn:
description:
- (deprecated) Enable or disable task warnings.
- This feature is deprecated and will be removed in 2.14.
- As of version 2.11, this option is now disabled by default.
type: bool
default: no
version_added: "1.8"
stdin:
description:
- Set the stdin of the command directly to the specified value.
type: str
version_added: "2.4"
stdin_add_newline:
type: bool
default: yes
description:
- If set to C(yes), append a newline to stdin data.
version_added: "2.8"
strip_empty_ends:
description:
- Strip empty lines from the end of stdout/stderr in result.
version_added: "2.8"
type: bool
default: yes
notes:
- If you want to run a command through the shell (say you are using C(<), C(>), C(|), and so on),
you actually want the M(ansible.builtin.shell) module instead.
Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to
use the C(command) module when possible.
- C(creates), C(removes), and C(chdir) can be specified after the command.
For instance, if you only want to run a command if a certain file does not exist, use this.
- Check mode is supported when passing C(creates) or C(removes). If running in check mode and either of these are specified, the module will
check for the existence of the file and report the correct changed status. If these are not supplied, the task will be skipped.
- The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(ansible.builtin.shell) module instead.
- For Windows targets, use the M(ansible.windows.win_command) module instead.
- For rebooting systems, use the M(ansible.builtin.reboot) or M(ansible.windows.win_reboot) module.
seealso:
- module: ansible.builtin.raw
- module: ansible.builtin.script
- module: ansible.builtin.shell
- module: ansible.windows.win_command
author:
- Ansible Core Team
- Michael DeHaan
'''
EXAMPLES = r'''
- name: Return motd to registered var
ansible.builtin.command: cat /etc/motd
register: mymotd
# free-form (string) arguments, all arguments on one line
- name: Run command if /path/to/database does not exist (without 'args')
ansible.builtin.command: /usr/bin/make_database.sh db_user db_name creates=/path/to/database
# free-form (string) arguments, some arguments on separate lines with the 'args' keyword
# 'args' is a task keyword, passed at the same level as the module
- name: Run command if /path/to/database does not exist (with 'args' keyword)
ansible.builtin.command: /usr/bin/make_database.sh db_user db_name
args:
creates: /path/to/database
# 'cmd' is module parameter
- name: Run command if /path/to/database does not exist (with 'cmd' parameter)
ansible.builtin.command:
cmd: /usr/bin/make_database.sh db_user db_name
creates: /path/to/database
- name: Change the working directory to somedir/ and run the command as db_owner if /path/to/database does not exist
ansible.builtin.command: /usr/bin/make_database.sh db_user db_name
become: yes
become_user: db_owner
args:
chdir: somedir/
creates: /path/to/database
# argv (list) arguments, each argument on a separate line, 'args' keyword not necessary
# 'argv' is a parameter, indented one level from the module
- name: Use 'argv' to send a command as a list - leave 'command' empty
ansible.builtin.command:
argv:
- /usr/bin/make_database.sh
- Username with whitespace
- dbname with whitespace
creates: /path/to/database
- name: Safely use templated variable to run command. Always use the quote filter to avoid injection issues
ansible.builtin.command: cat {{ myfile|quote }}
register: myoutput
'''
RETURN = r'''
msg:
description: changed
returned: always
type: bool
sample: True
start:
description: The command execution start time.
returned: always
type: str
sample: '2017-09-29 22:03:48.083128'
end:
description: The command execution end time.
returned: always
type: str
sample: '2017-09-29 22:03:48.084657'
delta:
description: The command execution delta time.
returned: always
type: str
sample: '0:00:00.001529'
stdout:
description: The command standard output.
returned: always
type: str
sample: 'Clustering node rabbit@slave1 with rabbit@master …'
stderr:
description: The command standard error.
returned: always
type: str
sample: 'ls cannot access foo: No such file or directory'
cmd:
description: The command executed by the task.
returned: always
type: list
sample:
- echo
- hello
rc:
description: The command return code (0 means success).
returned: always
type: int
sample: 0
stdout_lines:
description: The command standard output split in lines.
returned: always
type: list
sample: [u'Clustering node rabbit@slave1 with rabbit@master …']
stderr_lines:
description: The command standard error split in lines.
returned: always
type: list
sample: [u'ls cannot access foo: No such file or directory', u'ls …']
'''
import datetime
import glob
import os
import shlex
import select
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils._text import to_native, to_bytes, to_text
from ansible.module_utils.common.collections import is_iterable
# Imports needed for Zuul things
import re
import subprocess
import traceback
import threading
from ansible.module_utils.basic import heuristic_log_sanitize
from ansible.module_utils.six import (
PY2,
PY3,
b,
binary_type,
string_types,
text_type,
)
from ansible.module_utils.six.moves import shlex_quote
LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log'
PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')
# Lists to save stdout/stderr log lines in as we collect them
_log_lines = []
_stderr_log_lines = []
class Console(object):
def __init__(self, log_uuid):
# The streamer currently will not ask us for output from
# loops. This flag uuid was set in the action plugin if this
# call was part of a loop. This avoids us leaving behind
# files that will never be read, but also means no other
# special-casing for any of this path.
if log_uuid == 'in-loop-ignore':
self.logfile_name = os.devnull
elif log_uuid == 'skip':
self.logfile_name = os.devnull
else:
self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid)
def __enter__(self):
self.logfile = open(self.logfile_name, 'ab', buffering=0)
return self
def __exit__(self, etype, value, tb):
self.logfile.close()
def addLine(self, ln):
# Note this format with deliminator is "inspired" by the old
# Jenkins format but with microsecond resolution instead of
# millisecond. It is kept so log parsing/formatting remains
# consistent.
ts = str(datetime.datetime.now()).encode('utf-8')
if not isinstance(ln, bytes):
try:
ln = ln.encode('utf-8')
except Exception:
ln = repr(ln).encode('utf-8') + b'\n'
outln = b'%s | %s' % (ts, ln)
self.logfile.write(outln)
def follow(stdout, stderr, log_uuid):
newline_warning = False
with Console(log_uuid) as console:
rselect = list(s for s in (stdout, stderr) if s is not None)
while True:
if not rselect:
break
rready, _, __ = select.select(rselect, [], [])
for fd in rready:
line = fd.readline()
if not line:
rselect.remove(fd)
continue
if fd == stdout:
_log_lines.append(line)
else:
_stderr_log_lines.append(line)
if not line[-1] != b'\n':
line += b'\n'
newline_warning = True
console.addLine(line)
if newline_warning:
console.addLine('[Zuul] No trailing newline\n')
# Taken from ansible/module_utils/basic.py ... forking the method for now
# so that we can dive in and figure out how to make appropriate hook points
def zuul_run_command(self, args, zuul_log_id, zuul_ansible_split_streams, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None,
use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict',
expand_user_and_vars=True, pass_fds=None, before_communicate_callback=None, ignore_invalid_cwd=True):
'''
Execute a command, returns rc, stdout, and stderr.
:arg args: is the command to run
* If args is a list, the command will be run with shell=False.
* If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False
* If args is a string and use_unsafe_shell=True it runs with shell=True.
:kw check_rc: Whether to call fail_json in case of non zero RC.
Default False
:kw close_fds: See documentation for subprocess.Popen(). Default True
:kw executable: See documentation for subprocess.Popen(). Default None
:kw data: If given, information to write to the stdin of the command
:kw binary_data: If False, append a newline to the data. Default False
:kw path_prefix: If given, additional path to find the command in.
This adds to the PATH environment variable so helper commands in
the same directory can also be found
:kw cwd: If given, working directory to run the command inside
:kw use_unsafe_shell: See `args` parameter. Default False
:kw prompt_regex: Regex string (not a compiled regex) which can be
used to detect prompts in the stdout which would otherwise cause
the execution to hang (especially if no input data is specified)
:kw environ_update: dictionary to *update* environ variables with
:kw umask: Umask to be used when running the command. Default None
:kw encoding: Since we return native strings, on python3 we need to
know the encoding to use to transform from bytes to text. If you
want to always get bytes back, use encoding=None. The default is
"utf-8". This does not affect transformation of strings given as
args.
:kw errors: Since we return native strings, on python3 we need to
transform stdout and stderr from bytes to text. If the bytes are
undecodable in the ``encoding`` specified, then use this error
handler to deal with them. The default is ``surrogate_or_strict``
which means that the bytes will be decoded using the
surrogateescape error handler if available (available on all
python3 versions we support) otherwise a UnicodeError traceback
will be raised. This does not affect transformations of strings
given as args.
:kw expand_user_and_vars: When ``use_unsafe_shell=False`` this argument
dictates whether ``~`` is expanded in paths and environment variables
are expanded before running the command. When ``True`` a string such as
``$SHELL`` will be expanded regardless of escaping. When ``False`` and
``use_unsafe_shell=False`` no path or variable expansion will be done.
:kw pass_fds: When running on Python 3 this argument
dictates which file descriptors should be passed
to an underlying ``Popen`` constructor. On Python 2, this will
set ``close_fds`` to False.
:kw before_communicate_callback: This function will be called
after ``Popen`` object will be created
but before communicating to the process.
(``Popen`` object will be passed to callback as a first argument)
:kw ignore_invalid_cwd: This flag indicates whether an invalid ``cwd``
(non-existent or not a directory) should be ignored or should raise
an exception.
:returns: A 3-tuple of return code (integer), stdout (native string),
and stderr (native string). On python2, stdout and stderr are both
byte strings. On python3, stdout and stderr are text strings converted
according to the encoding and errors parameters. If you want byte
strings on python3, use encoding=None to turn decoding to text off.
'''
# used by clean args later on
self._clean = None
if not isinstance(args, (list, binary_type, text_type)):
msg = "Argument 'args' to run_command must be list or string"
self.fail_json(rc=257, cmd=args, msg=msg)
shell = False
if use_unsafe_shell:
# stringify args for unsafe/direct shell usage
if isinstance(args, list):
args = b" ".join([to_bytes(shlex_quote(x), errors='surrogate_or_strict') for x in args])
else:
args = to_bytes(args, errors='surrogate_or_strict')
# not set explicitly, check if set by controller
if executable:
executable = to_bytes(executable, errors='surrogate_or_strict')
args = [executable, b'-c', args]
elif self._shell not in (None, '/bin/sh'):
args = [to_bytes(self._shell, errors='surrogate_or_strict'), b'-c', args]
else:
shell = True
else:
# ensure args are a list
if isinstance(args, (binary_type, text_type)):
# On python2.6 and below, shlex has problems with text type
# On python3, shlex needs a text type.
if PY2:
args = to_bytes(args, errors='surrogate_or_strict')
elif PY3:
args = to_text(args, errors='surrogateescape')
args = shlex.split(args)
# expand ``~`` in paths, and all environment vars
if expand_user_and_vars:
args = [to_bytes(os.path.expanduser(os.path.expandvars(x)), errors='surrogate_or_strict') for x in args if x is not None]
else:
args = [to_bytes(x, errors='surrogate_or_strict') for x in args if x is not None]
prompt_re = None
if prompt_regex:
if isinstance(prompt_regex, text_type):
if PY3:
prompt_regex = to_bytes(prompt_regex, errors='surrogateescape')
elif PY2:
prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict')
try:
prompt_re = re.compile(prompt_regex, re.MULTILINE)
except re.error:
self.fail_json(msg="invalid prompt regular expression given to run_command")
rc = 0
msg = None
st_in = None
env = os.environ.copy()
# We can set this from both an attribute and per call
env.update(self.run_command_environ_update or {})
env.update(environ_update or {})
if path_prefix:
path = env.get('PATH', '')
if path:
env['PATH'] = "%s:%s" % (path_prefix, path)
else:
env['PATH'] = path_prefix
# If using test-module.py and explode, the remote lib path will resemble:
# /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py
# If using ansible or ansible-playbook with a remote system:
# /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py
# Clean out python paths set by ansiballz
if 'PYTHONPATH' in env:
pypaths = [x for x in env['PYTHONPATH'].split(':')
if x and
not x.endswith('/ansible_modlib.zip') and
not x.endswith('/debug_dir')]
if pypaths and any(pypaths):
env['PYTHONPATH'] = ':'.join(pypaths)
if data:
st_in = subprocess.PIPE
def preexec():
self._restore_signal_handlers()
if umask:
os.umask(umask)
# ZUUL: merge stdout/stderr depending on config
stderr = subprocess.PIPE if zuul_ansible_split_streams else subprocess.STDOUT
kwargs = dict(
executable=executable,
shell=shell,
close_fds=close_fds,
stdin=st_in,
stdout=subprocess.PIPE,
stderr=stderr,
preexec_fn=preexec,
env=env,
)
if PY3 and pass_fds:
kwargs["pass_fds"] = pass_fds
elif PY2 and pass_fds:
kwargs['close_fds'] = False
# make sure we're in the right working directory
if cwd:
cwd = to_bytes(os.path.abspath(os.path.expanduser(cwd)), errors='surrogate_or_strict')
if os.path.isdir(cwd):
kwargs['cwd'] = cwd
elif not ignore_invalid_cwd:
self.fail_json(msg="Provided cwd is not a valid directory: %s" % cwd)
t = None
fail_json_kwargs = None
try:
if self._debug:
self.log('Executing <%s>: %s',
zuul_log_id, self._clean_args(args))
# ZUUL: Replaced the execution loop with the zuul_runner run function
cmd = subprocess.Popen(args, **kwargs)
if before_communicate_callback:
before_communicate_callback(cmd)
if self.no_log:
t = None
else:
t = threading.Thread(target=follow, args=(cmd.stdout, cmd.stderr, zuul_log_id))
t.daemon = True
t.start()
# ZUUL: Our log thread will catch the output so don't do that here.
if data:
if not binary_data:
data += '\n'
if isinstance(data, text_type):
data = to_bytes(data)
cmd.stdin.write(data)
cmd.stdin.close()
# ZUUL: If the console log follow thread *is* stuck in readline,
# we can't close stdout (attempting to do so raises an
# exception) , so this is disabled.
# cmd.stdout.close()
# cmd.stderr.close()
rc = cmd.wait()
# ZUUL: Give the thread that is writing the console log up to
# 10 seconds to catch up and exit. If it hasn't done so by
# then, it is very likely stuck in readline() because it
# spawed a child that is holding stdout or stderr open.
if t:
t.join(10)
with Console(zuul_log_id) as console:
if t.is_alive():
console.addLine("[Zuul] standard output/error still open "
"after child exited")
# ZUUL: stdout and stderr are in the console log file
# ZUUL: return the saved log lines so we can ship them back
stdout = b('').join(_log_lines)
stderr = b('').join(_stderr_log_lines)
else:
stdout = b('')
stderr = b('')
except (OSError, IOError) as e:
self.log("Error Executing CMD:%s Exception:%s" % (self._clean_args(args), to_native(e)))
# ZUUL: store fail_json_kwargs and fail later in finally
fail_json_kwargs = dict(rc=e.errno, stdout=b'', stderr=b'', msg=to_native(e), cmd=self._clean_args(args))
except Exception as e:
self.log("Error Executing CMD:%s Exception:%s" % (self._clean_args(args), to_native(traceback.format_exc())))
# ZUUL: store fail_json_kwargs and fail later in finally
fail_json_kwargs = dict(rc=257, stdout=b'', stderr=b'', msg=to_native(e), exception=traceback.format_exc(), cmd=self._clean_args(args))
finally:
with Console(zuul_log_id) as console:
if t and t.is_alive():
console.addLine("[Zuul] standard output/error still open "
"after child exited")
if fail_json_kwargs:
# we hit an exception and need to use the rc from
# fail_json_kwargs
rc = fail_json_kwargs['rc']
console.addLine("[Zuul] Task exit code: %s\n" % rc)
if fail_json_kwargs:
self.fail_json(**fail_json_kwargs)
if rc != 0 and check_rc:
msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values)
self.fail_json(cmd=self._clean_args(args), rc=rc, stdout=stdout, stderr=stderr, msg=msg)
if encoding is not None:
return (rc, to_native(stdout, encoding=encoding, errors=errors),
to_native(stderr, encoding=encoding, errors=errors))
return (rc, stdout, stderr)
def check_command(module, commandline):
arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group',
'ln': 'state=link', 'mkdir': 'state=directory',
'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'}
commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri',
'svn': 'subversion', 'service': 'service',
'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt',
'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template',
'dnf': 'dnf', 'zypper': 'zypper'}
become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun', 'machinectl']
if isinstance(commandline, list):
command = commandline[0]
else:
command = commandline.split()[0]
command = os.path.basename(command)
disable_suffix = "If you need to use '{cmd}' because the {mod} module is insufficient you can add" \
" 'warn: false' to this command task or set 'command_warnings=False' in" \
" the defaults section of ansible.cfg to get rid of this message."
substitutions = {'mod': None, 'cmd': command}
if command in arguments:
msg = "Consider using the {mod} module with {subcmd} rather than running '{cmd}'. " + disable_suffix
substitutions['mod'] = 'file'
substitutions['subcmd'] = arguments[command]
module.warn(msg.format(**substitutions))
if command in commands:
msg = "Consider using the {mod} module rather than running '{cmd}'. " + disable_suffix
substitutions['mod'] = commands[command]
module.warn(msg.format(**substitutions))
if command in become:
module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,))
def main():
# the command module is the one ansible module that does not take key=value args
# hence don't copy this one if you are looking to build others!
# NOTE: ensure splitter.py is kept in sync for exceptions
module = AnsibleModule(
argument_spec=dict(
_raw_params=dict(),
_uses_shell=dict(type='bool', default=False),
argv=dict(type='list', elements='str'),
chdir=dict(type='path'),
executable=dict(),
creates=dict(type='path'),
removes=dict(type='path'),
# The default for this really comes from the action plugin
warn=dict(type='bool', default=False, removed_in_version='2.14', removed_from_collection='ansible.builtin'),
stdin=dict(required=False),
stdin_add_newline=dict(type='bool', default=True),
strip_empty_ends=dict(type='bool', default=True),
zuul_log_id=dict(type='str'),
zuul_ansible_split_streams=dict(type='bool'),
),
supports_check_mode=True,
)
shell = module.params['_uses_shell']
chdir = module.params['chdir']
executable = module.params['executable']
args = module.params['_raw_params']
argv = module.params['argv']
creates = module.params['creates']
removes = module.params['removes']
warn = module.params['warn']
stdin = module.params['stdin']
stdin_add_newline = module.params['stdin_add_newline']
strip = module.params['strip_empty_ends']
zuul_log_id = module.params['zuul_log_id']
zuul_ansible_split_streams = module.params["zuul_ansible_split_streams"]
# we promissed these in 'always' ( _lines get autoaded on action plugin)
r = {'changed': False, 'stdout': '', 'stderr': '', 'rc': None, 'cmd': None, 'start': None, 'end': None, 'delta': None, 'msg': ''}
if not shell and executable:
module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable)
executable = None
if not zuul_log_id:
module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params)
if (not args or args.strip() == '') and not argv:
r['rc'] = 256
r['msg'] = "no command given"
module.fail_json(**r)
if args and argv:
r['rc'] = 256
r['msg'] = "only command or argv can be given, not both"
module.fail_json(**r)
if not shell and args:
args = shlex.split(args)
args = args or argv
# All args must be strings
if is_iterable(args, include_strings=False):
args = [to_native(arg, errors='surrogate_or_strict', nonstring='simplerepr') for arg in args]
r['cmd'] = args
if warn:
# nany telling you to use module instead!
check_command(module, args)
if chdir:
chdir = to_bytes(chdir, errors='surrogate_or_strict')
try:
os.chdir(chdir)
except (IOError, OSError) as e:
r['msg'] = 'Unable to change directory before execution: %s' % to_text(e)
module.fail_json(**r)
# check_mode partial support, since it only really works in checking creates/removes
if module.check_mode:
shoulda = "Would"
else:
shoulda = "Did"
# special skips for idempotence if file exists (assumes command creates)
if creates:
if glob.glob(creates):
r['msg'] = "%s not run command since '%s' exists" % (shoulda, creates)
r['stdout'] = "skipped, since %s exists" % creates # TODO: deprecate
r['rc'] = 0
# special skips for idempotence if file does not exist (assumes command removes)
if not r['msg'] and removes:
if not glob.glob(removes):
r['msg'] = "%s not run command since '%s' does not exist" % (shoulda, removes)
r['stdout'] = "skipped, since %s does not exist" % removes # TODO: deprecate
r['rc'] = 0
if r['msg']:
module.exit_json(**r)
# actually executes command (or not ...)
if not module.check_mode:
r['start'] = datetime.datetime.now()
r['rc'], r['stdout'], r['stderr'] = zuul_run_command(module, args, zuul_log_id, zuul_ansible_split_streams, executable=executable, use_unsafe_shell=shell, encoding=None,
data=stdin, binary_data=(not stdin_add_newline))
r['end'] = datetime.datetime.now()
else:
# this is partial check_mode support, since we end up skipping if we get here
r['rc'] = 0
r['msg'] = "Command would have run if not in check mode"
r['skipped'] = True
r['changed'] = True
r['zuul_log_id'] = zuul_log_id
# convert to text for jsonization and usability
if r['start'] is not None and r['end'] is not None:
# these are datetime objects, but need them as strings to pass back
r['delta'] = to_text(r['end'] - r['start'])
r['end'] = to_text(r['end'])
r['start'] = to_text(r['start'])
if strip:
r['stdout'] = to_text(r['stdout']).rstrip("\r\n")
r['stderr'] = to_text(r['stderr']).rstrip("\r\n")
if r['rc'] != 0:
r['msg'] = 'non-zero return code'
module.fail_json(**r)
module.exit_json(**r)
if __name__ == '__main__':
main()

105
zuul/ansible/base/library/command.py Executable file → Normal file
View File

@ -24,23 +24,23 @@ module: command
short_description: Execute commands on targets
version_added: historical
description:
- The C(command) module takes the command name followed by a list of space-delimited arguments.
- The C(ansible.builtin.command) module takes the command name followed by a list of space-delimited arguments.
- The given command will be executed on all selected nodes.
- The command(s) will not be
processed through the shell, so variables like