Merge "Add new tox-remote job"
This commit is contained in:
commit
e2cda1ac3c
11
.zuul.yaml
11
.zuul.yaml
|
@ -41,6 +41,16 @@
|
|||
node_version: 8
|
||||
npm_command: build
|
||||
|
||||
- job:
|
||||
name: zuul-tox-remote
|
||||
parent: tox
|
||||
vars:
|
||||
tox_envlist: remote
|
||||
tox_environment:
|
||||
ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
|
||||
ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
|
||||
ZUUL_REMOTE_KEEP: "true"
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
|
@ -75,6 +85,7 @@
|
|||
- yarn.lock
|
||||
- web/.*
|
||||
- zuul-stream-functional
|
||||
- zuul-tox-remote
|
||||
- nodepool-zuul-functional:
|
||||
voting: false
|
||||
gate:
|
||||
|
|
|
@ -19,6 +19,7 @@ import asyncio
|
|||
import configparser
|
||||
from contextlib import contextmanager
|
||||
import datetime
|
||||
import errno
|
||||
import gc
|
||||
import hashlib
|
||||
from io import StringIO
|
||||
|
@ -54,6 +55,7 @@ import testtools.content
|
|||
import testtools.content_type
|
||||
from git.exc import NoSuchPathError
|
||||
import yaml
|
||||
import paramiko
|
||||
|
||||
import tests.fakegithub
|
||||
import zuul.driver.gerrit.gerritsource as gerritsource
|
||||
|
@ -1352,10 +1354,11 @@ class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
|
|||
if not host['host_vars'].get('ansible_connection'):
|
||||
host['host_vars']['ansible_connection'] = 'local'
|
||||
|
||||
hosts.append(dict(
|
||||
name='localhost',
|
||||
host_vars=dict(ansible_connection='local'),
|
||||
host_keys=[]))
|
||||
if not hosts:
|
||||
hosts.append(dict(
|
||||
name='localhost',
|
||||
host_vars=dict(ansible_connection='local'),
|
||||
host_keys=[]))
|
||||
return hosts
|
||||
|
||||
|
||||
|
@ -1567,6 +1570,7 @@ class FakeNodepool(object):
|
|||
log = logging.getLogger("zuul.test.FakeNodepool")
|
||||
|
||||
def __init__(self, host, port, chroot):
|
||||
self.host_keys = None
|
||||
self.client = kazoo.client.KazooClient(
|
||||
hosts='%s:%s%s' % (host, port, chroot))
|
||||
self.client.start()
|
||||
|
@ -1576,6 +1580,7 @@ class FakeNodepool(object):
|
|||
self.thread.daemon = True
|
||||
self.thread.start()
|
||||
self.fail_requests = set()
|
||||
self.remote_ansible = False
|
||||
|
||||
def stop(self):
|
||||
self._running = False
|
||||
|
@ -1639,13 +1644,17 @@ class FakeNodepool(object):
|
|||
def makeNode(self, request_id, node_type):
|
||||
now = time.time()
|
||||
path = '/nodepool/nodes/'
|
||||
remote_ip = os.environ.get('ZUUL_REMOTE_IPV4', '127.0.0.1')
|
||||
if self.remote_ansible and not self.host_keys:
|
||||
self.host_keys = self.keyscan(remote_ip)
|
||||
host_keys = self.host_keys or ["fake-key1", "fake-key2"]
|
||||
data = dict(type=node_type,
|
||||
cloud='test-cloud',
|
||||
provider='test-provider',
|
||||
region='test-region',
|
||||
az='test-az',
|
||||
interface_ip='127.0.0.1',
|
||||
public_ipv4='127.0.0.1',
|
||||
interface_ip=remote_ip,
|
||||
public_ipv4=remote_ip,
|
||||
private_ipv4=None,
|
||||
public_ipv6=None,
|
||||
allocated_to=request_id,
|
||||
|
@ -1654,8 +1663,10 @@ class FakeNodepool(object):
|
|||
created_time=now,
|
||||
updated_time=now,
|
||||
image_id=None,
|
||||
host_keys=["fake-key1", "fake-key2"],
|
||||
host_keys=host_keys,
|
||||
executor='fake-nodepool')
|
||||
if self.remote_ansible:
|
||||
data['connection_type'] = 'ssh'
|
||||
if 'fakeuser' in node_type:
|
||||
data['username'] = 'fakeuser'
|
||||
if 'windows' in node_type:
|
||||
|
@ -1703,6 +1714,55 @@ class FakeNodepool(object):
|
|||
except kazoo.exceptions.NoNodeError:
|
||||
self.log.debug("Node request %s %s disappeared" % (oid, data))
|
||||
|
||||
def keyscan(self, ip, port=22, timeout=60):
|
||||
'''
|
||||
Scan the IP address for public SSH keys.
|
||||
|
||||
Keys are returned formatted as: "<type> <base64_string>"
|
||||
'''
|
||||
addrinfo = socket.getaddrinfo(ip, port)[0]
|
||||
family = addrinfo[0]
|
||||
sockaddr = addrinfo[4]
|
||||
|
||||
keys = []
|
||||
key = None
|
||||
for count in iterate_timeout(timeout, "ssh access"):
|
||||
sock = None
|
||||
t = None
|
||||
try:
|
||||
sock = socket.socket(family, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
sock.connect(sockaddr)
|
||||
t = paramiko.transport.Transport(sock)
|
||||
t.start_client(timeout=timeout)
|
||||
key = t.get_remote_server_key()
|
||||
break
|
||||
except socket.error as e:
|
||||
if e.errno not in [
|
||||
errno.ECONNREFUSED, errno.EHOSTUNREACH, None]:
|
||||
self.log.exception(
|
||||
'Exception with ssh access to %s:' % ip)
|
||||
except Exception as e:
|
||||
self.log.exception("ssh-keyscan failure: %s", e)
|
||||
finally:
|
||||
try:
|
||||
if t:
|
||||
t.close()
|
||||
except Exception as e:
|
||||
self.log.exception('Exception closing paramiko: %s', e)
|
||||
try:
|
||||
if sock:
|
||||
sock.close()
|
||||
except Exception as e:
|
||||
self.log.exception('Exception closing socket: %s', e)
|
||||
|
||||
# Paramiko, at this time, seems to return only the ssh-rsa key, so
|
||||
# only the single key is placed into the list.
|
||||
if key:
|
||||
keys.append("%s %s" % (key.get_name(), key.get_base64()))
|
||||
|
||||
return keys
|
||||
|
||||
|
||||
class ChrootedKazooFixture(fixtures.Fixture):
|
||||
def __init__(self, test_id):
|
||||
|
@ -2042,7 +2102,9 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.setup_config()
|
||||
self.private_key_file = os.path.join(self.test_root, 'test_id_rsa')
|
||||
if not os.path.exists(self.private_key_file):
|
||||
src_private_key_file = os.path.join(FIXTURE_DIR, 'test_id_rsa')
|
||||
src_private_key_file = os.environ.get(
|
||||
'ZUUL_SSH_KEY',
|
||||
os.path.join(FIXTURE_DIR, 'test_id_rsa'))
|
||||
shutil.copy(src_private_key_file, self.private_key_file)
|
||||
shutil.copy('{}.pub'.format(src_private_key_file),
|
||||
'{}.pub'.format(self.private_key_file))
|
||||
|
@ -3015,6 +3077,10 @@ class AnsibleZuulTestCase(ZuulTestCase):
|
|||
'work', 'logs', 'job-output.txt')
|
||||
with open(path) as f:
|
||||
self.log.debug(f.read())
|
||||
path = os.path.join(self.test_root, build.uuid,
|
||||
'work', 'logs', 'job-output.json')
|
||||
with open(path) as f:
|
||||
self.log.debug(f.read())
|
||||
raise
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
file
|
|
@ -0,0 +1,17 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
post-review: true
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
|
@ -0,0 +1 @@
|
|||
test
|
4
tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-bad.yaml
vendored
Normal file
4
tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-bad.yaml
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
- hosts: all
|
||||
roles:
|
||||
- role: copy-test
|
||||
src_file: /opt/file
|
4
tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-good.yaml
vendored
Normal file
4
tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-good.yaml
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
- hosts: all
|
||||
roles:
|
||||
- role: copy-test
|
||||
src_file: file
|
|
@ -0,0 +1,9 @@
|
|||
- name: Create a destination directory for copied files
|
||||
tempfile:
|
||||
state: directory
|
||||
register: destdir
|
||||
|
||||
- name: Copy
|
||||
copy:
|
||||
src: "{{src_file}}"
|
||||
dest: "{{destdir.path}}/copy"
|
|
@ -0,0 +1,4 @@
|
|||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- noop
|
|
@ -0,0 +1,8 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- common-config
|
||||
untrusted-projects:
|
||||
- org/project
|
|
@ -0,0 +1,73 @@
|
|||
# Copyright 2018 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import textwrap
|
||||
|
||||
from tests.base import AnsibleZuulTestCase, FIXTURE_DIR
|
||||
|
||||
|
||||
class TestActionModules(AnsibleZuulTestCase):
|
||||
tenant_config_file = 'config/remote-action-modules/main.yaml'
|
||||
|
||||
def setUp(self):
|
||||
super(TestActionModules, self).setUp()
|
||||
self.fake_nodepool.remote_ansible = True
|
||||
|
||||
ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4')
|
||||
self.assertIsNotNone(ansible_remote)
|
||||
|
||||
# inject some files as forbidden sources
|
||||
fixture_dir = os.path.join(FIXTURE_DIR, 'bwrap-mounts')
|
||||
self.executor_server.execution_wrapper.bwrap_command.extend(
|
||||
['--ro-bind', fixture_dir, '/opt'])
|
||||
|
||||
def _run_job(self, job_name, result):
|
||||
# Keep the jobdir around so we can inspect contents if an
|
||||
# assert fails. It will be cleaned up anyway as it is contained
|
||||
# in a tmp dir which gets cleaned up after the test.
|
||||
self.executor_server.keep_jobdir = True
|
||||
|
||||
# Output extra ansible info so we might see errors.
|
||||
self.executor_server.verbose = True
|
||||
conf = textwrap.dedent(
|
||||
"""
|
||||
- job:
|
||||
name: {job_name}
|
||||
run: playbooks/{job_name}.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: controller
|
||||
label: whatever
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- {job_name}
|
||||
""".format(job_name=job_name))
|
||||
|
||||
file_dict = {'zuul.yaml': conf}
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
|
||||
files=file_dict)
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
job = self.getJobFromHistory(job_name)
|
||||
with self.jobLog(job):
|
||||
build = self.history[-1]
|
||||
self.assertEqual(build.result, result)
|
||||
|
||||
def test_copy_module(self):
|
||||
self._run_job('copy-good', 'SUCCESS')
|
||||
self._run_job('copy-bad', 'FAILURE')
|
6
tox.ini
6
tox.ini
|
@ -47,6 +47,12 @@ setenv =
|
|||
OS_TEST_PATH = ./tests/nodepool
|
||||
commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
|
||||
|
||||
[testenv:remote]
|
||||
setenv =
|
||||
OS_TEST_PATH = ./tests/remote
|
||||
passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS ZUUL_REMOTE_IPV4 ZUUL_SSH_KEY NODEPOOL_ZK_HOST
|
||||
commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
|
||||
|
||||
[flake8]
|
||||
# These are ignored intentionally in openstack-infra projects;
|
||||
# please don't submit patches that solely correct them or enable them.
|
||||
|
|
Loading…
Reference in New Issue