Merge "Add new tox-remote job"

.zuul.yaml

@@ -41,6 +41,16 @@
       node_version: 8
       npm_command: build
 
+- job:
+    name: zuul-tox-remote
+    parent: tox
+    vars:
+      tox_envlist: remote
+      tox_environment:
+        ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
+        ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
+        ZUUL_REMOTE_KEEP: "true"
+
 - project:
     check:
       jobs:
@@ -75,6 +85,7 @@
               - yarn.lock
               - web/.*
         - zuul-stream-functional
+        - zuul-tox-remote
         - nodepool-zuul-functional:
             voting: false
     gate:

tests/base.py

@@ -19,6 +19,7 @@ import asyncio
 import configparser
 from contextlib import contextmanager
 import datetime
+import errno
 import gc
 import hashlib
 from io import StringIO
@@ -54,6 +55,7 @@ import testtools.content
 import testtools.content_type
 from git.exc import NoSuchPathError
 import yaml
+import paramiko
 
 import tests.fakegithub
 import zuul.driver.gerrit.gerritsource as gerritsource
@@ -1352,10 +1354,11 @@ class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
             if not host['host_vars'].get('ansible_connection'):
                 host['host_vars']['ansible_connection'] = 'local'
 
-        hosts.append(dict(
-            name='localhost',
-            host_vars=dict(ansible_connection='local'),
-            host_keys=[]))
+        if not hosts:
+            hosts.append(dict(
+                name='localhost',
+                host_vars=dict(ansible_connection='local'),
+                host_keys=[]))
         return hosts
 
 
@@ -1567,6 +1570,7 @@ class FakeNodepool(object):
     log = logging.getLogger("zuul.test.FakeNodepool")
 
     def __init__(self, host, port, chroot):
+        self.host_keys = None
         self.client = kazoo.client.KazooClient(
             hosts='%s:%s%s' % (host, port, chroot))
         self.client.start()
@@ -1576,6 +1580,7 @@ class FakeNodepool(object):
         self.thread.daemon = True
         self.thread.start()
         self.fail_requests = set()
+        self.remote_ansible = False
 
     def stop(self):
         self._running = False
@@ -1639,13 +1644,17 @@ class FakeNodepool(object):
     def makeNode(self, request_id, node_type):
         now = time.time()
         path = '/nodepool/nodes/'
+        remote_ip = os.environ.get('ZUUL_REMOTE_IPV4', '127.0.0.1')
+        if self.remote_ansible and not self.host_keys:
+            self.host_keys = self.keyscan(remote_ip)
+        host_keys = self.host_keys or ["fake-key1", "fake-key2"]
         data = dict(type=node_type,
                     cloud='test-cloud',
                     provider='test-provider',
                     region='test-region',
                     az='test-az',
-                    interface_ip='127.0.0.1',
-                    public_ipv4='127.0.0.1',
+                    interface_ip=remote_ip,
+                    public_ipv4=remote_ip,
                     private_ipv4=None,
                     public_ipv6=None,
                     allocated_to=request_id,
@@ -1654,8 +1663,10 @@ class FakeNodepool(object):
                     created_time=now,
                     updated_time=now,
                     image_id=None,
-                    host_keys=["fake-key1", "fake-key2"],
+                    host_keys=host_keys,
                     executor='fake-nodepool')
+        if self.remote_ansible:
+            data['connection_type'] = 'ssh'
         if 'fakeuser' in node_type:
             data['username'] = 'fakeuser'
         if 'windows' in node_type:
@@ -1703,6 +1714,55 @@ class FakeNodepool(object):
         except kazoo.exceptions.NoNodeError:
             self.log.debug("Node request %s %s disappeared" % (oid, data))
 
+    def keyscan(self, ip, port=22, timeout=60):
+        '''
+        Scan the IP address for public SSH keys.
+
+        Keys are returned formatted as: "<type> <base64_string>"
+        '''
+        addrinfo = socket.getaddrinfo(ip, port)[0]
+        family = addrinfo[0]
+        sockaddr = addrinfo[4]
+
+        keys = []
+        key = None
+        for count in iterate_timeout(timeout, "ssh access"):
+            sock = None
+            t = None
+            try:
+                sock = socket.socket(family, socket.SOCK_STREAM)
+                sock.settimeout(timeout)
+                sock.connect(sockaddr)
+                t = paramiko.transport.Transport(sock)
+                t.start_client(timeout=timeout)
+                key = t.get_remote_server_key()
+                break
+            except socket.error as e:
+                if e.errno not in [
+                        errno.ECONNREFUSED, errno.EHOSTUNREACH, None]:
+                    self.log.exception(
+                        'Exception with ssh access to %s:' % ip)
+            except Exception as e:
+                self.log.exception("ssh-keyscan failure: %s", e)
+            finally:
+                try:
+                    if t:
+                        t.close()
+                except Exception as e:
+                    self.log.exception('Exception closing paramiko: %s', e)
+                try:
+                    if sock:
+                        sock.close()
+                except Exception as e:
+                    self.log.exception('Exception closing socket: %s', e)
+
+        # Paramiko, at this time, seems to return only the ssh-rsa key, so
+        # only the single key is placed into the list.
+        if key:
+            keys.append("%s %s" % (key.get_name(), key.get_base64()))
+
+        return keys
+
 
 class ChrootedKazooFixture(fixtures.Fixture):
     def __init__(self, test_id):
@@ -2042,7 +2102,9 @@ class ZuulTestCase(BaseTestCase):
         self.setup_config()
         self.private_key_file = os.path.join(self.test_root, 'test_id_rsa')
         if not os.path.exists(self.private_key_file):
-            src_private_key_file = os.path.join(FIXTURE_DIR, 'test_id_rsa')
+            src_private_key_file = os.environ.get(
+                'ZUUL_SSH_KEY',
+                os.path.join(FIXTURE_DIR, 'test_id_rsa'))
             shutil.copy(src_private_key_file, self.private_key_file)
             shutil.copy('{}.pub'.format(src_private_key_file),
                         '{}.pub'.format(self.private_key_file))
@@ -3015,6 +3077,10 @@ class AnsibleZuulTestCase(ZuulTestCase):
                                 'work', 'logs', 'job-output.txt')
             with open(path) as f:
                 self.log.debug(f.read())
+            path = os.path.join(self.test_root, build.uuid,
+                                'work', 'logs', 'job-output.json')
+            with open(path) as f:
+                self.log.debug(f.read())
             raise
 
 

tests/fixtures/bwrap-mounts/file

@@ -0,0 +1 @@
+file

tests/fixtures/config/remote-action-modules/git/common-config/zuul.yaml

@@ -0,0 +1,17 @@
+- pipeline:
+    name: check
+    manager: independent
+    post-review: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null

tests/fixtures/config/remote-action-modules/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-bad.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: copy-test
+      src_file: /opt/file

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-good.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: copy-test
+      src_file: file

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/roles/copy-test/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Create a destination directory for copied files
+  tempfile:
+    state: directory
+  register: destdir
+
+- name: Copy
+  copy:
+    src: "{{src_file}}"
+    dest: "{{destdir.path}}/copy"

tests/fixtures/config/remote-action-modules/git/org_project/zuul.yaml

@@ -0,0 +1,4 @@
+- project:
+    check:
+      jobs:
+        - noop

tests/fixtures/config/remote-action-modules/main.yaml

@@ -0,0 +1,8 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project

tests/remote/test_remote_action_modules.py

@@ -0,0 +1,73 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import textwrap
+
+from tests.base import AnsibleZuulTestCase, FIXTURE_DIR
+
+
+class TestActionModules(AnsibleZuulTestCase):
+    tenant_config_file = 'config/remote-action-modules/main.yaml'
+
+    def setUp(self):
+        super(TestActionModules, self).setUp()
+        self.fake_nodepool.remote_ansible = True
+
+        ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4')
+        self.assertIsNotNone(ansible_remote)
+
+        # inject some files as forbidden sources
+        fixture_dir = os.path.join(FIXTURE_DIR, 'bwrap-mounts')
+        self.executor_server.execution_wrapper.bwrap_command.extend(
+            ['--ro-bind', fixture_dir, '/opt'])
+
+    def _run_job(self, job_name, result):
+        # Keep the jobdir around so we can inspect contents if an
+        # assert fails. It will be cleaned up anyway as it is contained
+        # in a tmp dir which gets cleaned up after the test.
+        self.executor_server.keep_jobdir = True
+
+        # Output extra ansible info so we might see errors.
+        self.executor_server.verbose = True
+        conf = textwrap.dedent(
+            """
+            - job:
+                name: {job_name}
+                run: playbooks/{job_name}.yaml
+                nodeset:
+                  nodes:
+                    - name: controller
+                      label: whatever
+
+            - project:
+                check:
+                  jobs:
+                    - {job_name}
+            """.format(job_name=job_name))
+
+        file_dict = {'zuul.yaml': conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        job = self.getJobFromHistory(job_name)
+        with self.jobLog(job):
+            build = self.history[-1]
+            self.assertEqual(build.result, result)
+
+    def test_copy_module(self):
+        self._run_job('copy-good', 'SUCCESS')
+        self._run_job('copy-bad', 'FAILURE')

tox.ini

@@ -47,6 +47,12 @@ setenv =
    OS_TEST_PATH = ./tests/nodepool
 commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
 
+[testenv:remote]
+setenv =
+   OS_TEST_PATH = ./tests/remote
+passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS ZUUL_REMOTE_IPV4 ZUUL_SSH_KEY NODEPOOL_ZK_HOST
+commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
+
 [flake8]
 # These are ignored intentionally in openstack-infra projects;
 # please don't submit patches that solely correct them or enable them.