Add new tox-remote job

We need tests that really use ansible against a remote node. Our
current ansible tests are not sufficient for this goal as they run
against localhost. For being able to test restrictions in untrusted
jobs we need tests that run ansible via ssh against a remote node.

This adds a new tox-remote job and a new class of tests that run via
ssh against the interface ip of the test node.

Co-Authored-By: James E. Blair <jeblair@redhat.com>
Change-Id: Iacf670d992bb051560a0c46c313beaa6721489c4

.zuul.yaml

@@ -41,6 +41,16 @@
       node_version: 8
       npm_command: build
 
+- job:
+    name: zuul-tox-remote
+    parent: tox
+    vars:
+      tox_envlist: remote
+      tox_environment:
+        ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
+        ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
+        ZUUL_REMOTE_KEEP: "true"
+
 - project:
     check:
       jobs:
@@ -75,6 +85,7 @@
               - yarn.lock
               - web/.*
         - zuul-stream-functional
+        - zuul-tox-remote
         - nodepool-zuul-functional:
             voting: false
     gate:

tests/base.py

@@ -19,6 +19,7 @@ import asyncio
 import configparser
 from contextlib import contextmanager
 import datetime
+import errno
 import gc
 import hashlib
 from io import StringIO
@@ -54,6 +55,7 @@ import testtools.content
 import testtools.content_type
 from git.exc import NoSuchPathError
 import yaml
+import paramiko
 
 import tests.fakegithub
 import zuul.driver.gerrit.gerritsource as gerritsource
@@ -1352,10 +1354,11 @@ class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
             if not host['host_vars'].get('ansible_connection'):
                 host['host_vars']['ansible_connection'] = 'local'
 
-        hosts.append(dict(
-            name='localhost',
-            host_vars=dict(ansible_connection='local'),
-            host_keys=[]))
+        if not hosts:
+            hosts.append(dict(
+                name='localhost',
+                host_vars=dict(ansible_connection='local'),
+                host_keys=[]))
         return hosts
 
 
@@ -1567,6 +1570,7 @@ class FakeNodepool(object):
     log = logging.getLogger("zuul.test.FakeNodepool")
 
     def __init__(self, host, port, chroot):
+        self.host_keys = None
         self.client = kazoo.client.KazooClient(
             hosts='%s:%s%s' % (host, port, chroot))
         self.client.start()
@@ -1576,6 +1580,7 @@ class FakeNodepool(object):
         self.thread.daemon = True
         self.thread.start()
         self.fail_requests = set()
+        self.remote_ansible = False
 
     def stop(self):
         self._running = False
@@ -1639,13 +1644,17 @@ class FakeNodepool(object):
     def makeNode(self, request_id, node_type):
         now = time.time()
         path = '/nodepool/nodes/'
+        remote_ip = os.environ.get('ZUUL_REMOTE_IPV4', '127.0.0.1')
+        if self.remote_ansible and not self.host_keys:
+            self.host_keys = self.keyscan(remote_ip)
+        host_keys = self.host_keys or ["fake-key1", "fake-key2"]
         data = dict(type=node_type,
                     cloud='test-cloud',
                     provider='test-provider',
                     region='test-region',
                     az='test-az',
-                    interface_ip='127.0.0.1',
-                    public_ipv4='127.0.0.1',
+                    interface_ip=remote_ip,
+                    public_ipv4=remote_ip,
                     private_ipv4=None,
                     public_ipv6=None,
                     allocated_to=request_id,
@@ -1654,8 +1663,10 @@ class FakeNodepool(object):
                     created_time=now,
                     updated_time=now,
                     image_id=None,
-                    host_keys=["fake-key1", "fake-key2"],
+                    host_keys=host_keys,
                     executor='fake-nodepool')
+        if self.remote_ansible:
+            data['connection_type'] = 'ssh'
         if 'fakeuser' in node_type:
             data['username'] = 'fakeuser'
         if 'windows' in node_type:
@@ -1703,6 +1714,55 @@ class FakeNodepool(object):
         except kazoo.exceptions.NoNodeError:
             self.log.debug("Node request %s %s disappeared" % (oid, data))
 
+    def keyscan(self, ip, port=22, timeout=60):
+        '''
+        Scan the IP address for public SSH keys.
+
+        Keys are returned formatted as: "<type> <base64_string>"
+        '''
+        addrinfo = socket.getaddrinfo(ip, port)[0]
+        family = addrinfo[0]
+        sockaddr = addrinfo[4]
+
+        keys = []
+        key = None
+        for count in iterate_timeout(timeout, "ssh access"):
+            sock = None
+            t = None
+            try:
+                sock = socket.socket(family, socket.SOCK_STREAM)
+                sock.settimeout(timeout)
+                sock.connect(sockaddr)
+                t = paramiko.transport.Transport(sock)
+                t.start_client(timeout=timeout)
+                key = t.get_remote_server_key()
+                break
+            except socket.error as e:
+                if e.errno not in [
+                        errno.ECONNREFUSED, errno.EHOSTUNREACH, None]:
+                    self.log.exception(
+                        'Exception with ssh access to %s:' % ip)
+            except Exception as e:
+                self.log.exception("ssh-keyscan failure: %s", e)
+            finally:
+                try:
+                    if t:
+                        t.close()
+                except Exception as e:
+                    self.log.exception('Exception closing paramiko: %s', e)
+                try:
+                    if sock:
+                        sock.close()
+                except Exception as e:
+                    self.log.exception('Exception closing socket: %s', e)
+
+        # Paramiko, at this time, seems to return only the ssh-rsa key, so
+        # only the single key is placed into the list.
+        if key:
+            keys.append("%s %s" % (key.get_name(), key.get_base64()))
+
+        return keys
+
 
 class ChrootedKazooFixture(fixtures.Fixture):
     def __init__(self, test_id):
@@ -2042,7 +2102,9 @@ class ZuulTestCase(BaseTestCase):
         self.setup_config()
         self.private_key_file = os.path.join(self.test_root, 'test_id_rsa')
         if not os.path.exists(self.private_key_file):
-            src_private_key_file = os.path.join(FIXTURE_DIR, 'test_id_rsa')
+            src_private_key_file = os.environ.get(
+                'ZUUL_SSH_KEY',
+                os.path.join(FIXTURE_DIR, 'test_id_rsa'))
             shutil.copy(src_private_key_file, self.private_key_file)
             shutil.copy('{}.pub'.format(src_private_key_file),
                         '{}.pub'.format(self.private_key_file))
@@ -3015,6 +3077,10 @@ class AnsibleZuulTestCase(ZuulTestCase):
                                 'work', 'logs', 'job-output.txt')
             with open(path) as f:
                 self.log.debug(f.read())
+            path = os.path.join(self.test_root, build.uuid,
+                                'work', 'logs', 'job-output.json')
+            with open(path) as f:
+                self.log.debug(f.read())
             raise
 
 

tests/fixtures/bwrap-mounts/file

@@ -0,0 +1 @@
+file

tests/fixtures/config/remote-action-modules/git/common-config/zuul.yaml

@@ -0,0 +1,17 @@
+- pipeline:
+    name: check
+    manager: independent
+    post-review: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null

tests/fixtures/config/remote-action-modules/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-bad.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: copy-test
+      src_file: /opt/file

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/copy-good.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: copy-test
+      src_file: file

tests/fixtures/config/remote-action-modules/git/org_project/playbooks/roles/copy-test/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Create a destination directory for copied files
+  tempfile:
+    state: directory
+  register: destdir
+
+- name: Copy
+  copy:
+    src: "{{src_file}}"
+    dest: "{{destdir.path}}/copy"

tests/fixtures/config/remote-action-modules/git/org_project/zuul.yaml

@@ -0,0 +1,4 @@
+- project:
+    check:
+      jobs:
+        - noop

tests/fixtures/config/remote-action-modules/main.yaml

@@ -0,0 +1,8 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project

tests/remote/test_remote_action_modules.py

@@ -0,0 +1,73 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import textwrap
+
+from tests.base import AnsibleZuulTestCase, FIXTURE_DIR
+
+
+class TestActionModules(AnsibleZuulTestCase):
+    tenant_config_file = 'config/remote-action-modules/main.yaml'
+
+    def setUp(self):
+        super(TestActionModules, self).setUp()
+        self.fake_nodepool.remote_ansible = True
+
+        ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4')
+        self.assertIsNotNone(ansible_remote)
+
+        # inject some files as forbidden sources
+        fixture_dir = os.path.join(FIXTURE_DIR, 'bwrap-mounts')
+        self.executor_server.execution_wrapper.bwrap_command.extend(
+            ['--ro-bind', fixture_dir, '/opt'])
+
+    def _run_job(self, job_name, result):
+        # Keep the jobdir around so we can inspect contents if an
+        # assert fails. It will be cleaned up anyway as it is contained
+        # in a tmp dir which gets cleaned up after the test.
+        self.executor_server.keep_jobdir = True
+
+        # Output extra ansible info so we might see errors.
+        self.executor_server.verbose = True
+        conf = textwrap.dedent(
+            """
+            - job:
+                name: {job_name}
+                run: playbooks/{job_name}.yaml
+                nodeset:
+                  nodes:
+                    - name: controller
+                      label: whatever
+
+            - project:
+                check:
+                  jobs:
+                    - {job_name}
+            """.format(job_name=job_name))
+
+        file_dict = {'zuul.yaml': conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        job = self.getJobFromHistory(job_name)
+        with self.jobLog(job):
+            build = self.history[-1]
+            self.assertEqual(build.result, result)
+
+    def test_copy_module(self):
+        self._run_job('copy-good', 'SUCCESS')
+        self._run_job('copy-bad', 'FAILURE')

tox.ini

@@ -47,6 +47,12 @@ setenv =
    OS_TEST_PATH = ./tests/nodepool
 commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
 
+[testenv:remote]
+setenv =
+   OS_TEST_PATH = ./tests/remote
+passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS ZUUL_REMOTE_IPV4 ZUUL_SSH_KEY NODEPOOL_ZK_HOST
+commands = python setup.py test --slowest --testr-args='--concurrency=1 {posargs}'
+
 [flake8]
 # These are ignored intentionally in openstack-infra projects;
 # please don't submit patches that solely correct them or enable them.