x/tobiko
Code Issues Proposed changes

Implement HTTP over SSH tunneling.

Browse Source 
Change-Id: I614b957bb23f332d45d3795eaa46dc8344376a53
This commit is contained in:
Federico Ressi 2019-10-24 07:31:03 +02:00
parent 5225b13540
commit b9173016eb
7 changed files with 189 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tobiko/common/_detail.py
View File

@ -17,6 +17,7 @@ import json
import itertools
from oslo_log import log
import six
import testtools
from testtools import content
import yaml
@ -35,13 +36,16 @@ def gather_details(source_dict, target_dict):
:param target_dict: A dictionary into which details will be gathered.
"""
for name, content_object in source_dict.items():
content_id = get_details_content_id(content_object)
new_name = get_unique_detail_name(name=name,
content_id=content_id,
target_dict=target_dict)
if new_name not in target_dict:
target_dict[new_name] = copy_details_content(
content_object=content_object, content_id=content_id)
try:
content_id = get_details_content_id(content_object)
new_name = get_unique_detail_name(name=name,
content_id=content_id,
target_dict=target_dict)
if new_name not in target_dict:
target_dict[new_name] = copy_details_content(
content_object=content_object, content_id=content_id)
except Exception:
LOG.exception('Error gathering details')
def get_unique_detail_name(name, content_id, target_dict):
@ -88,8 +92,13 @@ def get_text_to_get_bytes(get_text):
assert callable(get_text)
def get_bytes():
for t in get_text():
yield t.encode(errors='ignore')
text = get_text()
if text:
if isinstance(text, six.string_types):
yield text.encode(errors='ignore')
else:
for t in text:
yield t.encode(errors='ignore')
return get_bytes

5
tobiko/openstack/keystone/_session.py
View File

@ -19,6 +19,7 @@ from oslo_log import log
import tobiko
from tobiko.openstack.keystone import _credentials
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
@ -71,7 +72,9 @@ class KeystoneSessionFixture(tobiko.SharedFixture):
# api version parameter is not accepted
params.pop('api_version', None)
auth = loader.load_from_options(**params)
self.session = session = _session.Session(auth=auth, verify=False)
http_session = ssh.ssh_tunnel_http_session()
self.session = session = _session.Session(
auth=auth, verify=False, session=http_session)
self.credentials = credentials

6
tobiko/openstack/neutron/_extension.py
View File

@ -13,6 +13,8 @@
# under the License.
from __future__ import absolute_import
import collections
import tobiko
from tobiko.openstack.neutron import _client
@ -31,7 +33,9 @@ class NetworkingExtensionsFixture(tobiko.SharedFixture):
self.client = _client.get_neutron_client()
def get_networking_extensions(self):
extensions = self.client.list_extensions()['extensions']
extensions = self.client.list_extensions()
if isinstance(extensions, collections.Mapping):
extensions = extensions['extensions']
self.extensions = frozenset(e['alias'] for e in extensions)

11
tobiko/openstack/nova/_client.py
View File

@ -103,8 +103,15 @@ def get_console_output(server, timeout=None, interval=1., length=None,
client = nova_client(client)
start_time = time.time()
while True:
output = client.servers.get_console_output(server=server,
length=length)
try:
output = client.servers.get_console_output(server=server,
length=length)
except TypeError:
# For some reason it could happen resulting body cannot be
# translated to json object and it is converted to None
# on such case get_console_output would raise a TypeError
return None
if timeout is None or output:
break

4
tobiko/shell/ssh/__init__.py
View File

@ -18,6 +18,8 @@ from __future__ import absolute_import
from tobiko.shell.ssh import _config
from tobiko.shell.ssh import _client
from tobiko.shell.ssh import _command
from tobiko.shell.ssh import _http
SSHHostConfig = _config.SSHHostConfig
ssh_host_config = _config.ssh_host_config
@ -28,3 +30,5 @@ ssh_command = _command.ssh_command
ssh_proxy_client = _client.ssh_proxy_client
SSHConnectFailure = _client.SSHConnectFailure
gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters
ssh_tunnel_http_session = _http.ssh_tunnel_http_session

22
tobiko/shell/ssh/_client.py
View File

@ -21,6 +21,7 @@ import getpass
import os
import socket
import time
import subprocess
import paramiko
from oslo_log import log
@ -403,15 +404,22 @@ def ssh_connect(hostname, username=None, port=None, connection_interval=None,
return client, proxy_sock
def ssh_proxy_sock(hostname, port=None, command=None, client=None):
if client:
# I need a command to execute with proxy client
command = command or 'nc {hostname!r} {port!r}'
elif not command:
# Proxy sock is not required
return None
def ssh_proxy_sock(hostname, port=None, command=None, client=None,
source_address=None):
if not command:
if client:
# I need a command to execute with proxy client
options = []
if source_address:
options += ['-s', str(source_address)]
command = ['nc'] + options + ['{hostname!s}', '{port!s}']
elif not command:
# Proxy sock is not required
return None
# Apply connect parameters to proxy command
if not isinstance(command, six.string_types):
command = subprocess.list2cmdline(command)
command = command.format(hostname=hostname, port=(port or 22))
if client:
if isinstance(client, SSHClientFixture):

134
tobiko/shell/ssh/_http.py Normal file
View File

@ -0,0 +1,134 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import collections
import functools
import requests
from urllib3 import connection
from urllib3 import connectionpool
from urllib3 import poolmanager
from tobiko.shell.ssh import _client
def ssh_tunnel_http_session(ssh_client=None):
ssh_client = ssh_client or _client.ssh_proxy_client()
if ssh_client is None:
return None
session = requests.Session()
mount_ssh_tunnel_http_adapter(session=session, ssh_client=ssh_client)
return session
def mount_ssh_tunnel_http_adapter(session, ssh_client):
adapter = SSHTunnelHttpAdapter(ssh_client=ssh_client)
for scheme in list(session.adapters):
session.mount(scheme, adapter)
class SSHTunnelHttpAdapter(requests.adapters.HTTPAdapter):
"""The custom adapter used to set tunnel HTTP connections over SSH tunnel
"""
def __init__(self, ssh_client, *args, **kwargs):
self.ssh_client = ssh_client
super(SSHTunnelHttpAdapter, self).__init__(*args, **kwargs)
def init_poolmanager(self, connections, maxsize,
block=requests.adapters.DEFAULT_POOLBLOCK,
**pool_kwargs):
# save these values for pickling
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = SSHTunnelPoolManager(
num_pools=connections, maxsize=maxsize, block=block, strict=True,
ssh_client=self.ssh_client, **pool_kwargs)
class SSHTunnelPoolManager(poolmanager.PoolManager):
def __init__(self, *args, **kwargs):
super(SSHTunnelPoolManager, self).__init__(*args, **kwargs)
# Locally set the pool classes and keys so other PoolManagers can
# override them.
self.pool_classes_by_scheme = pool_classes_by_scheme
self.key_fn_by_scheme = key_fn_by_scheme.copy()
# pylint: disable=protected-access
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = poolmanager._key_fields + ('key_ssh_client',)
#: The namedtuple class used to construct keys for the connection pool.
#: All custom key schemes should include the fields in this key at a minimum.
SSHTunnelPoolKey = collections.namedtuple("SSHTunnelPoolKey", _key_fields)
#: A dictionary that maps a scheme to a callable that creates a pool key.
#: This can be used to alter the way pool keys are constructed, if desired.
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(poolmanager._default_key_normalizer,
SSHTunnelPoolKey),
"https": functools.partial(poolmanager._default_key_normalizer,
SSHTunnelPoolKey),
}
# pylint: enable=protected-access
class SSHTunnelHTTPConnection(connection.HTTPConnection):
def __init__(self, *args, **kw):
self.ssh_client = kw.pop('ssh_client')
assert self.ssh_client is not None
super(SSHTunnelHTTPConnection, self).__init__(*args, **kw)
def _new_conn(self):
""" Establish a socket connection and set nodelay settings on it.
:return: New socket connection.
"""
return _client.ssh_proxy_sock(hostname=self._dns_host,
port=self.port,
source_address=self.source_address,
client=self.ssh_client)
class SSHTunnelHTTPSConnection(SSHTunnelHTTPConnection,
connection.HTTPSConnection):
pass
class SSHTunnelHTTPConnectionPool(connectionpool.HTTPConnectionPool):
ConnectionCls = SSHTunnelHTTPConnection
class SSHTunnelHTTPSConnectionPool(connectionpool.HTTPSConnectionPool):
ConnectionCls = SSHTunnelHTTPSConnection
pool_classes_by_scheme = {"http": SSHTunnelHTTPConnectionPool,
"https": SSHTunnelHTTPSConnectionPool}