From 425fa52cbd0346aab6959a108897cb1803a4f12c Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Tue, 29 Oct 2019 12:18:27 +0100 Subject: [PATCH] Create new tobiko.http package Change-Id: I40a83e6304abe1a5be3043e0221719fbbaaa1c24 --- tobiko/http/__init__.py | 21 ++++ tobiko/http/_connection.py | 94 ++++++++++++++++++ tobiko/http/_session.py | 70 ++++++++++++++ tobiko/openstack/keystone/_session.py | 4 +- tobiko/shell/ssh/__init__.py | 4 +- tobiko/shell/ssh/_http.py | 134 -------------------------- 6 files changed, 189 insertions(+), 138 deletions(-) create mode 100644 tobiko/http/__init__.py create mode 100644 tobiko/http/_connection.py create mode 100644 tobiko/http/_session.py delete mode 100644 tobiko/shell/ssh/_http.py diff --git a/tobiko/http/__init__.py b/tobiko/http/__init__.py new file mode 100644 index 000000000..176f9dcc6 --- /dev/null +++ b/tobiko/http/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from tobiko.http import _session + +get_http_session = _session.get_http_session +setup_http_session = _session.setup_http_session diff --git a/tobiko/http/_connection.py b/tobiko/http/_connection.py new file mode 100644 index 000000000..524e19df1 --- /dev/null +++ b/tobiko/http/_connection.py @@ -0,0 +1,94 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from urllib3 import connection +from urllib3 import connectionpool + +import tobiko +from tobiko.shell import ssh + + +class HTTPConnection(connection.HTTPConnection): + + def __init__(self, *args, **kwargs): + #: Port forwarding address to redirect connection too if given + self.forward_address = kwargs.pop("forward_address", None) + super(HTTPConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ Establish a socket connection and set nodelay settings on it. + + :return: New socket connection. + """ + extra_kw = {} + if self.source_address: + extra_kw["source_address"] = self.source_address + + if self.socket_options: + extra_kw["socket_options"] = self.socket_options + + address = self.forward_address or (self._dns_host, self.port) + try: + conn = connection.connection.create_connection( + address, self.timeout, **extra_kw) + + except connection.SocketTimeout: + raise connection.ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) + + except connection.SocketError as e: + raise connection.NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + return conn + + +class HTTPSConnection(HTTPConnection, connection.HTTPSConnection): + pass + + +class HTTPConnectionPool(connectionpool.HTTPConnectionPool): + + ConnectionCls = HTTPConnection + + forwarder = None + ssh_client = None + + def __init__(self, host, port, ssh_client=None, **kwargs): + if ssh_client is None: + ssh_client = ssh.ssh_proxy_client() or False + self.ssh_client = ssh_client + if ssh_client: + self.forwarder = forwarder = ssh.SSHTunnelForwarderFixture( + ssh_client=ssh_client) + forward_address = forwarder.put_forwarding(host, port) + tobiko.setup_fixture(forwarder) + kwargs['forward_address'] = forward_address + + super(HTTPConnectionPool, self).__init__(host=host, + port=port, + **kwargs) + + +class HTTPSConnectionPool(HTTPConnectionPool, + connectionpool.HTTPSConnectionPool): + + ConnectionCls = HTTPSConnection diff --git a/tobiko/http/_session.py b/tobiko/http/_session.py new file mode 100644 index 000000000..735da529c --- /dev/null +++ b/tobiko/http/_session.py @@ -0,0 +1,70 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import collections +import functools + +import requests +from urllib3 import poolmanager + +from tobiko.http import _connection + + +def get_http_session(ssh_client=None): + return setup_http_session(session=requests.Session(), + ssh_client=ssh_client) + + +def setup_http_session(session, ssh_client=None): + for adapter in session.adapters.values(): + manager = adapter.poolmanager + manager.pool_classes_by_scheme = pool_classes_by_scheme.copy() + manager.key_fn_by_scheme = key_fn_by_scheme.copy() + manager.connection_pool_kw['ssh_client'] = ssh_client + return session + + +# pylint: disable=protected-access + +# All known keyword arguments that could be provided to the pool manager, its +# pools, or the underlying connections. This is used to construct a pool key. +_key_fields = poolmanager._key_fields + ('key_ssh_client',) + + +class PoolKey(collections.namedtuple("PoolKey", _key_fields)): + """The namedtuple class used to construct keys for the connection pool. + + All custom key schemes should include the fields in this key at a minimum. + """ + + +#: A dictionary that maps a scheme to a callable that creates a pool key. +#: This can be used to alter the way pool keys are constructed, if desired. +#: Each PoolManager makes a copy of this dictionary so they can be configured +#: globally here, or individually on the instance. +key_fn_by_scheme = { + "http": functools.partial(poolmanager._default_key_normalizer, + PoolKey), + "https": functools.partial(poolmanager._default_key_normalizer, + PoolKey), +} + +# pylint: enable=protected-access + + +pool_classes_by_scheme = {"http": _connection.HTTPConnectionPool, + "https": _connection.HTTPSConnectionPool} diff --git a/tobiko/openstack/keystone/_session.py b/tobiko/openstack/keystone/_session.py index 3eaeda5bb..42b8ba637 100644 --- a/tobiko/openstack/keystone/_session.py +++ b/tobiko/openstack/keystone/_session.py @@ -19,7 +19,7 @@ from oslo_log import log import tobiko from tobiko.openstack.keystone import _credentials -from tobiko.shell import ssh +from tobiko import http LOG = log.getLogger(__name__) @@ -74,7 +74,7 @@ class KeystoneSessionFixture(tobiko.SharedFixture): auth = loader.load_from_options(**params) self.session = session = _session.Session( auth=auth, verify=False) - ssh.setup_http_session_ssh_tunneling(session=session) + http.setup_http_session(session) self.credentials = credentials diff --git a/tobiko/shell/ssh/__init__.py b/tobiko/shell/ssh/__init__.py index 3c7181ddf..3cd0f5b40 100644 --- a/tobiko/shell/ssh/__init__.py +++ b/tobiko/shell/ssh/__init__.py @@ -18,7 +18,7 @@ from __future__ import absolute_import from tobiko.shell.ssh import _config from tobiko.shell.ssh import _client from tobiko.shell.ssh import _command -from tobiko.shell.ssh import _http +from tobiko.shell.ssh import _forward SSHHostConfig = _config.SSHHostConfig @@ -31,4 +31,4 @@ ssh_proxy_client = _client.ssh_proxy_client SSHConnectFailure = _client.SSHConnectFailure gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters -setup_http_session_ssh_tunneling = _http.setup_http_session_ssh_tunneling +SSHTunnelForwarderFixture = _forward.SSHTunnelForwarderFixture diff --git a/tobiko/shell/ssh/_http.py b/tobiko/shell/ssh/_http.py deleted file mode 100644 index f302c59c4..000000000 --- a/tobiko/shell/ssh/_http.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) 2019 Red Hat, Inc. -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from __future__ import absolute_import - -import collections -import functools - -import requests -from urllib3 import connection -from urllib3 import connectionpool -from urllib3 import poolmanager - -import tobiko -from tobiko.shell.ssh import _client -from tobiko.shell.ssh import _forward - - -def setup_http_session_ssh_tunneling(session=None, ssh_client=None): - session = session or requests.Session() - ssh_client = ssh_client or _client.ssh_proxy_client() - if ssh_client is not None: - for adapter in session.adapters.values(): - manager = adapter.poolmanager - manager.pool_classes_by_scheme = pool_classes_by_scheme.copy() - manager.key_fn_by_scheme = key_fn_by_scheme.copy() - manager.connection_pool_kw['ssh_client'] = ssh_client - return session - - -# pylint: disable=protected-access - -# All known keyword arguments that could be provided to the pool manager, its -# pools, or the underlying connections. This is used to construct a pool key. -_key_fields = poolmanager._key_fields + ('key_ssh_client',) - - -class SSHTunnelPoolKey( - collections.namedtuple("SSHTunnelPoolKey", _key_fields)): - """The namedtuple class used to construct keys for the connection pool. - - All custom key schemes should include the fields in this key at a minimum. - """ - - -#: A dictionary that maps a scheme to a callable that creates a pool key. -#: This can be used to alter the way pool keys are constructed, if desired. -#: Each PoolManager makes a copy of this dictionary so they can be configured -#: globally here, or individually on the instance. -key_fn_by_scheme = { - "http": functools.partial(poolmanager._default_key_normalizer, - SSHTunnelPoolKey), - "https": functools.partial(poolmanager._default_key_normalizer, - SSHTunnelPoolKey), -} - -# pylint: enable=protected-access - - -class SSHTunnelHTTPConnection(connection.HTTPConnection): - - def __init__(self, local_address, *args, **kwargs): - super(SSHTunnelHTTPConnection, self).__init__(*args, **kwargs) - self.local_address = local_address - - def _new_conn(self): - """ Establish a socket connection and set nodelay settings on it. - - :return: New socket connection. - """ - extra_kw = {} - if self.source_address: - extra_kw["source_address"] = self.source_address - - if self.socket_options: - extra_kw["socket_options"] = self.socket_options - - try: - conn = connection.connection.create_connection( - self.local_address, self.timeout, **extra_kw) - - except connection.SocketTimeout: - raise connection.ConnectTimeoutError( - self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) - - except connection.SocketError as e: - raise connection.NewConnectionError( - self, "Failed to establish a new connection: %s" % e - ) - - return conn - - -class SSHTunnelHTTPSConnection(SSHTunnelHTTPConnection, - connection.HTTPSConnection): - pass - - -class SSHTunnelHTTPConnectionPool(connectionpool.HTTPConnectionPool): - - ConnectionCls = SSHTunnelHTTPConnection - - def __init__(self, host, port, ssh_client, **kwargs): - self.forwarder = forwarder = _forward.SSHTunnelForwarderFixture( - ssh_client=ssh_client) - local_address = forwarder.put_forwarding(host, port) - tobiko.setup_fixture(forwarder) - super(SSHTunnelHTTPConnectionPool, self).__init__( - host=host, port=port, local_address=local_address, **kwargs) - - -class SSHTunnelHTTPSConnectionPool(SSHTunnelHTTPConnectionPool, - connectionpool.HTTPSConnectionPool): - - ConnectionCls = SSHTunnelHTTPSConnection - - -pool_classes_by_scheme = {"http": SSHTunnelHTTPConnectionPool, - "https": SSHTunnelHTTPSConnectionPool}