x/tobiko
Code Issues Proposed changes

Create new tobiko.http package

Browse Source 
Change-Id: I40a83e6304abe1a5be3043e0221719fbbaaa1c24
This commit is contained in:
Federico Ressi 2019-10-29 12:18:27 +01:00
parent b6791dab27
commit 425fa52cbd
6 changed files with 189 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tobiko/http/__init__.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
from tobiko.http import _session
get_http_session = _session.get_http_session
setup_http_session = _session.setup_http_session

94
tobiko/http/_connection.py Normal file
View File

@ -0,0 +1,94 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
from urllib3 import connection
from urllib3 import connectionpool
import tobiko
from tobiko.shell import ssh
class HTTPConnection(connection.HTTPConnection):
def __init__(self, *args, **kwargs):
#: Port forwarding address to redirect connection too if given
self.forward_address = kwargs.pop("forward_address", None)
super(HTTPConnection, self).__init__(*args, **kwargs)
def _new_conn(self):
""" Establish a socket connection and set nodelay settings on it.
:return: New socket connection.
"""
extra_kw = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
if self.socket_options:
extra_kw["socket_options"] = self.socket_options
address = self.forward_address or (self._dns_host, self.port)
try:
conn = connection.connection.create_connection(
address, self.timeout, **extra_kw)
except connection.SocketTimeout:
raise connection.ConnectTimeoutError(
self,
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
except connection.SocketError as e:
raise connection.NewConnectionError(
self, "Failed to establish a new connection: %s" % e
)
return conn
class HTTPSConnection(HTTPConnection, connection.HTTPSConnection):
pass
class HTTPConnectionPool(connectionpool.HTTPConnectionPool):
ConnectionCls = HTTPConnection
forwarder = None
ssh_client = None
def __init__(self, host, port, ssh_client=None, **kwargs):
if ssh_client is None:
ssh_client = ssh.ssh_proxy_client() or False
self.ssh_client = ssh_client
if ssh_client:
self.forwarder = forwarder = ssh.SSHTunnelForwarderFixture(
ssh_client=ssh_client)
forward_address = forwarder.put_forwarding(host, port)
tobiko.setup_fixture(forwarder)
kwargs['forward_address'] = forward_address
super(HTTPConnectionPool, self).__init__(host=host,
port=port,
**kwargs)
class HTTPSConnectionPool(HTTPConnectionPool,
connectionpool.HTTPSConnectionPool):
ConnectionCls = HTTPSConnection

70
tobiko/http/_session.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import collections
import functools
import requests
from urllib3 import poolmanager
from tobiko.http import _connection
def get_http_session(ssh_client=None):
return setup_http_session(session=requests.Session(),
ssh_client=ssh_client)
def setup_http_session(session, ssh_client=None):
for adapter in session.adapters.values():
manager = adapter.poolmanager
manager.pool_classes_by_scheme = pool_classes_by_scheme.copy()
manager.key_fn_by_scheme = key_fn_by_scheme.copy()
manager.connection_pool_kw['ssh_client'] = ssh_client
return session
# pylint: disable=protected-access
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = poolmanager._key_fields + ('key_ssh_client',)
class PoolKey(collections.namedtuple("PoolKey", _key_fields)):
"""The namedtuple class used to construct keys for the connection pool.
All custom key schemes should include the fields in this key at a minimum.
"""
#: A dictionary that maps a scheme to a callable that creates a pool key.
#: This can be used to alter the way pool keys are constructed, if desired.
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(poolmanager._default_key_normalizer,
PoolKey),
"https": functools.partial(poolmanager._default_key_normalizer,
PoolKey),
}
# pylint: enable=protected-access
pool_classes_by_scheme = {"http": _connection.HTTPConnectionPool,
"https": _connection.HTTPSConnectionPool}

4
tobiko/openstack/keystone/_session.py
View File

@ -19,7 +19,7 @@ from oslo_log import log
import tobiko import tobiko
from tobiko.openstack.keystone import _credentials from tobiko.openstack.keystone import _credentials
from tobiko.shell import ssh from tobiko import http
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
@ -74,7 +74,7 @@ class KeystoneSessionFixture(tobiko.SharedFixture):
auth = loader.load_from_options(**params) auth = loader.load_from_options(**params)
self.session = session = _session.Session( self.session = session = _session.Session(
auth=auth, verify=False) auth=auth, verify=False)
ssh.setup_http_session_ssh_tunneling(session=session) http.setup_http_session(session)
self.credentials = credentials self.credentials = credentials

4
tobiko/shell/ssh/__init__.py
View File

@ -18,7 +18,7 @@ from __future__ import absolute_import
from tobiko.shell.ssh import _config from tobiko.shell.ssh import _config
from tobiko.shell.ssh import _client from tobiko.shell.ssh import _client
from tobiko.shell.ssh import _command from tobiko.shell.ssh import _command
from tobiko.shell.ssh import _http from tobiko.shell.ssh import _forward
SSHHostConfig = _config.SSHHostConfig SSHHostConfig = _config.SSHHostConfig
@ -31,4 +31,4 @@ ssh_proxy_client = _client.ssh_proxy_client
SSHConnectFailure = _client.SSHConnectFailure SSHConnectFailure = _client.SSHConnectFailure
gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters
setup_http_session_ssh_tunneling = _http.setup_http_session_ssh_tunneling SSHTunnelForwarderFixture = _forward.SSHTunnelForwarderFixture

134
tobiko/shell/ssh/_http.py
View File

@ -1,134 +0,0 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import collections
import functools
import requests
from urllib3 import connection
from urllib3 import connectionpool
from urllib3 import poolmanager
import tobiko
from tobiko.shell.ssh import _client
from tobiko.shell.ssh import _forward
def setup_http_session_ssh_tunneling(session=None, ssh_client=None):
session = session or requests.Session()
ssh_client = ssh_client or _client.ssh_proxy_client()
if ssh_client is not None:
for adapter in session.adapters.values():
manager = adapter.poolmanager
manager.pool_classes_by_scheme = pool_classes_by_scheme.copy()
manager.key_fn_by_scheme = key_fn_by_scheme.copy()
manager.connection_pool_kw['ssh_client'] = ssh_client
return session
# pylint: disable=protected-access
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = poolmanager._key_fields + ('key_ssh_client',)
class SSHTunnelPoolKey(
collections.namedtuple("SSHTunnelPoolKey", _key_fields)):
"""The namedtuple class used to construct keys for the connection pool.
All custom key schemes should include the fields in this key at a minimum.
"""
#: A dictionary that maps a scheme to a callable that creates a pool key.
#: This can be used to alter the way pool keys are constructed, if desired.
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(poolmanager._default_key_normalizer,
SSHTunnelPoolKey),
"https": functools.partial(poolmanager._default_key_normalizer,
SSHTunnelPoolKey),
}
# pylint: enable=protected-access
class SSHTunnelHTTPConnection(connection.HTTPConnection):
def __init__(self, local_address, *args, **kwargs):
super(SSHTunnelHTTPConnection, self).__init__(*args, **kwargs)
self.local_address = local_address
def _new_conn(self):
""" Establish a socket connection and set nodelay settings on it.
:return: New socket connection.
"""
extra_kw = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
if self.socket_options:
extra_kw["socket_options"] = self.socket_options
try:
conn = connection.connection.create_connection(
self.local_address, self.timeout, **extra_kw)
except connection.SocketTimeout:
raise connection.ConnectTimeoutError(
self,
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
except connection.SocketError as e:
raise connection.NewConnectionError(
self, "Failed to establish a new connection: %s" % e
)
return conn
class SSHTunnelHTTPSConnection(SSHTunnelHTTPConnection,
connection.HTTPSConnection):
pass
class SSHTunnelHTTPConnectionPool(connectionpool.HTTPConnectionPool):
ConnectionCls = SSHTunnelHTTPConnection
def __init__(self, host, port, ssh_client, **kwargs):
self.forwarder = forwarder = _forward.SSHTunnelForwarderFixture(
ssh_client=ssh_client)
local_address = forwarder.put_forwarding(host, port)
tobiko.setup_fixture(forwarder)
super(SSHTunnelHTTPConnectionPool, self).__init__(
host=host, port=port, local_address=local_address, **kwargs)
class SSHTunnelHTTPSConnectionPool(SSHTunnelHTTPConnectionPool,
connectionpool.HTTPSConnectionPool):
ConnectionCls = SSHTunnelHTTPSConnection
pool_classes_by_scheme = {"http": SSHTunnelHTTPConnectionPool,
"https": SSHTunnelHTTPSConnectionPool}