support http/https proxy for discovery url
HTTP(S) proxy can be specified when creating the template. https://docs.openstack.org/magnum/latest/admin/magnum-proxy.html However, it is not being utilized when talking to a public etcd discovery service, which result in failed cluster creation. We need to be able to use HTTP(S) proxy when services are running behind a firewall. NOTE: this patch also updated some test cases to explicitly set the proper HTTP status code in the mock HTTP request as by default they are not be initialized, which caused those tests to fail. Change-Id: I13d86b0dc7c232a51149107f0412219388d8c2cd story: 2004664 (cherry picked from commitffc61816c8
) (cherry picked from commitf86de684d2
)
This commit is contained in:
parent
01aec17237
commit
2ba0cbce41
|
@ -71,6 +71,16 @@ EOF
|
|||
|
||||
fi
|
||||
|
||||
if [ -n "$HTTP_PROXY" ]; then
|
||||
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
|
||||
if [ -n "$HTTP_PROXY" -o "$HTTPS_PROXY" ]; then
|
||||
ETCD_DISCOVERY_PROTOCOL=$(python -c "from six.moves.urllib import parse as urlparse; print urlparse.urlparse('${ETCD_DISCOVERY_URL}').scheme")
|
||||
ETCD_DISCOVERY_HOSTNAME=$(python -c "from six.moves.urllib import parse as urlparse; print urlparse.urlparse('${ETCD_DISCOVERY_URL}').netloc.partition(':')[0]")
|
||||
# prints 1 if $ETCD_DISCOVERY_HOSTNAME is listed explicitly in $NO_PROXY, or $NO_PROXY is set to "*"
|
||||
ETCD_DISCOVERY_PROXY_BYPASS=$(NO_PROXY="${NO_PROXY}" python -c "import requests; print requests.utils.proxy_bypass('${ETCD_DISCOVERY_HOSTNAME}')")
|
||||
if [ $ETCD_DISCOVERY_PROXY_BYPASS == "0" ]; then
|
||||
if [ -n "$HTTP_PROXY" -a "$ETCD_DISCOVERY_PROTOCOL" == "http" ]; then
|
||||
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
|
||||
elif [ -n "$HTTPS_PROXY" -a "$ETCD_DISCOVERY_PROTOCOL" == "https" ]; then
|
||||
echo "ETCD_DISCOVERY_PROXY=$HTTPS_PROXY" >> /etc/etcd/etcd.conf
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
|
@ -97,7 +97,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
|
|||
extra_params['minions_to_remove'] = (
|
||||
scale_mgr.get_removal_nodes(hosts))
|
||||
|
||||
extra_params['discovery_url'] = self.get_discovery_url(cluster)
|
||||
extra_params['discovery_url'] = \
|
||||
self.get_discovery_url(cluster, cluster_template=cluster_template)
|
||||
osc = self.get_osc(context)
|
||||
extra_params['magnum_url'] = osc.magnum_url()
|
||||
|
||||
|
|
|
@ -90,7 +90,8 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition):
|
|||
|
||||
def get_params(self, context, cluster_template, cluster, **kwargs):
|
||||
extra_params = kwargs.pop('extra_params', {})
|
||||
extra_params['discovery_url'] = self.get_discovery_url(cluster)
|
||||
extra_params['discovery_url'] = \
|
||||
self.get_discovery_url(cluster, cluster_template=cluster_template)
|
||||
# HACK(apmelton) - This uses the user's bearer token, ideally
|
||||
# it should be replaced with an actual trust token with only
|
||||
# access to do what the template needs it to do.
|
||||
|
|
|
@ -15,8 +15,10 @@ import abc
|
|||
import ast
|
||||
|
||||
from oslo_log import log as logging
|
||||
import re
|
||||
import requests
|
||||
import six
|
||||
from six.moves.urllib import parse as urlparse
|
||||
|
||||
from magnum.common import clients
|
||||
from magnum.common import exception
|
||||
|
@ -299,7 +301,26 @@ class BaseTemplateDefinition(TemplateDefinition):
|
|||
size=int(value),
|
||||
discovery_url=discovery_url)
|
||||
|
||||
def get_discovery_url(self, cluster):
|
||||
def get_proxies(self, url, cluster_template):
|
||||
proxies = dict()
|
||||
if cluster_template is None:
|
||||
return proxies
|
||||
hostname = urlparse.urlparse(url).netloc.partition(":")[0]
|
||||
if hasattr(cluster_template, 'no_proxy') and \
|
||||
cluster_template.no_proxy and \
|
||||
(cluster_template.no_proxy == '*' or
|
||||
re.search('\\b%s\\b' % re.escape(hostname),
|
||||
cluster_template.no_proxy, re.I)):
|
||||
LOG.debug('Bypass proxy, because discovery hostname is listed in'
|
||||
' cluster template no_proxy variable')
|
||||
else:
|
||||
if hasattr(cluster_template, 'http_proxy'):
|
||||
proxies['http'] = cluster_template.http_proxy
|
||||
if hasattr(cluster_template, 'https_proxy'):
|
||||
proxies['https'] = cluster_template.https_proxy
|
||||
return proxies
|
||||
|
||||
def get_discovery_url(self, cluster, cluster_template=None):
|
||||
if hasattr(cluster, 'discovery_url') and cluster.discovery_url:
|
||||
if getattr(cluster, 'master_count', None) is not None:
|
||||
self.validate_discovery_url(cluster.discovery_url,
|
||||
|
@ -312,7 +333,14 @@ class BaseTemplateDefinition(TemplateDefinition):
|
|||
CONF.cluster.etcd_discovery_service_endpoint_format %
|
||||
{'size': cluster.master_count})
|
||||
try:
|
||||
discovery_url = requests.get(discovery_endpoint).text
|
||||
proxies = self.get_proxies(discovery_endpoint,
|
||||
cluster_template)
|
||||
discovery_request = requests.get(discovery_endpoint,
|
||||
proxies=proxies)
|
||||
if discovery_request.status_code != requests.codes.ok:
|
||||
raise exception.GetDiscoveryUrlFailed(
|
||||
discovery_endpoint=discovery_endpoint)
|
||||
discovery_url = discovery_request.text
|
||||
except req_exceptions.RequestException as err:
|
||||
LOG.error(six.text_type(err))
|
||||
raise exception.GetDiscoveryUrlFailed(
|
||||
|
|
|
@ -489,6 +489,7 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
self.cluster_template_dict['cluster_distro'] = 'coreos'
|
||||
self.cluster_dict['discovery_url'] = None
|
||||
mock_req = mock.MagicMock(text='http://tokentest/h1/h2/h3')
|
||||
mock_req.status_code = 200
|
||||
reqget.return_value = mock_req
|
||||
cluster_template = objects.ClusterTemplate(
|
||||
self.context, **self.cluster_template_dict)
|
||||
|
@ -692,6 +693,7 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
mock_req = mock.MagicMock(text='https://address/token')
|
||||
mock_req.status_code = 200
|
||||
reqget.return_value = mock_req
|
||||
|
||||
(template_path,
|
||||
|
@ -755,7 +757,8 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'../../common/templates/environments/disable_floating_ip.yaml',
|
||||
],
|
||||
env_files)
|
||||
reqget.assert_called_once_with('http://etcd/test?size=1')
|
||||
reqget.assert_called_once_with('http://etcd/test?size=1', proxies={
|
||||
'http': 'http_proxy', 'https': 'https_proxy'})
|
||||
|
||||
@patch('magnum.common.short_id.generate_id')
|
||||
@patch('heatclient.common.template_utils.get_template_contents')
|
||||
|
|
|
@ -433,6 +433,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
|
|||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_resp.status_code = 200
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.master_count = 10
|
||||
|
@ -441,7 +442,92 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
|
|||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
discovery_url = k8s_def.get_discovery_url(mock_cluster)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=10')
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=10',
|
||||
proxies={})
|
||||
self.assertEqual(expected_discovery_url, mock_cluster.discovery_url)
|
||||
self.assertEqual(expected_discovery_url, discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_k8s_get_discovery_url_proxy(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.master_count = 10
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = 'localhost,127.0.0.1'
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
discovery_url = k8s_def.get_discovery_url(mock_cluster,
|
||||
mock_cluster_template)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=10', proxies={
|
||||
'http': 'http_proxy', 'https': 'https_proxy'})
|
||||
self.assertEqual(expected_discovery_url, mock_cluster.discovery_url)
|
||||
self.assertEqual(expected_discovery_url, discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_k8s_get_discovery_url_no_proxy(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.master_count = 10
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = 'localhost,127.0.0.1,etcd'
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
discovery_url = k8s_def.get_discovery_url(mock_cluster,
|
||||
mock_cluster_template)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=10',
|
||||
proxies={})
|
||||
self.assertEqual(expected_discovery_url, mock_cluster.discovery_url)
|
||||
self.assertEqual(expected_discovery_url, discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_k8s_get_discovery_url_no_proxy_wildcard(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.master_count = 10
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = '*'
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
discovery_url = k8s_def.get_discovery_url(mock_cluster,
|
||||
mock_cluster_template)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=10',
|
||||
proxies={})
|
||||
self.assertEqual(expected_discovery_url, mock_cluster.discovery_url)
|
||||
self.assertEqual(expected_discovery_url, discovery_url)
|
||||
|
||||
|
@ -470,6 +556,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
|
|||
def test_k8s_get_discovery_url_not_found(self, mock_get):
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.text = ''
|
||||
mock_resp.status_code = 200
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
fake_cluster = mock.MagicMock()
|
||||
|
@ -841,6 +928,7 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
|
|||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_resp.status_code = 200
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.discovery_url = None
|
||||
|
@ -848,7 +936,84 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
|
|||
swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition()
|
||||
discovery_url = swarm_def.get_discovery_url(mock_cluster)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=1')
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=1', proxies={})
|
||||
self.assertEqual(mock_cluster.discovery_url, expected_discovery_url)
|
||||
self.assertEqual(discovery_url, expected_discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_swarm_get_discovery_url_proxy(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = 'localhost,127.0.0.1'
|
||||
|
||||
swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition()
|
||||
discovery_url = swarm_def.get_discovery_url(mock_cluster,
|
||||
mock_cluster_template)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=1', proxies={
|
||||
'http': 'http_proxy', 'https': 'https_proxy'})
|
||||
self.assertEqual(mock_cluster.discovery_url, expected_discovery_url)
|
||||
self.assertEqual(discovery_url, expected_discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_swarm_get_discovery_url_no_proxy(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = 'etcd,localhost,127.0.0.1'
|
||||
|
||||
swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition()
|
||||
discovery_url = swarm_def.get_discovery_url(mock_cluster)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=1', proxies={})
|
||||
self.assertEqual(mock_cluster.discovery_url, expected_discovery_url)
|
||||
self.assertEqual(discovery_url, expected_discovery_url)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_swarm_get_discovery_url_no_proxy_wildcard(self, mock_get):
|
||||
CONF.set_override('etcd_discovery_service_endpoint_format',
|
||||
'http://etcd/test?size=%(size)d',
|
||||
group='cluster')
|
||||
expected_discovery_url = 'http://etcd/token'
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = expected_discovery_url
|
||||
mock_get.return_value = mock_resp
|
||||
mock_cluster = mock.MagicMock()
|
||||
mock_cluster.discovery_url = None
|
||||
|
||||
mock_cluster_template = mock.MagicMock()
|
||||
mock_cluster_template.http_proxy = 'http_proxy'
|
||||
mock_cluster_template.https_proxy = 'https_proxy'
|
||||
mock_cluster_template.no_proxy = '*'
|
||||
|
||||
swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition()
|
||||
discovery_url = swarm_def.get_discovery_url(mock_cluster)
|
||||
|
||||
mock_get.assert_called_once_with('http://etcd/test?size=1', proxies={})
|
||||
self.assertEqual(mock_cluster.discovery_url, expected_discovery_url)
|
||||
self.assertEqual(discovery_url, expected_discovery_url)
|
||||
|
||||
|
@ -856,6 +1021,7 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
|
|||
def test_swarm_get_discovery_url_not_found(self, mock_get):
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.text = ''
|
||||
mock_resp.status_code = 200
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
fake_cluster = mock.MagicMock()
|
||||
|
|
Loading…
Reference in New Issue