From e68f1d85f8b3aaf010f085cea2245eb9428af02c Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Sun, 31 May 2020 14:23:35 +1200 Subject: [PATCH] Fix proxy issue for etcd and k8s When the cloud is behind a proxy, podman needs to access the dockerhub via proxy to pull the image, so the proxy settings need to be exported to etcd systemd file as well. We're setting the heat-params as environment file for k8s components already. Besides, because CIDR of fixed subnet vary for different clusters, so the subnet CIDR should be added into NO_PROXY list. Otherwise, it will affect the communication between etcd members and also the communication between k8s components. Task: 39990 Story: 2007768 Change-Id: I4dba79e04abe38b9806e847348d3dd77ef96bee5 (cherry picked from commit b2e3f2346b8550f71e1ed2c737e82aa6050bcfec) --- .../templates/kubernetes/fragments/configure-etcd.sh | 1 + magnum/drivers/heat/k8s_template_def.py | 9 ++++++++- .../conductor/handlers/test_k8s_cluster_conductor.py | 10 +++++----- magnum/tests/unit/drivers/test_template_definition.py | 2 ++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh index c6c28fcf97..b7e5ff8983 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh @@ -58,6 +58,7 @@ After=network-online.target Wants=network-online.target [Service] +EnvironmentFile=/etc/sysconfig/heat-params ExecStartPre=mkdir -p /var/lib/etcd ExecStartPre=-/bin/podman rm etcd ExecStart=/bin/podman run \\ diff --git a/magnum/drivers/heat/k8s_template_def.py b/magnum/drivers/heat/k8s_template_def.py index 7a60792175..d5f3194c22 100644 --- a/magnum/drivers/heat/k8s_template_def.py +++ b/magnum/drivers/heat/k8s_template_def.py @@ -119,6 +119,7 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): cluster_attr=None) self.add_output('kube_masters_private', cluster_attr=None) + self.default_subnet_cidr = '10.0.0.0/24' def get_nodegroup_param_maps(self, master_params=None, worker_params=None): master_params = master_params or dict() @@ -205,6 +206,12 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): extra_params['fixed_subnet_cidr'] = neutron.get_subnet( context, subnet_id, "id", "cidr") + if cluster_template.no_proxy: + extra_params["no_proxy"] = ( + cluster_template.no_proxy + "," + ( + extra_params.get('fixed_subnet_cidr') or + self.default_subnet_cidr)) + return extra_params def get_params(self, context, cluster_template, cluster, **kwargs): @@ -283,7 +290,7 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): def _set_master_lb_allowed_cidrs(self, context, cluster, extra_params): if extra_params.get("master_lb_allowed_cidrs"): subnet_cidr = (cluster.labels.get("fixed_subnet_cidr") or - "10.0.0.0/24") + self.default_subnet_cidr) if extra_params.get("fixed_subnet"): subnet_cidr = neutron.get_subnet(context, extra_params["fixed_subnet"], diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 3ef39c33ba..3b9a64a5a5 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -335,7 +335,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'influx_grafana_dashboard_enabled': 'True', 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', - 'no_proxy': 'no_proxy', + 'no_proxy': 'no_proxy,20.200.0.0/16', 'username': 'fake_user', 'cluster_uuid': self.cluster_dict['uuid'], 'magnum_url': self.mock_osc.magnum_url.return_value, @@ -487,7 +487,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'master_flavor': 'master_flavor_id', 'minion_flavor': 'flavor_id', 'network_driver': 'network_driver', - 'no_proxy': 'no_proxy', + 'no_proxy': 'no_proxy,20.200.0.0/16', 'number_of_masters': 1, 'number_of_minions': 1, 'region_name': 'RegionOne', @@ -748,7 +748,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'etcd_volume_size': None, 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', - 'no_proxy': 'no_proxy', + 'no_proxy': 'no_proxy,20.200.0.0/16', 'flannel_network_cidr': '10.101.0.0/16', 'flannel_network_subnetlen': '26', 'flannel_backend': 'vxlan', @@ -862,7 +862,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'etcd_volume_size': None, 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', - 'no_proxy': 'no_proxy', + 'no_proxy': 'no_proxy,20.200.0.0/16', 'nodes_affinity_policy': 'soft-anti-affinity', 'flannel_network_cidr': '10.101.0.0/16', 'flannel_network_subnetlen': '26', @@ -1103,7 +1103,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'etcd_volume_type': '', 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', - 'no_proxy': 'no_proxy', + 'no_proxy': 'no_proxy,20.200.0.0/16', 'flannel_network_cidr': '10.101.0.0/16', 'flannel_network_subnetlen': '26', 'flannel_backend': 'vxlan', diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index ef9d929c29..f1baf6fbeb 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -424,6 +424,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): mock_cluster_template.network_driver = 'flannel' external_network_id = '17e4e301-b7f3-4996-b3dd-97b3a700174b' mock_cluster_template.external_network_id = external_network_id + mock_cluster_template.no_proxy = "" mock_cluster = mock.MagicMock() fixed_network_name = 'fixed_network' mock_get_fixed_network_name.return_value = fixed_network_name @@ -956,6 +957,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): mock_cluster_template.network_driver = 'calico' external_network_id = '17e4e301-b7f3-4996-b3dd-97b3a700174b' mock_cluster_template.external_network_id = external_network_id + mock_cluster_template.no_proxy = "" mock_cluster = mock.MagicMock() fixed_network_name = 'fixed_network' mock_cluster.fixed_network = fixed_network_name