From 2cc57c538627ae51c8b23ff33705c4f4fe01cc35 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Wed, 9 May 2018 21:29:01 +1200 Subject: [PATCH] Use Octavia for LoadBalancer type service In the OpenStack deployment with Octavia service enabled, the octavia service should be used not only for master nodes high availability, but also for k8s LoadBalancer type service implementation as well. Change-Id: Ib61f59507510253794a4780a91e49aa6682c8039 Closes-Bug: #1770133 --- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params.yaml | 1 + .../fragments/write-kube-os-config.sh | 1 + magnum/drivers/heat/k8s_template_def.py | 3 +++ .../fragments/write-heat-params-master.yaml | 1 + .../templates/fragments/write-heat-params.yaml | 1 + .../k8s_coreos_v1/templates/kubecluster.yaml | 8 ++++++++ .../k8s_coreos_v1/templates/kubemaster.yaml | 7 +++++++ .../k8s_coreos_v1/templates/kubeminion.yaml | 7 +++++++ .../templates/kubecluster.yaml | 8 ++++++++ .../templates/kubemaster.yaml | 7 +++++++ .../templates/kubeminion.yaml | 7 +++++++ .../templates/kubecluster.yaml | 8 ++++++++ .../templates/kubemaster.yaml | 7 +++++++ .../templates/kubeminion_software_configs.yaml | 7 +++++++ .../handlers/test_k8s_cluster_conductor.py | 13 +++++++++++++ .../unit/drivers/test_template_definition.py | 18 ++++++++++++++---- 17 files changed, 101 insertions(+), 4 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index a3f38cebdf..10f24ccec3 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -69,3 +69,4 @@ write_files: KUBEAPI_OPTIONS="$KUBEAPI_OPTIONS" KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS" KUBESCHEDULER_OPTIONS="$KUBESCHEDULER_OPTIONS" + OCTAVIA_ENABLED="$OCTAVIA_ENABLED" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index f32eb9f751..80bf41d949 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -48,3 +48,4 @@ write_files: DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" KUBELET_OPTIONS="$KUBELET_OPTIONS" KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS" + OCTAVIA_ENABLED="$OCTAVIA_ENABLED" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh index 12266d0a6d..8db5575bb0 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh @@ -16,6 +16,7 @@ password=$TRUSTEE_PASSWORD trust-id=$TRUST_ID ca-file=/etc/kubernetes/ca-bundle.crt [LoadBalancer] +use-octavia=$OCTAVIA_ENABLED subnet-id=$CLUSTER_SUBNET create-monitor=yes monitor-delay=1m diff --git a/magnum/drivers/heat/k8s_template_def.py b/magnum/drivers/heat/k8s_template_def.py index 17f70da55a..c7e12be8ad 100644 --- a/magnum/drivers/heat/k8s_template_def.py +++ b/magnum/drivers/heat/k8s_template_def.py @@ -12,6 +12,7 @@ from oslo_config import cfg +from magnum.common import keystone from magnum.drivers.heat import template_def CONF = cfg.CONF @@ -105,6 +106,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): extra_params['loadbalancing_protocol'] = 'HTTP' extra_params['kubernetes_port'] = 8080 + extra_params['octavia_enabled'] = keystone.is_octavia_enabled() + label_list = ['flannel_network_cidr', 'flannel_backend', 'flannel_network_subnetlen', 'system_pods_initial_delay', diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml index f89810a52b..fd379d57c1 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml @@ -49,3 +49,4 @@ write_files: KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION" DNS_SERVICE_IP="$DNS_SERVICE_IP" DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" + OCTAVIA_ENABLED="$OCTAVIA_ENABLED" diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml index 31c861c540..3a40a76e54 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml @@ -47,3 +47,4 @@ write_files: CONTAINER_RUNTIME="$CONTAINER_RUNTIME" DNS_SERVICE_IP="$DNS_SERVICE_IP" DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" + OCTAVIA_ENABLED="$OCTAVIA_ENABLED" diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml index 2f1122d360..a8ddfd8b5a 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml @@ -307,6 +307,12 @@ parameters: - allowed_values: ["affinity", "anti-affinity", "soft-affinity", "soft-anti-affinity"] + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: ###################################################################### @@ -491,6 +497,7 @@ resources: dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} nodes_server_group_id: {get_resource: nodes_server_group} + octavia_enabled: {get_param: octavia_enabled} ###################################################################### # @@ -547,6 +554,7 @@ resources: dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} nodes_server_group_id: {get_resource: nodes_server_group} + octavia_enabled: {get_param: octavia_enabled} outputs: diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml index c44244afdf..a25a1d5aee 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml @@ -237,6 +237,12 @@ parameters: type: string description: ID of the server group for kubernetes cluster nodes. + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: master_wait_handle: @@ -322,6 +328,7 @@ resources: "$ETCD_LB_VIP": {get_param: etcd_lb_vip} "$DNS_SERVICE_IP": {get_param: dns_service_ip} "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} + "$OCTAVIA_ENABLED": {get_param: octavia_enabled} add_ext_ca_certs: type: OS::Heat::SoftwareConfig diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml index 8d720ca3cb..749c00ab45 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml @@ -164,6 +164,12 @@ parameters: type: string description: ID of the server group for kubernetes cluster nodes. + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: minion_wait_handle: @@ -222,6 +228,7 @@ resources: "$CONTAINER_RUNTIME": {get_param: container_runtime} "$DNS_SERVICE_IP": {get_param: dns_service_ip} "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} + "$OCTAVIA_ENABLED": {get_param: octavia_enabled} add_ext_ca_certs: type: OS::Heat::SoftwareConfig diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 143bb6b1b9..0ef65ff8b9 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -462,6 +462,12 @@ parameters: additional options to be passed to the scheduler default: "" + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: ###################################################################### @@ -679,6 +685,7 @@ resources: kubeproxy_options: {get_param: kubeproxy_options} kubecontroller_options: {get_param: kubecontroller_options} kubescheduler_options: {get_param: kubescheduler_options} + octavia_enabled: {get_param: octavia_enabled} ###################################################################### # @@ -754,6 +761,7 @@ resources: pods_network_cidr: {get_param: pods_network_cidr} kubelet_options: {get_param: kubelet_options} kubeproxy_options: {get_param: kubeproxy_options} + octavia_enabled: {get_param: octavia_enabled} outputs: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 5546109844..f32339d9b1 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -350,6 +350,12 @@ parameters: description: > additional options to be passed to the scheduler + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: master_wait_handle: @@ -453,6 +459,7 @@ resources: "$KUBECONTROLLER_OPTIONS": {get_param: kubecontroller_options} "$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options} "$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options} + "$OCTAVIA_ENABLED": {get_param: octavia_enabled} install_openstack_ca: type: OS::Heat::SoftwareConfig diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 3103d83b27..341380c923 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -259,6 +259,12 @@ parameters: description: > additional options to be passed to the kube proxy + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: minion_wait_handle: @@ -327,6 +333,7 @@ resources: $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain} $KUBELET_OPTIONS: {get_param: kubelet_options} $KUBEPROXY_OPTIONS: {get_param: kubeproxy_options} + $OCTAVIA_ENABLED: {get_param: octavia_enabled} install_openstack_ca: type: OS::Heat::SoftwareConfig diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index d702da1159..d57534e07c 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -360,6 +360,12 @@ parameters: - allowed_values: ["affinity", "anti-affinity", "soft-affinity", "soft-anti-affinity"] + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: api_lb: @@ -535,6 +541,7 @@ resources: etcd_lb_vip: {get_attr: [etcd_lb, address]} openstack_ca: {get_param: openstack_ca} nodes_server_group_id: {get_resource: nodes_server_group} + octavia_enabled: {get_param: octavia_enabled} ###################################################################### # @@ -625,6 +632,7 @@ resources: container_infra_prefix: {get_param: container_infra_prefix} wc_curl_cli: {get_attr: [minion_wait_handle, curl_cli]} openstack_ca: {get_param: openstack_ca} + octavia_enabled: {get_param: octavia_enabled} ###################################################################### # diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml index e2071ddf62..ee42e99bdb 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml @@ -250,6 +250,12 @@ parameters: type: string description: ID of the server group for kubernetes cluster nodes. + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: ###################################################################### @@ -321,6 +327,7 @@ resources: "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix} "$ENABLE_CINDER": "False" "$ETCD_LB_VIP": {get_param: etcd_lb_vip} + "$OCTAVIA_ENABLED": {get_param: octavia_enabled} install_openstack_ca: type: OS::Heat::SoftwareConfig diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml index 3c787e3125..e9abf11718 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml @@ -177,6 +177,12 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + octavia_enabled: + type: boolean + description: > + whether or not to use Octavia for LoadBalancer type service. + default: False + resources: ###################################################################### @@ -231,6 +237,7 @@ resources: $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $ENABLE_CINDER: "False" + $OCTAVIA_ENABLED: {get_param: octavia_enabled} install_openstack_ca: type: OS::Heat::SoftwareConfig diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index a63a38ed1e..01c0240f4e 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -125,6 +125,13 @@ class TestClusterConductorWithK8s(base.TestCase): self.mock_osc.keystone.return_value = self.mock_keystone self.mock_osc_class.return_value = self.mock_osc + octavia_patcher = mock.patch( + 'magnum.common.keystone.is_octavia_enabled' + ) + self.mock_enable_octavia = octavia_patcher.start() + self.mock_enable_octavia.return_value = False + self.addCleanup(octavia_patcher.stop) + @patch('requests.get') @patch('magnum.objects.ClusterTemplate.get_by_uuid') @patch('magnum.drivers.common.driver.Driver.get_driver') @@ -263,6 +270,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } if missing_attr is not None: expected.pop(mapping[missing_attr], None) @@ -370,6 +378,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } self.assertEqual(expected, definition) @@ -464,6 +473,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } self.assertEqual(expected, definition) self.assertEqual( @@ -551,6 +561,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } self.assertEqual(expected, definition) self.assertEqual( @@ -633,6 +644,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } self.assertEqual(expected, definition) self.assertEqual( @@ -816,6 +828,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kubecontroller_options': '--kubecontroller', 'kubescheduler_options': '--kubescheduler', 'kubeproxy_options': '--kubeproxy', + 'octavia_enabled': False, } self.assertEqual(expected, definition) self.assertEqual( diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 644eece18e..27a75ca66d 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -219,6 +219,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): def get_definition(self): return k8sa_dr.Driver().get_template_definition() + @mock.patch('magnum.common.keystone.is_octavia_enabled') @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.drivers.k8s_fedora_atomic_v1.template_def' '.AtomicK8sTemplateDefinition.get_discovery_url') @@ -227,7 +228,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): @mock.patch('magnum.drivers.heat.template_def.TemplateDefinition' '.get_output') def test_k8s_get_params(self, mock_get_output, mock_get_params, - mock_get_discovery_url, mock_osc_class): + mock_get_discovery_url, mock_osc_class, + mock_enable_octavia): + mock_enable_octavia.return_value = False mock_context = mock.MagicMock() mock_context.auth_token = 'AUTH_TOKEN' mock_cluster_template = mock.MagicMock() @@ -349,12 +352,15 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'calico_ipv4pool': calico_ipv4pool, 'pods_network_cidr': pods_network_cidr, 'ingress_controller': ingress_controller, - 'ingress_controller_role': ingress_controller_role}} + 'ingress_controller_role': ingress_controller_role, + 'octavia_enabled': False, + }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, **expected_kwargs) + @mock.patch('magnum.common.keystone.is_octavia_enabled') @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.drivers.heat.template_def' '.BaseTemplateDefinition.get_discovery_url') @@ -363,7 +369,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): @mock.patch('magnum.drivers.heat.template_def.TemplateDefinition' '.get_output') def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params, - mock_get_discovery_url, mock_osc_class): + mock_get_discovery_url, mock_osc_class, + mock_enable_octavia): + mock_enable_octavia.return_value = False mock_context = mock.MagicMock() mock_context.auth_token = 'AUTH_TOKEN' mock_cluster_template = mock.MagicMock() @@ -487,7 +495,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'calico_ipv4pool': calico_ipv4pool, 'pods_network_cidr': pods_network_cidr, 'ingress_controller': ingress_controller, - 'ingress_controller_role': ingress_controller_role}} + 'ingress_controller_role': ingress_controller_role, + 'octavia_enabled': False, + }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster,