Browse Source

Add option to tag Neutron resources created by us

If we consider a K8s cluster running on OpenStack VM's, which is a
perfect use case for Kuryr-Kubernetes, we can easily imagine creating
multiple clusters in a single OpenStack public or private cloud. In such
use case those K8s clusters may come and go. As Kuryr is creating some
OpenStack resources, such as ports, networks, subnets, floating IP's or
SG's, it's useful to have a way of identifying those Kuryr-created
resources to delete them along with the K8s cluster that used them.

This commit makes that possible by adding an option to add tag to all
Neutron resources created by Kuryr.

Change-Id: If75028e17d13ec62fb414fa9797ee7ac02d948d1
Implements: blueprint kuryr-resources-tagging
Michał Dulko 2 months ago
parent
commit
88e38e8e94

+ 7
- 0
kuryr_kubernetes/config.py View File

@@ -205,6 +205,13 @@ neutron_defaults = [
205 205
                 help=_("A mapping of default subnets for certain driverType "
206 206
                        "in a form of <driverType>:<SUBNET-ID>"),
207 207
                 default={}),
208
+    cfg.ListOpt('resource_tags',
209
+                help=_("List of tags that will be applied to all OpenStack "
210
+                       "(Neutron and Octavia) resources created by Kuryr. "
211
+                       "This can be used to identify and garbage-collect "
212
+                       "them when Kubernetes cluster Kuryr was serving is no "
213
+                       "longer needed."),
214
+                default=[])
208 215
 ]
209 216
 
210 217
 octavia_defaults = [

+ 3
- 0
kuryr_kubernetes/controller/drivers/lbaasv2.py View File

@@ -29,6 +29,7 @@ from oslo_utils import timeutils
29 29
 from kuryr_kubernetes import clients
30 30
 from kuryr_kubernetes import config
31 31
 from kuryr_kubernetes.controller.drivers import base
32
+from kuryr_kubernetes.controller.drivers import utils as c_utils
32 33
 from kuryr_kubernetes import exceptions as k_exc
33 34
 from kuryr_kubernetes.objects import lbaas as obj_lbaas
34 35
 from kuryr_kubernetes import utils
@@ -119,6 +120,7 @@ class LBaaSv2Driver(base.LBaaSDriver):
119 120
                     },
120 121
                 })
121 122
             sg_id = sg['security_group']['id']
123
+            c_utils.tag_neutron_resources('security-groups', [sg_id])
122 124
             loadbalancer.security_groups.append(sg_id)
123 125
             vip_port = self._get_vip_port(loadbalancer)
124 126
             neutron.update_port(
@@ -290,6 +292,7 @@ class LBaaSv2Driver(base.LBaaSDriver):
290 292
                         },
291 293
                     })
292 294
                 sg_id = sg['security_group']['id']
295
+                c_utils.tag_neutron_resources('security-groups', [sg_id])
293 296
                 loadbalancer.security_groups.append(sg_id)
294 297
                 vip_port = self._get_vip_port(loadbalancer)
295 298
                 neutron.update_port(

+ 2
- 0
kuryr_kubernetes/controller/drivers/namespace_security_groups.py View File

@@ -21,6 +21,7 @@ from kuryr_kubernetes import clients
21 21
 from kuryr_kubernetes import config
22 22
 from kuryr_kubernetes import constants
23 23
 from kuryr_kubernetes.controller.drivers import base
24
+from kuryr_kubernetes.controller.drivers import utils
24 25
 from kuryr_kubernetes import exceptions
25 26
 
26 27
 from neutronclient.common import exceptions as n_exc
@@ -107,6 +108,7 @@ class NamespacePodSecurityGroupsDriver(base.PodSecurityGroupsDriver):
107 108
                         "project_id": project_id
108 109
                     }
109 110
                 }).get('security_group')
111
+            utils.tag_neutron_resources('security-groups', [sg['id']])
110 112
             neutron.create_security_group_rule(
111 113
                 {
112 114
                     "security_group_rule": {

+ 3
- 0
kuryr_kubernetes/controller/drivers/namespace_subnet.py View File

@@ -19,6 +19,7 @@ from oslo_log import log as logging
19 19
 from kuryr_kubernetes import clients
20 20
 from kuryr_kubernetes import constants
21 21
 from kuryr_kubernetes.controller.drivers import default_subnet
22
+from kuryr_kubernetes.controller.drivers import utils as c_utils
22 23
 from kuryr_kubernetes import exceptions
23 24
 from kuryr_kubernetes import utils
24 25
 
@@ -121,6 +122,7 @@ class NamespacePodSubnetDriver(default_subnet.DefaultPodSubnetDriver):
121 122
                         "project_id": project_id
122 123
                     }
123 124
                 }).get('network')
125
+            c_utils.tag_neutron_resources('networks', [neutron_net['id']])
124 126
 
125 127
             # create a subnet within that network
126 128
             neutron_subnet = neutron.create_subnet(
@@ -134,6 +136,7 @@ class NamespacePodSubnetDriver(default_subnet.DefaultPodSubnetDriver):
134 136
                         "project_id": project_id
135 137
                     }
136 138
                 }).get('subnet')
139
+            c_utils.tag_neutron_resources('subnets', [neutron_subnet['id']])
137 140
 
138 141
             # connect the subnet to the router
139 142
             neutron.add_interface_router(router_id,

+ 2
- 0
kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py View File

@@ -19,6 +19,7 @@ from oslo_log import log as logging
19 19
 
20 20
 from kuryr_kubernetes import clients
21 21
 from kuryr_kubernetes.controller.drivers import nested_vif
22
+from kuryr_kubernetes.controller.drivers import utils
22 23
 from kuryr_kubernetes import exceptions as k_exc
23 24
 from kuryr_kubernetes import os_vif_util as ovu
24 25
 
@@ -37,6 +38,7 @@ class NestedMacvlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
37 38
                                      security_groups)
38 39
         vm_port = self._get_parent_port(neutron, pod)
39 40
         container_port = neutron.create_port(req).get('port')
41
+        utils.tag_neutron_resources('ports', [container_port['id']])
40 42
 
41 43
         container_mac = container_port['mac_address']
42 44
         container_ips = frozenset(entry['ip_address'] for entry in

+ 4
- 1
kuryr_kubernetes/controller/drivers/nested_vlan_vif.py View File

@@ -44,6 +44,7 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
44 44
 
45 45
         rq = self._get_port_request(pod, project_id, subnets, security_groups)
46 46
         port = neutron.create_port(rq).get('port')
47
+        utils.tag_neutron_resources('ports', [port['id']])
47 48
         vlan_id = self._add_subport(neutron, trunk_id, port['id'])
48 49
 
49 50
         return ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
@@ -78,12 +79,14 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
78 79
             LOG.error("There are no vlan ids available to create subports")
79 80
             return []
80 81
 
81
-        bulk_port_rq = {'ports': [port_rq for _ in range(len(subports_info))]}
82
+        bulk_port_rq = {'ports': [port_rq] * len(subports_info)}
82 83
         try:
83 84
             ports = neutron.create_port(bulk_port_rq).get('ports')
84 85
         except n_exc.NeutronClientException:
85 86
             LOG.exception("Error creating bulk ports: %s", bulk_port_rq)
86 87
             raise
88
+        utils.tag_neutron_resources('ports', [port['id'] for port in ports])
89
+
87 90
         for index, port in enumerate(ports):
88 91
             subports_info[index]['port_id'] = port['id']
89 92
 

+ 1
- 0
kuryr_kubernetes/controller/drivers/network_policy.py View File

@@ -167,6 +167,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver):
167 167
             # Create initial security group
168 168
             sg = self.neutron.create_security_group(body=security_group_body)
169 169
             sg_id = sg['security_group']['id']
170
+            driver_utils.tag_neutron_resources('security-groups', [sg_id])
170 171
             i_rules, e_rules = self.parse_network_policy_rules(policy, sg_id)
171 172
             for i_rule in i_rules:
172 173
                 sgr_id = driver_utils.create_security_group_rule(i_rule)

+ 3
- 1
kuryr_kubernetes/controller/drivers/neutron_vif.py View File

@@ -37,6 +37,7 @@ class NeutronPodVIFDriver(base.PodVIFDriver):
37 37
 
38 38
         rq = self._get_port_request(pod, project_id, subnets, security_groups)
39 39
         port = neutron.create_port(rq).get('port')
40
+        utils.tag_neutron_resources('ports', [port['id']])
40 41
         vif_plugin = self._get_vif_plugin(port)
41 42
 
42 43
         return ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
@@ -48,12 +49,13 @@ class NeutronPodVIFDriver(base.PodVIFDriver):
48 49
         rq = self._get_port_request(pod, project_id, subnets, security_groups,
49 50
                                     unbound=True)
50 51
 
51
-        bulk_port_rq = {'ports': [rq for _ in range(num_ports)]}
52
+        bulk_port_rq = {'ports': [rq] * num_ports}
52 53
         try:
53 54
             ports = neutron.create_port(bulk_port_rq).get('ports')
54 55
         except n_exc.NeutronClientException:
55 56
             LOG.exception("Error creating bulk ports: %s", bulk_port_rq)
56 57
             raise
58
+        utils.tag_neutron_resources('ports', [port['id'] for port in ports])
57 59
 
58 60
         vif_plugin = self._get_vif_plugin(ports[0])
59 61
 

+ 8
- 5
kuryr_kubernetes/controller/drivers/public_ip.py View File

@@ -13,10 +13,13 @@
13 13
 #    License for the specific language governing permissions and limitations
14 14
 #    under the License.
15 15
 import abc
16
-from kuryr_kubernetes import clients
16
+import six
17
+
17 18
 from neutronclient.common import exceptions as n_exc
18 19
 from oslo_log import log as logging
19
-import six
20
+
21
+from kuryr_kubernetes import clients
22
+from kuryr_kubernetes.controller.drivers import utils
20 23
 
21 24
 LOG = logging.getLogger(__name__)
22 25
 
@@ -132,13 +135,13 @@ class FipPubIpDriver(BasePubIpDriver):
132 135
             request['floatingip']['description'] = description
133 136
 
134 137
         try:
135
-            response = neutron.create_floatingip(request)
138
+            fip = neutron.create_floatingip(request).get('floatingip')
136 139
         except n_exc.NeutronClientException:
137 140
             LOG.exception("Failed to create floating IP - netid=%s ",
138 141
                           pub_net_id)
139 142
             raise
140
-        return response['floatingip']['id'], response[
141
-            'floatingip']['floating_ip_address']
143
+        utils.tag_neutron_resources('networks', [fip['id']])
144
+        return fip['id'], fip['floating_ip_address']
142 145
 
143 146
     def free_ip(self, res_id):
144 147
         neutron = clients.get_neutron_client()

+ 1
- 0
kuryr_kubernetes/controller/drivers/sriov.py View File

@@ -50,6 +50,7 @@ class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
50 50
                                     subnets, security_groups)
51 51
 
52 52
         port = neutron.create_port(rq).get('port')
53
+        c_utils.tag_neutron_resources('ports', [port['id']])
53 54
         vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
54 55
         vif.physnet = physnet
55 56
 

+ 13
- 0
kuryr_kubernetes/controller/drivers/utils.py View File

@@ -379,3 +379,16 @@ def get_namespace_subnet_cidr(namespace):
379 379
         LOG.exception("Kubernetes Client Exception.")
380 380
         raise
381 381
     return net_crd['spec']['subnetCIDR']
382
+
383
+
384
+def tag_neutron_resources(resource, res_ids):
385
+    tags = CONF.neutron_defaults.resource_tags
386
+    if tags:
387
+        neutron = clients.get_neutron_client()
388
+        for res_id in res_ids:
389
+            try:
390
+                neutron.replace_tag(resource, res_id, body={"tags": tags})
391
+            except n_exc.NeutronClientException:
392
+                LOG.warning("Failed to tag %s %s with %s. Ignoring, but this "
393
+                            "is still unexpected.", resource, res_id, tags,
394
+                            exc_info=True)

+ 2
- 2
kuryr_kubernetes/tests/unit/controller/drivers/test_neutron_vif.py View File

@@ -39,7 +39,7 @@ class NeutronPodVIFDriver(test_base.TestCase):
39 39
         project_id = mock.sentinel.project_id
40 40
         subnets = mock.sentinel.subnets
41 41
         security_groups = mock.sentinel.security_groups
42
-        port = mock.sentinel.port
42
+        port = {'id': '910b1183-1f4a-450a-a298-0e80ad06ec8b'}
43 43
         port_request = mock.sentinel.port_request
44 44
         vif = mock.sentinel.vif
45 45
         vif_plugin = mock.sentinel.vif_plugin
@@ -72,7 +72,7 @@ class NeutronPodVIFDriver(test_base.TestCase):
72 72
 
73 73
         port_request = mock.sentinel.port_request
74 74
         m_driver._get_port_request.return_value = port_request
75
-        port = mock.sentinel.port
75
+        port = {'id': '910b1183-1f4a-450a-a298-0e80ad06ec8b'}
76 76
         vif_plugin = mock.sentinel.vif_plugin
77 77
         vif = mock.sentinel.vif
78 78
         bulk_rq = {'ports': [port_request for _ in range(num_ports)]}

Loading…
Cancel
Save