Add PodResources service client

PodResources client could be used by sriov cni to obtain devices
allocated for container by sriov device-plugin.

KubeletPodResources service is still in alpha, so it should be
explicitly enabled in kubelet feature-gates:
    kubelet --feature-gates KubeletPodResources=true

New config option 'kubelet_root_dir' added to 'sriov' section
that defaults to kubelet default root-dir '/var/lib/kulelet'.
In case kubelet started with non-default root directory passed
via '--root-dir' option, the same value should be configured
in 'kubelet_root_dir'.
Note that if sriov binding driver will be used inside container
'kubelet_root_dir'/pod-resources directory should be mounted
to this container in order to allow communication with kubelet
via gRPC protocol over the unix domain socket.

Partial-Bug: 1826865
Depends-On: https://review.openstack.org/#/c/652629
Change-Id: Icf088b839db079efe9c7647c31be4ead867ed32b
Signed-off-by: Ilya Maximets <i.maximets@samsung.com>

cni.Dockerfile

@@ -5,7 +5,7 @@ ARG UPPER_CONSTRAINTS_FILE="https://git.openstack.org/cgit/openstack/requirement
 ARG OSLO_LOCK_PATH=/var/kuryr-lock
 
 RUN yum install -y epel-release https://rdoproject.org/repos/rdo-release.rpm \
-    && yum install -y --setopt=tsflags=nodocs python-pip iproute bridge-utils openvswitch sudo \
+    && yum install -y --setopt=tsflags=nodocs python-pip iproute bridge-utils openvswitch sudo libstdc++ \
     && yum install -y --setopt=tsflags=nodocs gcc python-devel git
 
 COPY . /opt/kuryr-kubernetes

cni_py3.Dockerfile

@@ -5,7 +5,7 @@ ARG UPPER_CONSTRAINTS_FILE="https://git.openstack.org/cgit/openstack/requirement
 ARG OSLO_LOCK_PATH=/var/kuryr-lock
 
 RUN dnf update -y \
-    && dnf install -y --setopt=tsflags=nodocs python36 iproute bridge-utils openvswitch sudo \
+    && dnf install -y --setopt=tsflags=nodocs python36 iproute bridge-utils openvswitch sudo libstdc++ \
     && dnf install -y --setopt=tsflags=nodocs gcc git
 
 COPY . /opt/kuryr-kubernetes

controller.Dockerfile

@@ -4,7 +4,7 @@ LABEL authors="Antoni Segura Puimedon<toni@kuryr.org>, Michał Dulko<mdulko@redh
 ARG UPPER_CONSTRAINTS_FILE="https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt"
 
 RUN yum install -y epel-release \
-    && yum install -y --setopt=tsflags=nodocs python-pip \
+    && yum install -y --setopt=tsflags=nodocs python-pip libstdc++ \
     && yum install -y --setopt=tsflags=nodocs gcc python-devel git
 
 COPY . /opt/kuryr-kubernetes

controller_py3.Dockerfile

@@ -4,7 +4,7 @@ LABEL authors="Antoni Segura Puimedon<toni@kuryr.org>, Michał Dulko<mdulko@redh
 ARG UPPER_CONSTRAINTS_FILE="https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt"
 
 RUN dnf update -y \
-    && dnf install -y --setopt=tsflags=nodocs python36 \
+    && dnf install -y --setopt=tsflags=nodocs python36 libstdc++ \
     && dnf install -y --setopt=tsflags=nodocs gcc git
 
 COPY . /opt/kuryr-kubernetes

kuryr_kubernetes/clients.py

@@ -20,11 +20,13 @@ from openstack import connection
 
 from kuryr_kubernetes import config
 from kuryr_kubernetes import k8s_client
+from kuryr_kubernetes.pod_resources import client as pr_client
 
 _clients = {}
 _NEUTRON_CLIENT = 'neutron-client'
 _KUBERNETES_CLIENT = 'kubernetes-client'
 _OPENSTACKSDK = 'openstacksdk'
+_POD_RESOURCES_CLIENT = 'pod-resources-client'
 
 
 def get_neutron_client():
@@ -43,6 +45,10 @@ def get_kubernetes_client():
     return _clients[_KUBERNETES_CLIENT]
 
 
+def get_pod_resources_client():
+    return _clients[_POD_RESOURCES_CLIENT]
+
+
 def setup_clients():
     setup_neutron_client()
     setup_kubernetes_client()
@@ -70,3 +76,8 @@ def setup_openstacksdk():
     session = utils.get_keystone_session('neutron', auth_plugin)
     conn = connection.Connection(session=session)
     _clients[_OPENSTACKSDK] = conn
+
+
+def setup_pod_resources_client():
+    root_dir = config.CONF.sriov.kubelet_root_dir
+    _clients[_POD_RESOURCES_CLIENT] = pr_client.PodResourcesClient(root_dir)

kuryr_kubernetes/config.py

@@ -250,6 +250,9 @@ nested_vif_driver_opts = [
 DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
 DEFAULT_DEVICE_MAPPINGS = []
 sriov_opts = [
+    cfg.StrOpt('kubelet_root_dir',
+               help=_("The root directory of the Kubelet daemon"),
+               default='/var/lib/kubelet'),
     cfg.DictOpt('default_physnet_subnets',
                 help=_("A mapping of default subnets for certain physnets "
                        "in a form of physnet-name:<SUBNET-ID>"),

kuryr_kubernetes/pod_resources/api.proto

@@ -0,0 +1,41 @@
+// Generated from kubernetes/pkg/kubelet/apis/podresources/v1alpha1/api.proto
+// by removing the 'gogoproto' dependencies.
+// To regenerate api_pb2.py and api_pb2_grpc.py use:
+//    python -m grpc_tools.protoc -I./  --python_out=. --grpc_python_out=. kuryr_kubernetes/pod_resources/api.proto
+
+syntax = 'proto3';
+
+package v1alpha1;
+
+// PodResourcesLister is a service provided by the kubelet that provides information about the
+// node resources consumed by pods and containers on the node
+service PodResourcesLister {
+    rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {}
+}
+
+// ListPodResourcesRequest is the request made to the PodResourcesLister service
+message ListPodResourcesRequest {}
+
+// ListPodResourcesResponse is the response returned by List function
+message ListPodResourcesResponse {
+    repeated PodResources pod_resources = 1;
+}
+
+// PodResources contains information about the node resources assigned to a pod
+message PodResources {
+    string name = 1;
+    string namespace = 2;
+    repeated ContainerResources containers = 3;
+}
+
+// ContainerResources contains information about the resources assigned to a container
+message ContainerResources {
+    string name = 1;
+    repeated ContainerDevices devices = 2;
+}
+
+// ContainerDevices contains information about the devices assigned to a container
+message ContainerDevices {
+    string resource_name = 1;
+    repeated string device_ids = 2;
+}

kuryr_kubernetes/pod_resources/api_pb2.py

@@ -0,0 +1,273 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: kuryr_kubernetes/pod_resources/api.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='kuryr_kubernetes/pod_resources/api.proto',
+  package='v1alpha1',
+  syntax='proto3',
+  serialized_options=None,
+  serialized_pb=_b('\n(kuryr_kubernetes/pod_resources/api.proto\x12\x08v1alpha1\"\x19\n\x17ListPodResourcesRequest\"I\n\x18ListPodResourcesResponse\x12-\n\rpod_resources\x18\x01 \x03(\x0b\x32\x16.v1alpha1.PodResources\"a\n\x0cPodResources\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x30\n\ncontainers\x18\x03 \x03(\x0b\x32\x1c.v1alpha1.ContainerResources\"O\n\x12\x43ontainerResources\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x1a.v1alpha1.ContainerDevices\"=\n\x10\x43ontainerDevices\x12\x15\n\rresource_name\x18\x01 \x01(\t\x12\x12\n\ndevice_ids\x18\x02 \x03(\t2e\n\x12PodResourcesLister\x12O\n\x04List\x12!.v1alpha1.ListPodResourcesRequest\x1a\".v1alpha1.ListPodResourcesResponse\"\x00\x62\x06proto3')
+)
+
+
+
+
+_LISTPODRESOURCESREQUEST = _descriptor.Descriptor(
+  name='ListPodResourcesRequest',
+  full_name='v1alpha1.ListPodResourcesRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=54,
+  serialized_end=79,
+)
+
+
+_LISTPODRESOURCESRESPONSE = _descriptor.Descriptor(
+  name='ListPodResourcesResponse',
+  full_name='v1alpha1.ListPodResourcesResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='pod_resources', full_name='v1alpha1.ListPodResourcesResponse.pod_resources', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=81,
+  serialized_end=154,
+)
+
+
+_PODRESOURCES = _descriptor.Descriptor(
+  name='PodResources',
+  full_name='v1alpha1.PodResources',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='v1alpha1.PodResources.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='namespace', full_name='v1alpha1.PodResources.namespace', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='containers', full_name='v1alpha1.PodResources.containers', index=2,
+      number=3, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=156,
+  serialized_end=253,
+)
+
+
+_CONTAINERRESOURCES = _descriptor.Descriptor(
+  name='ContainerResources',
+  full_name='v1alpha1.ContainerResources',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='v1alpha1.ContainerResources.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='devices', full_name='v1alpha1.ContainerResources.devices', index=1,
+      number=2, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=255,
+  serialized_end=334,
+)
+
+
+_CONTAINERDEVICES = _descriptor.Descriptor(
+  name='ContainerDevices',
+  full_name='v1alpha1.ContainerDevices',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='resource_name', full_name='v1alpha1.ContainerDevices.resource_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='device_ids', full_name='v1alpha1.ContainerDevices.device_ids', index=1,
+      number=2, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=336,
+  serialized_end=397,
+)
+
+_LISTPODRESOURCESRESPONSE.fields_by_name['pod_resources'].message_type = _PODRESOURCES
+_PODRESOURCES.fields_by_name['containers'].message_type = _CONTAINERRESOURCES
+_CONTAINERRESOURCES.fields_by_name['devices'].message_type = _CONTAINERDEVICES
+DESCRIPTOR.message_types_by_name['ListPodResourcesRequest'] = _LISTPODRESOURCESREQUEST
+DESCRIPTOR.message_types_by_name['ListPodResourcesResponse'] = _LISTPODRESOURCESRESPONSE
+DESCRIPTOR.message_types_by_name['PodResources'] = _PODRESOURCES
+DESCRIPTOR.message_types_by_name['ContainerResources'] = _CONTAINERRESOURCES
+DESCRIPTOR.message_types_by_name['ContainerDevices'] = _CONTAINERDEVICES
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ListPodResourcesRequest = _reflection.GeneratedProtocolMessageType('ListPodResourcesRequest', (_message.Message,), dict(
+  DESCRIPTOR = _LISTPODRESOURCESREQUEST,
+  __module__ = 'kuryr_kubernetes.pod_resources.api_pb2'
+  # @@protoc_insertion_point(class_scope:v1alpha1.ListPodResourcesRequest)
+  ))
+_sym_db.RegisterMessage(ListPodResourcesRequest)
+
+ListPodResourcesResponse = _reflection.GeneratedProtocolMessageType('ListPodResourcesResponse', (_message.Message,), dict(
+  DESCRIPTOR = _LISTPODRESOURCESRESPONSE,
+  __module__ = 'kuryr_kubernetes.pod_resources.api_pb2'
+  # @@protoc_insertion_point(class_scope:v1alpha1.ListPodResourcesResponse)
+  ))
+_sym_db.RegisterMessage(ListPodResourcesResponse)
+
+PodResources = _reflection.GeneratedProtocolMessageType('PodResources', (_message.Message,), dict(
+  DESCRIPTOR = _PODRESOURCES,
+  __module__ = 'kuryr_kubernetes.pod_resources.api_pb2'
+  # @@protoc_insertion_point(class_scope:v1alpha1.PodResources)
+  ))
+_sym_db.RegisterMessage(PodResources)
+
+ContainerResources = _reflection.GeneratedProtocolMessageType('ContainerResources', (_message.Message,), dict(
+  DESCRIPTOR = _CONTAINERRESOURCES,
+  __module__ = 'kuryr_kubernetes.pod_resources.api_pb2'
+  # @@protoc_insertion_point(class_scope:v1alpha1.ContainerResources)
+  ))
+_sym_db.RegisterMessage(ContainerResources)
+
+ContainerDevices = _reflection.GeneratedProtocolMessageType('ContainerDevices', (_message.Message,), dict(
+  DESCRIPTOR = _CONTAINERDEVICES,
+  __module__ = 'kuryr_kubernetes.pod_resources.api_pb2'
+  # @@protoc_insertion_point(class_scope:v1alpha1.ContainerDevices)
+  ))
+_sym_db.RegisterMessage(ContainerDevices)
+
+
+
+_PODRESOURCESLISTER = _descriptor.ServiceDescriptor(
+  name='PodResourcesLister',
+  full_name='v1alpha1.PodResourcesLister',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  serialized_start=399,
+  serialized_end=500,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='List',
+    full_name='v1alpha1.PodResourcesLister.List',
+    index=0,
+    containing_service=None,
+    input_type=_LISTPODRESOURCESREQUEST,
+    output_type=_LISTPODRESOURCESRESPONSE,
+    serialized_options=None,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_PODRESOURCESLISTER)
+
+DESCRIPTOR.services_by_name['PodResourcesLister'] = _PODRESOURCESLISTER
+
+# @@protoc_insertion_point(module_scope)

kuryr_kubernetes/pod_resources/api_pb2_grpc.py

@@ -0,0 +1,48 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+from kuryr_kubernetes.pod_resources import api_pb2 as kuryr__kubernetes_dot_pod__resources_dot_api__pb2
+
+
+class PodResourcesListerStub(object):
+  """PodResourcesLister is a service provided by the kubelet that provides information about the
+  node resources consumed by pods and containers on the node
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.List = channel.unary_unary(
+        '/v1alpha1.PodResourcesLister/List',
+        request_serializer=kuryr__kubernetes_dot_pod__resources_dot_api__pb2.ListPodResourcesRequest.SerializeToString,
+        response_deserializer=kuryr__kubernetes_dot_pod__resources_dot_api__pb2.ListPodResourcesResponse.FromString,
+        )
+
+
+class PodResourcesListerServicer(object):
+  """PodResourcesLister is a service provided by the kubelet that provides information about the
+  node resources consumed by pods and containers on the node
+  """
+
+  def List(self, request, context):
+    # missing associated documentation comment in .proto file
+    pass
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_PodResourcesListerServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'List': grpc.unary_unary_rpc_method_handler(
+          servicer.List,
+          request_deserializer=kuryr__kubernetes_dot_pod__resources_dot_api__pb2.ListPodResourcesRequest.FromString,
+          response_serializer=kuryr__kubernetes_dot_pod__resources_dot_api__pb2.ListPodResourcesResponse.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'v1alpha1.PodResourcesLister', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

kuryr_kubernetes/pod_resources/client.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2019 Samsung Electronics Co.,Ltd
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log
+
+import grpc
+
+from kuryr_kubernetes.pod_resources import api_pb2
+from kuryr_kubernetes.pod_resources import api_pb2_grpc
+
+LOG = log.getLogger(__name__)
+
+POD_RESOURCES_SOCKET = '/pod-resources/kubelet.sock'
+
+
+class PodResourcesClient(object):
+
+    def __init__(self, kubelet_root_dir):
+        socket = 'unix:' + kubelet_root_dir + POD_RESOURCES_SOCKET
+        LOG.debug("Creating PodResourcesClient on socket: %s", socket)
+        self._channel = grpc.insecure_channel(socket)
+        self._stub = api_pb2_grpc.PodResourcesListerStub(self._channel)
+
+    def list(self):
+        try:
+            response = self._stub.List(api_pb2.ListPodResourcesRequest())
+            LOG.debug("PodResourceResponse: %s", response)
+            return response
+        except grpc.RpcError as e:
+            LOG.error("ListPodResourcesRequest failed: %s", e)
+            raise

lower-constraints.txt

@@ -30,6 +30,7 @@ Flask==0.12.3
 future==0.16.0
 futurist==1.6.0
 greenlet==0.4.13
+grpcio==1.12.0
 hacking==0.12.0
 idna==2.6
 imagesize==1.0.0
@@ -86,6 +87,7 @@ pep8==1.5.7
 pika==0.10.0
 pika-pool==0.1.3
 prettytable==0.7.2
+protobuf==3.5.2
 psutil==5.4.3
 pycparser==2.18
 pyflakes==0.8.1

requirements.txt

@@ -23,3 +23,5 @@ pyroute2>=0.5.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
 retrying!=1.3.0,>=1.2.3 # Apache-2.0
 six>=1.10.0 # MIT
 stevedore>=1.20.0 # Apache-2.0
+grpcio>=1.12.0 # Apache-2.0
+protobuf>=3.5.2 # 3-Clause BSD

tox.ini

@@ -59,7 +59,7 @@ show-source = true
 enable-extensions = H106,H203
 
 # TODO(dougw) neutron/tests/unit/vmware exclusion is a temporary services split hack
-exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,.ropeproject,rally-scenarios,neutron/tests/unit/vmware*,releasenotes
+exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,.ropeproject,rally-scenarios,neutron/tests/unit/vmware*,releasenotes,kuryr_kubernetes/pod_resources/api_pb2*
 
 [testenv:pylint]
 basepython = python3