From e56f5594e469bdec933c6a4bd6c1ae8a02d5a90c Mon Sep 17 00:00:00 2001 From: mbecker Date: Tue, 14 Feb 2023 15:56:03 +0100 Subject: [PATCH] Add gpu support for k8s/openshift pods This adds the option to request GPUs for kubernetes and openshift pods. Since the resource name depends on the GPU vendor and the cluster installation, this option is left for the user to define it in the node pool. To leverage the ability of some schedulers to use fractional GPUs, the actual GPU value is read as a string. For GPUs, requests and limits cannot be decoupled (cf. https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/), so the same value will be used for requests and limits. Change-Id: Ibe33b06c374a431f164080edb34c3a501c360df7 --- doc/source/kubernetes.rst | 19 ++++++++++++++ doc/source/openshift-pods.rst | 19 ++++++++++++++ doc/source/openshift.rst | 19 ++++++++++++++ nodepool/driver/kubernetes/config.py | 4 +++ nodepool/driver/kubernetes/provider.py | 3 +++ nodepool/driver/openshift/config.py | 4 +++ nodepool/driver/openshift/provider.py | 3 +++ nodepool/driver/openshiftpods/config.py | 2 ++ .../kubernetes-default-resources.yaml | 4 +++ .../fixtures/openshift-default-resources.yaml | 4 +++ nodepool/tests/unit/test_driver_kubernetes.py | 26 +++++++++++++++++++ nodepool/tests/unit/test_driver_openshift.py | 26 +++++++++++++++++++ .../notes/pod-gpu-0edcd573dd813244.yaml | 5 ++++ 13 files changed, 138 insertions(+) create mode 100644 releasenotes/notes/pod-gpu-0edcd573dd813244.yaml diff --git a/doc/source/kubernetes.rst b/doc/source/kubernetes.rst index 09b9e0736..ec81267f5 100644 --- a/doc/source/kubernetes.rst +++ b/doc/source/kubernetes.rst @@ -298,6 +298,25 @@ Selecting the kubernetes driver adds the following options to the label type; specifies the ephemeral-storage limit in MB for the pod. + .. attr:: gpu + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the amount of gpu allocated to the pod. + This will be used to set both requests and limits to the same + value, based on how kubernetes assigns gpu resources: + https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/. + + .. attr:: gpu-resource + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the custom schedulable resource + associated with the installed gpu that is available + in the cluster. + .. attr:: env :type: list :default: [] diff --git a/doc/source/openshift-pods.rst b/doc/source/openshift-pods.rst index 1d39d33ef..4a90144fb 100644 --- a/doc/source/openshift-pods.rst +++ b/doc/source/openshift-pods.rst @@ -209,6 +209,25 @@ Selecting the openshift pods driver adds the following options to the Specifies the ephemeral-storage limit in MB for the pod. + .. attr:: gpu + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the amount of gpu allocated to the pod. + This will be used to set both requests and limits to the same + value, based on how kubernetes assigns gpu resources: + https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/. + + .. attr:: gpu-resource + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the custom schedulable resource + associated with the installed gpu that is available + in the cluster. + .. attr:: python-path :type: str :default: auto diff --git a/doc/source/openshift.rst b/doc/source/openshift.rst index 3345e04be..55fd18339 100644 --- a/doc/source/openshift.rst +++ b/doc/source/openshift.rst @@ -296,6 +296,25 @@ Selecting the openshift driver adds the following options to the label type; specifies the ephemeral-storage limit in MB for the pod. + .. attr:: gpu + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the amount of gpu allocated to the pod. + This will be used to set both requests and limits to the same + value, based on how kubernetes assigns gpu resources: + https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/. + + .. attr:: gpu-resource + :type: str + + Only used by the + :value:`providers.[openshift].pools.labels.type.pod` + label type; specifies the custom schedulable resource + associated with the installed gpu that is available + in the cluster. + .. attr:: env :type: list :default: [] diff --git a/nodepool/driver/kubernetes/config.py b/nodepool/driver/kubernetes/config.py index cab87f9eb..46e9adc26 100644 --- a/nodepool/driver/kubernetes/config.py +++ b/nodepool/driver/kubernetes/config.py @@ -72,6 +72,8 @@ class KubernetesPool(ConfigPool): 'memory-limit', default_memory_limit) pl.storage_limit = label.get( 'storage-limit', default_storage_limit) + pl.gpu = label.get('gpu') + pl.gpu_resource = label.get('gpu-resource') pl.env = label.get('env', []) pl.node_selector = label.get('node-selector') pl.privileged = label.get('privileged') @@ -126,6 +128,8 @@ class KubernetesProviderConfig(ProviderConfig): 'cpu-limit': int, 'memory-limit': int, 'storage-limit': int, + 'gpu': str, + 'gpu-resource': str, 'env': [env_var], 'node-selector': dict, 'privileged': bool, diff --git a/nodepool/driver/kubernetes/provider.py b/nodepool/driver/kubernetes/provider.py index a8196e640..84d27a9cd 100644 --- a/nodepool/driver/kubernetes/provider.py +++ b/nodepool/driver/kubernetes/provider.py @@ -332,6 +332,9 @@ class KubernetesProvider(Provider, QuotaSupport): limits['memory'] = '%dMi' % int(label.memory_limit) if label.storage_limit: limits['ephemeral-storage'] = '%dM' % int(label.storage_limit) + if label.gpu_resource and label.gpu: + requests[label.gpu_resource] = label.gpu + limits[label.gpu_resource] = label.gpu resources = {} if requests: resources['requests'] = requests diff --git a/nodepool/driver/openshift/config.py b/nodepool/driver/openshift/config.py index 3cac55e51..25487a14b 100644 --- a/nodepool/driver/openshift/config.py +++ b/nodepool/driver/openshift/config.py @@ -70,6 +70,8 @@ class OpenshiftPool(ConfigPool): 'memory-limit', default_memory_limit) pl.storage_limit = label.get( 'storage-limit', default_storage_limit) + pl.gpu = label.get('gpu') + pl.gpu_resource = label.get('gpu-resource') pl.python_path = label.get('python-path', 'auto') pl.shell_type = label.get('shell-type') pl.env = label.get('env', []) @@ -126,6 +128,8 @@ class OpenshiftProviderConfig(ProviderConfig): 'cpu-limit': int, 'memory-limit': int, 'storage-limit': int, + 'gpu': str, + 'gpu-resource': str, 'python-path': str, 'shell-type': str, 'env': [env_var], diff --git a/nodepool/driver/openshift/provider.py b/nodepool/driver/openshift/provider.py index 1e7e61e9d..7cd62a430 100644 --- a/nodepool/driver/openshift/provider.py +++ b/nodepool/driver/openshift/provider.py @@ -248,6 +248,9 @@ class OpenshiftProvider(Provider, QuotaSupport): limits['memory'] = '%dMi' % int(label.memory_limit) if label.storage_limit: limits['ephemeral-storage'] = '%dM' % int(label.storage_limit) + if label.gpu_resource and label.gpu: + requests[label.gpu_resource] = label.gpu + limits[label.gpu_resource] = label.gpu resources = {} if requests: resources['requests'] = requests diff --git a/nodepool/driver/openshiftpods/config.py b/nodepool/driver/openshiftpods/config.py index c1390d48b..00603b3ad 100644 --- a/nodepool/driver/openshiftpods/config.py +++ b/nodepool/driver/openshiftpods/config.py @@ -60,6 +60,8 @@ class OpenshiftPodsProviderConfig(OpenshiftProviderConfig): 'cpu-limit': int, 'memory-limit': int, 'storage-limit': int, + 'gpu': str, + 'gpu-resource': str, 'python-path': str, 'shell-type': str, 'env': [env_var], diff --git a/nodepool/tests/fixtures/kubernetes-default-resources.yaml b/nodepool/tests/fixtures/kubernetes-default-resources.yaml index b5dae654d..0f2a4e4d4 100644 --- a/nodepool/tests/fixtures/kubernetes-default-resources.yaml +++ b/nodepool/tests/fixtures/kubernetes-default-resources.yaml @@ -35,3 +35,7 @@ providers: - name: pod-custom-storage type: pod storage: 20 + - name: pod-custom-gpu + type: pod + gpu-resource: gpu-vendor.example/example-gpu + gpu: '1' diff --git a/nodepool/tests/fixtures/openshift-default-resources.yaml b/nodepool/tests/fixtures/openshift-default-resources.yaml index b04e4d9ae..cd080c4e3 100644 --- a/nodepool/tests/fixtures/openshift-default-resources.yaml +++ b/nodepool/tests/fixtures/openshift-default-resources.yaml @@ -35,3 +35,7 @@ providers: - name: pod-custom-storage type: pod storage: 20 + - name: pod-custom-gpu + type: pod + gpu-resource: gpu-vendor.example/example-gpu + gpu: '1' diff --git a/nodepool/tests/unit/test_driver_kubernetes.py b/nodepool/tests/unit/test_driver_kubernetes.py index cc43b8749..4a2b40616 100644 --- a/nodepool/tests/unit/test_driver_kubernetes.py +++ b/nodepool/tests/unit/test_driver_kubernetes.py @@ -259,6 +259,7 @@ class TestDriverKubernetes(tests.DBTestCase): req.node_types.append('pod-custom-cpu') req.node_types.append('pod-custom-mem') req.node_types.append('pod-custom-storage') + req.node_types.append('pod-custom-gpu') self.zk.storeNodeRequest(req) self.log.debug("Waiting for request %s", req.id) @@ -270,6 +271,7 @@ class TestDriverKubernetes(tests.DBTestCase): node_cust_cpu = self.zk.getNode(req.nodes[1]) node_cust_mem = self.zk.getNode(req.nodes[2]) node_cust_storage = self.zk.getNode(req.nodes[3]) + node_cust_gpu = self.zk.getNode(req.nodes[4]) resources_default = { 'instances': 1, @@ -295,12 +297,20 @@ class TestDriverKubernetes(tests.DBTestCase): 'ram': 1024, 'ephemeral-storage': 20, } + resources_cust_gpu = { + 'instances': 1, + 'cores': 2, + 'ram': 1024, + 'ephemeral-storage': 10, + 'gpu': 1 + } self.assertDictEqual(resources_default, node_default.resources) self.assertDictEqual(resources_cust_cpu, node_cust_cpu.resources) self.assertDictEqual(resources_cust_mem, node_cust_mem.resources) self.assertDictEqual(resources_cust_storage, node_cust_storage.resources) + self.assertDictEqual(resources_cust_gpu, node_cust_gpu.resources) ns, pod = self.fake_k8s_client._pod_requests[0] self.assertEqual(pod['spec']['containers'][0]['resources'], { @@ -358,6 +368,22 @@ class TestDriverKubernetes(tests.DBTestCase): }, }) + ns, pod = self.fake_k8s_client._pod_requests[3] + self.assertEqual(pod['spec']['containers'][0]['resources'], { + 'limits': { + 'cpu': 2, + 'ephemeral-storage': '10M', + 'memory': '1024Mi', + 'gpu-vendor.example/example-gpu': 1 + }, + 'requests': { + 'cpu': 2, + 'ephemeral-storage': '10M', + 'memory': '1024Mi', + 'gpu-vendor.example/example-gpu': 1 + }, + }) + for node in (node_default, node_cust_cpu, node_cust_mem): node.state = zk.DELETING self.zk.storeNode(node) diff --git a/nodepool/tests/unit/test_driver_openshift.py b/nodepool/tests/unit/test_driver_openshift.py index b2cbc277e..db7243bc4 100644 --- a/nodepool/tests/unit/test_driver_openshift.py +++ b/nodepool/tests/unit/test_driver_openshift.py @@ -271,6 +271,7 @@ class TestDriverOpenshift(tests.DBTestCase): req.node_types.append('pod-custom-cpu') req.node_types.append('pod-custom-mem') req.node_types.append('pod-custom-storage') + req.node_types.append('pod-custom-gpu') self.zk.storeNodeRequest(req) self.log.debug("Waiting for request %s", req.id) @@ -282,6 +283,7 @@ class TestDriverOpenshift(tests.DBTestCase): node_cust_cpu = self.zk.getNode(req.nodes[1]) node_cust_mem = self.zk.getNode(req.nodes[2]) node_cust_storage = self.zk.getNode(req.nodes[3]) + node_cust_gpu = self.zk.getNode(req.nodes[4]) resources_default = { 'instances': 1, @@ -307,12 +309,20 @@ class TestDriverOpenshift(tests.DBTestCase): 'ram': 1024, 'ephemeral-storage': 20, } + resources_cust_gpu = { + 'instances': 1, + 'cores': 2, + 'ram': 1024, + 'ephemeral-storage': 20, + 'gpu': 1, + } self.assertDictEqual(resources_default, node_default.resources) self.assertDictEqual(resources_cust_cpu, node_cust_cpu.resources) self.assertDictEqual(resources_cust_mem, node_cust_mem.resources) self.assertDictEqual(resources_cust_storage, node_cust_storage.resources) + self.assertDictEqual(resources_cust_gpu, node_cust_gpu.resources) ns, pod = self.fake_k8s_client._pod_requests[0] self.assertEqual(pod['spec']['containers'][0]['resources'], { @@ -370,6 +380,22 @@ class TestDriverOpenshift(tests.DBTestCase): }, }) + ns, pod = self.fake_k8s_client._pod_requests[3] + self.assertEqual(pod['spec']['containers'][0]['resources'], { + 'limits': { + 'cpu': 2, + 'ephemeral-storage': '10M', + 'memory': '1024Mi', + 'gpu-vendor.example/example-gpu': 1 + }, + 'requests': { + 'cpu': 2, + 'ephemeral-storage': '10M', + 'memory': '1024Mi', + 'gpu-vendor.example/example-gpu': 1 + }, + }) + for node in (node_default, node_cust_cpu, node_cust_mem): node.state = zk.DELETING self.zk.storeNode(node) diff --git a/releasenotes/notes/pod-gpu-0edcd573dd813244.yaml b/releasenotes/notes/pod-gpu-0edcd573dd813244.yaml new file mode 100644 index 000000000..afc284de3 --- /dev/null +++ b/releasenotes/notes/pod-gpu-0edcd573dd813244.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for requesting gpu resources + in kubernetes and openshift drivers.