From 19be1a2e26ccfea538479e75001108e1164148fd Mon Sep 17 00:00:00 2001 From: Benjamin Schanzel Date: Fri, 14 Aug 2020 15:22:09 +0200 Subject: [PATCH] OpenShift/k8s Provider: Basic Support for k8s nodeSelectors This adds support to specify node selectors on Pod node labels. They are used by the k8s scheduler to place a Pod on specific nodes with corresponding labels. This allows to place a build node/Pod on k8s nodes with certain capabilities (e.g. storage types, number of CPU cores, etc.) Change-Id: Ic00a84181c8ef66189e4259ef6434dc62b81c3c6 --- doc/source/configuration.rst | 22 +++++++++++++++++++ nodepool/driver/kubernetes/config.py | 5 ++++- nodepool/driver/kubernetes/provider.py | 17 +++++++++----- nodepool/driver/openshift/config.py | 5 ++++- nodepool/driver/openshift/provider.py | 19 +++++++++++----- nodepool/driver/openshiftpods/config.py | 1 + .../tests/fixtures/config_validate/good.yaml | 4 ++++ ...ovider-node-selector-6a16173ac11af61e.yaml | 6 +++++ 8 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/k8s-provider-node-selector-6a16173ac11af61e.yaml diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 9a67cecee..a1141352c 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -1423,6 +1423,14 @@ Selecting the kubernetes driver adds the following options to the The value of the environment variable passed to the Pod. + .. attr:: node-selector + :type: dict + + Only used by the + :value:`providers.[kubernetes].pools.labels.type.pod` label type; + A map of key-value pairs to ensure the Kubernetes scheduler + places the Pod on a node with specific node labels. + Openshift Driver ---------------- @@ -1594,6 +1602,14 @@ Selecting the openshift driver adds the following options to the The value of the environment variable passed to the Pod. + .. attr:: node-selector + :type: dict + + Only used by the + :value:`providers.[openshift].labels.type.pod` label type; + A map of key-value pairs to ensure the OpenShift scheduler + places the Pod on a node with specific node labels. + Openshift Pods Driver --------------------- @@ -1724,6 +1740,12 @@ Selecting the openshift pods driver adds the following options to the The value of the environment variable passed to the Pod. + .. attr:: node-selector + :type: dict + + A map of key-value pairs to ensure the OpenShift scheduler + places the Pod on a node with specific node labels. + AWS EC2 Driver -------------- diff --git a/nodepool/driver/kubernetes/config.py b/nodepool/driver/kubernetes/config.py index 3163cd0a1..aaf6c986e 100644 --- a/nodepool/driver/kubernetes/config.py +++ b/nodepool/driver/kubernetes/config.py @@ -31,7 +31,8 @@ class KubernetesLabel(ConfigValue): other.image == self.image and other.cpu == self.cpu and other.memory == self.memory and - other.env == self.env) + other.env == self.env and + other.node_selector == self.node_selector) return False def __repr__(self): @@ -63,6 +64,7 @@ class KubernetesPool(ConfigPool): pl.cpu = label.get('cpu') pl.memory = label.get('memory') pl.env = label.get('env', []) + pl.node_selector = label.get('node-selector') pl.pool = self self.labels[pl.name] = pl full_config.labels[label['name']].pools.append(self) @@ -113,6 +115,7 @@ class KubernetesProviderConfig(ProviderConfig): 'cpu': int, 'memory': int, 'env': [env_var], + 'node-selector': dict, } pool = ConfigPool.getCommonSchemaDict() diff --git a/nodepool/driver/kubernetes/provider.py b/nodepool/driver/kubernetes/provider.py index 3395ae89f..94b5a629f 100644 --- a/nodepool/driver/kubernetes/provider.py +++ b/nodepool/driver/kubernetes/provider.py @@ -274,7 +274,7 @@ class KubernetesProvider(Provider, QuotaSupport): return resource def createPod(self, node, pool, label): - spec_body = { + container_body = { 'name': label.name, 'image': label.image, 'imagePullPolicy': label.image_pull, @@ -285,22 +285,27 @@ class KubernetesProvider(Provider, QuotaSupport): } if label.cpu or label.memory: - spec_body['resources'] = {} + container_body['resources'] = {} for rtype in ('requests', 'limits'): rbody = {} if label.cpu: rbody['cpu'] = int(label.cpu) if label.memory: rbody['memory'] = '%dMi' % int(label.memory) - spec_body['resources'][rtype] = rbody + container_body['resources'][rtype] = rbody + + spec_body = { + 'containers': [container_body] + } + + if label.node_selector: + spec_body['nodeSelector'] = label.node_selector pod_body = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': label.name}, - 'spec': { - 'containers': [spec_body], - }, + 'spec': spec_body, 'restartPolicy': 'Never', } diff --git a/nodepool/driver/openshift/config.py b/nodepool/driver/openshift/config.py index f1a98b7c8..e50a4f307 100644 --- a/nodepool/driver/openshift/config.py +++ b/nodepool/driver/openshift/config.py @@ -32,7 +32,8 @@ class OpenshiftLabel(ConfigValue): other.cpu == self.cpu and other.memory == self.memory and other.python_path == self.python_path and - other.env == self.env) + other.env == self.env and + other.node_selector == self.node_selector) return False def __repr__(self): @@ -64,6 +65,7 @@ class OpenshiftPool(ConfigPool): pl.memory = label.get('memory') pl.python_path = label.get('python-path', 'auto') pl.env = label.get('env', []) + pl.node_selector = label.get('node-selector') pl.pool = self self.labels[pl.name] = pl full_config.labels[label['name']].pools.append(self) @@ -115,6 +117,7 @@ class OpenshiftProviderConfig(ProviderConfig): 'memory': int, 'python-path': str, 'env': [env_var], + 'node-selector': dict, } pool = ConfigPool.getCommonSchemaDict() diff --git a/nodepool/driver/openshift/provider.py b/nodepool/driver/openshift/provider.py index 60efa5c3f..8228a9407 100644 --- a/nodepool/driver/openshift/provider.py +++ b/nodepool/driver/openshift/provider.py @@ -204,7 +204,7 @@ class OpenshiftProvider(Provider): def createPod(self, project, pod_name, label): self.log.debug("%s: creating pod in project %s" % (pod_name, project)) - spec_body = { + container_body = { 'name': label.name, 'image': label.image, 'imagePullPolicy': label.image_pull, @@ -214,23 +214,30 @@ class OpenshiftProvider(Provider): 'env': label.env, } if label.cpu or label.memory: - spec_body['resources'] = {} + container_body['resources'] = {} for rtype in ('requests', 'limits'): rbody = {} if label.cpu: rbody['cpu'] = int(label.cpu) if label.memory: rbody['memory'] = '%dMi' % int(label.memory) - spec_body['resources'][rtype] = rbody + container_body['resources'][rtype] = rbody + + spec_body = { + 'containers': [container_body] + } + + if label.node_selector: + spec_body['nodeSelector'] = label.node_selector + pod_body = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': pod_name}, - 'spec': { - 'containers': [spec_body], - }, + 'spec': spec_body, 'restartPolicy': 'Never', } + self.k8s_client.create_namespaced_pod(project, pod_body) def waitForPod(self, project, pod_name): diff --git a/nodepool/driver/openshiftpods/config.py b/nodepool/driver/openshiftpods/config.py index ff54fd828..6cc531560 100644 --- a/nodepool/driver/openshiftpods/config.py +++ b/nodepool/driver/openshiftpods/config.py @@ -57,6 +57,7 @@ class OpenshiftPodsProviderConfig(OpenshiftProviderConfig): 'memory': int, 'python-path': str, 'env': [env_var], + 'node-selector': dict } pool = ConfigPool.getCommonSchemaDict() diff --git a/nodepool/tests/fixtures/config_validate/good.yaml b/nodepool/tests/fixtures/config_validate/good.yaml index bc8807322..acf446002 100644 --- a/nodepool/tests/fixtures/config_validate/good.yaml +++ b/nodepool/tests/fixtures/config_validate/good.yaml @@ -143,6 +143,8 @@ providers: value: hello - name: BAR value: world + node-selector: + storageType: ssd - name: openshift driver: openshift @@ -163,6 +165,8 @@ providers: value: hello - name: BAR value: world + node-selector: + storageType: ssd - name: ec2-us-east-2 driver: aws diff --git a/releasenotes/notes/k8s-provider-node-selector-6a16173ac11af61e.yaml b/releasenotes/notes/k8s-provider-node-selector-6a16173ac11af61e.yaml new file mode 100644 index 000000000..3937807b9 --- /dev/null +++ b/releasenotes/notes/k8s-provider-node-selector-6a16173ac11af61e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Basic support for specifying k8s/OpenShift `nodeSelectors` on Pod node + labels. This allows to schedule a Pod on k8s nodes with specific labels, + e.g., having certain capabilities.