zuul
/
nodepool
Code Issues Proposed changes

Merge "Add scheduler, volumes, and labels to k8s/openshift"

This commit is contained in:
Zuul 2023-03-13 08:32:16 +00:00 committed by Gerrit Code Review
parent ccb530a166 9bf44b4a4c
commit 7ffb3f1fd6
17 changed files with 433 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
doc/source/kubernetes.rst
View File

@ -188,6 +188,15 @@ Selecting the kubernetes driver adds the following options to the
The ImagePullPolicy, can be IfNotPresent, Always or Never.
.. attr:: labels
:type: dict
A dictionary of additional values to be added to the
namespace or pod metadata. The value of this field is
added to the `metadata.labels` field in Kubernetes. Note
that this field contains arbitrary key/value pairs and is
unrelated to the concept of labels in Nodepool.
.. attr:: python-path
:type: str
:default: auto
@ -262,6 +271,15 @@ Selecting the kubernetes driver adds the following options to the
A map of key-value pairs to ensure the Kubernetes scheduler
places the Pod on a node with specific node labels.
.. attr:: scheduler-name
:type: str
Only used by the
:value:`providers.[kubernetes].pools.labels.type.pod`
label type. Sets the `schedulerName` field on the
container. Normally left unset for the Kubernetes
default.
.. attr:: privileged
:type: bool
@ -269,3 +287,21 @@ Selecting the kubernetes driver adds the following options to the
:value:`providers.[kubernetes].pools.labels.type.pod`
label type. Sets the `securityContext.privileged` flag on
the container. Normally left unset for the Kubernetes default.
.. attr:: volumes
:type: list
Only used by the
:value:`providers.[kubernetes].pools.labels.type.pod`
label type. Sets the `volumes` field on the pod. If
supplied, this should be a list of Kubernetes Pod Volume
definitions.
.. attr:: volume-mounts
:type: list
Only used by the
:value:`providers.[kubernetes].pools.labels.type.pod`
label type. Sets the `volumeMounts` flag on the
container. If supplied, this should be a list of
Kubernetes Container VolumeMount definitions.

28
doc/source/openshift-pods.rst
View File

@ -123,6 +123,15 @@ Selecting the openshift pods driver adds the following options to the
image-pull-secrets:
- name: registry-secret
.. attr:: labels
:type: dict
A dictionary of additional values to be added to the
namespace or pod metadata. The value of this field is
added to the `metadata.labels` field in OpenShift. Note
that this field contains arbitrary key/value pairs and is
unrelated to the concept of labels in Nodepool.
.. attr:: cpu
:type: int
@ -182,8 +191,27 @@ Selecting the openshift pods driver adds the following options to the
A map of key-value pairs to ensure the OpenShift scheduler
places the Pod on a node with specific node labels.
.. attr:: scheduler-name
:type: str
Sets the `schedulerName` field on the container. Normally
left unset for the OpenShift default.
.. attr:: privileged
:type: bool
Sets the `securityContext.privileged` flag on the
container. Normally left unset for the OpenShift default.
.. attr:: volumes
:type: list
Sets the `volumes` field on the pod. If supplied, this
should be a list of OpenShift Pod Volume definitions.
.. attr:: volume-mounts
:type: list
Sets the `volumeMounts` flag on the container. If
supplied, this should be a list of OpenShift Container
VolumeMount definitions.

36
doc/source/openshift.rst
View File

@ -159,6 +159,15 @@ Selecting the openshift driver adds the following options to the
image-pull-secrets:
- name: registry-secret
.. attr:: labels
:type: dict
A dictionary of additional values to be added to the
namespace or pod metadata. The value of this field is
added to the `metadata.labels` field in OpenShift. Note
that this field contains arbitrary key/value pairs and is
unrelated to the concept of labels in Nodepool.
.. attr:: python-path
:type: str
:default: auto
@ -226,6 +235,15 @@ Selecting the openshift driver adds the following options to the
A map of key-value pairs to ensure the OpenShift scheduler
places the Pod on a node with specific node labels.
.. attr:: scheduler-name
:type: str
Only used by the
:value:`providers.[openshift].pools.labels.type.pod`
label type. Sets the `schedulerName` field on the
container. Normally left unset for the OpenShift
default.
.. attr:: privileged
:type: bool
@ -233,3 +251,21 @@ Selecting the openshift driver adds the following options to the
:value:`providers.[openshift].pools.labels.type.pod`
label type. Sets the `securityContext.privileged` flag on
the container. Normally left unset for the OpenShift default.
.. attr:: volumes
:type: list
Only used by the
:value:`providers.[openshift].pools.labels.type.pod`
label type. Sets the `volumes` field on the pod. If
supplied, this should be a list of OpenShift Pod Volume
definitions.
.. attr:: volume-mounts
:type: list
Only used by the
:value:`providers.[openshift].pools.labels.type.pod`
label type. Sets the `volumeMounts` flag on the
container. If supplied, this should be a list of
OpenShift Container VolumeMount definitions.

8
nodepool/driver/kubernetes/config.py
View File

@ -57,6 +57,10 @@ class KubernetesPool(ConfigPool):
pl.env = label.get('env', [])
pl.node_selector = label.get('node-selector')
pl.privileged = label.get('privileged')
pl.scheduler_name = label.get('scheduler-name')
pl.volumes = label.get('volumes')
pl.volume_mounts = label.get('volume-mounts')
pl.labels = label.get('labels')
pl.pool = self
self.labels[pl.name] = pl
full_config.labels[label['name']].pools.append(self)
@ -104,6 +108,10 @@ class KubernetesProviderConfig(ProviderConfig):
'env': [env_var],
'node-selector': dict,
'privileged': bool,
'scheduler-name': str,
'volumes': list,
'volume-mounts': list,
'labels': dict,
}
pool = ConfigPool.getCommonSchemaDict()

2
nodepool/driver/kubernetes/handler.py
View File

@ -32,7 +32,7 @@ class K8SLauncher(NodeLauncher):
self.log.debug("Creating resource")
if self.label.type == "namespace":
resource = self.handler.manager.createNamespace(
self.node, self.handler.pool.name)
self.node, self.handler.pool.name, self.label)
else:
resource = self.handler.manager.createPod(
self.node, self.handler.pool.name, self.label)

48
nodepool/driver/kubernetes/provider.py
View File

@ -155,23 +155,30 @@ class KubernetesProvider(Provider, QuotaSupport):
break
time.sleep(1)
def createNamespace(self, node, pool, restricted_access=False):
def createNamespace(self, node, pool, label, restricted_access=False):
name = node.id
namespace = "%s-%s" % (pool, name)
user = "zuul-worker"
self.log.debug("%s: creating namespace" % namespace)
k8s_labels = {}
if label.labels:
k8s_labels.update(label.labels)
k8s_labels.update({
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
'nodepool_node_label': label.name,
})
# Create the namespace
ns_body = {
'apiVersion': 'v1',
'kind': 'Namespace',
'metadata': {
'name': namespace,
'labels': {
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
}
'labels': k8s_labels,
}
}
proj = self.k8s_client.create_namespace(ns_body)
@ -330,28 +337,43 @@ class KubernetesProvider(Provider, QuotaSupport):
if label.node_selector:
spec_body['nodeSelector'] = label.node_selector
if label.scheduler_name:
spec_body['schedulerName'] = label.scheduler_name
if label.volumes:
spec_body['volumes'] = label.volumes
if label.volume_mounts:
container_body['volumeMounts'] = label.volume_mounts
if label.privileged is not None:
container_body['securityContext'] = {
'privileged': label.privileged,
}
k8s_labels = {}
if label.labels:
k8s_labels.update(label.labels)
k8s_labels.update({
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
'nodepool_node_label': label.name,
})
pod_body = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': label.name,
'labels': {
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
'nodepool_node_label': label.name,
}
'labels': k8s_labels,
},
'spec': spec_body,
'restartPolicy': 'Never',
}
resource = self.createNamespace(node, pool, restricted_access=True)
resource = self.createNamespace(node, pool, label,
restricted_access=True)
namespace = resource['namespace']
self.k8s_client.create_namespaced_pod(namespace, pod_body)

8
nodepool/driver/openshift/config.py
View File

@ -53,6 +53,10 @@ class OpenshiftPool(ConfigPool):
pl.env = label.get('env', [])
pl.node_selector = label.get('node-selector')
pl.privileged = label.get('privileged')
pl.scheduler_name = label.get('scheduler-name')
pl.volumes = label.get('volumes')
pl.volume_mounts = label.get('volume-mounts')
pl.labels = label.get('labels')
pl.pool = self
self.labels[pl.name] = pl
full_config.labels[label['name']].pools.append(self)
@ -101,6 +105,10 @@ class OpenshiftProviderConfig(ProviderConfig):
'env': [env_var],
'node-selector': dict,
'privileged': bool,
'scheduler-name': str,
'volumes': list,
'volume-mounts': list,
'labels': dict,
}
pool = ConfigPool.getCommonSchemaDict()

4
nodepool/driver/openshift/handler.py
View File

@ -31,12 +31,14 @@ class OpenshiftLauncher(NodeLauncher):
def _launchLabel(self):
self.log.debug("Creating resource")
project = "%s-%s" % (self.handler.pool.name, self.node.id)
self.node.external_id = self.handler.manager.createProject(project)
self.node.external_id = self.handler.manager.createProject(
self.node, self.handler.pool.name, project, self.label)
self.zk.storeNode(self.node)
resource = self.handler.manager.prepareProject(project)
if self.label.type == "pod":
self.handler.manager.createPod(
self.node, self.handler.pool.name,
project, self.label.name, self.label)
self.handler.manager.waitForPod(project, self.label.name)
resource['pod'] = self.label.name

40
nodepool/driver/openshift/provider.py
View File

@ -125,14 +125,26 @@ class OpenshiftProvider(Provider, QuotaSupport):
break
time.sleep(1)
def createProject(self, project):
def createProject(self, node, pool, project, label):
self.log.debug("%s: creating project" % project)
# Create the project
k8s_labels = {}
if label.labels:
k8s_labels.update(label.labels)
k8s_labels.update({
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
'nodepool_node_label': label.name,
})
proj_body = {
'apiVersion': 'project.openshift.io/v1',
'kind': 'ProjectRequest',
'metadata': {
'name': project,
'labels': k8s_labels,
}
}
projects = self.os_client.resources.get(
@ -211,7 +223,7 @@ class OpenshiftProvider(Provider, QuotaSupport):
self.log.info("%s: project created" % project)
return resource
def createPod(self, project, pod_name, label):
def createPod(self, node, pool, project, pod_name, label):
self.log.debug("%s: creating pod in project %s" % (pod_name, project))
container_body = {
'name': label.name,
@ -239,15 +251,37 @@ class OpenshiftProvider(Provider, QuotaSupport):
if label.node_selector:
spec_body['nodeSelector'] = label.node_selector
if label.scheduler_name:
spec_body['schedulerName'] = label.scheduler_name
if label.volumes:
spec_body['volumes'] = label.volumes
if label.volume_mounts:
container_body['volumeMounts'] = label.volume_mounts
if label.privileged is not None:
container_body['securityContext'] = {
'privileged': label.privileged,
}
k8s_labels = {}
if label.labels:
k8s_labels.update(label.labels)
k8s_labels.update({
'nodepool_node_id': node.id,
'nodepool_provider_name': self.provider.name,
'nodepool_pool_name': pool,
'nodepool_node_label': label.name,
})
pod_body = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {'name': pod_name},
'metadata': {
'name': pod_name,
'labels': k8s_labels,
},
'spec': spec_body,
'restartPolicy': 'Never',
}

4
nodepool/driver/openshiftpods/config.py
View File

@ -61,6 +61,10 @@ class OpenshiftPodsProviderConfig(OpenshiftProviderConfig):
'env': [env_var],
'node-selector': dict,
'privileged': bool,
'scheduler-name': str,
'volumes': list,
'volume-mounts': list,
'labels': dict,
}
pool = ConfigPool.getCommonSchemaDict()

3
nodepool/driver/openshiftpods/handler.py
View File

@ -25,7 +25,8 @@ class OpenshiftPodLauncher(OpenshiftLauncher):
self.log.debug("Creating resource")
pod_name = "%s-%s" % (self.label.name, self.node.id)
project = self.handler.pool.name
self.handler.manager.createPod(project, pod_name, self.label)
self.handler.manager.createPod(self.node, self.handler.pool.name,
project, pod_name, self.label)
self.node.external_id = "%s-%s" % (project, pod_name)
self.node.interface_ip = pod_name
self.zk.storeNode(self.node)

20
nodepool/tests/fixtures/config_validate/good.yaml vendored
View File

@ -158,6 +158,16 @@ providers:
node-selector:
storageType: ssd
privileged: true
volumes:
- name: my-csi-inline-vol
csi:
driver: inline.storage.kubernetes.io
volume-mounts:
- mountPath: "/data"
name: my-csi-inline-vol
scheduler-name: niftyScheduler
labels:
environment: qa
- name: openshift
driver: openshift
@ -181,6 +191,16 @@ providers:
node-selector:
storageType: ssd
privileged: true
volumes:
- name: my-csi-inline-vol
csi:
driver: inline.storage.kubernetes.io
volume-mounts:
- mountPath: "/data"
name: my-csi-inline-vol
scheduler-name: niftyScheduler
labels:
environment: qa
- name: ec2-us-east-2
driver: aws

17
nodepool/tests/fixtures/kubernetes.yaml vendored
View File

@ -14,6 +14,7 @@ tenant-resource-limits:
labels:
- name: pod-fedora
- name: pod-extra
- name: kubernetes-namespace
providers:
@ -31,3 +32,19 @@ providers:
- name: pod-fedora
type: pod
image: docker.io/fedora:28
- name: pod-extra
type: pod
image: docker.io/fedora:28
labels:
environment: qa
privileged: true
node-selector:
storageType: ssd
scheduler-name: myscheduler
volumes:
- name: my-csi-inline-vol
csi:
driver: inline.storage.kubernetes.io
volume-mounts:
- name: my-csi-inline-vol
mountPath: /data

21
nodepool/tests/fixtures/openshift.yaml vendored
View File

@ -14,6 +14,7 @@ tenant-resource-limits:
labels:
- name: pod-fedora
- name: pod-extra
- name: openshift-project
- name: pod-fedora-secret
@ -32,10 +33,26 @@ providers:
- name: pod-fedora
type: pod
image: docker.io/fedora:28
python-path: '/usr/bin/python3'
shell-type: csh
- name: pod-fedora-secret
type: pod
image: docker.io/fedora:28
image-pull-secrets:
- name: registry-secret
- name: pod-extra
type: pod
image: docker.io/fedora:28
python-path: '/usr/bin/python3'
shell-type: csh
labels:
environment: qa
privileged: true
node-selector:
storageType: ssd
scheduler-name: myscheduler
volumes:
- name: my-csi-inline-vol
csi:
driver: inline.storage.kubernetes.io
volume-mounts:
- name: my-csi-inline-vol
mountPath: /data

83
nodepool/tests/unit/test_driver_kubernetes.py
View File

@ -24,6 +24,7 @@ from nodepool.zk import zookeeper as zk
class FakeCoreClient(object):
def __init__(self):
self.namespaces = []
self._pod_requests = []
class FakeApi:
class configuration:
@ -73,7 +74,7 @@ class FakeCoreClient(object):
return FakeSecret
def create_namespaced_pod(self, ns, pod_body):
return
self._pod_requests.append((ns, pod_body))
def read_namespaced_pod(self, name, ns):
class FakePod:
@ -109,6 +110,7 @@ class TestDriverKubernetes(tests.DBTestCase):
fake_get_client))
def test_kubernetes_machine(self):
# Test a pod with default values
configfile = self.setup_config('kubernetes.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.start()
@ -133,12 +135,91 @@ class TestDriverKubernetes(tests.DBTestCase):
{'key1': 'value1', 'key2': 'value2'})
self.assertEqual(node.cloud, 'admin-cluster.local')
self.assertEqual(node.host_id, 'k8s-default-pool-abcd-1234')
ns, pod = self.fake_k8s_client._pod_requests[0]
self.assertEqual(pod['metadata'], {
'name': 'pod-fedora',
'labels': {
'nodepool_node_id': '0000000000',
'nodepool_provider_name': 'kubespray',
'nodepool_pool_name': 'main',
'nodepool_node_label': 'pod-fedora'
},
})
self.assertEqual(pod['spec'], {
'containers': [{
'name': 'pod-fedora',
'image': 'docker.io/fedora:28',
'imagePullPolicy': 'IfNotPresent',
'command': ['/bin/sh', '-c'],
'args': ['while true; do sleep 30; done;'],
'env': []
}],
})
node.state = zk.DELETING
self.zk.storeNode(node)
self.waitForNodeDeletion(node)
def test_kubernetes_machine_extra(self):
# Test a pod with lots of extra settings
configfile = self.setup_config('kubernetes.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.start()
req = zk.NodeRequest()
req.state = zk.REQUESTED
req.tenant_name = 'tenant-1'
req.node_types.append('pod-extra')
self.zk.storeNodeRequest(req)
self.log.debug("Waiting for request %s", req.id)
req = self.waitForNodeRequest(req)
self.assertEqual(req.state, zk.FULFILLED)
self.assertNotEqual(req.nodes, [])
node = self.zk.getNode(req.nodes[0])
self.assertEqual(node.allocated_to, req.id)
self.assertEqual(node.state, zk.READY)
self.assertIsNotNone(node.launcher)
self.assertEqual(node.connection_type, 'kubectl')
self.assertEqual(node.connection_port.get('token'), 'fake-token')
self.assertEqual(node.attributes,
{'key1': 'value1', 'key2': 'value2'})
self.assertEqual(node.cloud, 'admin-cluster.local')
self.assertEqual(node.host_id, 'k8s-default-pool-abcd-1234')
ns, pod = self.fake_k8s_client._pod_requests[0]
self.assertEqual(pod['metadata'], {
'name': 'pod-extra',
'labels': {
'environment': 'qa',
'nodepool_node_id': '0000000000',
'nodepool_provider_name': 'kubespray',
'nodepool_pool_name': 'main',
'nodepool_node_label': 'pod-extra'
},
})
self.assertEqual(pod['spec'], {
'containers': [{
'args': ['while true; do sleep 30; done;'],
'command': ['/bin/sh', '-c'],
'env': [],
'image': 'docker.io/fedora:28',
'imagePullPolicy': 'IfNotPresent',
'name': 'pod-extra',
'securityContext': {'privileged': True},
'volumeMounts': [{
'mountPath': '/data',
'name': 'my-csi-inline-vol'
}],
}],
'nodeSelector': {'storageType': 'ssd'},
'schedulerName': 'myscheduler',
'volumes': [{
'csi': {'driver': 'inline.storage.kubernetes.io'},
'name': 'my-csi-inline-vol'
}],
})
def test_kubernetes_native(self):
configfile = self.setup_config('kubernetes.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)

94
nodepool/tests/unit/test_driver_openshift.py
View File

@ -92,6 +92,9 @@ class FakeOpenshiftClient(object):
class FakeCoreClient(object):
def __init__(self):
self._pod_requests = []
def create_namespaced_service_account(self, ns, sa_body):
return
@ -108,7 +111,7 @@ class FakeCoreClient(object):
return FakeSecret
def create_namespaced_pod(self, ns, pod_body):
return
self._pod_requests.append((ns, pod_body))
def read_namespaced_pod(self, name, ns):
class FakePod:
@ -136,6 +139,7 @@ class TestDriverOpenshift(tests.DBTestCase):
fake_get_client))
def test_openshift_machine(self):
# Test a pod with default values
configfile = self.setup_config('openshift.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.start()
@ -149,6 +153,61 @@ class TestDriverOpenshift(tests.DBTestCase):
req = self.waitForNodeRequest(req)
self.assertEqual(req.state, zk.FULFILLED)
self.assertNotEqual(req.nodes, [])
node = self.zk.getNode(req.nodes[0])
self.assertEqual(node.allocated_to, req.id)
self.assertEqual(node.state, zk.READY)
self.assertIsNotNone(node.launcher)
self.assertEqual(node.connection_type, 'kubectl')
self.assertEqual(node.connection_port.get('token'), 'fake-token')
self.assertEqual(node.python_path, 'auto')
self.assertEqual(node.shell_type, None)
self.assertEqual(node.attributes,
{'key1': 'value1', 'key2': 'value2'})
self.assertEqual(node.cloud, 'admin-cluster.local')
self.assertIsNone(node.host_id)
ns, pod = self.fake_k8s_client._pod_requests[0]
self.assertEqual(pod['metadata'], {
'name': 'pod-fedora',
'labels': {
'nodepool_node_id': '0000000000',
'nodepool_provider_name': 'openshift',
'nodepool_pool_name': 'main',
'nodepool_node_label': 'pod-fedora'
},
})
self.assertEqual(pod['spec'], {
'containers': [{
'name': 'pod-fedora',
'image': 'docker.io/fedora:28',
'imagePullPolicy': 'IfNotPresent',
'command': ['/bin/sh', '-c'],
'args': ['while true; do sleep 30; done;'],
'env': []
}],
'imagePullSecrets': [],
})
node.state = zk.DELETING
self.zk.storeNode(node)
self.waitForNodeDeletion(node)
def test_openshift_machine_extra(self):
# Test a pod with lots of extra settings
configfile = self.setup_config('openshift.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.start()
req = zk.NodeRequest()
req.state = zk.REQUESTED
req.tenant_name = 'tenant-1'
req.node_types.append('pod-extra')
self.zk.storeNodeRequest(req)
self.log.debug("Waiting for request %s", req.id)
req = self.waitForNodeRequest(req)
self.assertEqual(req.state, zk.FULFILLED)
self.assertNotEqual(req.nodes, [])
node = self.zk.getNode(req.nodes[0])
self.assertEqual(node.allocated_to, req.id)
@ -162,6 +221,39 @@ class TestDriverOpenshift(tests.DBTestCase):
{'key1': 'value1', 'key2': 'value2'})
self.assertEqual(node.cloud, 'admin-cluster.local')
self.assertIsNone(node.host_id)
ns, pod = self.fake_k8s_client._pod_requests[0]
self.assertEqual(pod['metadata'], {
'name': 'pod-extra',
'labels': {
'environment': 'qa',
'nodepool_node_id': '0000000000',
'nodepool_provider_name': 'openshift',
'nodepool_pool_name': 'main',
'nodepool_node_label': 'pod-extra'
},
})
self.assertEqual(pod['spec'], {
'containers': [{
'args': ['while true; do sleep 30; done;'],
'command': ['/bin/sh', '-c'],
'env': [],
'image': 'docker.io/fedora:28',
'imagePullPolicy': 'IfNotPresent',
'name': 'pod-extra',
'securityContext': {'privileged': True},
'volumeMounts': [{
'mountPath': '/data',
'name': 'my-csi-inline-vol'
}],
}],
'nodeSelector': {'storageType': 'ssd'},
'schedulerName': 'myscheduler',
'imagePullSecrets': [],
'volumes': [{
'csi': {'driver': 'inline.storage.kubernetes.io'},
'name': 'my-csi-inline-vol'
}],
})
node.state = zk.DELETING
self.zk.storeNode(node)

4
releasenotes/notes/pod-volumes-schedulers-labels-e04764f014b07424.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Added support for specifying the scheduler name, additional metadata, and volume mounts in Kubernetes and OpenShift drivers.