zuul
/
nodepool
Code Issues Proposed changes

Merge "Add python-path option to node"

This commit is contained in:
Zuul 2019-05-10 16:10:56 +00:00 committed by Gerrit Code Review
parent 1714b6aae3 76aa62230c
commit e0359d1c99
21 changed files with 93 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
doc/source/configuration.rst
View File

@ -309,6 +309,12 @@ Options
The username that a consumer should use when connecting to the
node.
.. attr:: python-path
:type: string
:default: /usr/bin/python2
The path of the default python interpreter.
.. attr:: providers
:type: list
@ -682,6 +688,12 @@ Selecting the OpenStack driver adds the following options to the
The username that a consumer should use when connecting to the
node.
.. attr:: python-path
:type: str
:default: /usr/bin/python2
The path of the default python interpreter.
.. attr:: connection-type
:type: str
@ -1071,6 +1083,12 @@ Selecting the static driver adds the following options to the
The username nodepool will use to validate it can connect to the
node.
.. attr:: python-path
:type: str
:default: /usr/bin/python2
The path of the default python interpreter.
.. attr:: max-parallel-jobs
:type: int
:default: 1
@ -1188,6 +1206,12 @@ Selecting the kubernetes driver adds the following options to the
:value:`providers.[kubernetes].labels.type.pod` label type;
specifies the image name used by the pod.
.. attr:: python-path
:type: str
:default: /usr/bin/python2
The path of the default python interpreter.
Openshift Driver
----------------
@ -1309,6 +1333,12 @@ Selecting the openshift driver adds the following options to the
The ImagePullPolicy, can be IfNotPresent, Always or Never.
.. attr:: python-path
:type: str
:default: /usr/bin/python2
The path of the default python interpreter.
.. attr:: cpu
:type: int
@ -1451,6 +1481,12 @@ section of the configuration.
The username that a consumer should use when connecting to the node.
.. attr:: python-path
:type: str
:default: /usr/bin/python2
The path of the default python interpreter.
.. attr:: connection-type
:type: str

8
nodepool/builder.py
View File

@ -879,6 +879,7 @@ class BuildWorker(BaseWorker):
build_data.builder_id = self._builder_id
build_data.builder = self._hostname
build_data.username = diskimage.username
build_data.python_path = diskimage.python_path
if self._statsd:
pipeline = self._statsd.pipeline()
@ -991,7 +992,7 @@ class UploadWorker(BaseWorker):
self._config = new_config
def _uploadImage(self, build_id, upload_id, image_name, images, provider,
username):
username, python_path):
'''
Upload a local DIB image build to a provider.
@ -1002,6 +1003,7 @@ class UploadWorker(BaseWorker):
that available for uploading.
:param provider: The provider from the parsed config file.
:param username:
:param python_path:
'''
start_time = time.time()
timestamp = int(start_time)
@ -1073,6 +1075,7 @@ class UploadWorker(BaseWorker):
data.external_name = ext_image_name
data.format = image.extension
data.username = username
data.python_path = python_path
return data
@ -1170,13 +1173,14 @@ class UploadWorker(BaseWorker):
data = zk.ImageUpload()
data.state = zk.UPLOADING
data.username = build.username
data.python_path = build.python_path
upnum = self._zk.storeImageUpload(
image.name, build.id, provider.name, data)
data = self._uploadImage(build.id, upnum, image.name,
local_images, provider,
build.username)
build.username, build.python_path)
# Set final state
self._zk.storeImageUpload(image.name, build.id,

1
nodepool/cmd/config_validator.py
View File

@ -44,6 +44,7 @@ class ConfigValidator:
'rebuild-age': int,
'env-vars': {str: str},
'username': str,
'python-path': str,
'build-timeout': int,
}

3
nodepool/config.py
View File

@ -118,6 +118,7 @@ class Config(ConfigValue):
d.image_types = set(diskimage.get('formats', []))
d.pause = bool(diskimage.get('pause', False))
d.username = diskimage.get('username', 'zuul')
d.python_path = diskimage.get('python-path', '/usr/bin/python2')
d.build_timeout = diskimage.get('build-timeout', (8 * 60 * 60))
self.diskimages[d.name] = d
@ -180,6 +181,7 @@ class DiskImage(ConfigValue):
self.image_types = None
self.pause = False
self.username = None
self.python_path = None
self.build_timeout = None
def __eq__(self, other):
@ -192,6 +194,7 @@ class DiskImage(ConfigValue):
other.image_types == self.image_types and
other.pause == self.pause and
other.username == self.username and
other.python_path == self.python_path and
other.build_timeout == self.build_timeout)
return False

3
nodepool/driver/aws/config.py
View File

@ -34,6 +34,7 @@ class ProviderCloudImage(ConfigValue):
return (self.name == other.name
and self.image_id == other.image_id
and self.username == other.username
and self.python_path == other.python_path
and self.connection_type == other.connection_type
and self.connection_port == other.connection_port)
return False
@ -189,6 +190,7 @@ class AwsProviderConfig(ProviderConfig):
i.name = image['name']
i.image_id = image.get('image-id', None)
i.username = image.get('username', None)
i.python_path = image.get('python-path', '/usr/bin/python2')
i.connection_type = image.get('connection-type', 'ssh')
i.connection_port = image.get(
'connection-port',
@ -224,6 +226,7 @@ class AwsProviderConfig(ProviderConfig):
'connection-port': int,
'image-id': str,
'username': str,
'python-path': str,
}
provider = ProviderConfig.getCommonSchemaDict()

1
nodepool/driver/aws/handler.py
View File

@ -101,6 +101,7 @@ class AwsInstanceLauncher(NodeLauncher):
self.node.public_ipv4 = server_ip
self.node.host_keys = keys
self.node.username = self.label.cloud_image.username
self.node.python_path = self.label.cloud_image.python_path
self.zk.storeNode(self.node)
self.log.info("Instance %s is ready", instance_id)

3
nodepool/driver/kubernetes/config.py
View File

@ -27,6 +27,7 @@ class KubernetesLabel(ConfigValue):
return (other.name == self.name and
other.type == self.type and
other.image_pull == self.image_pull and
other.python_path == self.python_path and
other.image == self.image)
return False
@ -55,6 +56,7 @@ class KubernetesPool(ConfigPool):
pl.type = label['type']
pl.image = label.get('image')
pl.image_pull = label.get('image-pull', 'IfNotPresent')
pl.python_path = label.get('python-path', '/usr/bin/python2')
pl.pool = self
self.labels[pl.name] = pl
full_config.labels[label['name']].pools.append(self)
@ -96,6 +98,7 @@ class KubernetesProviderConfig(ProviderConfig):
v.Required('type'): str,
'image': str,
'image-pull': str,
'python-path': str,
}
pool = ConfigPool.getCommonSchemaDict()

1
nodepool/driver/kubernetes/handler.py
View File

@ -39,6 +39,7 @@ class K8SLauncher(NodeLauncher):
self.node, self.handler.pool.name, self.label)
self.node.state = zk.READY
self.node.python_path = self.label.python_path
# NOTE: resource access token may be encrypted here
self.node.connection_port = resource
if self.label.type == "namespace":

5
nodepool/driver/openshift/config.py
View File

@ -30,7 +30,8 @@ class OpenshiftLabel(ConfigValue):
other.image_pull == self.image_pull and
other.image == self.image and
other.cpu == self.cpu and
other.memory == self.memory)
other.memory == self.memory and
other.python_path == self.python_path)
return False
def __repr__(self):
@ -60,6 +61,7 @@ class OpenshiftPool(ConfigPool):
pl.image_pull = label.get('image-pull', 'IfNotPresent')
pl.cpu = label.get('cpu')
pl.memory = label.get('memory')
pl.python_path = label.get('python-path', '/usr/bin/python2')
pl.pool = self
self.labels[pl.name] = pl
full_config.labels[label['name']].pools.append(self)
@ -104,6 +106,7 @@ class OpenshiftProviderConfig(ProviderConfig):
'image-pull': str,
'cpu': int,
'memory': int,
'python-path': str,
}
pool = {

1
nodepool/driver/openshift/handler.py
View File

@ -47,6 +47,7 @@ class OpenShiftLauncher(NodeLauncher):
self.node.connection_type = "project"
self.node.state = zk.READY
self.node.python_path = self.label.python_path
# NOTE: resource access token may be encrypted here
self.node.connection_port = resource
self.zk.storeNode(self.node)

4
nodepool/driver/openstack/config.py
View File

@ -52,6 +52,7 @@ class ProviderCloudImage(ConfigValue):
self.image_id = None
self.image_name = None
self.username = None
self.python_path = None
self.connection_type = None
self.connection_port = None
@ -62,6 +63,7 @@ class ProviderCloudImage(ConfigValue):
self.image_id == other.image_id and
self.image_name == other.image_name and
self.username == other.username and
self.python_path == other.python_path and
self.connection_type == other.connection_type and
self.connection_port == other.connection_port)
return False
@ -319,6 +321,7 @@ class OpenStackProviderConfig(ProviderConfig):
i.image_id = image.get('image-id', None)
i.image_name = image.get('image-name', None)
i.username = image.get('username', None)
i.python_path = image.get('python-path', '/usr/bin/python2')
i.connection_type = image.get('connection-type', 'ssh')
i.connection_port = image.get(
'connection-port',
@ -348,6 +351,7 @@ class OpenStackProviderConfig(ProviderConfig):
v.Exclusive('image-id', 'cloud-image-name-or-id'): str,
v.Exclusive('image-name', 'cloud-image-name-or-id'): str,
'username': str,
'python-path': str,
}
pool_label_main = {

4
nodepool/driver/openstack/handler.py
View File

@ -88,6 +88,7 @@ class OpenStackNodeLauncher(NodeLauncher):
upload_id=cloud_image.id)
image_name = diskimage.name
username = cloud_image.username
python_path = cloud_image.python_path
connection_type = diskimage.connection_type
connection_port = diskimage.connection_port
@ -105,6 +106,7 @@ class OpenStackNodeLauncher(NodeLauncher):
image_id = self.label.cloud_image.name
image_name = self.label.cloud_image.name
username = self.label.cloud_image.username
python_path = self.label.cloud_image.python_path
connection_type = self.label.cloud_image.connection_type
connection_port = self.label.cloud_image.connection_port
@ -158,6 +160,8 @@ class OpenStackNodeLauncher(NodeLauncher):
self.node.resources = resources.quota['compute']
if username:
self.node.username = username
self.node.python_path = python_path
self.node.connection_type = connection_type
self.node.connection_port = connection_port

2
nodepool/driver/static/config.py
View File

@ -58,6 +58,7 @@ class StaticPool(ConfigPool):
'connection-type': node.get('connection-type', 'ssh'),
'username': node.get('username', 'zuul'),
'max-parallel-jobs': int(node.get('max-parallel-jobs', 1)),
'python-path': node.get('python-path', '/usr/bin/python2'),
})
if isinstance(node['labels'], str):
for label in node['labels'].split():
@ -106,6 +107,7 @@ class StaticProviderConfig(ProviderConfig):
'connection-port': int,
'connection-type': str,
'max-parallel-jobs': int,
'python-path': str,
}
pool = ConfigPool.getCommonSchemaDict()
pool.update({

1
nodepool/driver/static/provider.py
View File

@ -143,6 +143,7 @@ class StaticNodeProvider(Provider):
node.interface_ip = static_node["name"]
node.connection_port = static_node["connection-port"]
node.connection_type = static_node["connection-type"]
node.python_path = static_node["python-path"]
nodeutils.set_node_ip(node)
node.host_keys = host_keys
self.zk.storeNode(node)

3
nodepool/tests/fixtures/config_validate/good.yaml vendored
View File

@ -85,6 +85,7 @@ providers:
config-drive: true
- name: windows-unmanaged
username: winzuul
python-path: A:/python3.7.exe
connection-type: winrm
connection-port: 5986
pools:
@ -144,6 +145,7 @@ providers:
- name: openshift-pod
type: pod
image: docker.io/fedora:28
python-path: /usr/bin/python3
memory: 512
cpu: 2
@ -183,6 +185,7 @@ diskimages:
release: trusty
rebuild-age: 3600
build-timeout: 3600
python-path: /bin/python3.6
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

1
nodepool/tests/fixtures/node_no_min_ready.yaml vendored
View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
python-path: /usr/bin/python3
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

1
nodepool/tests/fixtures/node_unmanaged_image.yaml vendored
View File

@ -23,6 +23,7 @@ providers:
rate: 0.0001
cloud-images:
- name: fake-image
python-path: /usr/bin/python3
- name: fake-image-windows
username: zuul
connection-type: winrm

1
nodepool/tests/fixtures/openshift.yaml vendored
View File

@ -19,3 +19,4 @@ providers:
- name: pod-fedora
type: pod
image: docker.io/fedora:28
python-path: '/usr/bin/python3'

1
nodepool/tests/unit/test_driver_openshift.py
View File

@ -120,6 +120,7 @@ class TestDriverOpenshift(tests.DBTestCase):
self.assertIsNotNone(node.launcher)
self.assertEqual(node.connection_type, 'kubectl')
self.assertEqual(node.connection_port.get('token'), 'fake-token')
self.assertEqual(node.python_path, '/usr/bin/python3')
node.state = zk.DELETING
self.zk.storeNode(node)

4
nodepool/tests/unit/test_launcher.py
View File

@ -66,6 +66,7 @@ class TestLauncher(tests.DBTestCase):
self.assertEqual(node.username, "zuul")
self.assertEqual(node.connection_type, 'ssh')
self.assertEqual(node.connection_port, 22)
self.assertEqual(node.python_path, '/usr/bin/python3')
p = "{path}/{id}".format(
path=self.zk._imageUploadPath(image.image_name,
image.build_id,
@ -146,6 +147,7 @@ class TestLauncher(tests.DBTestCase):
self.assertEqual(nodes[1].type, ['fake-label1'])
self.assertEqual(nodes[2].type, ['fake-label4'])
self.assertEqual(nodes[3].type, ['fake-label2'])
self.assertEqual(nodes[0].python_path, '/usr/bin/python2')
def _test_node_assignment_at_quota(self,
config,
@ -1313,6 +1315,7 @@ class TestLauncher(tests.DBTestCase):
nodes = self.waitForNodes('fake-label')
self.assertEqual(len(nodes), 1)
self.assertIsNone(nodes[0].username)
self.assertEqual(nodes[0].python_path, '/usr/bin/python3')
nodes = self.waitForNodes('fake-label-windows')
self.assertEqual(len(nodes), 1)
@ -1320,6 +1323,7 @@ class TestLauncher(tests.DBTestCase):
self.assertEqual('winrm', nodes[0].connection_type)
self.assertEqual(5986, nodes[0].connection_port)
self.assertEqual(nodes[0].host_keys, [])
self.assertEqual(nodes[0].python_path, '/usr/bin/python2')
nodes = self.waitForNodes('fake-label-arbitrary-port')
self.assertEqual(len(nodes), 1)

14
nodepool/zk.py
View File

@ -283,6 +283,7 @@ class ImageBuild(BaseModel):
self.builder = None # Hostname
self.builder_id = None # Unique ID
self.username = None
self.python_path = None
def __repr__(self):
d = self.toDict()
@ -315,6 +316,7 @@ class ImageBuild(BaseModel):
if len(self.formats):
d['formats'] = ','.join(self.formats)
d['username'] = self.username
d['python_path'] = self.python_path
return d
@staticmethod
@ -332,6 +334,7 @@ class ImageBuild(BaseModel):
o.builder = d.get('builder')
o.builder_id = d.get('builder_id')
o.username = d.get('username', 'zuul')
o.python_path = d.get('python_path', '/usr/bin/python2')
# Only attempt the split on non-empty string
if d.get('formats', ''):
o.formats = d.get('formats', '').split(',')
@ -345,13 +348,14 @@ class ImageUpload(BaseModel):
VALID_STATES = set([UPLOADING, READY, DELETING, FAILED])
def __init__(self, build_id=None, provider_name=None, image_name=None,
upload_id=None, username=None):
upload_id=None, username=None, python_path=None):
super(ImageUpload, self).__init__(upload_id)
self.build_id = build_id
self.provider_name = provider_name
self.image_name = image_name
self.format = None
self.username = username
self.python_path = python_path
self.external_id = None # Provider ID of the image
self.external_name = None # Provider name of the image
@ -383,6 +387,7 @@ class ImageUpload(BaseModel):
d['external_name'] = self.external_name
d['format'] = self.format
d['username'] = self.username
d['python_path'] = self.python_path
return d
@staticmethod
@ -404,6 +409,7 @@ class ImageUpload(BaseModel):
o.external_name = d.get('external_name')
o.format = d.get('format')
o.username = d.get('username', 'zuul')
o.python_path = d.get('python_path', '/usr/bin/python2')
return o
@ -541,6 +547,7 @@ class Node(BaseModel):
self.hold_expiration = None
self.resources = None
self.attributes = None
self.python_path = None
def __repr__(self):
d = self.toDict()
@ -578,7 +585,8 @@ class Node(BaseModel):
self.host_keys == other.host_keys and
self.hold_expiration == other.hold_expiration and
self.resources == other.resources and
self.attributes == other.attributes)
self.attributes == other.attributes and
self.python_path == other.python_path)
else:
return False
@ -627,6 +635,7 @@ class Node(BaseModel):
d['hold_expiration'] = self.hold_expiration
d['resources'] = self.resources
d['attributes'] = self.attributes
d['python_path'] = self.python_path
return d
@staticmethod
@ -690,6 +699,7 @@ class Node(BaseModel):
self.hold_expiration = hold_expiration
self.resources = d.get('resources')
self.attributes = d.get('attributes')
self.python_path = d.get('python_path')
class ZooKeeper(object):