zuul/zuul
Code Issues Proposed changes

Unify provider validation and deserialization schemas

Browse Source 
We are going to add a lot of configuration options related to
providers, and there will be some duplication.  Each Section
may have a number of attributes that will act as defaults for
labels, flavors, and images.  Labels, Flavors, and Images may
set those same values.  Providers may override them.

We need to validate each of these values in each of those places.
Then we also need to serialize and deserialize them to share
between the schedulers and launchers.

The more modular and reusable the schemas are, the easier it
will be to understand and maintain this.  To that end, this
change introduces a method of unifying the validation and
deserialization schemas.  This will allow us to use the same
voluptuous schema to validate the YAML as well as set the
attributes on the python object.

It allows mutating the key names so that "foo-bar" in YAML is
translated to "foo_bar" in python.  It also allows us to
set the python attribute to None if no value is provided without
also allowing None in the YAML input.

Change-Id: I11932d2d7da8ecd26010319db5be04c19d64756f
This commit is contained in:
James E. Blair 2024-07-23 14:25:07 -07:00
parent e09725fddc
commit d92378d473
6 changed files with 386 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/fixtures/layouts/nodepool-image.yaml vendored
View File

@ -131,6 +131,7 @@
instance-type: t3.medium
images:
- name: debian
image-id: ami-1e749f67
- name: debian-local
- section:

4
zuul/driver/aws/awsendpoint.py
View File

@ -552,7 +552,7 @@ class AwsProviderEndpoint(BaseProviderEndpoint):
else:
quota = self._getQuotaForInstanceType(
flavor.instance_type,
SPOT if flavor.use_spot else ON_DEMAND)
SPOT if flavor.market_type == 'spot' else ON_DEMAND)
# TODO
# if label.volume_type:
# quota.add(self._getQuotaForVolumeType(
@ -1403,7 +1403,7 @@ class AwsProviderEndpoint(BaseProviderEndpoint):
# args['BlockDeviceMappings'] = [mapping]
# TODO
# if label.use_spot:
# if flavor.market_type == 'spot':
# args['InstanceMarketOptions'] = {
# 'MarketType': 'spot',
# 'SpotOptions': {

161
zuul/driver/aws/awsprovider.py
View File

@ -15,6 +15,11 @@
import logging
import math
import zuul.provider.schema as provider_schema
from zuul.lib.voluputil import (
Required, Optional, Nullable, discriminate, assemble, RequiredExclusive,
)
import voluptuous as vs
from zuul.driver.aws.awsendpoint import (
@ -37,30 +42,105 @@ from zuul.provider import (
class AwsProviderImage(BaseProviderImage):
aws_image_filters = {
'name': str,
'values': [str],
}
aws_cloud_schema = vs.Schema({
vs.Exclusive(Required('image-id'), 'spec'): str,
vs.Exclusive(Required('image-filters'), 'spec'): [aws_image_filters],
Required('type'): 'cloud',
})
cloud_schema = vs.All(
assemble(
BaseProviderImage.schema,
aws_cloud_schema,
),
RequiredExclusive('image_id', 'image_filters',
msg=('Provide either '
'"image-filters", or "image-id" keys'))
)
zuul_schema = assemble(
BaseProviderImage.schema,
vs.Schema({'type': 'zuul'}),
)
inheritable_schema = BaseProviderImage.inheritable_schema
schema = vs.Union(
cloud_schema, zuul_schema,
discriminant=discriminate(
lambda val, alt: val['type'] == alt['type']))
def __init__(self, config):
self.image_id = None
self.image_filters = None
super().__init__(config)
self.image_id = config.get('image-id')
self.image_filters = config.get('image-filters')
class AwsProviderFlavor(BaseProviderFlavor):
def __init__(self, config):
super().__init__(config)
self.instance_type = config['instance-type']
self.volume_type = config.get('volume-type')
self.dedicated_host = config.get('dedicated-host', False)
self.ebs_optimized = bool(config.get('ebs-optimized', False))
# TODO
self.use_spot = False
aws_flavor_schema = vs.Schema({
Required('instance-type'): str,
Optional('volume-type'): Nullable(str),
Optional('dedicated-host', default=False): bool,
Optional('ebs-optimized', default=False): bool,
Optional('market-type', default='on-demand'): vs.Any(
'on-demand', 'spot'),
})
inheritable_schema = assemble(
BaseProviderFlavor.inheritable_schema,
provider_schema.cloud_flavor,
)
schema = assemble(
BaseProviderFlavor.schema,
provider_schema.cloud_flavor,
aws_flavor_schema,
)
class AwsProviderLabel(BaseProviderLabel):
def __init__(self, config):
super().__init__(config)
inheritable_schema = assemble(
BaseProviderLabel.inheritable_schema,
provider_schema.ssh_label,
)
schema = assemble(
BaseProviderLabel.schema,
provider_schema.ssh_label,
)
class AwsProviderSchema(BaseProviderSchema):
def getLabelSchema(self):
return AwsProviderLabel.schema
def getImageSchema(self):
return AwsProviderImage.schema
def getFlavorSchema(self):
return AwsProviderFlavor.schema
def getProviderSchema(self):
schema = super().getProviderSchema()
object_storage = {
vs.Required('bucket-name'): str,
}
aws_provider_schema = vs.Schema({
Required('region'): str,
Optional('object-storage'): Nullable(object_storage),
})
return assemble(
schema,
aws_provider_schema,
AwsProviderImage.inheritable_schema,
AwsProviderFlavor.inheritable_schema,
AwsProviderLabel.inheritable_schema,
)
class AwsProvider(BaseProvider, subclass_id='aws'):
log = logging.getLogger("zuul.AwsProvider")
schema = AwsProviderSchema().getProviderSchema()
def __init__(self, *args, **kw):
super().__init__(*args, **kw)
@ -83,12 +163,6 @@ class AwsProvider(BaseProvider, subclass_id='aws'):
self._set(_endpoint=self.getEndpoint())
return self._endpoint
def parseConfig(self, config):
data = super().parseConfig(config)
data['region'] = config['region']
data['object_storage'] = config.get('object-storage')
return data
def parseImage(self, image_config):
return AwsProviderImage(image_config)
@ -219,54 +293,3 @@ class AwsProvider(BaseProvider, subclass_id='aws'):
def deleteImage(self, external_id):
self.endpoint.deleteImage(external_id)
class AwsProviderSchema(BaseProviderSchema):
def getImageSchema(self):
base_schema = super().getImageSchema()
# This is AWS syntax, so we allow upper or lower case
image_filters = {
vs.Any('Name', 'name'): str,
vs.Any('Values', 'values'): [str]
}
cloud_schema = base_schema.extend({
'image-id': str,
'image-filters': [image_filters],
})
def validator(data):
if data.get('type') == 'cloud':
return cloud_schema(data)
return base_schema(data)
return validator
def getFlavorSchema(self):
base_schema = super().getLabelSchema()
schema = base_schema.extend({
vs.Required('instance-type'): str,
'volume-type': str,
'dedicated-host': bool,
'ebs-optimized': bool,
})
return schema
def getLabelSchema(self):
base_schema = super().getLabelSchema()
return base_schema
def getProviderSchema(self):
# TODO: validate tag values are strings
schema = super().getProviderSchema()
object_storage = {
vs.Required('bucket-name'): str,
}
schema = schema.extend({
vs.Required('region'): str,
'object-storage': object_storage,
})
return schema

148
zuul/lib/voluputil.py Normal file
View File

@ -0,0 +1,148 @@
# Copyright 2024 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This adds some helpers that are useful for mutating hyphenated YAML
# structures into underscored python dicts.
import voluptuous as vs
UNDEFINED = object()
def assemble(*schemas):
"""Merge any number of voluptuous schemas into a single schema. The
input schemas must all be dictionary-based.
"""
ret = vs.Schema({})
for x in schemas:
ret = ret.extend(x.schema)
return ret
def discriminate(comparator):
"""Chose among multiple schemas in a Union using a supplied comparator
function. This does some extra work to find the interesting part
of nested schemas. It follows nested schemas down and picks the
first one of each. This makes it suitable for use with All
combined with RequiredExclusive. This is intended to be passed to
the discriminant argument of Union.
The comparator is called with the following arguments:
:param object val: The input data to be validated.
:param Schema s: The first schema with values likely to be comparable.
"""
def disc(val, alt):
ret = []
for a in alt:
s = a
while hasattr(s, 'validators'):
s = s.validators[0]
while hasattr(s, 'schema'):
s = s.schema
if comparator(val, s):
ret.append(a)
return ret
return disc
class RequiredExclusive:
"""A validator to require that at least one of a set of alternates is
present. Use this in combination with Exclusive and All (since
Exclusive doesn't imply Required). Example:
All(
{Exclusive(...), Exclusive(...)},
RequiredExclusive(...)
)
"""
def __init__(self, *attributes, msg=None):
self.schema = vs.Schema({
# Use the mutated names for the requirements
vs.Required(
vs.Any(*attributes),
msg=msg,
): object,
object: object,
})
def __call__(self, v):
return self.schema(v)
class Required(vs.Required):
"""Require an attribute and mutate its name
This mutates the name of the attribute to lowercase it and replace
hyphens with underscares. This enables the output of the
validator to be used directly in setting python object attributes.
"""
def __init__(self, schema, default=UNDEFINED, output=None):
if not isinstance(schema, str):
raise Exception("Only strings are supported")
super().__init__(schema, default=default)
if output is None:
output = str(schema).replace('-', '_').lower()
self.output = output
def __call__(self, data):
# Superclass ensures that data==schema
super().__call__(data)
# Return our mutated form
return self.output
class Optional(vs.Optional):
"""Mark an attribute optional and mutate its name
This mutates the name of the attribute to lowercase it and replace
hyphens with underscares. This enables the output of the
validator to be used directly in setting python object attributes.
This works in conjuction with Nullable.
"""
def __init__(self, schema, default=UNDEFINED, output=None):
if not isinstance(schema, str):
raise Exception("Only strings are supported")
super().__init__(schema, default=default)
if output is None:
output = str(schema).replace('-', '_').lower()
self.output = output
def __call__(self, data):
# Superclass ensures that data==schema
super().__call__(data)
# Return our mutated form
return self.output
class Nullable:
"""Set the output value to None when no input is supplied.
When used with Optional, if no input value is supplied, this will
set the output to None without also accepting None as an input
value.
"""
def __init__(self, schema):
self.schema = vs.Schema(schema)
def __call__(self, v):
if v is UNDEFINED:
return None
return self.schema(v)

140
zuul/provider/__init__.py
View File

@ -17,20 +17,27 @@ import json
import math
import urllib.parse
from zuul.lib.voluputil import Required, Optional, Nullable, assemble
from zuul import model
from zuul.driver.util import QuotaInformation
from zuul.zk import zkobject
import zuul.provider.schema as provider_schema
import voluptuous as vs
class BaseProviderImage(metaclass=abc.ABCMeta):
inheritable_schema = assemble(
provider_schema.common_image,
)
schema = assemble(
provider_schema.common_image,
provider_schema.base_image,
)
def __init__(self, config):
self.project_canonical_name = config['project_canonical_name']
self.name = config['name']
self.branch = config['branch']
self.type = config['type']
# TODO: get formats from configuration
self.__dict__.update(self.schema(config))
# TODO: generate this automatically from config
self.formats = set(['raw'])
@property
@ -43,21 +50,23 @@ class BaseProviderImage(metaclass=abc.ABCMeta):
class BaseProviderFlavor(metaclass=abc.ABCMeta):
inheritable_schema = assemble()
schema = assemble(
provider_schema.base_flavor,
)
def __init__(self, config):
self.project_canonical_name = config['project_canonical_name']
self.name = config['name']
self.public_ipv4 = config.get('public-ipv4', False)
self.__dict__.update(self.schema(config))
class BaseProviderLabel(metaclass=abc.ABCMeta):
inheritable_schema = assemble()
schema = assemble(
provider_schema.base_label,
)
def __init__(self, config):
self.project_canonical_name = config['project_canonical_name']
self.name = config['name']
self.flavor = config.get('flavor')
self.image = config.get('image')
self.min_ready = config.get('min-ready', 0)
self.tags = config.get('tags', {})
self.key_name = config.get('key-name')
self.__dict__.update(self.schema(config))
class BaseProviderEndpoint(metaclass=abc.ABCMeta):
@ -75,9 +84,38 @@ class BaseProviderEndpoint(metaclass=abc.ABCMeta):
self.connection = connection
class BaseProviderSchema(metaclass=abc.ABCMeta):
def getLabelSchema(self):
return BaseProviderLabel.schema
def getImageSchema(self):
return BaseProviderImage.schema
def getFlavorSchema(self):
return BaseProviderFlavor.schema
def getProviderSchema(self):
schema = vs.Schema({
'_source_context': model.SourceContext,
'_start_mark': model.ZuulMark,
Required('name'): str,
Required('section'): str,
Required('labels'): [self.getLabelSchema()],
Required('images'): [self.getImageSchema()],
Required('flavors'): [self.getFlavorSchema()],
Required('abstract'): vs.Maybe(bool),
Required('parent'): vs.Maybe(str),
Required('connection'): str,
Optional('boot-timeout'): Nullable(int),
Optional('launch-timeout'): Nullable(int),
})
return schema
class BaseProvider(zkobject.PolymorphicZKObjectMixin,
zkobject.ShardedZKObject):
"""Base class for provider."""
schema = BaseProviderSchema().getProviderSchema()
def __init__(self, *args):
super().__init__()
@ -86,6 +124,8 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
config = config.copy()
config.pop('_source_context')
config.pop('_start_mark')
parsed_config = self.parseConfig(config)
parsed_config.pop('connection')
self._set(
driver=driver,
connection=connection,
@ -93,7 +133,7 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
tenant_name=tenant_name,
canonical_name=canonical_name,
config=config,
**self.parseConfig(config),
**parsed_config,
)
@classmethod
@ -115,15 +155,18 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
driver=connection.driver)
return obj
def getProviderSchema(self):
return self.schema
def parseConfig(self, config):
return dict(
name=config['name'],
section_name=config['section'],
description=config.get('description'),
schema = self.getProviderSchema()
ret = schema(config)
ret.update(dict(
images=self.parseImages(config),
flavors=self.parseFlavors(config),
labels=self.parseLabels(config),
)
))
return ret
def deserialize(self, raw, context):
data = super().deserialize(raw, context)
@ -408,58 +451,3 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
:param external_id str or dict: The external id of the server
"""
pass
class BaseProviderSchema(metaclass=abc.ABCMeta):
def getLabelSchema(self):
schema = vs.Schema({
vs.Required('project_canonical_name'): str,
vs.Required('name'): str,
'description': str,
'image': str,
'flavor': str,
'tags': dict,
'key-name': str,
})
return schema
def getImageSchema(self):
schema = vs.Schema({
vs.Required('project_canonical_name'): str,
vs.Required('name'): str,
vs.Required('branch'): str,
'description': str,
'username': str,
'connection-type': str,
'connection-port': int,
'python-path': str,
'shell-type': str,
'type': str,
})
return schema
def getFlavorSchema(self):
schema = vs.Schema({
vs.Required('project_canonical_name'): str,
vs.Required('name'): str,
'description': str,
'public-ipv4': bool,
})
return schema
def getProviderSchema(self):
schema = vs.Schema({
'_source_context': model.SourceContext,
'_start_mark': model.ZuulMark,
vs.Required('name'): str,
vs.Required('section'): str,
vs.Required('labels'): [self.getLabelSchema()],
vs.Required('images'): [self.getImageSchema()],
vs.Required('flavors'): [self.getFlavorSchema()],
'abstract': bool,
'parent': str,
'connection': str,
'boot-timeout': int,
'launch-timeout': int,
})
return schema

79
zuul/provider/schema.py Normal file
View File

@ -0,0 +1,79 @@
# Copyright 2024 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This file contains provider-related schema chunks that can be reused
# by multiple drivers. When adding new configuration options, if they
# can be used by more than one driver, add them here instead of in the
# driver.
import voluptuous as vs
from zuul.lib.voluputil import Required, Optional, Nullable
# Labels
# The label attributes that can appear in a section/provider label or
# a standalone label (but not in the section body).
base_label = vs.Schema({
Required('project_canonical_name'): str,
Required('name'): str,
Optional('description'): Nullable(str),
Optional('image'): Nullable(str),
Optional('flavor'): Nullable(str),
Optional('tags', default=dict): {str: str},
})
# Label attributes that are common to any kind of ssh-based driver.
ssh_label = vs.Schema({
Optional('key-name'): Nullable(str),
})
# Images
# The image attributes which can appear either in the main body of the
# section stanza, or in a section/provider image, or in a standalone
# image.
common_image = vs.Schema({
Optional('username'): Nullable(str),
Optional('connection-type'): Nullable(str),
Optional('connection-port'): Nullable(int),
Optional('python-path'): Nullable(str),
Optional('shell-type'): Nullable(str),
})
# The image attributes that, in addition to those above, can appear in
# a section/provider image or a standalone image (but not in the
# section body).
base_image = vs.Schema({
Required('project_canonical_name'): str,
Required('name'): str,
Optional('description'): Nullable(str),
Required('branch'): str,
Required('type'): vs.Any('cloud', 'zuul'),
})
# Flavors
# The flavor attributes that can appear in a section/provider flavor or
# a standalone flavor (but not in the section body).
base_flavor = vs.Schema({
Required('project_canonical_name'): str,
Required('name'): str,
Optional('description'): Nullable(str),
})
# Flavor attributes that are common to any kind of cloud driver.
cloud_flavor = vs.Schema({
Optional('public-ipv4', default=False): bool,
Optional('public-ipv6', default=False): bool,
})