fd9d6aa80c
This does the following: * Retries node requests without penalty if they hit an unexpected quota error. The new node state "TEMPFAIL" is used for this. * Calculates total provider quota in a method similar to nodepool and caches it and updates it every 5 minutes. * Keeps track of realtime zuul quota usage by way of a custom TreeCache that adds and removes quota as it sees events. This lets us know real-time usage without needing to iterate over all nodes. * Avoids starting a node state machine if it is expected that the node will put the provider over quota. This is similar, but not quite the same as nodepool's idea of a paused provider. * Attempts to keep the percentage quota used of all providers roughly equivalent while still including some randomization. This is still likely not a final allocation algorithm. This has some issues that will need to be resolved in subsequent commits: * It is possible for a node request to be starved if multiple launchers are running for a provider, and there are requests of different sizes for that provider (the large one may never run if the small ones are able to fit in the last bit of quota). * Because we record quota failures as node objects in ZK, if a node request continually hits quota errors, our node records in ZK will grow without bound. Despite these issues, this change is useful for continued testing of the system, and since it is mainly focused with adding quota support and only minimally changes the node/request handling algorithms, it stands on its own. Change-Id: I6d65db2b103cc3f0aec7e46a6a63a393399c91eb
285 lines
8.3 KiB
Python
285 lines
8.3 KiB
Python
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
# Copyright 2022, 2024 Acme Gating, LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import logging
|
|
import uuid
|
|
|
|
from zuul.driver.openstack.openstackendpoint import OpenstackProviderEndpoint
|
|
|
|
|
|
class FakeOpenstackObject:
|
|
def __init__(self, **kw):
|
|
self.__dict__.update(kw)
|
|
self.__kw = list(kw.keys())
|
|
|
|
def _get_dict(self):
|
|
data = {}
|
|
for k in self.__kw:
|
|
data[k] = getattr(self, k)
|
|
return data
|
|
|
|
def __contains__(self, key):
|
|
return key in self.__dict__
|
|
|
|
def __getitem__(self, key, default=None):
|
|
return getattr(self, key, default)
|
|
|
|
def __setitem__(self, key, value):
|
|
setattr(self, key, value)
|
|
|
|
def get(self, key, default=None):
|
|
return getattr(self, key, default)
|
|
|
|
def set(self, key, value):
|
|
setattr(self, key, value)
|
|
|
|
|
|
class FakeOpenstackFlavor(FakeOpenstackObject):
|
|
pass
|
|
|
|
|
|
class FakeOpenstackServer(FakeOpenstackObject):
|
|
pass
|
|
|
|
|
|
class FakeOpenstackLocation(FakeOpenstackObject):
|
|
pass
|
|
|
|
|
|
class FakeOpenstackImage(FakeOpenstackObject):
|
|
pass
|
|
|
|
|
|
class FakeOpenstackNetwork(FakeOpenstackObject):
|
|
pass
|
|
|
|
|
|
class FakeOpenstackFloatingIp(FakeOpenstackObject):
|
|
def _fake_toDict(self):
|
|
return {
|
|
'version': 4,
|
|
'addr': self.floating_ip_address,
|
|
'OS-EXT-IPS:type': 'floating',
|
|
}
|
|
|
|
|
|
class FakeOpenstackCloud:
|
|
log = logging.getLogger("zuul.FakeOpenstackCloud")
|
|
|
|
def __init__(self,
|
|
needs_floating_ip=False,
|
|
auto_attach_floating_ip=True,
|
|
):
|
|
self._fake_needs_floating_ip = needs_floating_ip
|
|
self._fake_auto_attach_floating_ip = auto_attach_floating_ip
|
|
self.servers = []
|
|
self.volumes = []
|
|
self.images = []
|
|
self.floating_ips = []
|
|
self.flavors = [
|
|
FakeOpenstackFlavor(
|
|
id='425e3203150e43d6b22792f86752533d',
|
|
name='Fake Flavor',
|
|
ram=8192,
|
|
vcpus=4,
|
|
)
|
|
]
|
|
self.networks = [
|
|
FakeOpenstackNetwork(
|
|
id=uuid.uuid4().hex,
|
|
name='fake-network',
|
|
)
|
|
]
|
|
|
|
def _getConnection(self):
|
|
return FakeOpenstackConnection(self)
|
|
|
|
|
|
class FakeOpenstackResponse:
|
|
def __init__(self, data):
|
|
self._data = data
|
|
self.links = []
|
|
|
|
def json(self):
|
|
return self._data
|
|
|
|
|
|
class FakeOpenstackSession:
|
|
def __init__(self, cloud):
|
|
self.cloud = cloud
|
|
|
|
def get(self, uri, headers, params):
|
|
if uri == '/servers/detail':
|
|
server_list = []
|
|
for server in self.cloud.servers:
|
|
data = server._get_dict()
|
|
data['hostId'] = data.pop('host_id')
|
|
data['OS-EXT-AZ:availability_zone'] = data.pop('location').zone
|
|
data['os-extended-volumes:volumes_attached'] =\
|
|
data.pop('volumes')
|
|
server_list.append(data)
|
|
return FakeOpenstackResponse({'servers': server_list})
|
|
|
|
|
|
class FakeOpenstackConfig:
|
|
pass
|
|
|
|
|
|
class FakeOpenstackConnection:
|
|
log = logging.getLogger("zuul.FakeOpenstackConnection")
|
|
|
|
def __init__(self, cloud):
|
|
self.cloud = cloud
|
|
self.compute = FakeOpenstackSession(cloud)
|
|
self.config = FakeOpenstackConfig()
|
|
self.config.config = {}
|
|
self.config.config['image_format'] = 'qcow2'
|
|
self.config.config['region_name'] = 'region1'
|
|
|
|
def _needs_floating_ip(self, server, nat_destination):
|
|
return self.cloud._fake_needs_floating_ip
|
|
|
|
def _has_floating_ips(self):
|
|
return False
|
|
|
|
def list_flavors(self, get_extra=False):
|
|
return self.cloud.flavors
|
|
|
|
def list_volumes(self):
|
|
return self.cloud.volumes
|
|
|
|
def get_network(self, name_or_id, filters=None):
|
|
for x in self.cloud.networks:
|
|
if x.id == name_or_id or x.name == name_or_id:
|
|
return x
|
|
|
|
def list_servers(self):
|
|
return self.cloud.servers
|
|
|
|
def create_server(self, wait=None, name=None, image=None,
|
|
flavor=None, config_drive=None, key_name=None,
|
|
nics=None, meta=None):
|
|
location = FakeOpenstackLocation(zone=None)
|
|
if self.cloud._fake_needs_floating_ip:
|
|
addresses = dict(
|
|
public=[],
|
|
private=[dict(version=4, addr='198.51.100.1')]
|
|
)
|
|
interface_ip = '198.51.100.1'
|
|
else:
|
|
addresses = dict(
|
|
public=[dict(version=4, addr='198.51.100.1'),
|
|
dict(version=6, addr='2001:db8::1')],
|
|
private=[dict(version=4, addr='198.51.100.1')]
|
|
)
|
|
interface_ip = '198.51.100.1'
|
|
|
|
args = dict(
|
|
id=uuid.uuid4().hex,
|
|
name=name,
|
|
host_id='fake_host_id',
|
|
location=location,
|
|
volumes=[],
|
|
status='ACTIVE',
|
|
addresses=addresses,
|
|
interface_ip=interface_ip,
|
|
flavor=flavor,
|
|
)
|
|
server = FakeOpenstackServer(**args)
|
|
self.cloud.servers.append(server)
|
|
return server
|
|
|
|
def delete_server(self, name_or_id):
|
|
for x in self.cloud.servers:
|
|
if x.id == name_or_id:
|
|
self.cloud.servers.remove(x)
|
|
return
|
|
|
|
def create_image(self, wait=None, name=None, filename=None,
|
|
is_public=None, md5=None, sha256=None,
|
|
timeout=None, **meta):
|
|
image = FakeOpenstackImage(
|
|
id=uuid.uuid4().hex,
|
|
name=name,
|
|
filename=filename,
|
|
is_public=is_public,
|
|
md5=md5,
|
|
sha256=sha256,
|
|
status='ACTIVE',
|
|
)
|
|
self.cloud.images.append(image)
|
|
return image
|
|
|
|
def delete_image(self, name_or_id):
|
|
for x in self.cloud.servers:
|
|
if x.id == name_or_id:
|
|
self.cloud.servers.remove(x)
|
|
return
|
|
|
|
def create_floating_ip(self, server, wait=None):
|
|
args = dict(
|
|
id=uuid.uuid4().hex,
|
|
floating_ip_address='fake',
|
|
status='ACTIVE',
|
|
)
|
|
if self.cloud._fake_auto_attach_floating_ip:
|
|
args['port_id'] = 'fake'
|
|
fip = FakeOpenstackFloatingIp(**args)
|
|
self.cloud.floating_ips.append(fip)
|
|
if self.cloud._fake_auto_attach_floating_ip:
|
|
server['addresses']['public'].append(fip._fake_toDict())
|
|
return fip
|
|
|
|
def list_floating_ips(self):
|
|
return self.cloud.floating_ips
|
|
|
|
def delete_floating_ip(self, name_or_id):
|
|
for x in self.cloud.floating_ips:
|
|
if x.id == name_or_id:
|
|
self.cloud.floating_ips.remove(x)
|
|
return
|
|
|
|
def _attach_ip_to_server(self, server, floating_ip, skip_attach=False):
|
|
if floating_ip.get('port_id'):
|
|
raise Exception("Attaching already attached fip")
|
|
floating_ip['port_id'] = 'fake'
|
|
server['addresses']['public'].append(floating_ip._fake_toDict())
|
|
|
|
def get_compute_limits(self):
|
|
return FakeOpenstackObject(
|
|
max_total_cores=self.cloud.max_cores,
|
|
max_total_instances=self.cloud.max_instances,
|
|
max_total_ram_size=self.cloud.max_ram,
|
|
total_cores_used=4 * len(self.cloud.servers),
|
|
total_instances_used=len(self.cloud.servers),
|
|
total_ram_used=8192 * len(self.cloud.servers),
|
|
)
|
|
|
|
def get_volume_limits(self):
|
|
return FakeOpenstackObject(
|
|
absolute=dict(
|
|
maxTotalVolumes=self.cloud.max_volumes,
|
|
maxTotalVolumeGigabytes=self.cloud.max_volume_gb,
|
|
))
|
|
|
|
|
|
class FakeOpenstackProviderEndpoint(OpenstackProviderEndpoint):
|
|
def _getClient(self):
|
|
return self._fake_cloud._getConnection()
|
|
|
|
def _expandServer(self, server):
|
|
return server
|