Store runtime related system attributes together
This change introduces a new class for storing runtime related system attributes that are coming from the Zuul config. The system attributes will be updated when the config changes instead of directly reading them from the config on demand. This gives us a clear interface for updating those values and storing them in Zookeeper later on. System config attributes currently are: * [scheduler].relative_priority * [scheduler].max_hold_expiration * [scheduler].default_hold_expiration * [scheduler].default_ansible_version * [web].root * [web].status_url * [web].websocket_url Change-Id: Iefc7857eddc3e0c068b10d01489aa6ae75a6374b
This commit is contained in:
Simon Westphahl
parent
c30eca959a
commit
8aa2ad2db0
|
|
@ -1553,7 +1553,7 @@ class TenantParser(object):
|
|||
conf['exclude-unprotected-branches']
|
||||
if conf.get('admin-rules') is not None:
|
||||
tenant.authorization_rules = conf['admin-rules']
|
||||
tenant.web_root = conf.get('web-root', self.scheduler.web_root)
|
||||
tenant.web_root = conf.get('web-root', self.scheduler.globals.web_root)
|
||||
if tenant.web_root and not tenant.web_root.endswith('/'):
|
||||
tenant.web_root += '/'
|
||||
tenant.allowed_triggers = conf.get('allowed-triggers')
|
||||
|
|
|
|||
|
|
@ -629,7 +629,7 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
return False
|
||||
build_set = item.current_build_set
|
||||
log.debug("Requesting nodes for change %s", item.change)
|
||||
if self.sched.use_relative_priority:
|
||||
if self.sched.globals.use_relative_priority:
|
||||
relative_priority = item.getNodePriority()
|
||||
else:
|
||||
relative_priority = 0
|
||||
|
|
@ -1296,7 +1296,8 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
if failing_reasons:
|
||||
log.debug("%s is a failing item because %s" %
|
||||
(item, failing_reasons))
|
||||
if item.live and not dequeued and self.sched.use_relative_priority:
|
||||
if (item.live and not dequeued
|
||||
and self.sched.globals.use_relative_priority):
|
||||
priority = item.getNodePriority()
|
||||
for node_request in item.current_build_set.node_requests.values():
|
||||
if node_request.fulfilled:
|
||||
|
|
|
|||
|
|
@ -4387,6 +4387,84 @@ class ProjectMetadata(object):
|
|||
self.queue_name = None
|
||||
|
||||
|
||||
class SystemAttributes:
|
||||
"""Global system attributes from the Zuul config.
|
||||
|
||||
Those runtime related settings are expected to be consistent on
|
||||
all schedulers and will be synchronized via Zookeeper.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.use_relative_priority = False
|
||||
self.max_hold_expiration = 0
|
||||
self.default_hold_expiration = 0
|
||||
self.default_ansible_version = None
|
||||
self.web_root = None
|
||||
self.web_status_url = ""
|
||||
self.websocket_url = None
|
||||
|
||||
@classmethod
|
||||
def fromConfig(cls, config):
|
||||
sys_attrs = cls()
|
||||
sys_attrs.updateFromConfig(config)
|
||||
return sys_attrs
|
||||
|
||||
def updateFromConfig(self, config):
|
||||
"""Set runtime related system attributes from config."""
|
||||
self.use_relative_priority = False
|
||||
if config.has_option('scheduler', 'relative_priority'):
|
||||
if config.getboolean('scheduler', 'relative_priority'):
|
||||
self.use_relative_priority = True
|
||||
|
||||
max_hold = get_default(config, 'scheduler', 'max_hold_expiration', 0)
|
||||
default_hold = get_default(
|
||||
config, 'scheduler', 'default_hold_expiration', max_hold)
|
||||
|
||||
# If the max hold is not infinite, we need to make sure that
|
||||
# our default value does not exceed it.
|
||||
if (max_hold and default_hold != max_hold
|
||||
and (default_hold == 0 or default_hold > max_hold)):
|
||||
default_hold = max_hold
|
||||
self.max_hold_expiration = max_hold
|
||||
self.default_hold_expiration = default_hold
|
||||
|
||||
# Reload the ansible manager in case the default ansible version
|
||||
# changed.
|
||||
self.default_ansible_version = get_default(
|
||||
config, 'scheduler', 'default_ansible_version', None)
|
||||
|
||||
web_root = get_default(config, 'web', 'root', None)
|
||||
if web_root:
|
||||
web_root = urllib.parse.urljoin(web_root, 't/{tenant.name}/')
|
||||
self.web_root = web_root
|
||||
|
||||
self.web_status_url = get_default(config, 'web', 'status_url', '')
|
||||
self.websocket_url = get_default(config, 'web', 'websocket_url', None)
|
||||
|
||||
def toDict(self):
|
||||
return {
|
||||
"use_relative_priority": self.use_relative_priority,
|
||||
"max_hold_expiration": self.max_hold_expiration,
|
||||
"default_hold_expiration": self.default_hold_expiration,
|
||||
"default_ansible_version": self.default_ansible_version,
|
||||
"web_root": self.web_root,
|
||||
"web_status_url": self.web_status_url,
|
||||
"websocket_url": self.websocket_url,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def fromDict(cls, data):
|
||||
sys_attrs = cls()
|
||||
sys_attrs.use_relative_priority = data["use_relative_priority"]
|
||||
sys_attrs.max_hold_expiration = data["max_hold_expiration"]
|
||||
sys_attrs.default_hold_expiration = data["default_hold_expiration"]
|
||||
sys_attrs.default_ansible_version = data["default_ansible_version"]
|
||||
sys_attrs.web_root = data["web_root"]
|
||||
sys_attrs.web_status_url = data["web_status_url"]
|
||||
sys_attrs.websocket_url = data["websocket_url"]
|
||||
return sys_attrs
|
||||
|
||||
|
||||
# TODO(ianw) : this would clearly be better if it recorded the
|
||||
# original file and made line-relative comments, however the contexts
|
||||
# the subclasses are raised in don't have that info currently, so this
|
||||
|
|
|
|||
|
|
@ -148,8 +148,7 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
|
|||
return ret
|
||||
|
||||
def _formatItemReportEnqueue(self, item, with_jobs=True):
|
||||
status_url = get_default(self.connection.sched.config,
|
||||
'web', 'status_url', '')
|
||||
status_url = self.connection.sched.globals.web_status_url
|
||||
if status_url:
|
||||
status_url = item.formatUrlPattern(status_url)
|
||||
|
||||
|
|
@ -159,8 +158,7 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
|
|||
status_url=status_url)
|
||||
|
||||
def _formatItemReportStart(self, item, with_jobs=True):
|
||||
status_url = get_default(self.connection.sched.config,
|
||||
'web', 'status_url', '')
|
||||
status_url = self.connection.sched.globals.web_status_url
|
||||
if status_url:
|
||||
status_url = item.formatUrlPattern(status_url)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import sys
|
|||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import urllib
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
|
||||
|
|
@ -67,6 +66,7 @@ from zuul.model import (
|
|||
TenantReconfigureEvent,
|
||||
TimeDataBase,
|
||||
UnparsedAbideConfig,
|
||||
SystemAttributes,
|
||||
STATE_FAILED,
|
||||
)
|
||||
from zuul.zk import ZooKeeperClient
|
||||
|
|
@ -224,19 +224,10 @@ class Scheduler(threading.Thread):
|
|||
else:
|
||||
self.zuul_version = zuul_version.release_string
|
||||
self.last_reconfigured = None
|
||||
self.use_relative_priority = False
|
||||
if self.config.has_option('scheduler', 'relative_priority'):
|
||||
if self.config.getboolean('scheduler', 'relative_priority'):
|
||||
self.use_relative_priority = True
|
||||
web_root = get_default(self.config, 'web', 'root', None)
|
||||
if web_root:
|
||||
web_root = urllib.parse.urljoin(web_root, 't/{tenant.name}/')
|
||||
self.web_root = web_root
|
||||
|
||||
default_ansible_version = get_default(
|
||||
self.config, 'scheduler', 'default_ansible_version', None)
|
||||
self.globals = SystemAttributes.fromConfig(self.config)
|
||||
self.ansible_manager = AnsibleManager(
|
||||
default_version=default_ansible_version)
|
||||
default_version=self.globals.default_ansible_version)
|
||||
|
||||
if not testonly:
|
||||
self.executor = self._executor_client_class(self.config, self)
|
||||
|
|
@ -681,25 +672,15 @@ class Scheduler(threading.Thread):
|
|||
request.reason = reason
|
||||
request.max_count = count
|
||||
|
||||
max_hold = get_default(
|
||||
self.config, 'scheduler', 'max_hold_expiration', 0)
|
||||
default_hold = get_default(
|
||||
self.config, 'scheduler', 'default_hold_expiration', max_hold)
|
||||
|
||||
# If the max hold is not infinite, we need to make sure that
|
||||
# our default value does not exceed it.
|
||||
if max_hold and default_hold != max_hold and (default_hold == 0 or
|
||||
default_hold > max_hold):
|
||||
default_hold = max_hold
|
||||
|
||||
# Set node_hold_expiration to default if no value is supplied
|
||||
if node_hold_expiration is None:
|
||||
node_hold_expiration = default_hold
|
||||
node_hold_expiration = self.globals.default_hold_expiration
|
||||
|
||||
# Reset node_hold_expiration to max if it exceeds the max
|
||||
elif max_hold and (node_hold_expiration == 0 or
|
||||
node_hold_expiration > max_hold):
|
||||
node_hold_expiration = max_hold
|
||||
elif self.globals.max_hold_expiration and (
|
||||
node_hold_expiration == 0 or
|
||||
node_hold_expiration > self.globals.max_hold_expiration):
|
||||
node_hold_expiration = self.globals.max_hold_expiration
|
||||
|
||||
request.node_expiration = node_hold_expiration
|
||||
|
||||
|
|
@ -906,16 +887,14 @@ class Scheduler(threading.Thread):
|
|||
# a request
|
||||
reconfigured_tenants = []
|
||||
with self.layout_lock:
|
||||
self.config = self._zuul_app.config
|
||||
self.log.info("Reconfiguration beginning (smart=%s)", event.smart)
|
||||
start = time.monotonic()
|
||||
|
||||
# Reload the ansible manager in case the default ansible version
|
||||
# changed.
|
||||
default_ansible_version = get_default(
|
||||
self.config, 'scheduler', 'default_ansible_version', None)
|
||||
# Update runtime related system attributes from config
|
||||
self.config = self._zuul_app.config
|
||||
self.globals = SystemAttributes.fromConfig(self.config)
|
||||
self.ansible_manager = AnsibleManager(
|
||||
default_version=default_ansible_version)
|
||||
default_version=self.globals.default_ansible_version)
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
|
|
@ -1961,9 +1940,7 @@ class Scheduler(threading.Thread):
|
|||
def formatStatusJSON(self, tenant_name):
|
||||
# TODOv3(jeblair): use tenants
|
||||
data = {}
|
||||
|
||||
data['zuul_version'] = self.zuul_version
|
||||
websocket_url = get_default(self.config, 'web', 'websocket_url', None)
|
||||
|
||||
data['trigger_event_queue'] = {}
|
||||
data['trigger_event_queue']['length'] = len(
|
||||
|
|
@ -2002,7 +1979,7 @@ class Scheduler(threading.Thread):
|
|||
result_event_queues = self.pipeline_result_events[tenant_name]
|
||||
management_event_queues = self.pipeline_management_events[tenant_name]
|
||||
for pipeline in tenant.layout.pipelines.values():
|
||||
status = pipeline.formatStatusJSON(websocket_url)
|
||||
status = pipeline.formatStatusJSON(self.globals.websocket_url)
|
||||
status['trigger_events'] = len(
|
||||
trigger_event_queues[pipeline.name])
|
||||
status['result_events'] = len(
|
||||
|
|
|
|||
Loading…
Reference in New Issue