Make the ConfigLoader work independently of the Scheduler
This is an early preparation step for removing the RPC calls between zuul-web and the scheduler. We want to format the status JSON and do the job freezing (job freezing API) directly in zuul-web without utilising the scheduler via RPC. In order to make this work, zuul-web must instantiate a ConfigLoader. Currently this would require a scheduler instance which is not available in zuul-web, thus we have to make this parameter optional. Change-Id: I41214086aaa9d822ab888baf001972d2846528be
This commit is contained in:
parent
2c900c2c4a
commit
3029b16489
|
@ -57,7 +57,7 @@ class BaseClientTestCase(BaseTestCase):
|
|||
|
||||
|
||||
class TestTenantValidationClient(BaseClientTestCase):
|
||||
config_with_zk = False
|
||||
config_with_zk = True
|
||||
|
||||
def test_client_tenant_conf_check(self):
|
||||
self.config.set(
|
||||
|
|
|
@ -11,18 +11,52 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from configparser import ConfigParser
|
||||
|
||||
import fixtures
|
||||
import logging
|
||||
import textwrap
|
||||
import testtools
|
||||
from collections import defaultdict
|
||||
from configparser import ConfigParser
|
||||
|
||||
from zuul import model
|
||||
from zuul.configloader import AuthorizationRuleParser, safe_load_yaml
|
||||
from zuul.lib.ansible import AnsibleManager
|
||||
from zuul.configloader import (
|
||||
AuthorizationRuleParser, ConfigLoader, safe_load_yaml
|
||||
)
|
||||
from zuul.model import Abide, MergeRequest, SourceContext
|
||||
from zuul.zk.locks import tenant_read_lock
|
||||
|
||||
from tests.base import ZuulTestCase
|
||||
from zuul.model import MergeRequest, SourceContext
|
||||
|
||||
|
||||
class TestConfigLoader(ZuulTestCase):
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
|
||||
def test_update_system_config(self):
|
||||
"""Test if the system config can be updated without a scheduler."""
|
||||
sched = self.scheds.first.sched
|
||||
|
||||
# Get the current system config before instantiating a ConfigLoader.
|
||||
unparsed_abide, zuul_globals = sched.system_config_cache.get()
|
||||
ansible_manager = AnsibleManager(
|
||||
default_version=zuul_globals.default_ansible_version)
|
||||
loader = ConfigLoader(
|
||||
sched.connections, self.zk_client, zuul_globals, sched.statsd,
|
||||
keystorage=sched.keystore)
|
||||
abide = Abide()
|
||||
loader.loadTPCs(abide, unparsed_abide)
|
||||
loader.loadAdminRules(abide, unparsed_abide)
|
||||
|
||||
for tenant_name in unparsed_abide.tenants:
|
||||
tlock = tenant_read_lock(self.zk_client, tenant_name)
|
||||
# Consider all caches valid (min. ltime -1)
|
||||
min_ltimes = defaultdict(lambda: defaultdict(lambda: -1))
|
||||
with tlock:
|
||||
tenant = loader.loadTenant(
|
||||
abide, tenant_name, ansible_manager, unparsed_abide,
|
||||
min_ltimes=min_ltimes)
|
||||
|
||||
self.assertEqual(tenant.name, tenant_name)
|
||||
|
||||
|
||||
class TenantParserTestCase(ZuulTestCase):
|
||||
|
|
|
@ -31,6 +31,7 @@ import urllib.parse
|
|||
import zuul.rpcclient
|
||||
import zuul.cmd
|
||||
from zuul.lib.config import get_default
|
||||
from zuul.model import SystemAttributes
|
||||
from zuul.zk import ZooKeeperClient
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
|
||||
|
@ -830,9 +831,12 @@ class Client(zuul.cmd.ZuulApp):
|
|||
self.connections = connections
|
||||
self.unparsed_config_cache = None
|
||||
|
||||
sched = SchedulerConfig(self.config, self.connections)
|
||||
zk_client = ZooKeeperClient.fromConfig(self.config)
|
||||
zk_client.connect()
|
||||
zuul_globals = SystemAttributes.fromConfig(self.config)
|
||||
loader = configloader.ConfigLoader(
|
||||
sched.connections, sched, None, None)
|
||||
self.connections, zk_client, zuul_globals)
|
||||
sched = SchedulerConfig(self.config, self.connections)
|
||||
tenant_config, script = sched._checkTenantSourceConf(self.config)
|
||||
unparsed_abide = loader.readConfig(tenant_config, from_script=script)
|
||||
try:
|
||||
|
|
|
@ -33,6 +33,7 @@ import zuul.manager.serial
|
|||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.lib.re2util import filter_allowed_disallowed
|
||||
from zuul.lib.varnames import check_varnames
|
||||
from zuul.zk.config_cache import UnparsedConfigCache
|
||||
from zuul.zk.semaphore import SemaphoreHandler
|
||||
|
||||
|
||||
|
@ -1447,13 +1448,17 @@ class ParseContext(object):
|
|||
|
||||
|
||||
class TenantParser(object):
|
||||
def __init__(self, connections, scheduler, merger, keystorage):
|
||||
def __init__(self, connections, zk_client, scheduler, merger, keystorage,
|
||||
zuul_globals, statsd):
|
||||
self.log = logging.getLogger("zuul.TenantParser")
|
||||
self.connections = connections
|
||||
self.zk_client = zk_client
|
||||
self.scheduler = scheduler
|
||||
self.merger = merger
|
||||
self.keystorage = keystorage
|
||||
self.unparsed_config_cache = self.scheduler.unparsed_config_cache
|
||||
self.globals = zuul_globals
|
||||
self.statsd = statsd
|
||||
self.unparsed_config_cache = UnparsedConfigCache(self.zk_client)
|
||||
|
||||
classes = vs.Any('pipeline', 'job', 'semaphore', 'project',
|
||||
'project-template', 'nodeset', 'secret', 'queue')
|
||||
|
@ -1534,7 +1539,7 @@ class TenantParser(object):
|
|||
tenant.authorization_rules = conf['admin-rules']
|
||||
if conf.get('authentication-realm') is not None:
|
||||
tenant.default_auth_realm = conf['authentication-realm']
|
||||
tenant.web_root = conf.get('web-root', self.scheduler.globals.web_root)
|
||||
tenant.web_root = conf.get('web-root', self.globals.web_root)
|
||||
if tenant.web_root and not tenant.web_root.endswith('/'):
|
||||
tenant.web_root += '/'
|
||||
tenant.allowed_triggers = conf.get('allowed-triggers')
|
||||
|
@ -1603,8 +1608,7 @@ class TenantParser(object):
|
|||
tenant.layout = self._parseLayout(
|
||||
tenant, parsed_config, loading_errors, layout_uuid)
|
||||
tenant.semaphore_handler = SemaphoreHandler(
|
||||
self.scheduler.zk_client, self.scheduler.statsd,
|
||||
tenant.name, tenant.layout
|
||||
self.zk_client, self.statsd, tenant.name, tenant.layout
|
||||
)
|
||||
|
||||
return tenant
|
||||
|
@ -1791,7 +1795,10 @@ class TenantParser(object):
|
|||
|
||||
extra_config_files = abide.getExtraConfigFiles(project.name)
|
||||
extra_config_dirs = abide.getExtraConfigDirs(project.name)
|
||||
ltime = self.scheduler.zk_client.getCurrentLtime()
|
||||
if not self.merger:
|
||||
raise RuntimeError(
|
||||
"Cannot load config files without a merger client.")
|
||||
ltime = self.zk_client.getCurrentLtime()
|
||||
job = self.merger.getFiles(
|
||||
project.source.connection.connection_name,
|
||||
project.name, branch,
|
||||
|
@ -2205,8 +2212,12 @@ class TenantParser(object):
|
|||
|
||||
self._addLayoutItems(layout, tenant, data)
|
||||
|
||||
for pipeline in layout.pipelines.values():
|
||||
pipeline.manager._postConfig(layout)
|
||||
# Only call the postConfig hook if we have a scheduler as this will
|
||||
# change data in ZooKeeper. In case we are in a zuul-web context,
|
||||
# we don't want to do that.
|
||||
if self.scheduler:
|
||||
for pipeline in layout.pipelines.values():
|
||||
pipeline.manager._postConfig(layout)
|
||||
|
||||
return layout
|
||||
|
||||
|
@ -2214,13 +2225,17 @@ class TenantParser(object):
|
|||
class ConfigLoader(object):
|
||||
log = logging.getLogger("zuul.ConfigLoader")
|
||||
|
||||
def __init__(self, connections, scheduler, merger, keystorage):
|
||||
def __init__(self, connections, zk_client, zuul_globals, statsd=None,
|
||||
scheduler=None, merger=None, keystorage=None):
|
||||
self.connections = connections
|
||||
self.zk_client = zk_client
|
||||
self.globals = zuul_globals
|
||||
self.scheduler = scheduler
|
||||
self.merger = merger
|
||||
self.keystorage = keystorage
|
||||
self.tenant_parser = TenantParser(connections, scheduler,
|
||||
merger, self.keystorage)
|
||||
self.tenant_parser = TenantParser(
|
||||
connections, zk_client, scheduler, merger, keystorage,
|
||||
zuul_globals, statsd)
|
||||
self.admin_rule_parser = AuthorizationRuleParser()
|
||||
|
||||
def expandConfigPath(self, config_path):
|
||||
|
|
|
@ -65,13 +65,15 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
self.ref_filters = []
|
||||
# Cached dynamic layouts (layout uuid -> layout)
|
||||
self._layout_cache = {}
|
||||
self.sql = self.sched.sql
|
||||
# A small local cache to avoid hitting the ZK-based connection
|
||||
# change cache for multiple hits in the same pipeline run.
|
||||
self._change_cache = {}
|
||||
# Current ZK context when the pipeline is locked
|
||||
self.current_context = None
|
||||
|
||||
if sched:
|
||||
self.sql = sched.sql
|
||||
|
||||
def __str__(self):
|
||||
return "<%s %s>" % (self.__class__.__name__, self.pipeline.name)
|
||||
|
||||
|
@ -896,7 +898,8 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
# Late import to break an import loop
|
||||
import zuul.configloader
|
||||
loader = zuul.configloader.ConfigLoader(
|
||||
self.sched.connections, self.sched, None, None)
|
||||
self.sched.connections, self.sched.zk_client, self.sched.globals,
|
||||
self.sched.statsd, self.sched)
|
||||
|
||||
log.debug("Loading dynamic layout")
|
||||
|
||||
|
|
|
@ -768,7 +768,8 @@ class Scheduler(threading.Thread):
|
|||
self.primeSystemConfig()
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd, self,
|
||||
self.merger, self.keystore)
|
||||
new_tenants = (set(self.unparsed_abide.tenants)
|
||||
- self.abide.tenants.keys())
|
||||
|
||||
|
@ -993,7 +994,8 @@ class Scheduler(threading.Thread):
|
|||
# Consider all caches valid (min. ltime -1)
|
||||
min_ltimes = defaultdict(lambda: defaultdict(lambda: -1))
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd, self,
|
||||
self.merger, self.keystore)
|
||||
with self.layout_lock:
|
||||
self.log.debug("Updating local layout of tenant %s ", tenant_name)
|
||||
layout_state = self.tenant_layout_state.get(tenant_name)
|
||||
|
@ -1044,7 +1046,8 @@ class Scheduler(threading.Thread):
|
|||
start = time.monotonic()
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd,
|
||||
self, self.merger, self.keystore)
|
||||
tenant_config, script = self._checkTenantSourceConf(self.config)
|
||||
unparsed_abide = loader.readConfig(tenant_config,
|
||||
from_script=script)
|
||||
|
@ -1098,7 +1101,8 @@ class Scheduler(threading.Thread):
|
|||
default_version=self.globals.default_ansible_version)
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd,
|
||||
self, self.merger, self.keystore)
|
||||
tenant_config, script = self._checkTenantSourceConf(self.config)
|
||||
old_unparsed_abide = self.unparsed_abide
|
||||
self.unparsed_abide = loader.readConfig(
|
||||
|
@ -1188,7 +1192,8 @@ class Scheduler(threading.Thread):
|
|||
branch_cache_min_ltimes[connection_name] = ltime
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd,
|
||||
self, self.merger, self.keystore)
|
||||
old_tenant = self.abide.tenants.get(event.tenant_name)
|
||||
loader.loadTPCs(self.abide, self.unparsed_abide,
|
||||
[event.tenant_name])
|
||||
|
@ -1655,7 +1660,8 @@ class Scheduler(threading.Thread):
|
|||
def primeSystemConfig(self):
|
||||
with self.layout_lock:
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd,
|
||||
self, self.merger, self.keystore)
|
||||
tenant_config, script = self._checkTenantSourceConf(self.config)
|
||||
self.unparsed_abide = loader.readConfig(
|
||||
tenant_config, from_script=script)
|
||||
|
@ -1670,7 +1676,8 @@ class Scheduler(threading.Thread):
|
|||
self.ansible_manager = AnsibleManager(
|
||||
default_version=self.globals.default_ansible_version)
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
self.connections, self.zk_client, self.globals, self.statsd,
|
||||
self, self.merger, self.keystore)
|
||||
loader.loadTPCs(self.abide, self.unparsed_abide)
|
||||
loader.loadAdminRules(self.abide, self.unparsed_abide)
|
||||
|
||||
|
|
Loading…
Reference in New Issue