Merge "Required SQL reporters"
This commit is contained in:
commit
bbe3bb0297
|
@ -28,9 +28,7 @@ name=opendev
|
||||||
driver=git
|
driver=git
|
||||||
baseurl=https://opendev.org
|
baseurl=https://opendev.org
|
||||||
|
|
||||||
[connection "mysql"]
|
[database]
|
||||||
name=mysql
|
|
||||||
driver=sql
|
|
||||||
dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
|
dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
|
||||||
|
|
||||||
[web]
|
[web]
|
||||||
|
|
|
@ -17,11 +17,9 @@
|
||||||
success:
|
success:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
mysql:
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
mysql:
|
|
||||||
|
|
||||||
- pipeline:
|
- pipeline:
|
||||||
name: gate
|
name: gate
|
||||||
|
@ -48,8 +46,6 @@
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 2
|
Verified: 2
|
||||||
submit: true
|
submit: true
|
||||||
mysql:
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -2
|
Verified: -2
|
||||||
mysql:
|
|
||||||
|
|
|
@ -148,6 +148,9 @@ service in Zuul, and a connection to Gerrit.
|
||||||
user=zuul
|
user=zuul
|
||||||
sshkey=/home/zuul/.ssh/id_rsa
|
sshkey=/home/zuul/.ssh/id_rsa
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=mysql+pymysql://zuul:secret@mysql/zuul
|
||||||
|
|
||||||
See :ref:`components` and :ref:`connections` for more details.
|
See :ref:`components` and :ref:`connections` for more details.
|
||||||
|
|
||||||
The following tells Zuul to read its configuration from and operate on
|
The following tells Zuul to read its configuration from and operate on
|
||||||
|
|
|
@ -5,6 +5,7 @@ Admin Reference
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
tenants
|
tenants
|
||||||
|
database
|
||||||
connections
|
connections
|
||||||
drivers/index
|
drivers/index
|
||||||
client
|
client
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
:title: Database Configuration
|
||||||
|
|
||||||
|
.. _database:
|
||||||
|
|
||||||
|
Database Configuration
|
||||||
|
======================
|
||||||
|
|
||||||
|
The database configuration is located in the ``database`` section of
|
||||||
|
``zuul.conf``:
|
||||||
|
|
||||||
|
.. code-block:: ini
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=<URI>
|
||||||
|
|
||||||
|
The following options can be defined in this section.
|
||||||
|
|
||||||
|
.. attr:: database
|
||||||
|
|
||||||
|
.. attr:: dburi
|
||||||
|
:required:
|
||||||
|
|
||||||
|
Database connection information in the form of a URI understood
|
||||||
|
by SQLAlchemy. See `The SQLAlchemy manual
|
||||||
|
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
|
||||||
|
for more information.
|
||||||
|
|
||||||
|
The driver will automatically set up the database creating and managing
|
||||||
|
the necessary tables. Therefore the provided user should have sufficient
|
||||||
|
permissions to manage the database. For example:
|
||||||
|
|
||||||
|
.. code-block:: sql
|
||||||
|
|
||||||
|
GRANT ALL ON my_database TO 'my_user'@'%';
|
||||||
|
|
||||||
|
.. attr:: pool_recycle
|
||||||
|
:default: 1
|
||||||
|
|
||||||
|
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
|
||||||
|
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
|
||||||
|
for more information.
|
||||||
|
|
||||||
|
.. attr:: table_prefix
|
||||||
|
:default: ''
|
||||||
|
|
||||||
|
The string to prefix the table names. This makes it possible to run
|
||||||
|
several zuul deployments against the same database. This can be useful
|
||||||
|
if you rely on external databases which are not under your control.
|
||||||
|
The default is to have no prefix.
|
|
@ -5,6 +5,10 @@
|
||||||
SQL
|
SQL
|
||||||
===
|
===
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
This driver is deprecated, use :attr:`database` configuration instead.
|
||||||
|
|
||||||
The SQL driver supports reporters only. Only one connection per
|
The SQL driver supports reporters only. Only one connection per
|
||||||
database is permitted.
|
database is permitted.
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
upgrade:
|
||||||
|
- |
|
||||||
|
The components zuul-scheduler and zuul-web now require database
|
||||||
|
configuration. There is now only one supported database connection. If
|
||||||
|
multiple sql connections have been configured only the first one will be
|
||||||
|
used.
|
||||||
|
|
||||||
|
deprecations:
|
||||||
|
- |
|
||||||
|
Defining database connections using :attr:`<sql connection>` is now
|
||||||
|
deprecated. Refer to :ref:`database` how to configure the database now.
|
|
@ -68,6 +68,7 @@ from git.util import IterableList
|
||||||
import yaml
|
import yaml
|
||||||
import paramiko
|
import paramiko
|
||||||
|
|
||||||
|
from zuul.driver.sql.sqlconnection import DatabaseSession
|
||||||
from zuul.model import Change
|
from zuul.model import Change
|
||||||
from zuul.rpcclient import RPCClient
|
from zuul.rpcclient import RPCClient
|
||||||
|
|
||||||
|
@ -76,7 +77,7 @@ from zuul.driver.git import GitDriver
|
||||||
from zuul.driver.smtp import SMTPDriver
|
from zuul.driver.smtp import SMTPDriver
|
||||||
from zuul.driver.github import GithubDriver
|
from zuul.driver.github import GithubDriver
|
||||||
from zuul.driver.timer import TimerDriver
|
from zuul.driver.timer import TimerDriver
|
||||||
from zuul.driver.sql import SQLDriver
|
from zuul.driver.sql import SQLDriver, sqlconnection
|
||||||
from zuul.driver.bubblewrap import BubblewrapDriver
|
from zuul.driver.bubblewrap import BubblewrapDriver
|
||||||
from zuul.driver.nullwrap import NullwrapDriver
|
from zuul.driver.nullwrap import NullwrapDriver
|
||||||
from zuul.driver.mqtt import MQTTDriver
|
from zuul.driver.mqtt import MQTTDriver
|
||||||
|
@ -317,12 +318,19 @@ class GitlabDriverMock(GitlabDriver):
|
||||||
return connection
|
return connection
|
||||||
|
|
||||||
|
|
||||||
|
class SQLDriverMock(SQLDriver):
|
||||||
|
|
||||||
|
def getConnection(self, name, config):
|
||||||
|
return FakeSqlConnection(self, name, config)
|
||||||
|
|
||||||
|
|
||||||
class TestConnectionRegistry(ConnectionRegistry):
|
class TestConnectionRegistry(ConnectionRegistry):
|
||||||
def __init__(self, changes: Dict[str, Dict[str, Change]],
|
def __init__(self, changes: Dict[str, Dict[str, Change]],
|
||||||
config: ConfigParser, additional_event_queues,
|
config: ConfigParser, additional_event_queues,
|
||||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||||
git_url_with_auth: bool,
|
git_url_with_auth: bool,
|
||||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
add_cleanup: Callable[[Callable[[], None]], None],
|
||||||
|
fake_sql: bool):
|
||||||
self.connections = OrderedDict()
|
self.connections = OrderedDict()
|
||||||
self.drivers = {}
|
self.drivers = {}
|
||||||
|
|
||||||
|
@ -336,7 +344,10 @@ class TestConnectionRegistry(ConnectionRegistry):
|
||||||
rpcclient, git_url_with_auth))
|
rpcclient, git_url_with_auth))
|
||||||
self.registerDriver(SMTPDriver())
|
self.registerDriver(SMTPDriver())
|
||||||
self.registerDriver(TimerDriver())
|
self.registerDriver(TimerDriver())
|
||||||
self.registerDriver(SQLDriver())
|
if fake_sql:
|
||||||
|
self.registerDriver(SQLDriverMock())
|
||||||
|
else:
|
||||||
|
self.registerDriver(SQLDriver())
|
||||||
self.registerDriver(BubblewrapDriver())
|
self.registerDriver(BubblewrapDriver())
|
||||||
self.registerDriver(NullwrapDriver())
|
self.registerDriver(NullwrapDriver())
|
||||||
self.registerDriver(MQTTDriver())
|
self.registerDriver(MQTTDriver())
|
||||||
|
@ -2961,6 +2972,37 @@ class FakeBuild(object):
|
||||||
return repos
|
return repos
|
||||||
|
|
||||||
|
|
||||||
|
class FakeZuulDatabaseSession(DatabaseSession):
|
||||||
|
|
||||||
|
def __exit__(self, etype, value, tb):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def FakeOrmSessionFactory():
|
||||||
|
class FakeBackendSession:
|
||||||
|
def add(self, *args, **kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return FakeBackendSession()
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSqlConnection(sqlconnection.SQLConnection):
|
||||||
|
|
||||||
|
def __init__(self, driver, connection_name, connection_config):
|
||||||
|
connection_config['dburi'] = 'postgresql://example.com'
|
||||||
|
super().__init__(driver, connection_name, connection_config)
|
||||||
|
self.session = FakeOrmSessionFactory
|
||||||
|
|
||||||
|
def onLoad(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def onStop(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def getSession(self):
|
||||||
|
return FakeZuulDatabaseSession(self)
|
||||||
|
|
||||||
|
|
||||||
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
|
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
|
||||||
result = None
|
result = None
|
||||||
|
|
||||||
|
@ -3659,12 +3701,12 @@ class ZuulWebFixture(fixtures.Fixture):
|
||||||
additional_event_queues, upstream_root: str,
|
additional_event_queues, upstream_root: str,
|
||||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||||
add_cleanup: Callable[[Callable[[], None]], None],
|
add_cleanup: Callable[[Callable[[], None]], None],
|
||||||
test_root, info=None, zk_hosts=None):
|
test_root, info=None, zk_hosts=None, fake_sql=True):
|
||||||
super(ZuulWebFixture, self).__init__()
|
super(ZuulWebFixture, self).__init__()
|
||||||
self.gearman_server_port = gearman_server_port
|
self.gearman_server_port = gearman_server_port
|
||||||
self.connections = TestConnectionRegistry(
|
self.connections = TestConnectionRegistry(
|
||||||
changes, config, additional_event_queues, upstream_root, rpcclient,
|
changes, config, additional_event_queues, upstream_root, rpcclient,
|
||||||
poller_events, git_url_with_auth, add_cleanup)
|
poller_events, git_url_with_auth, add_cleanup, fake_sql)
|
||||||
self.connections.configure(
|
self.connections.configure(
|
||||||
config,
|
config,
|
||||||
include_drivers=[zuul.driver.sql.SQLDriver,
|
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||||
|
@ -3919,6 +3961,7 @@ class SchedulerTestApp:
|
||||||
additional_event_queues, upstream_root: str,
|
additional_event_queues, upstream_root: str,
|
||||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||||
source_only: bool,
|
source_only: bool,
|
||||||
|
fake_sql: bool,
|
||||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
add_cleanup: Callable[[Callable[[], None]], None]):
|
||||||
|
|
||||||
self.log = log
|
self.log = log
|
||||||
|
@ -3940,7 +3983,7 @@ class SchedulerTestApp:
|
||||||
self.connections = TestConnectionRegistry(
|
self.connections = TestConnectionRegistry(
|
||||||
self.changes, self.config, additional_event_queues,
|
self.changes, self.config, additional_event_queues,
|
||||||
upstream_root, rpcclient, poller_events,
|
upstream_root, rpcclient, poller_events,
|
||||||
git_url_with_auth, add_cleanup)
|
git_url_with_auth, add_cleanup, fake_sql)
|
||||||
self.connections.configure(self.config, source_only=source_only)
|
self.connections.configure(self.config, source_only=source_only)
|
||||||
|
|
||||||
self.sched.registerConnections(self.connections)
|
self.sched.registerConnections(self.connections)
|
||||||
|
@ -3989,12 +4032,13 @@ class SchedulerTestManager:
|
||||||
changes: Dict[str, Dict[str, Change]], additional_event_queues,
|
changes: Dict[str, Dict[str, Change]], additional_event_queues,
|
||||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||||
git_url_with_auth: bool, source_only: bool,
|
git_url_with_auth: bool, source_only: bool,
|
||||||
|
fake_sql: bool,
|
||||||
add_cleanup: Callable[[Callable[[], None]], None])\
|
add_cleanup: Callable[[Callable[[], None]], None])\
|
||||||
-> SchedulerTestApp:
|
-> SchedulerTestApp:
|
||||||
app = SchedulerTestApp(log, config, zk_config, changes,
|
app = SchedulerTestApp(log, config, zk_config, changes,
|
||||||
additional_event_queues, upstream_root,
|
additional_event_queues, upstream_root,
|
||||||
rpcclient, poller_events, git_url_with_auth,
|
rpcclient, poller_events, git_url_with_auth,
|
||||||
source_only, add_cleanup)
|
source_only, fake_sql, add_cleanup)
|
||||||
self.instances.append(app)
|
self.instances.append(app)
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
@ -4102,6 +4146,7 @@ class ZuulTestCase(BaseTestCase):
|
||||||
git_url_with_auth: bool = False
|
git_url_with_auth: bool = False
|
||||||
log_console_port: int = 19885
|
log_console_port: int = 19885
|
||||||
source_only: bool = False
|
source_only: bool = False
|
||||||
|
fake_sql: bool = True
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
"""Allows to access fake connections the old way, e.g., using
|
"""Allows to access fake connections the old way, e.g., using
|
||||||
|
@ -4233,7 +4278,7 @@ class ZuulTestCase(BaseTestCase):
|
||||||
executor_connections = TestConnectionRegistry(
|
executor_connections = TestConnectionRegistry(
|
||||||
self.changes, self.config, self.additional_event_queues,
|
self.changes, self.config, self.additional_event_queues,
|
||||||
self.upstream_root, self.rpcclient, self.poller_events,
|
self.upstream_root, self.rpcclient, self.poller_events,
|
||||||
self.git_url_with_auth, self.addCleanup)
|
self.git_url_with_auth, self.addCleanup, True)
|
||||||
executor_connections.configure(self.config,
|
executor_connections.configure(self.config,
|
||||||
source_only=self.source_only)
|
source_only=self.source_only)
|
||||||
self.executor_server = RecordingExecutorServer(
|
self.executor_server = RecordingExecutorServer(
|
||||||
|
@ -4252,7 +4297,7 @@ class ZuulTestCase(BaseTestCase):
|
||||||
self.log, self.config, self.zk_config, self.changes,
|
self.log, self.config, self.zk_config, self.changes,
|
||||||
self.additional_event_queues, self.upstream_root, self.rpcclient,
|
self.additional_event_queues, self.upstream_root, self.rpcclient,
|
||||||
self.poller_events, self.git_url_with_auth, self.source_only,
|
self.poller_events, self.git_url_with_auth, self.source_only,
|
||||||
self.addCleanup)
|
self.fake_sql, self.addCleanup)
|
||||||
|
|
||||||
if hasattr(self, 'fake_github'):
|
if hasattr(self, 'fake_github'):
|
||||||
self.additional_event_queues.append(
|
self.additional_event_queues.append(
|
||||||
|
@ -4343,6 +4388,7 @@ class ZuulTestCase(BaseTestCase):
|
||||||
# Make test_root persist after ansible run for .flag test
|
# Make test_root persist after ansible run for .flag test
|
||||||
config.set('executor', 'trusted_rw_paths', self.test_root)
|
config.set('executor', 'trusted_rw_paths', self.test_root)
|
||||||
self.setupAllProjectKeys(config)
|
self.setupAllProjectKeys(config)
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
def setupSimpleLayout(self, config: ConfigParser):
|
def setupSimpleLayout(self, config: ConfigParser):
|
||||||
|
@ -5345,7 +5391,21 @@ class SSLZuulTestCase(ZuulTestCase):
|
||||||
|
|
||||||
|
|
||||||
class ZuulDBTestCase(ZuulTestCase):
|
class ZuulDBTestCase(ZuulTestCase):
|
||||||
|
fake_sql = False
|
||||||
|
|
||||||
def setup_config(self, config_file: str):
|
def setup_config(self, config_file: str):
|
||||||
|
def _setup_fixture(config, section_name):
|
||||||
|
if (config.get(section_name, 'dburi') ==
|
||||||
|
'$MYSQL_FIXTURE_DBURI$'):
|
||||||
|
f = MySQLSchemaFixture()
|
||||||
|
self.useFixture(f)
|
||||||
|
config.set(section_name, 'dburi', f.dburi)
|
||||||
|
elif (config.get(section_name, 'dburi') ==
|
||||||
|
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||||
|
f = PostgresqlSchemaFixture()
|
||||||
|
self.useFixture(f)
|
||||||
|
config.set(section_name, 'dburi', f.dburi)
|
||||||
|
|
||||||
config = super(ZuulDBTestCase, self).setup_config(config_file)
|
config = super(ZuulDBTestCase, self).setup_config(config_file)
|
||||||
for section_name in config.sections():
|
for section_name in config.sections():
|
||||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||||
|
@ -5354,16 +5414,11 @@ class ZuulDBTestCase(ZuulTestCase):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if config.get(section_name, 'driver') == 'sql':
|
if config.get(section_name, 'driver') == 'sql':
|
||||||
if (config.get(section_name, 'dburi') ==
|
_setup_fixture(config, section_name)
|
||||||
'$MYSQL_FIXTURE_DBURI$'):
|
|
||||||
f = MySQLSchemaFixture()
|
if 'database' in config.sections():
|
||||||
self.useFixture(f)
|
_setup_fixture(config, 'database')
|
||||||
config.set(section_name, 'dburi', f.dburi)
|
|
||||||
elif (config.get(section_name, 'dburi') ==
|
|
||||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
|
||||||
f = PostgresqlSchemaFixture()
|
|
||||||
self.useFixture(f)
|
|
||||||
config.set(section_name, 'dburi', f.dburi)
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -7,15 +7,10 @@
|
||||||
success:
|
success:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
resultsdb_mysql: null
|
resultsdb_failures: null
|
||||||
resultsdb_mysql_failures: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
resultsdb_postgresql_failures: null
|
|
||||||
|
|
||||||
- pipeline:
|
- pipeline:
|
||||||
name: gate
|
name: gate
|
||||||
|
@ -30,15 +25,10 @@
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 2
|
Verified: 2
|
||||||
submit: true
|
submit: true
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -2
|
Verified: -2
|
||||||
resultsdb_mysql: null
|
resultsdb_failures: null
|
||||||
resultsdb_mysql_failures: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
resultsdb_postgresql_failures: null
|
|
||||||
start:
|
start:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 0
|
Verified: 0
|
||||||
|
@ -51,14 +41,8 @@
|
||||||
gerrit:
|
gerrit:
|
||||||
- event: ref-updated
|
- event: ref-updated
|
||||||
ref: ^refs/tags/.*$
|
ref: ^refs/tags/.*$
|
||||||
success:
|
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
resultsdb_mysql: null
|
resultsdb_failures: null
|
||||||
resultsdb_mysql_failures: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
resultsdb_postgresql_failures: null
|
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: base
|
name: base
|
||||||
|
|
|
@ -7,13 +7,9 @@
|
||||||
success:
|
success:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: base
|
name: base
|
||||||
|
|
|
@ -7,13 +7,9 @@
|
||||||
success:
|
success:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
|
|
||||||
- pipeline:
|
- pipeline:
|
||||||
name: gate
|
name: gate
|
||||||
|
|
|
@ -7,13 +7,9 @@
|
||||||
success:
|
success:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
failure:
|
failure:
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
resultsdb_mysql: null
|
|
||||||
resultsdb_postgresql: null
|
|
||||||
|
|
||||||
- pipeline:
|
- pipeline:
|
||||||
name: gate
|
name: gate
|
||||||
|
|
|
@ -5,11 +5,9 @@
|
||||||
gerrit:
|
gerrit:
|
||||||
- event: patchset-created
|
- event: patchset-created
|
||||||
success:
|
success:
|
||||||
resultsdb_mysql: null
|
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: 1
|
Verified: 1
|
||||||
failure:
|
failure:
|
||||||
resultsdb_mysql: null
|
|
||||||
gerrit:
|
gerrit:
|
||||||
Verified: -1
|
Verified: -1
|
||||||
|
|
||||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
||||||
client_id=zuul.example.com
|
client_id=zuul.example.com
|
||||||
issuer_id=zuul_operator
|
issuer_id=zuul_operator
|
||||||
secret=NoDanaOnlyZuul
|
secret=NoDanaOnlyZuul
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -43,3 +43,7 @@ realm=myOIDC2
|
||||||
client_id=zuul
|
client_id=zuul
|
||||||
issuer_id=http://oidc2
|
issuer_id=http://oidc2
|
||||||
scope=openid profile email special-scope
|
scope=openid profile email special-scope
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -30,3 +30,7 @@ client_id=zuul.example.com
|
||||||
issuer_id=zuul_operator
|
issuer_id=zuul_operator
|
||||||
secret=NoDanaOnlyZuul
|
secret=NoDanaOnlyZuul
|
||||||
max_validity_time=5
|
max_validity_time=5
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
||||||
client_id=zuul.example.com
|
client_id=zuul.example.com
|
||||||
issuer_id=zuul_operator
|
issuer_id=zuul_operator
|
||||||
secret=NoDanaOnlyZuul
|
secret=NoDanaOnlyZuul
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -24,3 +24,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -21,10 +21,10 @@ server=review.example.com
|
||||||
user=jenkins
|
user=jenkins
|
||||||
sshkey=fake_id_rsa1
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
[connection resultsdb]
|
[database]
|
||||||
driver=sql
|
|
||||||
dburi=$MYSQL_FIXTURE_DBURI$
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
||||||
[connection smtp]
|
[connection smtp]
|
||||||
driver=smtp
|
driver=smtp
|
||||||
server=localhost
|
server=localhost
|
||||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -37,3 +37,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -34,3 +34,7 @@ default_to=you@example.com
|
||||||
|
|
||||||
[web]
|
[web]
|
||||||
static_cache_expiry=1200
|
static_cache_expiry=1200
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -23,3 +23,6 @@ driver=elasticsearch
|
||||||
uri=localhost:9200
|
uri=localhost:9200
|
||||||
use_ssl=true
|
use_ssl=true
|
||||||
verify_certs=false
|
verify_certs=false
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
|
@ -46,3 +46,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -31,3 +31,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -18,3 +18,7 @@ webhook_token=0000000000000000000000000000000000000000
|
||||||
app_id=1
|
app_id=1
|
||||||
app_key=$APP_KEY_FIXTURE$
|
app_key=$APP_KEY_FIXTURE$
|
||||||
server=github.enterprise.io
|
server=github.enterprise.io
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -26,3 +26,7 @@ sshkey=/home/zuul/.ssh/id_rsa
|
||||||
driver=github
|
driver=github
|
||||||
sshkey=/home/zuul/.ssh/id_rsa
|
sshkey=/home/zuul/.ssh/id_rsa
|
||||||
server=github.enterprise.io
|
server=github.enterprise.io
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
||||||
driver=gitlab
|
driver=gitlab
|
||||||
server=gitlab
|
server=gitlab
|
||||||
api_token=0000000000000000000000000000000000000000
|
api_token=0000000000000000000000000000000000000000
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -36,3 +36,7 @@ default_to=you@example.com
|
||||||
[web]
|
[web]
|
||||||
static_cache_expiry=1200
|
static_cache_expiry=1200
|
||||||
root=https://zuul.example.com/
|
root=https://zuul.example.com/
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -21,5 +21,8 @@ sshkey=fake_id_rsa1
|
||||||
[connection mqtt]
|
[connection mqtt]
|
||||||
driver=mqtt
|
driver=mqtt
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
[web]
|
[web]
|
||||||
root=https://tenant.example.com/
|
root=https://tenant.example.com/
|
||||||
|
|
|
@ -17,3 +17,7 @@ driver=pagure
|
||||||
server=pagure
|
server=pagure
|
||||||
api_token=0000000000000000000000000000000000000000
|
api_token=0000000000000000000000000000000000000000
|
||||||
source_whitelist=::ffff:127.0.0.1
|
source_whitelist=::ffff:127.0.0.1
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
||||||
driver=pagure
|
driver=pagure
|
||||||
server=pagure
|
server=pagure
|
||||||
api_token=0000000000000000000000000000000000000000
|
api_token=0000000000000000000000000000000000000000
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -20,3 +20,7 @@ webhook_token=00000000000000000000000000000000000000000
|
||||||
driver=gerrit
|
driver=gerrit
|
||||||
server=review.example.com
|
server=review.example.com
|
||||||
user=jenkins
|
user=jenkins
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
|
|
@ -18,8 +18,7 @@ server=review.example.com
|
||||||
user=jenkins
|
user=jenkins
|
||||||
sshkey=fake_id_rsa1
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
[connection resultsdb_mysql]
|
[database]
|
||||||
driver=sql
|
|
||||||
dburi=mysql+pymysql://bad:creds@host/db
|
dburi=mysql+pymysql://bad:creds@host/db
|
||||||
|
|
||||||
[connection resultsdb_mysql_failures]
|
[connection resultsdb_mysql_failures]
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
[gearman]
|
||||||
|
server=127.0.0.1
|
||||||
|
|
||||||
|
[scheduler]
|
||||||
|
tenant_config=main.yaml
|
||||||
|
|
||||||
|
[merger]
|
||||||
|
git_dir=/tmp/zuul-test/merger-git
|
||||||
|
git_user_email=zuul@example.com
|
||||||
|
git_user_name=zuul
|
||||||
|
|
||||||
|
[executor]
|
||||||
|
git_dir=/tmp/zuul-test/executor-git
|
||||||
|
|
||||||
|
[connection gerrit]
|
||||||
|
driver=gerrit
|
||||||
|
server=review.example.com
|
||||||
|
user=jenkins
|
||||||
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
[connection resultsdb_failures]
|
||||||
|
driver=sql
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
|
@ -0,0 +1,26 @@
|
||||||
|
[gearman]
|
||||||
|
server=127.0.0.1
|
||||||
|
|
||||||
|
[scheduler]
|
||||||
|
tenant_config=main.yaml
|
||||||
|
|
||||||
|
[merger]
|
||||||
|
git_dir=/tmp/zuul-test/merger-git
|
||||||
|
git_user_email=zuul@example.com
|
||||||
|
git_user_name=zuul
|
||||||
|
|
||||||
|
[executor]
|
||||||
|
git_dir=/tmp/zuul-test/executor-git
|
||||||
|
|
||||||
|
[connection gerrit]
|
||||||
|
driver=gerrit
|
||||||
|
server=review.example.com
|
||||||
|
user=jenkins
|
||||||
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
|
[connection resultsdb_failures]
|
||||||
|
driver=sql
|
||||||
|
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -18,20 +18,10 @@ server=review.example.com
|
||||||
user=jenkins
|
user=jenkins
|
||||||
sshkey=fake_id_rsa1
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
[connection resultsdb_mysql]
|
[database]
|
||||||
driver=sql
|
|
||||||
dburi=$MYSQL_FIXTURE_DBURI$
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
table_prefix=prefix_
|
table_prefix=prefix_
|
||||||
|
|
||||||
[connection resultsdb_mysql_failures]
|
[connection resultsdb_failures]
|
||||||
driver=sql
|
driver=sql
|
||||||
dburi=$MYSQL_FIXTURE_DBURI$
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
[connection resultsdb_postgresql]
|
|
||||||
driver=sql
|
|
||||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
||||||
table_prefix=prefix_
|
|
||||||
|
|
||||||
[connection resultsdb_postgresql_failures]
|
|
||||||
driver=sql
|
|
||||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
|
@ -18,18 +18,11 @@ server=review.example.com
|
||||||
user=jenkins
|
user=jenkins
|
||||||
sshkey=fake_id_rsa1
|
sshkey=fake_id_rsa1
|
||||||
|
|
||||||
[connection resultsdb_mysql]
|
[database]
|
||||||
driver=sql
|
|
||||||
dburi=$MYSQL_FIXTURE_DBURI$
|
|
||||||
|
|
||||||
[connection resultsdb_mysql_failures]
|
|
||||||
driver=sql
|
|
||||||
dburi=$MYSQL_FIXTURE_DBURI$
|
|
||||||
|
|
||||||
[connection resultsdb_postgresql]
|
|
||||||
driver=sql
|
driver=sql
|
||||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||||
|
table_prefix=prefix_
|
||||||
|
|
||||||
[connection resultsdb_postgresql_failures]
|
[connection resultsdb_failures]
|
||||||
driver=sql
|
driver=sql
|
||||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -35,3 +35,6 @@ server=localhost
|
||||||
port=25
|
port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
|
@ -31,6 +31,9 @@ port=25
|
||||||
default_from=zuul@example.com
|
default_from=zuul@example.com
|
||||||
default_to=you@example.com
|
default_to=you@example.com
|
||||||
|
|
||||||
|
[database]
|
||||||
|
dburi=$MYSQL_FIXTURE_DBURI$
|
||||||
|
|
||||||
[web]
|
[web]
|
||||||
static_cache_expiry=1200
|
static_cache_expiry=1200
|
||||||
root=https://zuul.example.com/
|
root=https://zuul.example.com/
|
||||||
|
|
|
@ -11,14 +11,19 @@
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
import configparser
|
||||||
|
import os
|
||||||
|
import re
|
||||||
import textwrap
|
import textwrap
|
||||||
import time
|
import time
|
||||||
import types
|
import types
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|
||||||
from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
|
import zuul
|
||||||
|
from tests.base import ZuulTestCase, FIXTURE_DIR, \
|
||||||
|
PostgresqlSchemaFixture, MySQLSchemaFixture, ZuulDBTestCase, \
|
||||||
|
BaseTestCase, AnsibleZuulTestCase
|
||||||
|
|
||||||
|
|
||||||
def _get_reporter_from_connection_name(reporters, connection_name):
|
def _get_reporter_from_connection_name(reporters, connection_name):
|
||||||
|
@ -61,8 +66,8 @@ class TestConnections(ZuulTestCase):
|
||||||
'civoter')
|
'civoter')
|
||||||
|
|
||||||
|
|
||||||
class TestSQLConnection(ZuulDBTestCase):
|
class TestSQLConnectionMysql(ZuulDBTestCase):
|
||||||
config_file = 'zuul-sql-driver.conf'
|
config_file = 'zuul-sql-driver-mysql.conf'
|
||||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||||
expected_table_prefix = ''
|
expected_table_prefix = ''
|
||||||
|
|
||||||
|
@ -81,8 +86,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
||||||
|
|
||||||
def test_sql_tables_created(self):
|
def test_sql_tables_created(self):
|
||||||
"Test the tables for storing results are created properly"
|
"Test the tables for storing results are created properly"
|
||||||
self._sql_tables_created('resultsdb_mysql')
|
self._sql_tables_created('database')
|
||||||
self._sql_tables_created('resultsdb_postgresql')
|
|
||||||
|
|
||||||
def _sql_indexes_created(self, connection_name):
|
def _sql_indexes_created(self, connection_name):
|
||||||
connection = self.scheds.first.connections.connections[connection_name]
|
connection = self.scheds.first.connections.connections[connection_name]
|
||||||
|
@ -116,8 +120,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
||||||
|
|
||||||
def test_sql_indexes_created(self):
|
def test_sql_indexes_created(self):
|
||||||
"Test the indexes are created properly"
|
"Test the indexes are created properly"
|
||||||
self._sql_indexes_created('resultsdb_mysql')
|
self._sql_indexes_created('database')
|
||||||
self._sql_indexes_created('resultsdb_postgresql')
|
|
||||||
|
|
||||||
def test_sql_results(self):
|
def test_sql_results(self):
|
||||||
"Test results are entered into an sql table"
|
"Test results are entered into an sql table"
|
||||||
|
@ -225,8 +228,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
||||||
self.orderedRelease()
|
self.orderedRelease()
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
check_results('resultsdb_mysql')
|
check_results('database')
|
||||||
check_results('resultsdb_postgresql')
|
|
||||||
|
|
||||||
def test_sql_results_retry_builds(self):
|
def test_sql_results_retry_builds(self):
|
||||||
"Test that retry results are entered into an sql table correctly"
|
"Test that retry results are entered into an sql table correctly"
|
||||||
|
@ -306,8 +308,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
||||||
self.orderedRelease()
|
self.orderedRelease()
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
check_results('resultsdb_mysql')
|
check_results('database')
|
||||||
check_results('resultsdb_postgresql')
|
|
||||||
|
|
||||||
def test_multiple_sql_connections(self):
|
def test_multiple_sql_connections(self):
|
||||||
"Test putting results in different databases"
|
"Test putting results in different databases"
|
||||||
|
@ -353,36 +354,79 @@ class TestSQLConnection(ZuulDBTestCase):
|
||||||
tenant.layout.pipelines['check'].failure_actions,
|
tenant.layout.pipelines['check'].failure_actions,
|
||||||
connection_name_2
|
connection_name_2
|
||||||
)
|
)
|
||||||
|
self.assertIsNone(reporter2) # Explicit SQL reporters are ignored
|
||||||
|
|
||||||
conn = self.scheds.first.connections.\
|
|
||||||
connections[connection_name_2].engine.connect()
|
|
||||||
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
|
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
|
||||||
[reporter2.connection.zuul_buildset_table])).fetchall()
|
[reporter1.connection.zuul_buildset_table])).fetchall()
|
||||||
# The failure db should only have 1 buildset failed
|
# The failure db should only have 1 buildset failed
|
||||||
self.assertEqual(1, len(buildsets_resultsdb_failures))
|
self.assertEqual(2, len(buildsets_resultsdb_failures))
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
'check', buildsets_resultsdb_failures[0]['pipeline'])
|
'check', buildsets_resultsdb_failures[1]['pipeline'])
|
||||||
self.assertEqual('org/project',
|
self.assertEqual('org/project',
|
||||||
buildsets_resultsdb_failures[0]['project'])
|
buildsets_resultsdb_failures[1]['project'])
|
||||||
self.assertEqual(2,
|
self.assertEqual(2,
|
||||||
buildsets_resultsdb_failures[0]['change'])
|
buildsets_resultsdb_failures[1]['change'])
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
'1', buildsets_resultsdb_failures[0]['patchset'])
|
'1', buildsets_resultsdb_failures[1]['patchset'])
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
'FAILURE', buildsets_resultsdb_failures[0]['result'])
|
'FAILURE', buildsets_resultsdb_failures[1]['result'])
|
||||||
self.assertEqual('Build failed.',
|
self.assertEqual('Build failed.',
|
||||||
buildsets_resultsdb_failures[0]['message'])
|
buildsets_resultsdb_failures[1]['message'])
|
||||||
|
|
||||||
check_results('resultsdb_mysql', 'resultsdb_mysql_failures')
|
check_results('database', 'resultsdb_failures')
|
||||||
check_results('resultsdb_postgresql', 'resultsdb_postgresql_failures')
|
|
||||||
|
|
||||||
|
|
||||||
class TestSQLConnectionPrefix(TestSQLConnection):
|
class TestSQLConnectionPostgres(TestSQLConnectionMysql):
|
||||||
config_file = 'zuul-sql-driver-prefix.conf'
|
config_file = 'zuul-sql-driver-postgres.conf'
|
||||||
|
|
||||||
|
|
||||||
|
class TestSQLConnectionPrefixMysql(TestSQLConnectionMysql):
|
||||||
|
config_file = 'zuul-sql-driver-prefix-mysql.conf'
|
||||||
expected_table_prefix = 'prefix_'
|
expected_table_prefix = 'prefix_'
|
||||||
|
|
||||||
|
|
||||||
|
class TestSQLConnectionPrefixPostgres(TestSQLConnectionMysql):
|
||||||
|
config_file = 'zuul-sql-driver-prefix-postgres.conf'
|
||||||
|
expected_table_prefix = 'prefix_'
|
||||||
|
|
||||||
|
|
||||||
|
class TestRequiredSQLConnection(BaseTestCase):
|
||||||
|
config = None
|
||||||
|
connections = None
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
self.addCleanup(self.stop_connection)
|
||||||
|
|
||||||
|
def setup_connection(self, config_file):
|
||||||
|
self.config = configparser.ConfigParser()
|
||||||
|
self.config.read(os.path.join(FIXTURE_DIR, config_file))
|
||||||
|
|
||||||
|
# Setup databases
|
||||||
|
for section_name in self.config.sections():
|
||||||
|
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||||
|
section_name, re.I)
|
||||||
|
if not con_match:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if self.config.get(section_name, 'driver') == 'sql':
|
||||||
|
if (self.config.get(section_name, 'dburi') ==
|
||||||
|
'$MYSQL_FIXTURE_DBURI$'):
|
||||||
|
f = MySQLSchemaFixture()
|
||||||
|
self.useFixture(f)
|
||||||
|
self.config.set(section_name, 'dburi', f.dburi)
|
||||||
|
elif (self.config.get(section_name, 'dburi') ==
|
||||||
|
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||||
|
f = PostgresqlSchemaFixture()
|
||||||
|
self.useFixture(f)
|
||||||
|
self.config.set(section_name, 'dburi', f.dburi)
|
||||||
|
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||||
|
|
||||||
|
def stop_connection(self):
|
||||||
|
self.connections.stop()
|
||||||
|
|
||||||
|
|
||||||
class TestConnectionsBadSQL(ZuulDBTestCase):
|
class TestConnectionsBadSQL(ZuulDBTestCase):
|
||||||
config_file = 'zuul-sql-driver-bad.conf'
|
config_file = 'zuul-sql-driver-bad.conf'
|
||||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||||
|
|
|
@ -5370,28 +5370,34 @@ For CI problems and help debugging, contact ci@example.org"""
|
||||||
tenant.layout.pipelines['gate'].merge_failure_message)
|
tenant.layout.pipelines['gate'].merge_failure_message)
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
|
len(tenant.layout.pipelines['check'].merge_failure_actions), 2)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
|
len(tenant.layout.pipelines['gate'].merge_failure_actions), 3)
|
||||||
|
|
||||||
self.assertTrue(isinstance(
|
self.assertTrue(isinstance(
|
||||||
tenant.layout.pipelines['check'].merge_failure_actions[0],
|
tenant.layout.pipelines['check'].merge_failure_actions[1],
|
||||||
gerritreporter.GerritReporter))
|
gerritreporter.GerritReporter))
|
||||||
|
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
(
|
(
|
||||||
isinstance(tenant.layout.pipelines['gate'].
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
merge_failure_actions[0],
|
merge_failure_actions[0],
|
||||||
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||||
isinstance(tenant.layout.pipelines['gate'].
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
merge_failure_actions[1],
|
merge_failure_actions[1],
|
||||||
|
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
||||||
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
|
merge_failure_actions[2],
|
||||||
gerritreporter.GerritReporter)
|
gerritreporter.GerritReporter)
|
||||||
) or (
|
) or (
|
||||||
isinstance(tenant.layout.pipelines['gate'].
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
merge_failure_actions[0],
|
merge_failure_actions[0],
|
||||||
gerritreporter.GerritReporter) and
|
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||||
isinstance(tenant.layout.pipelines['gate'].
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
merge_failure_actions[1],
|
merge_failure_actions[1],
|
||||||
|
gerritreporter.GerritReporter) and
|
||||||
|
isinstance(tenant.layout.pipelines['gate'].
|
||||||
|
merge_failure_actions[2],
|
||||||
zuul.driver.smtp.smtpreporter.SMTPReporter)
|
zuul.driver.smtp.smtpreporter.SMTPReporter)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
@ -6140,8 +6140,8 @@ class TestProvidesRequiresBuildset(ZuulTestCase):
|
||||||
}])
|
}])
|
||||||
|
|
||||||
|
|
||||||
class TestProvidesRequires(ZuulDBTestCase):
|
class TestProvidesRequiresMysql(ZuulDBTestCase):
|
||||||
config_file = "zuul-sql-driver.conf"
|
config_file = "zuul-sql-driver-mysql.conf"
|
||||||
|
|
||||||
@simple_layout('layouts/provides-requires.yaml')
|
@simple_layout('layouts/provides-requires.yaml')
|
||||||
def test_provides_requires_shared_queue_fast(self):
|
def test_provides_requires_shared_queue_fast(self):
|
||||||
|
@ -6682,6 +6682,10 @@ class TestProvidesRequires(ZuulDBTestCase):
|
||||||
1, B.messages[0])
|
1, B.messages[0])
|
||||||
|
|
||||||
|
|
||||||
|
class TestProvidesRequiresPostgres(TestProvidesRequiresMysql):
|
||||||
|
config_file = "zuul-sql-driver-postgres.conf"
|
||||||
|
|
||||||
|
|
||||||
class TestForceMergeMissingTemplate(ZuulTestCase):
|
class TestForceMergeMissingTemplate(ZuulTestCase):
|
||||||
tenant_config_file = "config/force-merge-template/main.yaml"
|
tenant_config_file = "config/force-merge-template/main.yaml"
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,6 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import socket
|
import socket
|
||||||
|
@ -62,7 +61,8 @@ class BaseTestWeb(ZuulTestCase):
|
||||||
self.test_root,
|
self.test_root,
|
||||||
info=zuul.model.WebInfo.fromConfig(
|
info=zuul.model.WebInfo.fromConfig(
|
||||||
self.zuul_ini_config),
|
self.zuul_ini_config),
|
||||||
zk_hosts=self.zk_config))
|
zk_hosts=self.zk_config,
|
||||||
|
fake_sql=self.fake_sql))
|
||||||
|
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
|
|
||||||
|
@ -88,6 +88,7 @@ class BaseTestWeb(ZuulTestCase):
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
def get_url(self, url, *args, **kwargs):
|
def get_url(self, url, *args, **kwargs):
|
||||||
|
zuul.web.ZuulWebAPI._tenants.cache_clear()
|
||||||
return requests.get(
|
return requests.get(
|
||||||
urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
|
urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
|
||||||
|
|
||||||
|
@ -266,11 +267,7 @@ class TestWeb(BaseTestWeb):
|
||||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
req = urllib.request.Request(
|
data = self.get_url("api/tenants").json()
|
||||||
"http://127.0.0.1:%s/api/tenants" % self.port)
|
|
||||||
f = urllib.request.urlopen(req)
|
|
||||||
data = f.read().decode('utf8')
|
|
||||||
data = json.loads(data)
|
|
||||||
|
|
||||||
self.assertEqual('tenant-one', data[0]['name'])
|
self.assertEqual('tenant-one', data[0]['name'])
|
||||||
self.assertEqual(3, data[0]['projects'])
|
self.assertEqual(3, data[0]['projects'])
|
||||||
|
@ -1021,7 +1018,7 @@ class TestWebSecrets(BaseTestWeb):
|
||||||
|
|
||||||
class TestInfo(ZuulDBTestCase, BaseTestWeb):
|
class TestInfo(ZuulDBTestCase, BaseTestWeb):
|
||||||
|
|
||||||
config_file = 'zuul-sql-driver.conf'
|
config_file = 'zuul-sql-driver-mysql.conf'
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestInfo, self).setUp()
|
super(TestInfo, self).setUp()
|
||||||
|
@ -1148,7 +1145,7 @@ class TestGraphiteUrl(TestInfo):
|
||||||
|
|
||||||
|
|
||||||
class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||||
config_file = 'zuul-sql-driver.conf'
|
config_file = 'zuul-sql-driver-mysql.conf'
|
||||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||||
|
|
||||||
def test_web_list_builds(self):
|
def test_web_list_builds(self):
|
||||||
|
@ -1262,7 +1259,7 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||||
|
|
||||||
|
|
||||||
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
|
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
|
||||||
config_file = 'zuul-sql-driver.conf'
|
config_file = 'zuul-sql-driver-mysql.conf'
|
||||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||||
|
|
||||||
def test_artifacts(self):
|
def test_artifacts(self):
|
||||||
|
@ -2259,7 +2256,7 @@ class TestTenantScopedWebApiTokenWithExpiry(BaseTestWeb):
|
||||||
|
|
||||||
|
|
||||||
class TestHeldAttributeInBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
class TestHeldAttributeInBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||||
config_file = 'zuul-sql-driver.conf'
|
config_file = 'zuul-sql-driver-mysql.conf'
|
||||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||||
|
|
||||||
def test_autohold_and_retrieve_held_build_info(self):
|
def test_autohold_and_retrieve_held_build_info(self):
|
||||||
|
|
|
@ -164,9 +164,11 @@ class ZuulApp(object):
|
||||||
logging_config.setDebug()
|
logging_config.setDebug()
|
||||||
logging_config.apply()
|
logging_config.apply()
|
||||||
|
|
||||||
def configure_connections(self, source_only=False, include_drivers=None):
|
def configure_connections(self, source_only=False, include_drivers=None,
|
||||||
|
require_sql=False):
|
||||||
self.connections = zuul.lib.connections.ConnectionRegistry()
|
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||||
self.connections.configure(self.config, source_only, include_drivers)
|
self.connections.configure(self.config, source_only, include_drivers,
|
||||||
|
require_sql)
|
||||||
|
|
||||||
|
|
||||||
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):
|
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):
|
||||||
|
|
|
@ -160,7 +160,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
||||||
tls_key=zookeeper_tls_key,
|
tls_key=zookeeper_tls_key,
|
||||||
tls_ca=zookeeper_tls_ca)
|
tls_ca=zookeeper_tls_ca)
|
||||||
|
|
||||||
self.configure_connections()
|
self.configure_connections(require_sql=True)
|
||||||
self.sched.setExecutor(gearman)
|
self.sched.setExecutor(gearman)
|
||||||
self.sched.setMerger(merger)
|
self.sched.setMerger(merger)
|
||||||
self.sched.setNodepool(nodepool)
|
self.sched.setNodepool(nodepool)
|
||||||
|
|
|
@ -124,7 +124,8 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
|
||||||
include_drivers=[zuul.driver.sql.SQLDriver,
|
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||||
zuul.driver.github.GithubDriver,
|
zuul.driver.github.GithubDriver,
|
||||||
zuul.driver.pagure.PagureDriver,
|
zuul.driver.pagure.PagureDriver,
|
||||||
zuul.driver.gitlab.GitlabDriver])
|
zuul.driver.gitlab.GitlabDriver],
|
||||||
|
require_sql=True)
|
||||||
self.configure_authenticators()
|
self.configure_authenticators()
|
||||||
self._run()
|
self._run()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|
|
@ -24,6 +24,7 @@ import subprocess
|
||||||
|
|
||||||
import voluptuous as vs
|
import voluptuous as vs
|
||||||
|
|
||||||
|
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||||
from zuul import model
|
from zuul import model
|
||||||
from zuul.lib import yamlutil as yaml
|
from zuul.lib import yamlutil as yaml
|
||||||
import zuul.manager.dependent
|
import zuul.manager.dependent
|
||||||
|
@ -1272,11 +1273,24 @@ class PipelineParser(object):
|
||||||
reporter_set = []
|
reporter_set = []
|
||||||
allowed_reporters = self.pcontext.tenant.allowed_reporters
|
allowed_reporters = self.pcontext.tenant.allowed_reporters
|
||||||
if conf.get(conf_key):
|
if conf.get(conf_key):
|
||||||
|
if conf_key in ['success', 'failure', 'merge-failure']:
|
||||||
|
# SQL reporters are required (an implied SQL reporter is
|
||||||
|
# added to every pipeline, ...(1)
|
||||||
|
sql_reporter = self.pcontext.connections\
|
||||||
|
.getSqlReporter(pipeline)
|
||||||
|
sql_reporter.setAction(conf_key)
|
||||||
|
reporter_set.append(sql_reporter)
|
||||||
for reporter_name, params \
|
for reporter_name, params \
|
||||||
in conf.get(conf_key).items():
|
in conf.get(conf_key).items():
|
||||||
if allowed_reporters is not None and \
|
if allowed_reporters is not None and \
|
||||||
reporter_name not in allowed_reporters:
|
reporter_name not in allowed_reporters:
|
||||||
raise UnknownConnection(reporter_name)
|
raise UnknownConnection(reporter_name)
|
||||||
|
if type(self.pcontext.connections
|
||||||
|
.connections[reporter_name]) == SQLConnection:
|
||||||
|
# (1)... explicit SQL reporters are ignored)
|
||||||
|
self.log.warning("Ignoring SQL reporter configured in"
|
||||||
|
" pipeline %s" % pipeline.name)
|
||||||
|
continue
|
||||||
reporter = self.pcontext.connections.getReporter(
|
reporter = self.pcontext.connections.getReporter(
|
||||||
reporter_name, pipeline, params)
|
reporter_name, pipeline, params)
|
||||||
reporter.setAction(conf_key)
|
reporter.setAction(conf_key)
|
||||||
|
@ -1287,6 +1301,16 @@ class PipelineParser(object):
|
||||||
if not pipeline.merge_failure_actions:
|
if not pipeline.merge_failure_actions:
|
||||||
pipeline.merge_failure_actions = pipeline.failure_actions
|
pipeline.merge_failure_actions = pipeline.failure_actions
|
||||||
|
|
||||||
|
for conf_key, action in self.reporter_actions.items():
|
||||||
|
if conf_key in ['success', 'failure', 'merge-failure']\
|
||||||
|
and not getattr(pipeline, action):
|
||||||
|
# SQL reporters are required ... add SQL reporters to the
|
||||||
|
# rest of actions.
|
||||||
|
sql_reporter = self.pcontext.connections\
|
||||||
|
.getSqlReporter(pipeline)
|
||||||
|
sql_reporter.setAction(conf_key)
|
||||||
|
setattr(pipeline, action, [sql_reporter])
|
||||||
|
|
||||||
pipeline.disable_at = conf.get(
|
pipeline.disable_at = conf.get(
|
||||||
'disable-after-consecutive-failures', None)
|
'disable-after-consecutive-failures', None)
|
||||||
|
|
||||||
|
|
|
@ -22,28 +22,9 @@ class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
|
||||||
name = 'sql'
|
name = 'sql'
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.tenant_connections = {}
|
|
||||||
cpb.capabilities_registry.register_capabilities(
|
cpb.capabilities_registry.register_capabilities(
|
||||||
'job_history', True)
|
'job_history', True)
|
||||||
|
|
||||||
def reconfigure(self, tenant):
|
|
||||||
# NOTE(corvus): This stores the connection of the first
|
|
||||||
# reporter seen for each tenant; we should figure out how to
|
|
||||||
# support multiple connections for a tenant (how do we deal
|
|
||||||
# with pagination of queries across multiple connections), or
|
|
||||||
# otherwise, require there only be one connection in a tenant.
|
|
||||||
if tenant.name in self.tenant_connections:
|
|
||||||
del self.tenant_connections[tenant.name]
|
|
||||||
for pipeline in tenant.layout.pipelines.values():
|
|
||||||
reporters = (pipeline.start_actions + pipeline.success_actions
|
|
||||||
+ pipeline.failure_actions
|
|
||||||
+ pipeline.merge_failure_actions)
|
|
||||||
for reporter in reporters:
|
|
||||||
if not isinstance(reporter, sqlreporter.SQLReporter):
|
|
||||||
continue
|
|
||||||
self.tenant_connections[tenant.name] = reporter.connection
|
|
||||||
return
|
|
||||||
|
|
||||||
def registerScheduler(self, scheduler):
|
def registerScheduler(self, scheduler):
|
||||||
self.sched = scheduler
|
self.sched = scheduler
|
||||||
|
|
||||||
|
|
|
@ -190,6 +190,8 @@ class SQLConnection(BaseConnection):
|
||||||
self.connection = None
|
self.connection = None
|
||||||
self.tables_established = False
|
self.tables_established = False
|
||||||
self.table_prefix = self.connection_config.get('table_prefix', '')
|
self.table_prefix = self.connection_config.get('table_prefix', '')
|
||||||
|
self.log.info("Initializing SQL connection {} (prefix: {})".format(
|
||||||
|
connection_name, self.table_prefix))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.dburi = self.connection_config.get('dburi')
|
self.dburi = self.connection_config.get('dburi')
|
||||||
|
|
|
@ -17,6 +17,9 @@ import re
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from zuul import model
|
||||||
|
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||||
|
from zuul.driver.sql.sqlreporter import SQLReporter
|
||||||
import zuul.driver.zuul
|
import zuul.driver.zuul
|
||||||
import zuul.driver.gerrit
|
import zuul.driver.gerrit
|
||||||
import zuul.driver.git
|
import zuul.driver.git
|
||||||
|
@ -85,10 +88,18 @@ class ConnectionRegistry(object):
|
||||||
for driver in self.drivers.values():
|
for driver in self.drivers.values():
|
||||||
driver.stop()
|
driver.stop()
|
||||||
|
|
||||||
def configure(self, config, source_only=False, include_drivers=None):
|
def configure(self, config, source_only=False, include_drivers=None,
|
||||||
|
require_sql=False):
|
||||||
# Register connections from the config
|
# Register connections from the config
|
||||||
connections = OrderedDict()
|
connections = OrderedDict()
|
||||||
|
|
||||||
|
if 'database' in config.sections() and not source_only:
|
||||||
|
driver = self.drivers['sql']
|
||||||
|
con_config = dict(config.items('database'))
|
||||||
|
|
||||||
|
connection = driver.getConnection('database', con_config)
|
||||||
|
connections['database'] = connection
|
||||||
|
|
||||||
for section_name in config.sections():
|
for section_name in config.sections():
|
||||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||||
section_name, re.I)
|
section_name, re.I)
|
||||||
|
@ -126,6 +137,12 @@ class ConnectionRegistry(object):
|
||||||
|
|
||||||
connection = driver.getConnection(con_name, con_config)
|
connection = driver.getConnection(con_name, con_config)
|
||||||
connections[con_name] = connection
|
connections[con_name] = connection
|
||||||
|
if con_driver == 'sql' and 'database' not in connections:
|
||||||
|
# The [database] section was missing. To stay backwards
|
||||||
|
# compatible duplicate the first database connection to the
|
||||||
|
# connection named 'database'
|
||||||
|
connections['database'] = driver.getConnection(
|
||||||
|
'database', con_config)
|
||||||
|
|
||||||
# If the [gerrit] or [smtp] sections still exist, load them in as a
|
# If the [gerrit] or [smtp] sections still exist, load them in as a
|
||||||
# connection named 'gerrit' or 'smtp' respectfully
|
# connection named 'gerrit' or 'smtp' respectfully
|
||||||
|
@ -160,8 +177,38 @@ class ConnectionRegistry(object):
|
||||||
connections[driver.name] = DefaultConnection(
|
connections[driver.name] = DefaultConnection(
|
||||||
driver, driver.name, {})
|
driver, driver.name, {})
|
||||||
|
|
||||||
|
if require_sql:
|
||||||
|
if 'database' not in connections:
|
||||||
|
raise Exception("Database configuration is required")
|
||||||
|
|
||||||
self.connections = connections
|
self.connections = connections
|
||||||
|
|
||||||
|
def getSqlConnection(self) -> SQLConnection:
|
||||||
|
"""
|
||||||
|
Gets the SQL connection. This is either the connection
|
||||||
|
described in the [database] section, or the first configured
|
||||||
|
connection.
|
||||||
|
|
||||||
|
:return: The SQL connection.
|
||||||
|
|
||||||
|
"""
|
||||||
|
connection = self.connections.get('database')
|
||||||
|
if not connection:
|
||||||
|
raise Exception("No SQL connections")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
def getSqlReporter(self, pipeline: model.Pipeline) -> SQLReporter:
|
||||||
|
"""
|
||||||
|
Gets the SQL reporter. Such reporter is based on
|
||||||
|
`getSqlConnection`.
|
||||||
|
|
||||||
|
:param pipeline: Pipeline
|
||||||
|
:return: The SQL reporter
|
||||||
|
|
||||||
|
"""
|
||||||
|
connection = self.getSqlConnection()
|
||||||
|
return connection.driver.getReporter(connection, pipeline)
|
||||||
|
|
||||||
def getSource(self, connection_name):
|
def getSource(self, connection_name):
|
||||||
connection = self.connections[connection_name]
|
connection = self.connections[connection_name]
|
||||||
return connection.driver.getSource(connection)
|
return connection.driver.getSource(connection)
|
||||||
|
|
|
@ -2423,8 +2423,7 @@ class QueueItem(object):
|
||||||
self.log.debug("Checking DB for requirements")
|
self.log.debug("Checking DB for requirements")
|
||||||
requirements_tuple = tuple(sorted(requirements))
|
requirements_tuple = tuple(sorted(requirements))
|
||||||
if requirements_tuple not in self._cached_sql_results:
|
if requirements_tuple not in self._cached_sql_results:
|
||||||
sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
|
conn = self.pipeline.manager.sched.connections.getSqlConnection()
|
||||||
conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
|
|
||||||
if conn:
|
if conn:
|
||||||
builds = conn.getBuilds(
|
builds = conn.getBuilds(
|
||||||
tenant=self.pipeline.tenant.name,
|
tenant=self.pipeline.tenant.name,
|
||||||
|
|
|
@ -185,7 +185,6 @@ class RPCListener(RPCListenerBase):
|
||||||
'get_running_jobs',
|
'get_running_jobs',
|
||||||
'get_job_log_stream_address',
|
'get_job_log_stream_address',
|
||||||
'tenant_list',
|
'tenant_list',
|
||||||
'tenant_sql_connection',
|
|
||||||
'status_get',
|
'status_get',
|
||||||
'job_get',
|
'job_get',
|
||||||
'job_list',
|
'job_list',
|
||||||
|
@ -376,16 +375,6 @@ class RPCListener(RPCListenerBase):
|
||||||
'queue': queue_size})
|
'queue': queue_size})
|
||||||
job.sendWorkComplete(json.dumps(output))
|
job.sendWorkComplete(json.dumps(output))
|
||||||
|
|
||||||
def handle_tenant_sql_connection(self, job):
|
|
||||||
args = json.loads(job.arguments)
|
|
||||||
sql_driver = self.sched.connections.drivers['sql']
|
|
||||||
conn = sql_driver.tenant_connections.get(args['tenant'])
|
|
||||||
if conn:
|
|
||||||
name = conn.connection_name
|
|
||||||
else:
|
|
||||||
name = ''
|
|
||||||
job.sendWorkComplete(json.dumps(name))
|
|
||||||
|
|
||||||
def handle_status_get(self, job):
|
def handle_status_get(self, job):
|
||||||
args = json.loads(job.arguments)
|
args = json.loads(job.arguments)
|
||||||
output = self.sched.formatStatusJSON(args.get("tenant"))
|
output = self.sched.formatStatusJSON(args.get("tenant"))
|
||||||
|
|
|
@ -12,10 +12,10 @@
|
||||||
# implied.
|
# implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
import cherrypy
|
import cherrypy
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
|
from cachetools.func import ttl_cache
|
||||||
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
|
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
|
||||||
from ws4py.websocket import WebSocket
|
from ws4py.websocket import WebSocket
|
||||||
import codecs
|
import codecs
|
||||||
|
@ -702,11 +702,15 @@ class ZuulWebAPI(object):
|
||||||
admin_tenants = json.loads(job.data[0])
|
admin_tenants = json.loads(job.data[0])
|
||||||
return admin_tenants
|
return admin_tenants
|
||||||
|
|
||||||
|
@ttl_cache(ttl=60)
|
||||||
|
def _tenants(self):
|
||||||
|
job = self.rpc.submitJob('zuul:tenant_list', {})
|
||||||
|
return json.loads(job.data[0])
|
||||||
|
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||||
def tenants(self):
|
def tenants(self):
|
||||||
job = self.rpc.submitJob('zuul:tenant_list', {})
|
ret = self._tenants()
|
||||||
ret = json.loads(job.data[0])
|
|
||||||
resp = cherrypy.response
|
resp = cherrypy.response
|
||||||
resp.headers['Access-Control-Allow-Origin'] = '*'
|
resp.headers['Access-Control-Allow-Origin'] = '*'
|
||||||
return ret
|
return ret
|
||||||
|
@ -963,16 +967,8 @@ class ZuulWebAPI(object):
|
||||||
})
|
})
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def _get_connection(self, tenant):
|
def _get_connection(self):
|
||||||
# Ask the scheduler which sql connection to use for this tenant
|
return self.zuulweb.connections.connections['database']
|
||||||
job = self.rpc.submitJob('zuul:tenant_sql_connection',
|
|
||||||
{'tenant': tenant})
|
|
||||||
connection_name = json.loads(job.data[0])
|
|
||||||
|
|
||||||
if not connection_name:
|
|
||||||
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
|
||||||
|
|
||||||
return self.zuulweb.connections.connections[connection_name]
|
|
||||||
|
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
@cherrypy.tools.save_params()
|
@cherrypy.tools.save_params()
|
||||||
|
@ -981,7 +977,10 @@ class ZuulWebAPI(object):
|
||||||
branch=None, patchset=None, ref=None, newrev=None,
|
branch=None, patchset=None, ref=None, newrev=None,
|
||||||
uuid=None, job_name=None, voting=None, node_name=None,
|
uuid=None, job_name=None, voting=None, node_name=None,
|
||||||
result=None, final=None, held=None, limit=50, skip=0):
|
result=None, final=None, held=None, limit=50, skip=0):
|
||||||
connection = self._get_connection(tenant)
|
connection = self._get_connection()
|
||||||
|
|
||||||
|
if tenant not in [x['name'] for x in self._tenants()]:
|
||||||
|
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
||||||
|
|
||||||
# If final is None, we return all builds, both final and non-final
|
# If final is None, we return all builds, both final and non-final
|
||||||
if final is not None:
|
if final is not None:
|
||||||
|
@ -1001,7 +1000,7 @@ class ZuulWebAPI(object):
|
||||||
@cherrypy.tools.save_params()
|
@cherrypy.tools.save_params()
|
||||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||||
def build(self, tenant, uuid):
|
def build(self, tenant, uuid):
|
||||||
connection = self._get_connection(tenant)
|
connection = self._get_connection()
|
||||||
|
|
||||||
data = connection.getBuilds(tenant=tenant, uuid=uuid, limit=1)
|
data = connection.getBuilds(tenant=tenant, uuid=uuid, limit=1)
|
||||||
if not data:
|
if not data:
|
||||||
|
@ -1040,7 +1039,7 @@ class ZuulWebAPI(object):
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
@cherrypy.tools.save_params()
|
@cherrypy.tools.save_params()
|
||||||
def badge(self, tenant, project=None, pipeline=None, branch=None):
|
def badge(self, tenant, project=None, pipeline=None, branch=None):
|
||||||
connection = self._get_connection(tenant)
|
connection = self._get_connection()
|
||||||
|
|
||||||
buildsets = connection.getBuildsets(
|
buildsets = connection.getBuildsets(
|
||||||
tenant=tenant, project=project, pipeline=pipeline,
|
tenant=tenant, project=project, pipeline=pipeline,
|
||||||
|
@ -1063,7 +1062,7 @@ class ZuulWebAPI(object):
|
||||||
def buildsets(self, tenant, project=None, pipeline=None, change=None,
|
def buildsets(self, tenant, project=None, pipeline=None, change=None,
|
||||||
branch=None, patchset=None, ref=None, newrev=None,
|
branch=None, patchset=None, ref=None, newrev=None,
|
||||||
uuid=None, result=None, limit=50, skip=0):
|
uuid=None, result=None, limit=50, skip=0):
|
||||||
connection = self._get_connection(tenant)
|
connection = self._get_connection()
|
||||||
|
|
||||||
buildsets = connection.getBuildsets(
|
buildsets = connection.getBuildsets(
|
||||||
tenant=tenant, project=project, pipeline=pipeline, change=change,
|
tenant=tenant, project=project, pipeline=pipeline, change=change,
|
||||||
|
@ -1079,7 +1078,7 @@ class ZuulWebAPI(object):
|
||||||
@cherrypy.tools.save_params()
|
@cherrypy.tools.save_params()
|
||||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||||
def buildset(self, tenant, uuid):
|
def buildset(self, tenant, uuid):
|
||||||
connection = self._get_connection(tenant)
|
connection = self._get_connection()
|
||||||
|
|
||||||
data = connection.getBuildset(tenant, uuid)
|
data = connection.getBuildset(tenant, uuid)
|
||||||
if not data:
|
if not data:
|
||||||
|
|
Loading…
Reference in New Issue