Merge "Required SQL reporters"
This commit is contained in:
commit
bbe3bb0297
|
@ -28,9 +28,7 @@ name=opendev
|
|||
driver=git
|
||||
baseurl=https://opendev.org
|
||||
|
||||
[connection "mysql"]
|
||||
name=mysql
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
|
||||
|
||||
[web]
|
||||
|
|
|
@ -17,11 +17,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
mysql:
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
mysql:
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
@ -48,8 +46,6 @@
|
|||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
mysql:
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
mysql:
|
||||
|
|
|
@ -148,6 +148,9 @@ service in Zuul, and a connection to Gerrit.
|
|||
user=zuul
|
||||
sshkey=/home/zuul/.ssh/id_rsa
|
||||
|
||||
[database]
|
||||
dburi=mysql+pymysql://zuul:secret@mysql/zuul
|
||||
|
||||
See :ref:`components` and :ref:`connections` for more details.
|
||||
|
||||
The following tells Zuul to read its configuration from and operate on
|
||||
|
|
|
@ -5,6 +5,7 @@ Admin Reference
|
|||
:maxdepth: 3
|
||||
|
||||
tenants
|
||||
database
|
||||
connections
|
||||
drivers/index
|
||||
client
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
:title: Database Configuration
|
||||
|
||||
.. _database:
|
||||
|
||||
Database Configuration
|
||||
======================
|
||||
|
||||
The database configuration is located in the ``database`` section of
|
||||
``zuul.conf``:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[database]
|
||||
dburi=<URI>
|
||||
|
||||
The following options can be defined in this section.
|
||||
|
||||
.. attr:: database
|
||||
|
||||
.. attr:: dburi
|
||||
:required:
|
||||
|
||||
Database connection information in the form of a URI understood
|
||||
by SQLAlchemy. See `The SQLAlchemy manual
|
||||
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
|
||||
for more information.
|
||||
|
||||
The driver will automatically set up the database creating and managing
|
||||
the necessary tables. Therefore the provided user should have sufficient
|
||||
permissions to manage the database. For example:
|
||||
|
||||
.. code-block:: sql
|
||||
|
||||
GRANT ALL ON my_database TO 'my_user'@'%';
|
||||
|
||||
.. attr:: pool_recycle
|
||||
:default: 1
|
||||
|
||||
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
|
||||
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
|
||||
for more information.
|
||||
|
||||
.. attr:: table_prefix
|
||||
:default: ''
|
||||
|
||||
The string to prefix the table names. This makes it possible to run
|
||||
several zuul deployments against the same database. This can be useful
|
||||
if you rely on external databases which are not under your control.
|
||||
The default is to have no prefix.
|
|
@ -5,6 +5,10 @@
|
|||
SQL
|
||||
===
|
||||
|
||||
.. warning::
|
||||
|
||||
This driver is deprecated, use :attr:`database` configuration instead.
|
||||
|
||||
The SQL driver supports reporters only. Only one connection per
|
||||
database is permitted.
|
||||
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
The components zuul-scheduler and zuul-web now require database
|
||||
configuration. There is now only one supported database connection. If
|
||||
multiple sql connections have been configured only the first one will be
|
||||
used.
|
||||
|
||||
deprecations:
|
||||
- |
|
||||
Defining database connections using :attr:`<sql connection>` is now
|
||||
deprecated. Refer to :ref:`database` how to configure the database now.
|
|
@ -68,6 +68,7 @@ from git.util import IterableList
|
|||
import yaml
|
||||
import paramiko
|
||||
|
||||
from zuul.driver.sql.sqlconnection import DatabaseSession
|
||||
from zuul.model import Change
|
||||
from zuul.rpcclient import RPCClient
|
||||
|
||||
|
@ -76,7 +77,7 @@ from zuul.driver.git import GitDriver
|
|||
from zuul.driver.smtp import SMTPDriver
|
||||
from zuul.driver.github import GithubDriver
|
||||
from zuul.driver.timer import TimerDriver
|
||||
from zuul.driver.sql import SQLDriver
|
||||
from zuul.driver.sql import SQLDriver, sqlconnection
|
||||
from zuul.driver.bubblewrap import BubblewrapDriver
|
||||
from zuul.driver.nullwrap import NullwrapDriver
|
||||
from zuul.driver.mqtt import MQTTDriver
|
||||
|
@ -317,12 +318,19 @@ class GitlabDriverMock(GitlabDriver):
|
|||
return connection
|
||||
|
||||
|
||||
class SQLDriverMock(SQLDriver):
|
||||
|
||||
def getConnection(self, name, config):
|
||||
return FakeSqlConnection(self, name, config)
|
||||
|
||||
|
||||
class TestConnectionRegistry(ConnectionRegistry):
|
||||
def __init__(self, changes: Dict[str, Dict[str, Change]],
|
||||
config: ConfigParser, additional_event_queues,
|
||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||
git_url_with_auth: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
||||
add_cleanup: Callable[[Callable[[], None]], None],
|
||||
fake_sql: bool):
|
||||
self.connections = OrderedDict()
|
||||
self.drivers = {}
|
||||
|
||||
|
@ -336,7 +344,10 @@ class TestConnectionRegistry(ConnectionRegistry):
|
|||
rpcclient, git_url_with_auth))
|
||||
self.registerDriver(SMTPDriver())
|
||||
self.registerDriver(TimerDriver())
|
||||
self.registerDriver(SQLDriver())
|
||||
if fake_sql:
|
||||
self.registerDriver(SQLDriverMock())
|
||||
else:
|
||||
self.registerDriver(SQLDriver())
|
||||
self.registerDriver(BubblewrapDriver())
|
||||
self.registerDriver(NullwrapDriver())
|
||||
self.registerDriver(MQTTDriver())
|
||||
|
@ -2961,6 +2972,37 @@ class FakeBuild(object):
|
|||
return repos
|
||||
|
||||
|
||||
class FakeZuulDatabaseSession(DatabaseSession):
|
||||
|
||||
def __exit__(self, etype, value, tb):
|
||||
pass
|
||||
|
||||
|
||||
def FakeOrmSessionFactory():
|
||||
class FakeBackendSession:
|
||||
def add(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
return FakeBackendSession()
|
||||
|
||||
|
||||
class FakeSqlConnection(sqlconnection.SQLConnection):
|
||||
|
||||
def __init__(self, driver, connection_name, connection_config):
|
||||
connection_config['dburi'] = 'postgresql://example.com'
|
||||
super().__init__(driver, connection_name, connection_config)
|
||||
self.session = FakeOrmSessionFactory
|
||||
|
||||
def onLoad(self):
|
||||
pass
|
||||
|
||||
def onStop(self):
|
||||
pass
|
||||
|
||||
def getSession(self):
|
||||
return FakeZuulDatabaseSession(self)
|
||||
|
||||
|
||||
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
|
||||
result = None
|
||||
|
||||
|
@ -3659,12 +3701,12 @@ class ZuulWebFixture(fixtures.Fixture):
|
|||
additional_event_queues, upstream_root: str,
|
||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None],
|
||||
test_root, info=None, zk_hosts=None):
|
||||
test_root, info=None, zk_hosts=None, fake_sql=True):
|
||||
super(ZuulWebFixture, self).__init__()
|
||||
self.gearman_server_port = gearman_server_port
|
||||
self.connections = TestConnectionRegistry(
|
||||
changes, config, additional_event_queues, upstream_root, rpcclient,
|
||||
poller_events, git_url_with_auth, add_cleanup)
|
||||
poller_events, git_url_with_auth, add_cleanup, fake_sql)
|
||||
self.connections.configure(
|
||||
config,
|
||||
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||
|
@ -3919,6 +3961,7 @@ class SchedulerTestApp:
|
|||
additional_event_queues, upstream_root: str,
|
||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||
source_only: bool,
|
||||
fake_sql: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
||||
|
||||
self.log = log
|
||||
|
@ -3940,7 +3983,7 @@ class SchedulerTestApp:
|
|||
self.connections = TestConnectionRegistry(
|
||||
self.changes, self.config, additional_event_queues,
|
||||
upstream_root, rpcclient, poller_events,
|
||||
git_url_with_auth, add_cleanup)
|
||||
git_url_with_auth, add_cleanup, fake_sql)
|
||||
self.connections.configure(self.config, source_only=source_only)
|
||||
|
||||
self.sched.registerConnections(self.connections)
|
||||
|
@ -3989,12 +4032,13 @@ class SchedulerTestManager:
|
|||
changes: Dict[str, Dict[str, Change]], additional_event_queues,
|
||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||
git_url_with_auth: bool, source_only: bool,
|
||||
fake_sql: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None])\
|
||||
-> SchedulerTestApp:
|
||||
app = SchedulerTestApp(log, config, zk_config, changes,
|
||||
additional_event_queues, upstream_root,
|
||||
rpcclient, poller_events, git_url_with_auth,
|
||||
source_only, add_cleanup)
|
||||
source_only, fake_sql, add_cleanup)
|
||||
self.instances.append(app)
|
||||
return app
|
||||
|
||||
|
@ -4102,6 +4146,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
git_url_with_auth: bool = False
|
||||
log_console_port: int = 19885
|
||||
source_only: bool = False
|
||||
fake_sql: bool = True
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Allows to access fake connections the old way, e.g., using
|
||||
|
@ -4233,7 +4278,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
executor_connections = TestConnectionRegistry(
|
||||
self.changes, self.config, self.additional_event_queues,
|
||||
self.upstream_root, self.rpcclient, self.poller_events,
|
||||
self.git_url_with_auth, self.addCleanup)
|
||||
self.git_url_with_auth, self.addCleanup, True)
|
||||
executor_connections.configure(self.config,
|
||||
source_only=self.source_only)
|
||||
self.executor_server = RecordingExecutorServer(
|
||||
|
@ -4252,7 +4297,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.log, self.config, self.zk_config, self.changes,
|
||||
self.additional_event_queues, self.upstream_root, self.rpcclient,
|
||||
self.poller_events, self.git_url_with_auth, self.source_only,
|
||||
self.addCleanup)
|
||||
self.fake_sql, self.addCleanup)
|
||||
|
||||
if hasattr(self, 'fake_github'):
|
||||
self.additional_event_queues.append(
|
||||
|
@ -4343,6 +4388,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
# Make test_root persist after ansible run for .flag test
|
||||
config.set('executor', 'trusted_rw_paths', self.test_root)
|
||||
self.setupAllProjectKeys(config)
|
||||
|
||||
return config
|
||||
|
||||
def setupSimpleLayout(self, config: ConfigParser):
|
||||
|
@ -5345,7 +5391,21 @@ class SSLZuulTestCase(ZuulTestCase):
|
|||
|
||||
|
||||
class ZuulDBTestCase(ZuulTestCase):
|
||||
fake_sql = False
|
||||
|
||||
def setup_config(self, config_file: str):
|
||||
def _setup_fixture(config, section_name):
|
||||
if (config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
elif (config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
|
||||
config = super(ZuulDBTestCase, self).setup_config(config_file)
|
||||
for section_name in config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
|
@ -5354,16 +5414,11 @@ class ZuulDBTestCase(ZuulTestCase):
|
|||
continue
|
||||
|
||||
if config.get(section_name, 'driver') == 'sql':
|
||||
if (config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
elif (config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
_setup_fixture(config, section_name)
|
||||
|
||||
if 'database' in config.sections():
|
||||
_setup_fixture(config, 'database')
|
||||
|
||||
return config
|
||||
|
||||
|
||||
|
|
|
@ -7,15 +7,10 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
@ -30,15 +25,10 @@
|
|||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
|
@ -51,14 +41,8 @@
|
|||
gerrit:
|
||||
- event: ref-updated
|
||||
ref: ^refs/tags/.*$
|
||||
success:
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
|
||||
- job:
|
||||
name: base
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- job:
|
||||
name: base
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
|
|
@ -5,11 +5,9 @@
|
|||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
resultsdb_mysql: null
|
||||
gerrit:
|
||||
Verified: 1
|
||||
failure:
|
||||
resultsdb_mysql: null
|
||||
gerrit:
|
||||
Verified: -1
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
|||
client_id=zuul.example.com
|
||||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -43,3 +43,7 @@ realm=myOIDC2
|
|||
client_id=zuul
|
||||
issuer_id=http://oidc2
|
||||
scope=openid profile email special-scope
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ client_id=zuul.example.com
|
|||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
max_validity_time=5
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
|||
client_id=zuul.example.com
|
||||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -24,3 +24,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -21,10 +21,10 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -37,3 +37,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -34,3 +34,7 @@ default_to=you@example.com
|
|||
|
||||
[web]
|
||||
static_cache_expiry=1200
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -23,3 +23,6 @@ driver=elasticsearch
|
|||
uri=localhost:9200
|
||||
use_ssl=true
|
||||
verify_certs=false
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
|
|
@ -46,3 +46,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -31,3 +31,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -18,3 +18,7 @@ webhook_token=0000000000000000000000000000000000000000
|
|||
app_id=1
|
||||
app_key=$APP_KEY_FIXTURE$
|
||||
server=github.enterprise.io
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -26,3 +26,7 @@ sshkey=/home/zuul/.ssh/id_rsa
|
|||
driver=github
|
||||
sshkey=/home/zuul/.ssh/id_rsa
|
||||
server=github.enterprise.io
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
|||
driver=gitlab
|
||||
server=gitlab
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -36,3 +36,7 @@ default_to=you@example.com
|
|||
[web]
|
||||
static_cache_expiry=1200
|
||||
root=https://zuul.example.com/
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -21,5 +21,8 @@ sshkey=fake_id_rsa1
|
|||
[connection mqtt]
|
||||
driver=mqtt
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[web]
|
||||
root=https://tenant.example.com/
|
||||
|
|
|
@ -17,3 +17,7 @@ driver=pagure
|
|||
server=pagure
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
source_whitelist=::ffff:127.0.0.1
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
|||
driver=pagure
|
||||
server=pagure
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -20,3 +20,7 @@ webhook_token=00000000000000000000000000000000000000000
|
|||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -18,8 +18,7 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=mysql+pymysql://bad:creds@host/db
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
|
@ -0,0 +1,26 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[database]
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -18,20 +18,10 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_postgresql]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_postgresql_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -18,18 +18,11 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_postgresql]
|
||||
[database]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_postgresql_failures]
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -35,3 +35,6 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
|
|
@ -31,6 +31,9 @@ port=25
|
|||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[web]
|
||||
static_cache_expiry=1200
|
||||
root=https://zuul.example.com/
|
||||
|
|
|
@ -11,14 +11,19 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import configparser
|
||||
import os
|
||||
import re
|
||||
import textwrap
|
||||
import time
|
||||
import types
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
|
||||
import zuul
|
||||
from tests.base import ZuulTestCase, FIXTURE_DIR, \
|
||||
PostgresqlSchemaFixture, MySQLSchemaFixture, ZuulDBTestCase, \
|
||||
BaseTestCase, AnsibleZuulTestCase
|
||||
|
||||
|
||||
def _get_reporter_from_connection_name(reporters, connection_name):
|
||||
|
@ -61,8 +66,8 @@ class TestConnections(ZuulTestCase):
|
|||
'civoter')
|
||||
|
||||
|
||||
class TestSQLConnection(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
class TestSQLConnectionMysql(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
expected_table_prefix = ''
|
||||
|
||||
|
@ -81,8 +86,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
|
||||
def test_sql_tables_created(self):
|
||||
"Test the tables for storing results are created properly"
|
||||
self._sql_tables_created('resultsdb_mysql')
|
||||
self._sql_tables_created('resultsdb_postgresql')
|
||||
self._sql_tables_created('database')
|
||||
|
||||
def _sql_indexes_created(self, connection_name):
|
||||
connection = self.scheds.first.connections.connections[connection_name]
|
||||
|
@ -116,8 +120,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
|
||||
def test_sql_indexes_created(self):
|
||||
"Test the indexes are created properly"
|
||||
self._sql_indexes_created('resultsdb_mysql')
|
||||
self._sql_indexes_created('resultsdb_postgresql')
|
||||
self._sql_indexes_created('database')
|
||||
|
||||
def test_sql_results(self):
|
||||
"Test results are entered into an sql table"
|
||||
|
@ -225,8 +228,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
self.orderedRelease()
|
||||
self.waitUntilSettled()
|
||||
|
||||
check_results('resultsdb_mysql')
|
||||
check_results('resultsdb_postgresql')
|
||||
check_results('database')
|
||||
|
||||
def test_sql_results_retry_builds(self):
|
||||
"Test that retry results are entered into an sql table correctly"
|
||||
|
@ -306,8 +308,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
self.orderedRelease()
|
||||
self.waitUntilSettled()
|
||||
|
||||
check_results('resultsdb_mysql')
|
||||
check_results('resultsdb_postgresql')
|
||||
check_results('database')
|
||||
|
||||
def test_multiple_sql_connections(self):
|
||||
"Test putting results in different databases"
|
||||
|
@ -353,36 +354,79 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
tenant.layout.pipelines['check'].failure_actions,
|
||||
connection_name_2
|
||||
)
|
||||
self.assertIsNone(reporter2) # Explicit SQL reporters are ignored
|
||||
|
||||
conn = self.scheds.first.connections.\
|
||||
connections[connection_name_2].engine.connect()
|
||||
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
|
||||
[reporter2.connection.zuul_buildset_table])).fetchall()
|
||||
[reporter1.connection.zuul_buildset_table])).fetchall()
|
||||
# The failure db should only have 1 buildset failed
|
||||
self.assertEqual(1, len(buildsets_resultsdb_failures))
|
||||
self.assertEqual(2, len(buildsets_resultsdb_failures))
|
||||
|
||||
self.assertEqual(
|
||||
'check', buildsets_resultsdb_failures[0]['pipeline'])
|
||||
'check', buildsets_resultsdb_failures[1]['pipeline'])
|
||||
self.assertEqual('org/project',
|
||||
buildsets_resultsdb_failures[0]['project'])
|
||||
buildsets_resultsdb_failures[1]['project'])
|
||||
self.assertEqual(2,
|
||||
buildsets_resultsdb_failures[0]['change'])
|
||||
buildsets_resultsdb_failures[1]['change'])
|
||||
self.assertEqual(
|
||||
'1', buildsets_resultsdb_failures[0]['patchset'])
|
||||
'1', buildsets_resultsdb_failures[1]['patchset'])
|
||||
self.assertEqual(
|
||||
'FAILURE', buildsets_resultsdb_failures[0]['result'])
|
||||
'FAILURE', buildsets_resultsdb_failures[1]['result'])
|
||||
self.assertEqual('Build failed.',
|
||||
buildsets_resultsdb_failures[0]['message'])
|
||||
buildsets_resultsdb_failures[1]['message'])
|
||||
|
||||
check_results('resultsdb_mysql', 'resultsdb_mysql_failures')
|
||||
check_results('resultsdb_postgresql', 'resultsdb_postgresql_failures')
|
||||
check_results('database', 'resultsdb_failures')
|
||||
|
||||
|
||||
class TestSQLConnectionPrefix(TestSQLConnection):
|
||||
config_file = 'zuul-sql-driver-prefix.conf'
|
||||
class TestSQLConnectionPostgres(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-postgres.conf'
|
||||
|
||||
|
||||
class TestSQLConnectionPrefixMysql(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-prefix-mysql.conf'
|
||||
expected_table_prefix = 'prefix_'
|
||||
|
||||
|
||||
class TestSQLConnectionPrefixPostgres(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-prefix-postgres.conf'
|
||||
expected_table_prefix = 'prefix_'
|
||||
|
||||
|
||||
class TestRequiredSQLConnection(BaseTestCase):
|
||||
config = None
|
||||
connections = None
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.addCleanup(self.stop_connection)
|
||||
|
||||
def setup_connection(self, config_file):
|
||||
self.config = configparser.ConfigParser()
|
||||
self.config.read(os.path.join(FIXTURE_DIR, config_file))
|
||||
|
||||
# Setup databases
|
||||
for section_name in self.config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
section_name, re.I)
|
||||
if not con_match:
|
||||
continue
|
||||
|
||||
if self.config.get(section_name, 'driver') == 'sql':
|
||||
if (self.config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
self.config.set(section_name, 'dburi', f.dburi)
|
||||
elif (self.config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
self.config.set(section_name, 'dburi', f.dburi)
|
||||
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||
|
||||
def stop_connection(self):
|
||||
self.connections.stop()
|
||||
|
||||
|
||||
class TestConnectionsBadSQL(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver-bad.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
|
|
@ -5370,28 +5370,34 @@ For CI problems and help debugging, contact ci@example.org"""
|
|||
tenant.layout.pipelines['gate'].merge_failure_message)
|
||||
|
||||
self.assertEqual(
|
||||
len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
|
||||
len(tenant.layout.pipelines['check'].merge_failure_actions), 2)
|
||||
self.assertEqual(
|
||||
len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
|
||||
len(tenant.layout.pipelines['gate'].merge_failure_actions), 3)
|
||||
|
||||
self.assertTrue(isinstance(
|
||||
tenant.layout.pipelines['check'].merge_failure_actions[0],
|
||||
tenant.layout.pipelines['check'].merge_failure_actions[1],
|
||||
gerritreporter.GerritReporter))
|
||||
|
||||
self.assertTrue(
|
||||
(
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[0],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
||||
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[1],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[2],
|
||||
gerritreporter.GerritReporter)
|
||||
) or (
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[0],
|
||||
gerritreporter.GerritReporter) and
|
||||
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[1],
|
||||
gerritreporter.GerritReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[2],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter)
|
||||
)
|
||||
)
|
||||
|
|
|
@ -6140,8 +6140,8 @@ class TestProvidesRequiresBuildset(ZuulTestCase):
|
|||
}])
|
||||
|
||||
|
||||
class TestProvidesRequires(ZuulDBTestCase):
|
||||
config_file = "zuul-sql-driver.conf"
|
||||
class TestProvidesRequiresMysql(ZuulDBTestCase):
|
||||
config_file = "zuul-sql-driver-mysql.conf"
|
||||
|
||||
@simple_layout('layouts/provides-requires.yaml')
|
||||
def test_provides_requires_shared_queue_fast(self):
|
||||
|
@ -6682,6 +6682,10 @@ class TestProvidesRequires(ZuulDBTestCase):
|
|||
1, B.messages[0])
|
||||
|
||||
|
||||
class TestProvidesRequiresPostgres(TestProvidesRequiresMysql):
|
||||
config_file = "zuul-sql-driver-postgres.conf"
|
||||
|
||||
|
||||
class TestForceMergeMissingTemplate(ZuulTestCase):
|
||||
tenant_config_file = "config/force-merge-template/main.yaml"
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
import socket
|
||||
|
@ -62,7 +61,8 @@ class BaseTestWeb(ZuulTestCase):
|
|||
self.test_root,
|
||||
info=zuul.model.WebInfo.fromConfig(
|
||||
self.zuul_ini_config),
|
||||
zk_hosts=self.zk_config))
|
||||
zk_hosts=self.zk_config,
|
||||
fake_sql=self.fake_sql))
|
||||
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
|
||||
|
@ -88,6 +88,7 @@ class BaseTestWeb(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
def get_url(self, url, *args, **kwargs):
|
||||
zuul.web.ZuulWebAPI._tenants.cache_clear()
|
||||
return requests.get(
|
||||
urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
|
||||
|
||||
|
@ -266,11 +267,7 @@ class TestWeb(BaseTestWeb):
|
|||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:%s/api/tenants" % self.port)
|
||||
f = urllib.request.urlopen(req)
|
||||
data = f.read().decode('utf8')
|
||||
data = json.loads(data)
|
||||
data = self.get_url("api/tenants").json()
|
||||
|
||||
self.assertEqual('tenant-one', data[0]['name'])
|
||||
self.assertEqual(3, data[0]['projects'])
|
||||
|
@ -1021,7 +1018,7 @@ class TestWebSecrets(BaseTestWeb):
|
|||
|
||||
class TestInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
|
||||
def setUp(self):
|
||||
super(TestInfo, self).setUp()
|
||||
|
@ -1148,7 +1145,7 @@ class TestGraphiteUrl(TestInfo):
|
|||
|
||||
|
||||
class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_web_list_builds(self):
|
||||
|
@ -1262,7 +1259,7 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
|||
|
||||
|
||||
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_artifacts(self):
|
||||
|
@ -2259,7 +2256,7 @@ class TestTenantScopedWebApiTokenWithExpiry(BaseTestWeb):
|
|||
|
||||
|
||||
class TestHeldAttributeInBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_autohold_and_retrieve_held_build_info(self):
|
||||
|
|
|
@ -164,9 +164,11 @@ class ZuulApp(object):
|
|||
logging_config.setDebug()
|
||||
logging_config.apply()
|
||||
|
||||
def configure_connections(self, source_only=False, include_drivers=None):
|
||||
def configure_connections(self, source_only=False, include_drivers=None,
|
||||
require_sql=False):
|
||||
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||
self.connections.configure(self.config, source_only, include_drivers)
|
||||
self.connections.configure(self.config, source_only, include_drivers,
|
||||
require_sql)
|
||||
|
||||
|
||||
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):
|
||||
|
|
|
@ -160,7 +160,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
tls_key=zookeeper_tls_key,
|
||||
tls_ca=zookeeper_tls_ca)
|
||||
|
||||
self.configure_connections()
|
||||
self.configure_connections(require_sql=True)
|
||||
self.sched.setExecutor(gearman)
|
||||
self.sched.setMerger(merger)
|
||||
self.sched.setNodepool(nodepool)
|
||||
|
|
|
@ -124,7 +124,8 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
|
|||
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||
zuul.driver.github.GithubDriver,
|
||||
zuul.driver.pagure.PagureDriver,
|
||||
zuul.driver.gitlab.GitlabDriver])
|
||||
zuul.driver.gitlab.GitlabDriver],
|
||||
require_sql=True)
|
||||
self.configure_authenticators()
|
||||
self._run()
|
||||
except Exception:
|
||||
|
|
|
@ -24,6 +24,7 @@ import subprocess
|
|||
|
||||
import voluptuous as vs
|
||||
|
||||
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||
from zuul import model
|
||||
from zuul.lib import yamlutil as yaml
|
||||
import zuul.manager.dependent
|
||||
|
@ -1272,11 +1273,24 @@ class PipelineParser(object):
|
|||
reporter_set = []
|
||||
allowed_reporters = self.pcontext.tenant.allowed_reporters
|
||||
if conf.get(conf_key):
|
||||
if conf_key in ['success', 'failure', 'merge-failure']:
|
||||
# SQL reporters are required (an implied SQL reporter is
|
||||
# added to every pipeline, ...(1)
|
||||
sql_reporter = self.pcontext.connections\
|
||||
.getSqlReporter(pipeline)
|
||||
sql_reporter.setAction(conf_key)
|
||||
reporter_set.append(sql_reporter)
|
||||
for reporter_name, params \
|
||||
in conf.get(conf_key).items():
|
||||
if allowed_reporters is not None and \
|
||||
reporter_name not in allowed_reporters:
|
||||
raise UnknownConnection(reporter_name)
|
||||
if type(self.pcontext.connections
|
||||
.connections[reporter_name]) == SQLConnection:
|
||||
# (1)... explicit SQL reporters are ignored)
|
||||
self.log.warning("Ignoring SQL reporter configured in"
|
||||
" pipeline %s" % pipeline.name)
|
||||
continue
|
||||
reporter = self.pcontext.connections.getReporter(
|
||||
reporter_name, pipeline, params)
|
||||
reporter.setAction(conf_key)
|
||||
|
@ -1287,6 +1301,16 @@ class PipelineParser(object):
|
|||
if not pipeline.merge_failure_actions:
|
||||
pipeline.merge_failure_actions = pipeline.failure_actions
|
||||
|
||||
for conf_key, action in self.reporter_actions.items():
|
||||
if conf_key in ['success', 'failure', 'merge-failure']\
|
||||
and not getattr(pipeline, action):
|
||||
# SQL reporters are required ... add SQL reporters to the
|
||||
# rest of actions.
|
||||
sql_reporter = self.pcontext.connections\
|
||||
.getSqlReporter(pipeline)
|
||||
sql_reporter.setAction(conf_key)
|
||||
setattr(pipeline, action, [sql_reporter])
|
||||
|
||||
pipeline.disable_at = conf.get(
|
||||
'disable-after-consecutive-failures', None)
|
||||
|
||||
|
|
|
@ -22,28 +22,9 @@ class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
|
|||
name = 'sql'
|
||||
|
||||
def __init__(self):
|
||||
self.tenant_connections = {}
|
||||
cpb.capabilities_registry.register_capabilities(
|
||||
'job_history', True)
|
||||
|
||||
def reconfigure(self, tenant):
|
||||
# NOTE(corvus): This stores the connection of the first
|
||||
# reporter seen for each tenant; we should figure out how to
|
||||
# support multiple connections for a tenant (how do we deal
|
||||
# with pagination of queries across multiple connections), or
|
||||
# otherwise, require there only be one connection in a tenant.
|
||||
if tenant.name in self.tenant_connections:
|
||||
del self.tenant_connections[tenant.name]
|
||||
for pipeline in tenant.layout.pipelines.values():
|
||||
reporters = (pipeline.start_actions + pipeline.success_actions
|
||||
+ pipeline.failure_actions
|
||||
+ pipeline.merge_failure_actions)
|
||||
for reporter in reporters:
|
||||
if not isinstance(reporter, sqlreporter.SQLReporter):
|
||||
continue
|
||||
self.tenant_connections[tenant.name] = reporter.connection
|
||||
return
|
||||
|
||||
def registerScheduler(self, scheduler):
|
||||
self.sched = scheduler
|
||||
|
||||
|
|
|
@ -190,6 +190,8 @@ class SQLConnection(BaseConnection):
|
|||
self.connection = None
|
||||
self.tables_established = False
|
||||
self.table_prefix = self.connection_config.get('table_prefix', '')
|
||||
self.log.info("Initializing SQL connection {} (prefix: {})".format(
|
||||
connection_name, self.table_prefix))
|
||||
|
||||
try:
|
||||
self.dburi = self.connection_config.get('dburi')
|
||||
|
|
|
@ -17,6 +17,9 @@ import re
|
|||
from collections import OrderedDict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from zuul import model
|
||||
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||
from zuul.driver.sql.sqlreporter import SQLReporter
|
||||
import zuul.driver.zuul
|
||||
import zuul.driver.gerrit
|
||||
import zuul.driver.git
|
||||
|
@ -85,10 +88,18 @@ class ConnectionRegistry(object):
|
|||
for driver in self.drivers.values():
|
||||
driver.stop()
|
||||
|
||||
def configure(self, config, source_only=False, include_drivers=None):
|
||||
def configure(self, config, source_only=False, include_drivers=None,
|
||||
require_sql=False):
|
||||
# Register connections from the config
|
||||
connections = OrderedDict()
|
||||
|
||||
if 'database' in config.sections() and not source_only:
|
||||
driver = self.drivers['sql']
|
||||
con_config = dict(config.items('database'))
|
||||
|
||||
connection = driver.getConnection('database', con_config)
|
||||
connections['database'] = connection
|
||||
|
||||
for section_name in config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
section_name, re.I)
|
||||
|
@ -126,6 +137,12 @@ class ConnectionRegistry(object):
|
|||
|
||||
connection = driver.getConnection(con_name, con_config)
|
||||
connections[con_name] = connection
|
||||
if con_driver == 'sql' and 'database' not in connections:
|
||||
# The [database] section was missing. To stay backwards
|
||||
# compatible duplicate the first database connection to the
|
||||
# connection named 'database'
|
||||
connections['database'] = driver.getConnection(
|
||||
'database', con_config)
|
||||
|
||||
# If the [gerrit] or [smtp] sections still exist, load them in as a
|
||||
# connection named 'gerrit' or 'smtp' respectfully
|
||||
|
@ -160,8 +177,38 @@ class ConnectionRegistry(object):
|
|||
connections[driver.name] = DefaultConnection(
|
||||
driver, driver.name, {})
|
||||
|
||||
if require_sql:
|
||||
if 'database' not in connections:
|
||||
raise Exception("Database configuration is required")
|
||||
|
||||
self.connections = connections
|
||||
|
||||
def getSqlConnection(self) -> SQLConnection:
|
||||
"""
|
||||
Gets the SQL connection. This is either the connection
|
||||
described in the [database] section, or the first configured
|
||||
connection.
|
||||
|
||||
:return: The SQL connection.
|
||||
|
||||
"""
|
||||
connection = self.connections.get('database')
|
||||
if not connection:
|
||||
raise Exception("No SQL connections")
|
||||
return connection
|
||||
|
||||
def getSqlReporter(self, pipeline: model.Pipeline) -> SQLReporter:
|
||||
"""
|
||||
Gets the SQL reporter. Such reporter is based on
|
||||
`getSqlConnection`.
|
||||
|
||||
:param pipeline: Pipeline
|
||||
:return: The SQL reporter
|
||||
|
||||
"""
|
||||
connection = self.getSqlConnection()
|
||||
return connection.driver.getReporter(connection, pipeline)
|
||||
|
||||
def getSource(self, connection_name):
|
||||
connection = self.connections[connection_name]
|
||||
return connection.driver.getSource(connection)
|
||||
|
|
|
@ -2423,8 +2423,7 @@ class QueueItem(object):
|
|||
self.log.debug("Checking DB for requirements")
|
||||
requirements_tuple = tuple(sorted(requirements))
|
||||
if requirements_tuple not in self._cached_sql_results:
|
||||
sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
|
||||
conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
|
||||
conn = self.pipeline.manager.sched.connections.getSqlConnection()
|
||||
if conn:
|
||||
builds = conn.getBuilds(
|
||||
tenant=self.pipeline.tenant.name,
|
||||
|
|
|
@ -185,7 +185,6 @@ class RPCListener(RPCListenerBase):
|
|||
'get_running_jobs',
|
||||
'get_job_log_stream_address',
|
||||
'tenant_list',
|
||||
'tenant_sql_connection',
|
||||
'status_get',
|
||||
'job_get',
|
||||
'job_list',
|
||||
|
@ -376,16 +375,6 @@ class RPCListener(RPCListenerBase):
|
|||
'queue': queue_size})
|
||||
job.sendWorkComplete(json.dumps(output))
|
||||
|
||||
def handle_tenant_sql_connection(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
sql_driver = self.sched.connections.drivers['sql']
|
||||
conn = sql_driver.tenant_connections.get(args['tenant'])
|
||||
if conn:
|
||||
name = conn.connection_name
|
||||
else:
|
||||
name = ''
|
||||
job.sendWorkComplete(json.dumps(name))
|
||||
|
||||
def handle_status_get(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
output = self.sched.formatStatusJSON(args.get("tenant"))
|
||||
|
|
|
@ -12,10 +12,10 @@
|
|||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import cherrypy
|
||||
import socket
|
||||
|
||||
from cachetools.func import ttl_cache
|
||||
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
|
||||
from ws4py.websocket import WebSocket
|
||||
import codecs
|
||||
|
@ -702,11 +702,15 @@ class ZuulWebAPI(object):
|
|||
admin_tenants = json.loads(job.data[0])
|
||||
return admin_tenants
|
||||
|
||||
@ttl_cache(ttl=60)
|
||||
def _tenants(self):
|
||||
job = self.rpc.submitJob('zuul:tenant_list', {})
|
||||
return json.loads(job.data[0])
|
||||
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def tenants(self):
|
||||
job = self.rpc.submitJob('zuul:tenant_list', {})
|
||||
ret = json.loads(job.data[0])
|
||||
ret = self._tenants()
|
||||
resp = cherrypy.response
|
||||
resp.headers['Access-Control-Allow-Origin'] = '*'
|
||||
return ret
|
||||
|
@ -963,16 +967,8 @@ class ZuulWebAPI(object):
|
|||
})
|
||||
return ret
|
||||
|
||||
def _get_connection(self, tenant):
|
||||
# Ask the scheduler which sql connection to use for this tenant
|
||||
job = self.rpc.submitJob('zuul:tenant_sql_connection',
|
||||
{'tenant': tenant})
|
||||
connection_name = json.loads(job.data[0])
|
||||
|
||||
if not connection_name:
|
||||
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
||||
|
||||
return self.zuulweb.connections.connections[connection_name]
|
||||
def _get_connection(self):
|
||||
return self.zuulweb.connections.connections['database']
|
||||
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.save_params()
|
||||
|
@ -981,7 +977,10 @@ class ZuulWebAPI(object):
|
|||
branch=None, patchset=None, ref=None, newrev=None,
|
||||
uuid=None, job_name=None, voting=None, node_name=None,
|
||||
result=None, final=None, held=None, limit=50, skip=0):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
if tenant not in [x['name'] for x in self._tenants()]:
|
||||
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
||||
|
||||
# If final is None, we return all builds, both final and non-final
|
||||
if final is not None:
|
||||
|
@ -1001,7 +1000,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.tools.save_params()
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def build(self, tenant, uuid):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
data = connection.getBuilds(tenant=tenant, uuid=uuid, limit=1)
|
||||
if not data:
|
||||
|
@ -1040,7 +1039,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.expose
|
||||
@cherrypy.tools.save_params()
|
||||
def badge(self, tenant, project=None, pipeline=None, branch=None):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
buildsets = connection.getBuildsets(
|
||||
tenant=tenant, project=project, pipeline=pipeline,
|
||||
|
@ -1063,7 +1062,7 @@ class ZuulWebAPI(object):
|
|||
def buildsets(self, tenant, project=None, pipeline=None, change=None,
|
||||
branch=None, patchset=None, ref=None, newrev=None,
|
||||
uuid=None, result=None, limit=50, skip=0):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
buildsets = connection.getBuildsets(
|
||||
tenant=tenant, project=project, pipeline=pipeline, change=change,
|
||||
|
@ -1079,7 +1078,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.tools.save_params()
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def buildset(self, tenant, uuid):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
data = connection.getBuildset(tenant, uuid)
|
||||
if not data:
|
||||
|
|
Loading…
Reference in New Issue