zuul
/
zuul
Code Issues Proposed changes

Required SQL reporters 

On the way towards a fully scale out scheduler we need to move the
times database from the local filesystem into the SQL
database. Therefore we need to make at least one SQL connection
mandatory.

SQL reporters are required (an implied sql reporter is added to
every pipeline, explicit sql reporters ignored)

Change-Id: I30723f9b320b9f2937cc1d7ff3267519161bc380
Depends-On: https://review.opendev.org/621479
Story: 2007192
Task: 38329
This commit is contained in:
Jan Kubovy 2020-05-27 14:21:19 +02:00 committed by James E. Blair
parent 9e0c733aac
commit 9ab527971f
62 changed files with 514 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/examples/etc_zuul/zuul.conf
View File

@ -28,9 +28,7 @@ name=opendev
driver=git
baseurl=https://opendev.org
[connection "mysql"]
name=mysql
driver=sql
[database]
dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
[web]

4
doc/source/examples/zuul-config/zuul.d/pipelines.yaml
View File

@ -17,11 +17,9 @@
success:
gerrit:
Verified: 1
mysql:
failure:
gerrit:
Verified: -1
mysql:
- pipeline:
name: gate
@ -48,8 +46,6 @@
gerrit:
Verified: 2
submit: true
mysql:
failure:
gerrit:
Verified: -2
mysql:

3
doc/source/howtos/installation.rst
View File

@ -148,6 +148,9 @@ service in Zuul, and a connection to Gerrit.
user=zuul
sshkey=/home/zuul/.ssh/id_rsa
[database]
dburi=mysql+pymysql://zuul:secret@mysql/zuul
See :ref:`components` and :ref:`connections` for more details.
The following tells Zuul to read its configuration from and operate on

1
doc/source/reference/admin.rst
View File

@ -5,6 +5,7 @@ Admin Reference
:maxdepth: 3
tenants
database
connections
drivers/index
client

49
doc/source/reference/database.rst Normal file
View File

@ -0,0 +1,49 @@
:title: Database Configuration
.. _database:
Database Configuration
======================
The database configuration is located in the ``database`` section of
``zuul.conf``:
.. code-block:: ini
[database]
dburi=<URI>
The following options can be defined in this section.
.. attr:: database
.. attr:: dburi
:required:
Database connection information in the form of a URI understood
by SQLAlchemy. See `The SQLAlchemy manual
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
for more information.
The driver will automatically set up the database creating and managing
the necessary tables. Therefore the provided user should have sufficient
permissions to manage the database. For example:
.. code-block:: sql
GRANT ALL ON my_database TO 'my_user'@'%';
.. attr:: pool_recycle
:default: 1
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
for more information.
.. attr:: table_prefix
:default: ''
The string to prefix the table names. This makes it possible to run
several zuul deployments against the same database. This can be useful
if you rely on external databases which are not under your control.
The default is to have no prefix.

4
doc/source/reference/drivers/sql.rst
View File

@ -5,6 +5,10 @@
SQL
===
.. warning::
This driver is deprecated, use :attr:`database` configuration instead.
The SQL driver supports reporters only. Only one connection per
database is permitted.

12
releasenotes/notes/required-sql-7dd38b76d9f4273f.yaml Normal file
View File

@ -0,0 +1,12 @@
---
upgrade:
- |
The components zuul-scheduler and zuul-web now require database
configuration. There is now only one supported database connection. If
multiple sql connections have been configured only the first one will be
used.
deprecations:
- |
Defining database connections using :attr:`<sql connection>` is now
deprecated. Refer to :ref:`database` how to configure the database now.

93
tests/base.py
View File

@ -66,6 +66,7 @@ from git.exc import NoSuchPathError
import yaml
import paramiko
from zuul.driver.sql.sqlconnection import DatabaseSession
from zuul.model import Change
from zuul.rpcclient import RPCClient
@ -74,7 +75,7 @@ from zuul.driver.git import GitDriver
from zuul.driver.smtp import SMTPDriver
from zuul.driver.github import GithubDriver
from zuul.driver.timer import TimerDriver
from zuul.driver.sql import SQLDriver
from zuul.driver.sql import SQLDriver, sqlconnection
from zuul.driver.bubblewrap import BubblewrapDriver
from zuul.driver.nullwrap import NullwrapDriver
from zuul.driver.mqtt import MQTTDriver
@ -311,12 +312,19 @@ class GitlabDriverMock(GitlabDriver):
return connection
class SQLDriverMock(SQLDriver):
def getConnection(self, name, config):
return FakeSqlConnection(self, name, config)
class TestConnectionRegistry(ConnectionRegistry):
def __init__(self, changes: Dict[str, Dict[str, Change]],
config: ConfigParser, additional_event_queues,
upstream_root: str, rpcclient: RPCClient, poller_events,
git_url_with_auth: bool,
add_cleanup: Callable[[Callable[[], None]], None]):
add_cleanup: Callable[[Callable[[], None]], None],
fake_sql: bool):
self.connections = OrderedDict()
self.drivers = {}
@ -330,7 +338,10 @@ class TestConnectionRegistry(ConnectionRegistry):
rpcclient, git_url_with_auth))
self.registerDriver(SMTPDriver())
self.registerDriver(TimerDriver())
self.registerDriver(SQLDriver())
if fake_sql:
self.registerDriver(SQLDriverMock())
else:
self.registerDriver(SQLDriver())
self.registerDriver(BubblewrapDriver())
self.registerDriver(NullwrapDriver())
self.registerDriver(MQTTDriver())
@ -2889,6 +2900,37 @@ class FakeBuild(object):
return repos
class FakeZuulDatabaseSession(DatabaseSession):
def __exit__(self, etype, value, tb):
pass
def FakeOrmSessionFactory():
class FakeBackendSession:
def add(self, *args, **kwargs):
pass
return FakeBackendSession()
class FakeSqlConnection(sqlconnection.SQLConnection):
def __init__(self, driver, connection_name, connection_config):
connection_config['dburi'] = 'postgresql://example.com'
super().__init__(driver, connection_name, connection_config)
self.session = FakeOrmSessionFactory
def onLoad(self):
pass
def onStop(self):
pass
def getSession(self):
return FakeZuulDatabaseSession(self)
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
result = None
@ -3587,12 +3629,12 @@ class ZuulWebFixture(fixtures.Fixture):
additional_event_queues, upstream_root: str,
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
add_cleanup: Callable[[Callable[[], None]], None],
test_root, info=None, zk_hosts=None):
test_root, info=None, zk_hosts=None, fake_sql=True):
super(ZuulWebFixture, self).__init__()
self.gearman_server_port = gearman_server_port
self.connections = TestConnectionRegistry(
changes, config, additional_event_queues, upstream_root, rpcclient,
poller_events, git_url_with_auth, add_cleanup)
poller_events, git_url_with_auth, add_cleanup, fake_sql)
self.connections.configure(
config,
include_drivers=[zuul.driver.sql.SQLDriver,
@ -3847,6 +3889,7 @@ class SchedulerTestApp:
additional_event_queues, upstream_root: str,
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
source_only: bool,
fake_sql: bool,
add_cleanup: Callable[[Callable[[], None]], None]):
self.log = log
@ -3868,7 +3911,7 @@ class SchedulerTestApp:
self.connections = TestConnectionRegistry(
self.changes, self.config, additional_event_queues,
upstream_root, rpcclient, poller_events,
git_url_with_auth, add_cleanup)
git_url_with_auth, add_cleanup, fake_sql)
self.connections.configure(self.config, source_only=source_only)
self.sched.registerConnections(self.connections)
@ -3917,12 +3960,13 @@ class SchedulerTestManager:
changes: Dict[str, Dict[str, Change]], additional_event_queues,
upstream_root: str, rpcclient: RPCClient, poller_events,
git_url_with_auth: bool, source_only: bool,
fake_sql: bool,
add_cleanup: Callable[[Callable[[], None]], None])\
-> SchedulerTestApp:
app = SchedulerTestApp(log, config, zk_config, changes,
additional_event_queues, upstream_root,
rpcclient, poller_events, git_url_with_auth,
source_only, add_cleanup)
source_only, fake_sql, add_cleanup)
self.instances.append(app)
return app
@ -4030,6 +4074,7 @@ class ZuulTestCase(BaseTestCase):
git_url_with_auth: bool = False
log_console_port: int = 19885
source_only: bool = False
fake_sql: bool = True
def __getattr__(self, name):
"""Allows to access fake connections the old way, e.g., using
@ -4161,7 +4206,7 @@ class ZuulTestCase(BaseTestCase):
executor_connections = TestConnectionRegistry(
self.changes, self.config, self.additional_event_queues,
self.upstream_root, self.rpcclient, self.poller_events,
self.git_url_with_auth, self.addCleanup)
self.git_url_with_auth, self.addCleanup, True)
executor_connections.configure(self.config,
source_only=self.source_only)
self.executor_server = RecordingExecutorServer(
@ -4180,7 +4225,7 @@ class ZuulTestCase(BaseTestCase):
self.log, self.config, self.zk_config, self.changes,
self.additional_event_queues, self.upstream_root, self.rpcclient,
self.poller_events, self.git_url_with_auth, self.source_only,
self.addCleanup)
self.fake_sql, self.addCleanup)
if hasattr(self, 'fake_github'):
self.additional_event_queues.append(
@ -4271,6 +4316,7 @@ class ZuulTestCase(BaseTestCase):
# Make test_root persist after ansible run for .flag test
config.set('executor', 'trusted_rw_paths', self.test_root)
self.setupAllProjectKeys(config)
return config
def setupSimpleLayout(self, config: ConfigParser):
@ -5273,7 +5319,21 @@ class SSLZuulTestCase(ZuulTestCase):
class ZuulDBTestCase(ZuulTestCase):
fake_sql = False
def setup_config(self, config_file: str):
def _setup_fixture(config, section_name):
if (config.get(section_name, 'dburi') ==
'$MYSQL_FIXTURE_DBURI$'):
f = MySQLSchemaFixture()
self.useFixture(f)
config.set(section_name, 'dburi', f.dburi)
elif (config.get(section_name, 'dburi') ==
'$POSTGRESQL_FIXTURE_DBURI$'):
f = PostgresqlSchemaFixture()
self.useFixture(f)
config.set(section_name, 'dburi', f.dburi)
config = super(ZuulDBTestCase, self).setup_config(config_file)
for section_name in config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
@ -5282,16 +5342,11 @@ class ZuulDBTestCase(ZuulTestCase):
continue
if config.get(section_name, 'driver') == 'sql':
if (config.get(section_name, 'dburi') ==
'$MYSQL_FIXTURE_DBURI$'):
f = MySQLSchemaFixture()
self.useFixture(f)
config.set(section_name, 'dburi', f.dburi)
elif (config.get(section_name, 'dburi') ==
'$POSTGRESQL_FIXTURE_DBURI$'):
f = PostgresqlSchemaFixture()
self.useFixture(f)
config.set(section_name, 'dburi', f.dburi)
_setup_fixture(config, section_name)
if 'database' in config.sections():
_setup_fixture(config, 'database')
return config

22
tests/fixtures/config/sql-driver/git/common-config/zuul.yaml vendored
View File

@ -7,15 +7,10 @@
success:
gerrit:
Verified: 1
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
gerrit:
Verified: -1
resultsdb_mysql: null
resultsdb_mysql_failures: null
resultsdb_postgresql: null
resultsdb_postgresql_failures: null
resultsdb_failures: null
- pipeline:
name: gate
@ -30,15 +25,10 @@
gerrit:
Verified: 2
submit: true
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
gerrit:
Verified: -2
resultsdb_mysql: null
resultsdb_mysql_failures: null
resultsdb_postgresql: null
resultsdb_postgresql_failures: null
resultsdb_failures: null
start:
gerrit:
Verified: 0
@ -51,14 +41,8 @@
gerrit:
- event: ref-updated
ref: ^refs/tags/.*$
success:
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
resultsdb_mysql: null
resultsdb_mysql_failures: null
resultsdb_postgresql: null
resultsdb_postgresql_failures: null
resultsdb_failures: null
- job:
name: base

4
tests/fixtures/layouts/provides-requires-single-project.yaml vendored
View File

@ -7,13 +7,9 @@
success:
gerrit:
Verified: 1
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
gerrit:
Verified: -1
resultsdb_mysql: null
resultsdb_postgresql: null
- job:
name: base

4
tests/fixtures/layouts/provides-requires-two-jobs.yaml vendored
View File

@ -7,13 +7,9 @@
success:
gerrit:
Verified: 1
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
gerrit:
Verified: -1
resultsdb_mysql: null
resultsdb_postgresql: null
- pipeline:
name: gate

4
tests/fixtures/layouts/provides-requires.yaml vendored
View File

@ -7,13 +7,9 @@
success:
gerrit:
Verified: 1
resultsdb_mysql: null
resultsdb_postgresql: null
failure:
gerrit:
Verified: -1
resultsdb_mysql: null
resultsdb_postgresql: null
- pipeline:
name: gate

2
tests/fixtures/layouts/sql-build-error.yaml vendored
View File

@ -5,11 +5,9 @@
gerrit:
- event: patchset-created
success:
resultsdb_mysql: null
gerrit:
Verified: 1
failure:
resultsdb_mysql: null
gerrit:
Verified: -1

4
tests/fixtures/zuul-admin-web-no-override.conf vendored
View File

@ -29,3 +29,7 @@ realm=zuul.example.com
client_id=zuul.example.com
issuer_id=zuul_operator
secret=NoDanaOnlyZuul
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-admin-web-oidc.conf vendored
View File

@ -43,3 +43,7 @@ realm=myOIDC2
client_id=zuul
issuer_id=http://oidc2
scope=openid profile email special-scope
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-admin-web-token-expiry.conf vendored
View File

@ -30,3 +30,7 @@ client_id=zuul.example.com
issuer_id=zuul_operator
secret=NoDanaOnlyZuul
max_validity_time=5
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-admin-web.conf vendored
View File

@ -29,3 +29,7 @@ realm=zuul.example.com
client_id=zuul.example.com
issuer_id=zuul_operator
secret=NoDanaOnlyZuul
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-connections-cgit.conf vendored
View File

@ -25,3 +25,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-connections-gerrit-and-github.conf vendored
View File

@ -29,3 +29,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-connections-gitweb.conf vendored
View File

@ -24,3 +24,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-connections-merger.conf vendored
View File

@ -21,10 +21,10 @@ server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[connection resultsdb]
driver=sql
[database]
dburi=$MYSQL_FIXTURE_DBURI$
[connection smtp]
driver=smtp
server=localhost

4
tests/fixtures/zuul-connections-multiple-gerrits.conf vendored
View File

@ -30,3 +30,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-connections-same-gerrit.conf vendored
View File

@ -30,3 +30,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-crd-pagure.conf vendored
View File

@ -37,3 +37,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-default-ansible-version.conf vendored
View File

@ -34,3 +34,7 @@ default_to=you@example.com
[web]
static_cache_expiry=1200
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-disk-accounting.conf vendored
View File

@ -25,3 +25,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

3
tests/fixtures/zuul-elastic-driver.conf vendored
View File

@ -23,3 +23,6 @@ driver=elasticsearch
uri=localhost:9200
use_ssl=true
verify_certs=false
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-executor-ansible-callback.conf vendored
View File

@ -46,3 +46,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-executor-hostname.conf vendored
View File

@ -30,3 +30,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-gerrit-github.conf vendored
View File

@ -33,3 +33,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-gerrit-no-stream.conf vendored
View File

@ -31,3 +31,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-gerrit-web.conf vendored
View File

@ -33,3 +33,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-git-driver.conf vendored
View File

@ -29,3 +29,7 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-github-driver-enterprise.conf vendored
View File

@ -18,3 +18,7 @@ webhook_token=0000000000000000000000000000000000000000
app_id=1
app_key=$APP_KEY_FIXTURE$
server=github.enterprise.io
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-github-driver.conf vendored
View File

@ -26,3 +26,7 @@ sshkey=/home/zuul/.ssh/id_rsa
driver=github
sshkey=/home/zuul/.ssh/id_rsa
server=github.enterprise.io
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-gitlab-driver.conf vendored
View File

@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
driver=gitlab
server=gitlab
api_token=0000000000000000000000000000000000000000
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-hold-expiration.conf vendored
View File

@ -36,3 +36,7 @@ default_to=you@example.com
[web]
static_cache_expiry=1200
root=https://zuul.example.com/
[database]
dburi=$MYSQL_FIXTURE_DBURI$

3
tests/fixtures/zuul-mqtt-driver.conf vendored
View File

@ -21,5 +21,8 @@ sshkey=fake_id_rsa1
[connection mqtt]
driver=mqtt
[database]
dburi=$MYSQL_FIXTURE_DBURI$
[web]
root=https://tenant.example.com/

4
tests/fixtures/zuul-pagure-driver-whitelist.conf vendored
View File

@ -17,3 +17,7 @@ driver=pagure
server=pagure
api_token=0000000000000000000000000000000000000000
source_whitelist=::ffff:127.0.0.1
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-pagure-driver.conf vendored
View File

@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
driver=pagure
server=pagure
api_token=0000000000000000000000000000000000000000
[database]
dburi=$MYSQL_FIXTURE_DBURI$

4
tests/fixtures/zuul-push-reqs.conf vendored
View File

@ -20,3 +20,7 @@ webhook_token=00000000000000000000000000000000000000000
driver=gerrit
server=review.example.com
user=jenkins
[database]
dburi=$MYSQL_FIXTURE_DBURI$

3
tests/fixtures/zuul-sql-driver-bad.conf vendored
View File

@ -18,8 +18,7 @@ server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[connection resultsdb_mysql]
driver=sql
[database]
dburi=mysql+pymysql://bad:creds@host/db
[connection resultsdb_mysql_failures]

26
tests/fixtures/zuul-sql-driver-mysql.conf vendored Normal file
View File

@ -0,0 +1,26 @@
[gearman]
server=127.0.0.1
[scheduler]
tenant_config=main.yaml
[merger]
git_dir=/tmp/zuul-test/merger-git
git_user_email=zuul@example.com
git_user_name=zuul
[executor]
git_dir=/tmp/zuul-test/executor-git
[connection gerrit]
driver=gerrit
server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[database]
dburi=$MYSQL_FIXTURE_DBURI$
[connection resultsdb_failures]
driver=sql
dburi=$MYSQL_FIXTURE_DBURI$

26
tests/fixtures/zuul-sql-driver-postgres.conf vendored Normal file
View File

@ -0,0 +1,26 @@
[gearman]
server=127.0.0.1
[scheduler]
tenant_config=main.yaml
[merger]
git_dir=/tmp/zuul-test/merger-git
git_user_email=zuul@example.com
git_user_name=zuul
[executor]
git_dir=/tmp/zuul-test/executor-git
[connection gerrit]
driver=gerrit
server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[database]
dburi=$POSTGRESQL_FIXTURE_DBURI$
[connection resultsdb_failures]
driver=sql
dburi=$POSTGRESQL_FIXTURE_DBURI$

14
tests/fixtures/zuul-sql-driver-prefix.conf → tests/fixtures/zuul-sql-driver-prefix-mysql.conf vendored
View File

@ -18,20 +18,10 @@ server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[connection resultsdb_mysql]
driver=sql
[database]
dburi=$MYSQL_FIXTURE_DBURI$
table_prefix=prefix_
[connection resultsdb_mysql_failures]
[connection resultsdb_failures]
driver=sql
dburi=$MYSQL_FIXTURE_DBURI$
[connection resultsdb_postgresql]
driver=sql
dburi=$POSTGRESQL_FIXTURE_DBURI$
table_prefix=prefix_
[connection resultsdb_postgresql_failures]
driver=sql
dburi=$POSTGRESQL_FIXTURE_DBURI$

13
tests/fixtures/zuul-sql-driver.conf → tests/fixtures/zuul-sql-driver-prefix-postgres.conf vendored
View File

@ -18,18 +18,11 @@ server=review.example.com
user=jenkins
sshkey=fake_id_rsa1
[connection resultsdb_mysql]
driver=sql
dburi=$MYSQL_FIXTURE_DBURI$
[connection resultsdb_mysql_failures]
driver=sql
dburi=$MYSQL_FIXTURE_DBURI$
[connection resultsdb_postgresql]
[database]
driver=sql
dburi=$POSTGRESQL_FIXTURE_DBURI$
table_prefix=prefix_
[connection resultsdb_postgresql_failures]
[connection resultsdb_failures]
driver=sql
dburi=$POSTGRESQL_FIXTURE_DBURI$

3
tests/fixtures/zuul-winrm.conf vendored
View File

@ -35,3 +35,6 @@ server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$

3
tests/fixtures/zuul.conf vendored
View File

@ -31,6 +31,9 @@ port=25
default_from=zuul@example.com
default_to=you@example.com
[database]
dburi=$MYSQL_FIXTURE_DBURI$
[web]
static_cache_expiry=1200
root=https://zuul.example.com/

96
tests/unit/test_connection.py
View File

@ -11,13 +11,18 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import configparser
import os
import re
import textwrap
import time
import sqlalchemy as sa
from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
import zuul
from tests.base import ZuulTestCase, FIXTURE_DIR, \
PostgresqlSchemaFixture, MySQLSchemaFixture, ZuulDBTestCase, \
BaseTestCase, AnsibleZuulTestCase
def _get_reporter_from_connection_name(reporters, connection_name):
@ -60,8 +65,8 @@ class TestConnections(ZuulTestCase):
'civoter')
class TestSQLConnection(ZuulDBTestCase):
config_file = 'zuul-sql-driver.conf'
class TestSQLConnectionMysql(ZuulDBTestCase):
config_file = 'zuul-sql-driver-mysql.conf'
tenant_config_file = 'config/sql-driver/main.yaml'
expected_table_prefix = ''
@ -80,8 +85,7 @@ class TestSQLConnection(ZuulDBTestCase):
def test_sql_tables_created(self):
"Test the tables for storing results are created properly"
self._sql_tables_created('resultsdb_mysql')
self._sql_tables_created('resultsdb_postgresql')
self._sql_tables_created('database')
def _sql_indexes_created(self, connection_name):
connection = self.scheds.first.connections.connections[connection_name]
@ -115,8 +119,7 @@ class TestSQLConnection(ZuulDBTestCase):
def test_sql_indexes_created(self):
"Test the indexes are created properly"
self._sql_indexes_created('resultsdb_mysql')
self._sql_indexes_created('resultsdb_postgresql')
self._sql_indexes_created('database')
def test_sql_results(self):
"Test results are entered into an sql table"
@ -224,8 +227,7 @@ class TestSQLConnection(ZuulDBTestCase):
self.orderedRelease()
self.waitUntilSettled()
check_results('resultsdb_mysql')
check_results('resultsdb_postgresql')
check_results('database')
def test_sql_results_retry_builds(self):
"Test that retry results are entered into an sql table correctly"
@ -305,8 +307,7 @@ class TestSQLConnection(ZuulDBTestCase):
self.orderedRelease()
self.waitUntilSettled()
check_results('resultsdb_mysql')
check_results('resultsdb_postgresql')
check_results('database')
def test_multiple_sql_connections(self):
"Test putting results in different databases"
@ -352,36 +353,79 @@ class TestSQLConnection(ZuulDBTestCase):
tenant.layout.pipelines['check'].failure_actions,
connection_name_2
)
self.assertIsNone(reporter2) # Explicit SQL reporters are ignored
conn = self.scheds.first.connections.\
connections[connection_name_2].engine.connect()
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
[reporter2.connection.zuul_buildset_table])).fetchall()
[reporter1.connection.zuul_buildset_table])).fetchall()
# The failure db should only have 1 buildset failed
self.assertEqual(1, len(buildsets_resultsdb_failures))
self.assertEqual(2, len(buildsets_resultsdb_failures))
self.assertEqual(
'check', buildsets_resultsdb_failures[0]['pipeline'])
'check', buildsets_resultsdb_failures[1]['pipeline'])
self.assertEqual('org/project',
buildsets_resultsdb_failures[0]['project'])
buildsets_resultsdb_failures[1]['project'])
self.assertEqual(2,
buildsets_resultsdb_failures[0]['change'])
buildsets_resultsdb_failures[1]['change'])
self.assertEqual(
'1', buildsets_resultsdb_failures[0]['patchset'])
'1', buildsets_resultsdb_failures[1]['patchset'])
self.assertEqual(
'FAILURE', buildsets_resultsdb_failures[0]['result'])
'FAILURE', buildsets_resultsdb_failures[1]['result'])
self.assertEqual('Build failed.',
buildsets_resultsdb_failures[0]['message'])
buildsets_resultsdb_failures[1]['message'])
check_results('resultsdb_mysql', 'resultsdb_mysql_failures')
check_results('resultsdb_postgresql', 'resultsdb_postgresql_failures')
check_results('database', 'resultsdb_failures')
class TestSQLConnectionPrefix(TestSQLConnection):
config_file = 'zuul-sql-driver-prefix.conf'
class TestSQLConnectionPostgres(TestSQLConnectionMysql):
config_file = 'zuul-sql-driver-postgres.conf'
class TestSQLConnectionPrefixMysql(TestSQLConnectionMysql):
config_file = 'zuul-sql-driver-prefix-mysql.conf'
expected_table_prefix = 'prefix_'
class TestSQLConnectionPrefixPostgres(TestSQLConnectionMysql):
config_file = 'zuul-sql-driver-prefix-postgres.conf'
expected_table_prefix = 'prefix_'
class TestRequiredSQLConnection(BaseTestCase):
config = None
connections = None
def setUp(self):
super().setUp()
self.addCleanup(self.stop_connection)
def setup_connection(self, config_file):
self.config = configparser.ConfigParser()
self.config.read(os.path.join(FIXTURE_DIR, config_file))
# Setup databases
for section_name in self.config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
section_name, re.I)
if not con_match:
continue
if self.config.get(section_name, 'driver') == 'sql':
if (self.config.get(section_name, 'dburi') ==
'$MYSQL_FIXTURE_DBURI$'):
f = MySQLSchemaFixture()
self.useFixture(f)
self.config.set(section_name, 'dburi', f.dburi)
elif (self.config.get(section_name, 'dburi') ==
'$POSTGRESQL_FIXTURE_DBURI$'):
f = PostgresqlSchemaFixture()
self.useFixture(f)
self.config.set(section_name, 'dburi', f.dburi)
self.connections = zuul.lib.connections.ConnectionRegistry()
def stop_connection(self):
self.connections.stop()
class TestConnectionsBadSQL(ZuulDBTestCase):
config_file = 'zuul-sql-driver-bad.conf'
tenant_config_file = 'config/sql-driver/main.yaml'

16
tests/unit/test_scheduler.py
View File

@ -5369,28 +5369,34 @@ For CI problems and help debugging, contact ci@example.org"""
tenant.layout.pipelines['gate'].merge_failure_message)
self.assertEqual(
len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
len(tenant.layout.pipelines['check'].merge_failure_actions), 2)
self.assertEqual(
len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
len(tenant.layout.pipelines['gate'].merge_failure_actions), 3)
self.assertTrue(isinstance(
tenant.layout.pipelines['check'].merge_failure_actions[0],
tenant.layout.pipelines['check'].merge_failure_actions[1],
gerritreporter.GerritReporter))
self.assertTrue(
(
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[0],
zuul.driver.smtp.smtpreporter.SMTPReporter) and
zuul.driver.sql.sqlreporter.SQLReporter) and
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[1],
zuul.driver.smtp.smtpreporter.SMTPReporter) and
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[2],
gerritreporter.GerritReporter)
) or (
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[0],
gerritreporter.GerritReporter) and
zuul.driver.sql.sqlreporter.SQLReporter) and
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[1],
gerritreporter.GerritReporter) and
isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[2],
zuul.driver.smtp.smtpreporter.SMTPReporter)
)
)

8
tests/unit/test_v3.py
View File

@ -6140,8 +6140,8 @@ class TestProvidesRequiresBuildset(ZuulTestCase):
}])
class TestProvidesRequires(ZuulDBTestCase):
config_file = "zuul-sql-driver.conf"
class TestProvidesRequiresMysql(ZuulDBTestCase):
config_file = "zuul-sql-driver-mysql.conf"
@simple_layout('layouts/provides-requires.yaml')
def test_provides_requires_shared_queue_fast(self):
@ -6682,6 +6682,10 @@ class TestProvidesRequires(ZuulDBTestCase):
1, B.messages[0])
class TestProvidesRequiresPostgres(TestProvidesRequiresMysql):
config_file = "zuul-sql-driver-postgres.conf"
class TestForceMergeMissingTemplate(ZuulTestCase):
tenant_config_file = "config/force-merge-template/main.yaml"

19
tests/unit/test_web.py
View File

@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
import urllib.parse
import socket
@ -62,7 +61,8 @@ class BaseTestWeb(ZuulTestCase):
self.test_root,
info=zuul.model.WebInfo.fromConfig(
self.zuul_ini_config),
zk_hosts=self.zk_config))
zk_hosts=self.zk_config,
fake_sql=self.fake_sql))
self.executor_server.hold_jobs_in_build = True
@ -88,6 +88,7 @@ class BaseTestWeb(ZuulTestCase):
self.waitUntilSettled()
def get_url(self, url, *args, **kwargs):
zuul.web.ZuulWebAPI._tenants.cache_clear()
return requests.get(
urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
@ -266,11 +267,7 @@ class TestWeb(BaseTestWeb):
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
req = urllib.request.Request(
"http://127.0.0.1:%s/api/tenants" % self.port)
f = urllib.request.urlopen(req)
data = f.read().decode('utf8')
data = json.loads(data)
data = self.get_url("api/tenants").json()
self.assertEqual('tenant-one', data[0]['name'])
self.assertEqual(3, data[0]['projects'])
@ -1021,7 +1018,7 @@ class TestWebSecrets(BaseTestWeb):
class TestInfo(ZuulDBTestCase, BaseTestWeb):
config_file = 'zuul-sql-driver.conf'
config_file = 'zuul-sql-driver-mysql.conf'
def setUp(self):
super(TestInfo, self).setUp()
@ -1148,7 +1145,7 @@ class TestGraphiteUrl(TestInfo):
class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
config_file = 'zuul-sql-driver.conf'
config_file = 'zuul-sql-driver-mysql.conf'
tenant_config_file = 'config/sql-driver/main.yaml'
def test_web_list_builds(self):
@ -1262,7 +1259,7 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
config_file = 'zuul-sql-driver.conf'
config_file = 'zuul-sql-driver-mysql.conf'
tenant_config_file = 'config/sql-driver/main.yaml'
def test_artifacts(self):
@ -2259,7 +2256,7 @@ class TestTenantScopedWebApiTokenWithExpiry(BaseTestWeb):
class TestHeldAttributeInBuildInfo(ZuulDBTestCase, BaseTestWeb):
config_file = 'zuul-sql-driver.conf'
config_file = 'zuul-sql-driver-mysql.conf'
tenant_config_file = 'config/sql-driver/main.yaml'
def test_autohold_and_retrieve_held_build_info(self):

6
zuul/cmd/__init__.py
View File

@ -164,9 +164,11 @@ class ZuulApp(object):
logging_config.setDebug()
logging_config.apply()
def configure_connections(self, source_only=False, include_drivers=None):
def configure_connections(self, source_only=False, include_drivers=None,
require_sql=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
self.connections.configure(self.config, source_only, include_drivers)
self.connections.configure(self.config, source_only, include_drivers,
require_sql)
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):

2
zuul/cmd/scheduler.py
View File

@ -160,7 +160,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
tls_key=zookeeper_tls_key,
tls_ca=zookeeper_tls_ca)
self.configure_connections()
self.configure_connections(require_sql=True)
self.sched.setExecutor(gearman)
self.sched.setMerger(merger)
self.sched.setNodepool(nodepool)

3
zuul/cmd/web.py
View File

@ -124,7 +124,8 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
include_drivers=[zuul.driver.sql.SQLDriver,
zuul.driver.github.GithubDriver,
zuul.driver.pagure.PagureDriver,
zuul.driver.gitlab.GitlabDriver])
zuul.driver.gitlab.GitlabDriver],
require_sql=True)
self.configure_authenticators()
self._run()
except Exception:

24
zuul/configloader.py
View File

@ -24,6 +24,7 @@ import subprocess
import voluptuous as vs
from zuul.driver.sql.sqlconnection import SQLConnection
from zuul import model
from zuul.lib import yamlutil as yaml
import zuul.manager.dependent
@ -1272,11 +1273,24 @@ class PipelineParser(object):
reporter_set = []
allowed_reporters = self.pcontext.tenant.allowed_reporters
if conf.get(conf_key):
if conf_key in ['success', 'failure', 'merge-failure']:
# SQL reporters are required (an implied SQL reporter is
# added to every pipeline, ...(1)
sql_reporter = self.pcontext.connections\
.getSqlReporter(pipeline)
sql_reporter.setAction(conf_key)
reporter_set.append(sql_reporter)
for reporter_name, params \
in conf.get(conf_key).items():
if allowed_reporters is not None and \
reporter_name not in allowed_reporters:
raise UnknownConnection(reporter_name)
if type(self.pcontext.connections
.connections[reporter_name]) == SQLConnection:
# (1)... explicit SQL reporters are ignored)
self.log.warning("Ignoring SQL reporter configured in"
" pipeline %s" % pipeline.name)
continue
reporter = self.pcontext.connections.getReporter(
reporter_name, pipeline, params)
reporter.setAction(conf_key)
@ -1287,6 +1301,16 @@ class PipelineParser(object):
if not pipeline.merge_failure_actions:
pipeline.merge_failure_actions = pipeline.failure_actions
for conf_key, action in self.reporter_actions.items():
if conf_key in ['success', 'failure', 'merge-failure']\
and not getattr(pipeline, action):
# SQL reporters are required ... add SQL reporters to the
# rest of actions.
sql_reporter = self.pcontext.connections\
.getSqlReporter(pipeline)
sql_reporter.setAction(conf_key)
setattr(pipeline, action, [sql_reporter])
pipeline.disable_at = conf.get(
'disable-after-consecutive-failures', None)

19
zuul/driver/sql/__init__.py
View File

@ -22,28 +22,9 @@ class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
name = 'sql'
def __init__(self):
self.tenant_connections = {}
cpb.capabilities_registry.register_capabilities(
'job_history', True)
def reconfigure(self, tenant):
# NOTE(corvus): This stores the connection of the first
# reporter seen for each tenant; we should figure out how to
# support multiple connections for a tenant (how do we deal
# with pagination of queries across multiple connections), or
# otherwise, require there only be one connection in a tenant.
if tenant.name in self.tenant_connections:
del self.tenant_connections[tenant.name]
for pipeline in tenant.layout.pipelines.values():
reporters = (pipeline.start_actions + pipeline.success_actions
+ pipeline.failure_actions
+ pipeline.merge_failure_actions)
for reporter in reporters:
if not isinstance(reporter, sqlreporter.SQLReporter):
continue
self.tenant_connections[tenant.name] = reporter.connection
return
def registerScheduler(self, scheduler):
self.sched = scheduler

2
zuul/driver/sql/sqlconnection.py
View File

@ -190,6 +190,8 @@ class SQLConnection(BaseConnection):
self.connection = None
self.tables_established = False
self.table_prefix = self.connection_config.get('table_prefix', '')
self.log.info("Initializing SQL connection {} (prefix: {})".format(
connection_name, self.table_prefix))
try:
self.dburi = self.connection_config.get('dburi')

49
zuul/lib/connections.py
View File

@ -17,6 +17,9 @@ import re
from collections import OrderedDict
from urllib.parse import urlparse
from zuul import model
from zuul.driver.sql.sqlconnection import SQLConnection
from zuul.driver.sql.sqlreporter import SQLReporter
import zuul.driver.zuul
import zuul.driver.gerrit
import zuul.driver.git
@ -85,10 +88,18 @@ class ConnectionRegistry(object):
for driver in self.drivers.values():
driver.stop()
def configure(self, config, source_only=False, include_drivers=None):
def configure(self, config, source_only=False, include_drivers=None,
require_sql=False):
# Register connections from the config
connections = OrderedDict()
if 'database' in config.sections() and not source_only:
driver = self.drivers['sql']
con_config = dict(config.items('database'))
connection = driver.getConnection('database', con_config)
connections['database'] = connection
for section_name in config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
section_name, re.I)
@ -126,6 +137,12 @@ class ConnectionRegistry(object):
connection = driver.getConnection(con_name, con_config)
connections[con_name] = connection
if con_driver == 'sql' and 'database' not in connections:
# The [database] section was missing. To stay backwards
# compatible duplicate the first database connection to the
# connection named 'database'
connections['database'] = driver.getConnection(
'database', con_config)
# If the [gerrit] or [smtp] sections still exist, load them in as a
# connection named 'gerrit' or 'smtp' respectfully
@ -160,8 +177,38 @@ class ConnectionRegistry(object):
connections[driver.name] = DefaultConnection(
driver, driver.name, {})
if require_sql:
if 'database' not in connections:
raise Exception("Database configuration is required")
self.connections = connections
def getSqlConnection(self) -> SQLConnection:
"""
Gets the SQL connection. This is either the connection
described in the [database] section, or the first configured
connection.
:return: The SQL connection.
"""
connection = self.connections.get('database')
if not connection:
raise Exception("No SQL connections")
return connection
def getSqlReporter(self, pipeline: model.Pipeline) -> SQLReporter:
"""
Gets the SQL reporter. Such reporter is based on
`getSqlConnection`.
:param pipeline: Pipeline
:return: The SQL reporter
"""
connection = self.getSqlConnection()
return connection.driver.getReporter(connection, pipeline)
def getSource(self, connection_name):
connection = self.connections[connection_name]
return connection.driver.getSource(connection)

3
zuul/model.py
View File

@ -2423,8 +2423,7 @@ class QueueItem(object):
self.log.debug("Checking DB for requirements")
requirements_tuple = tuple(sorted(requirements))
if requirements_tuple not in self._cached_sql_results:
sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
conn = self.pipeline.manager.sched.connections.getSqlConnection()
if conn:
builds = conn.getBuilds(
tenant=self.pipeline.tenant.name,

11
zuul/rpclistener.py
View File

@ -185,7 +185,6 @@ class RPCListener(RPCListenerBase):
'get_running_jobs',
'get_job_log_stream_address',
'tenant_list',
'tenant_sql_connection',
'status_get',
'job_get',
'job_list',
@ -376,16 +375,6 @@ class RPCListener(RPCListenerBase):
'queue': queue_size})
job.sendWorkComplete(json.dumps(output))
def handle_tenant_sql_connection(self, job):
args = json.loads(job.arguments)
sql_driver = self.sched.connections.drivers['sql']
conn = sql_driver.tenant_connections.get(args['tenant'])
if conn:
name = conn.connection_name
else:
name = ''
job.sendWorkComplete(json.dumps(name))
def handle_status_get(self, job):
args = json.loads(job.arguments)
output = self.sched.formatStatusJSON(args.get("tenant"))

37
zuul/web/__init__.py
View File

@ -12,10 +12,10 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cherrypy
import socket
from cachetools.func import ttl_cache
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
from ws4py.websocket import WebSocket
import codecs
@ -702,11 +702,15 @@ class ZuulWebAPI(object):
admin_tenants = json.loads(job.data[0])
return admin_tenants
@ttl_cache(ttl=60)
def _tenants(self):
job = self.rpc.submitJob('zuul:tenant_list', {})
return json.loads(job.data[0])
@cherrypy.expose
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def tenants(self):
job = self.rpc.submitJob('zuul:tenant_list', {})
ret = json.loads(job.data[0])
ret = self._tenants()
resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'
return ret
@ -963,16 +967,8 @@ class ZuulWebAPI(object):
})
return ret
def _get_connection(self, tenant):
# Ask the scheduler which sql connection to use for this tenant
job = self.rpc.submitJob('zuul:tenant_sql_connection',
{'tenant': tenant})
connection_name = json.loads(job.data[0])
if not connection_name:
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
return self.zuulweb.connections.connections[connection_name]
def _get_connection(self):
return self.zuulweb.connections.connections['database']
@cherrypy.expose
@cherrypy.tools.save_params()
@ -981,7 +977,10 @@ class ZuulWebAPI(object):
branch=None, patchset=None, ref=None, newrev=None,
uuid=None, job_name=None, voting=None, node_name=None,
result=None, final=None, held=None, limit=50, skip=0):
connection = self._get_connection(tenant)
connection = self._get_connection()
if tenant not in [x['name'] for x in self._tenants()]:
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
# If final is None, we return all builds, both final and non-final
if final is not None:
@ -1001,7 +1000,7 @@ class ZuulWebAPI(object):
@cherrypy.tools.save_params()
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def build(self, tenant, uuid):
connection = self._get_connection(tenant)
connection = self._get_connection()
data = connection.getBuilds(tenant=tenant, uuid=uuid, limit=1)
if not data:
@ -1040,7 +1039,7 @@ class ZuulWebAPI(object):
@cherrypy.expose
@cherrypy.tools.save_params()
def badge(self, tenant, project=None, pipeline=None, branch=None):
connection = self._get_connection(tenant)
connection = self._get_connection()
buildsets = connection.getBuildsets(
tenant=tenant, project=project, pipeline=pipeline,
@ -1063,7 +1062,7 @@ class ZuulWebAPI(object):
def buildsets(self, tenant, project=None, pipeline=None, change=None,
branch=None, patchset=None, ref=None, newrev=None,
uuid=None, result=None, limit=50, skip=0):
connection = self._get_connection(tenant)
connection = self._get_connection()
buildsets = connection.getBuildsets(
tenant=tenant, project=project, pipeline=pipeline, change=change,
@ -1079,7 +1078,7 @@ class ZuulWebAPI(object):
@cherrypy.tools.save_params()
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def buildset(self, tenant, uuid):
connection = self._get_connection(tenant)
connection = self._get_connection()
data = connection.getBuildset(tenant, uuid)
if not data: