Port SQLAlchemy reporter to v3 driver structure
This completes the merge from master and re-enables the sqlalchemy driver tests. Change-Id: I8e6effb3f0052343d5c6c80e9edaa2a9ff360b4d
This commit is contained in:
parent
f8aec83b3b
commit
8284489e09
|
@ -53,7 +53,6 @@ from git.exc import NoSuchPathError
|
|||
|
||||
import zuul.driver.gerrit.gerritsource as gerritsource
|
||||
import zuul.driver.gerrit.gerritconnection as gerritconnection
|
||||
import zuul.connection.sql
|
||||
import zuul.scheduler
|
||||
import zuul.webapp
|
||||
import zuul.rpclistener
|
||||
|
@ -1918,8 +1917,8 @@ class AnsibleZuulTestCase(ZuulTestCase):
|
|||
|
||||
|
||||
class ZuulDBTestCase(ZuulTestCase):
|
||||
def setup_config(self, config_file='zuul-connections-same-gerrit.conf'):
|
||||
super(ZuulDBTestCase, self).setup_config(config_file)
|
||||
def setup_config(self):
|
||||
super(ZuulDBTestCase, self).setup_config()
|
||||
for section_name in self.config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
section_name, re.I)
|
||||
|
|
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-merge.yaml
vendored
Normal file
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-merge.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
vendored
Normal file
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
2
tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,38 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
source: gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
resultsdb:
|
||||
score: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
||||
resultsdb:
|
||||
score: -1
|
||||
resultsdb_failures:
|
||||
score: -1
|
||||
|
||||
- job:
|
||||
name: project-merge
|
||||
|
||||
- job:
|
||||
name: project-test1
|
||||
|
||||
- job:
|
||||
name: project-test2
|
||||
|
||||
- project:
|
||||
name: org/project
|
||||
check:
|
||||
jobs:
|
||||
- project-merge
|
||||
- project-test1:
|
||||
dependencies: project-merge
|
||||
- project-test2:
|
||||
dependencies: project-merge
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,8 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-repos:
|
||||
- common-config
|
||||
project-repos:
|
||||
- org/project
|
|
@ -1,27 +0,0 @@
|
|||
pipelines:
|
||||
- name: check
|
||||
manager: IndependentPipelineManager
|
||||
source:
|
||||
review_gerrit
|
||||
trigger:
|
||||
review_gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
review_gerrit:
|
||||
verified: 1
|
||||
resultsdb:
|
||||
score: 1
|
||||
failure:
|
||||
review_gerrit:
|
||||
verified: -1
|
||||
resultsdb:
|
||||
score: -1
|
||||
resultsdb_failures:
|
||||
score: -1
|
||||
|
||||
projects:
|
||||
- name: org/project
|
||||
check:
|
||||
- project-merge:
|
||||
- project-test1
|
||||
- project-test2
|
|
@ -33,12 +33,3 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
# TODOv3(jeblair): commented out until sqlalchemy conenction ported to
|
||||
# v3 driver syntax
|
||||
#[connection resultsdb] driver=sql
|
||||
#dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
#[connection resultsdb_failures]
|
||||
#driver=sql
|
||||
#dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
|
|
@ -2,34 +2,24 @@
|
|||
server=127.0.0.1
|
||||
|
||||
[zuul]
|
||||
layout_config=layout-connections-multiple-voters.yaml
|
||||
tenant_config=main.yaml
|
||||
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
|
||||
job_name_in_report=true
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/git
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
zuul_url=http://zuul.example.com/p
|
||||
|
||||
[connection review_gerrit]
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=none
|
||||
|
||||
[connection alt_voting_gerrit]
|
||||
driver=gerrit
|
||||
server=alt_review.example.com
|
||||
user=civoter
|
||||
sshkey=none
|
||||
|
||||
[connection outgoing_smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb]
|
||||
driver=sql
|
|
@ -0,0 +1,40 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[zuul]
|
||||
tenant_config=main.yaml
|
||||
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
|
||||
job_name_in_report=true
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
zuul_url=http://zuul.example.com/p
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[swift]
|
||||
authurl=https://identity.api.example.org/v2.0/
|
||||
user=username
|
||||
key=password
|
||||
tenant_name=" "
|
||||
|
||||
default_container=logs
|
||||
region_name=EXP
|
||||
logserver_prefix=http://logs.example.org/server.app/
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
|
@ -13,7 +13,6 @@
|
|||
# under the License.
|
||||
|
||||
import sqlalchemy as sa
|
||||
from unittest import skip
|
||||
|
||||
from tests.base import ZuulTestCase, ZuulDBTestCase
|
||||
|
||||
|
@ -57,45 +56,44 @@ class TestConnections(ZuulTestCase):
|
|||
self.assertEqual(B.patchsets[-1]['approvals'][0]['by']['username'],
|
||||
'civoter')
|
||||
|
||||
def _test_sql_tables_created(self, metadata_table=None):
|
||||
|
||||
class TestSQLConnection(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_sql_tables_created(self, metadata_table=None):
|
||||
"Test the tables for storing results are created properly"
|
||||
buildset_table = 'zuul_buildset'
|
||||
build_table = 'zuul_build'
|
||||
|
||||
insp = sa.engine.reflection.Inspector(
|
||||
self.connections['resultsdb'].engine)
|
||||
self.connections.connections['resultsdb'].engine)
|
||||
|
||||
self.assertEqual(9, len(insp.get_columns(buildset_table)))
|
||||
self.assertEqual(10, len(insp.get_columns(build_table)))
|
||||
|
||||
@skip("Disabled for early v3 development")
|
||||
def test_sql_tables_created(self):
|
||||
"Test the default table is created"
|
||||
self.config.set('zuul', 'layout_config',
|
||||
'tests/fixtures/layout-sql-reporter.yaml')
|
||||
self.sched.reconfigure(self.config)
|
||||
self._test_sql_tables_created()
|
||||
|
||||
def _test_sql_results(self):
|
||||
def test_sql_results(self):
|
||||
"Test results are entered into an sql table"
|
||||
# Grab the sa tables
|
||||
tenant = self.sched.abide.tenants.get('tenant-one')
|
||||
reporter = _get_reporter_from_connection_name(
|
||||
self.sched.layout.pipelines['check'].success_actions,
|
||||
tenant.layout.pipelines['check'].success_actions,
|
||||
'resultsdb'
|
||||
)
|
||||
|
||||
# Add a success result
|
||||
A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Add a failed result for a negative score
|
||||
B = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'B')
|
||||
self.worker.addFailTest('project-test1', B)
|
||||
self.fake_review_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
|
||||
|
||||
self.executor_server.failJob('project-test1', B)
|
||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
conn = self.connections['resultsdb'].engine.connect()
|
||||
conn = self.connections.connections['resultsdb'].engine.connect()
|
||||
result = conn.execute(
|
||||
sa.sql.select([reporter.connection.zuul_buildset_table]))
|
||||
|
||||
|
@ -122,7 +120,7 @@ class TestConnections(ZuulTestCase):
|
|||
# Check the first result, which should be the project-merge job
|
||||
self.assertEqual('project-merge', buildset0_builds[0]['job_name'])
|
||||
self.assertEqual("SUCCESS", buildset0_builds[0]['result'])
|
||||
self.assertEqual('http://logs.example.com/1/1/check/project-merge/0',
|
||||
self.assertEqual('https://server/job/project-merge/0/',
|
||||
buildset0_builds[0]['log_url'])
|
||||
|
||||
self.assertEqual('check', buildset1['pipeline'])
|
||||
|
@ -144,42 +142,33 @@ class TestConnections(ZuulTestCase):
|
|||
# which failed
|
||||
self.assertEqual('project-test1', buildset1_builds[-2]['job_name'])
|
||||
self.assertEqual("FAILURE", buildset1_builds[-2]['result'])
|
||||
self.assertEqual('http://logs.example.com/2/1/check/project-test1/4',
|
||||
self.assertEqual('https://server/job/project-test1/0/',
|
||||
buildset1_builds[-2]['log_url'])
|
||||
|
||||
@skip("Disabled for early v3 development")
|
||||
def test_sql_results(self):
|
||||
"Test results are entered into the default sql table"
|
||||
self.config.set('zuul', 'layout_config',
|
||||
'tests/fixtures/layout-sql-reporter.yaml')
|
||||
self.sched.reconfigure(self.config)
|
||||
self._test_sql_results()
|
||||
|
||||
@skip("Disabled for early v3 development")
|
||||
def test_multiple_sql_connections(self):
|
||||
"Test putting results in different databases"
|
||||
self.config.set('zuul', 'layout_config',
|
||||
self.updateConfigLayout(
|
||||
'tests/fixtures/layout-sql-reporter.yaml')
|
||||
self.sched.reconfigure(self.config)
|
||||
|
||||
# Add a successful result
|
||||
A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Add a failed result
|
||||
B = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'B')
|
||||
self.worker.addFailTest('project-test1', B)
|
||||
self.fake_review_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
|
||||
self.executor_server.failJob('project-test1', B)
|
||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Grab the sa tables for resultsdb
|
||||
tenant = self.sched.abide.tenants.get('tenant-one')
|
||||
reporter1 = _get_reporter_from_connection_name(
|
||||
self.sched.layout.pipelines['check'].success_actions,
|
||||
tenant.layout.pipelines['check'].success_actions,
|
||||
'resultsdb'
|
||||
)
|
||||
|
||||
conn = self.connections['resultsdb'].engine.connect()
|
||||
conn = self.connections.connections['resultsdb'].engine.connect()
|
||||
buildsets_resultsdb = conn.execute(sa.sql.select(
|
||||
[reporter1.connection.zuul_buildset_table])).fetchall()
|
||||
# Should have been 2 buildset reported to the resultsdb (both success
|
||||
|
@ -196,11 +185,12 @@ class TestConnections(ZuulTestCase):
|
|||
|
||||
# Grab the sa tables for resultsdb_failures
|
||||
reporter2 = _get_reporter_from_connection_name(
|
||||
self.sched.layout.pipelines['check'].failure_actions,
|
||||
tenant.layout.pipelines['check'].failure_actions,
|
||||
'resultsdb_failures'
|
||||
)
|
||||
|
||||
conn = self.connections['resultsdb_failures'].engine.connect()
|
||||
conn = self.connections.connections['resultsdb_failures'].\
|
||||
engine.connect()
|
||||
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
|
||||
[reporter2.connection.zuul_buildset_table])).fetchall()
|
||||
# The failure db should only have 1 buildset failed
|
||||
|
@ -217,10 +207,9 @@ class TestConnections(ZuulTestCase):
|
|||
|
||||
|
||||
class TestConnectionsBadSQL(ZuulDBTestCase):
|
||||
def setup_config(self, config_file='zuul-connections-bad-sql.conf'):
|
||||
super(TestConnectionsBadSQL, self).setup_config(config_file)
|
||||
config_file = 'zuul-sql-driver-bad.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
@skip("Disabled for early v3 development")
|
||||
def test_unable_to_connect(self):
|
||||
"Test the SQL reporter fails gracefully when unable to connect"
|
||||
self.config.set('zuul', 'layout_config',
|
||||
|
@ -229,8 +218,8 @@ class TestConnectionsBadSQL(ZuulDBTestCase):
|
|||
|
||||
# Trigger a reporter. If no errors are raised, the reporter has been
|
||||
# disabled correctly
|
||||
A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
# Copyright 2016 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from zuul.driver import Driver, ConnectionInterface, ReporterInterface
|
||||
import sqlconnection
|
||||
import sqlreporter
|
||||
|
||||
|
||||
class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
|
||||
name = 'sql'
|
||||
|
||||
def registerScheduler(self, scheduler):
|
||||
self.sched = scheduler
|
||||
|
||||
def getConnection(self, name, config):
|
||||
return sqlconnection.SQLConnection(self, name, config)
|
||||
|
||||
def getReporter(self, connection, config=None):
|
||||
return sqlreporter.SQLReporter(self, connection, config)
|
||||
|
||||
def getReporterSchema(self):
|
||||
return sqlreporter.getSchema()
|
|
@ -29,9 +29,10 @@ class SQLConnection(BaseConnection):
|
|||
driver_name = 'sql'
|
||||
log = logging.getLogger("connection.sql")
|
||||
|
||||
def __init__(self, connection_name, connection_config):
|
||||
def __init__(self, driver, connection_name, connection_config):
|
||||
|
||||
super(SQLConnection, self).__init__(connection_name, connection_config)
|
||||
super(SQLConnection, self).__init__(driver, connection_name,
|
||||
connection_config)
|
||||
|
||||
self.dburi = None
|
||||
self.engine = None
|
|
@ -25,10 +25,10 @@ class SQLReporter(BaseReporter):
|
|||
name = 'sql'
|
||||
log = logging.getLogger("zuul.reporter.mysql.SQLReporter")
|
||||
|
||||
def __init__(self, reporter_config={}, sched=None, connection=None):
|
||||
def __init__(self, driver, connection, config={}):
|
||||
super(SQLReporter, self).__init__(
|
||||
reporter_config, sched, connection)
|
||||
self.result_score = reporter_config.get('score', None)
|
||||
driver, connection, config)
|
||||
self.result_score = config.get('score', None)
|
||||
|
||||
def report(self, source, pipeline, item):
|
||||
"""Create an entry into a database."""
|
||||
|
@ -37,13 +37,12 @@ class SQLReporter(BaseReporter):
|
|||
self.log.warn("SQL reporter (%s) is disabled " % self)
|
||||
return
|
||||
|
||||
if self.sched.config.has_option('zuul', 'url_pattern'):
|
||||
url_pattern = self.sched.config.get('zuul', 'url_pattern')
|
||||
if self.driver.sched.config.has_option('zuul', 'url_pattern'):
|
||||
url_pattern = self.driver.sched.config.get('zuul', 'url_pattern')
|
||||
else:
|
||||
url_pattern = None
|
||||
|
||||
score = self.reporter_config['score']\
|
||||
if 'score' in self.reporter_config else 0
|
||||
score = self.config.get('score', 0)
|
||||
|
||||
with self.connection.engine.begin() as conn:
|
||||
buildset_ins = self.connection.zuul_buildset_table.insert().values(
|
||||
|
@ -60,7 +59,7 @@ class SQLReporter(BaseReporter):
|
|||
buildset_ins_result = conn.execute(buildset_ins)
|
||||
build_inserts = []
|
||||
|
||||
for job in pipeline.getJobs(item):
|
||||
for job in item.getJobs():
|
||||
build = item.current_build_set.getBuild(job.name)
|
||||
if not build:
|
||||
# build hasn't began. The sql reporter can only send back
|
|
@ -20,6 +20,7 @@ import zuul.driver.gerrit
|
|||
import zuul.driver.git
|
||||
import zuul.driver.smtp
|
||||
import zuul.driver.timer
|
||||
import zuul.driver.sql
|
||||
from zuul.connection import BaseConnection
|
||||
|
||||
|
||||
|
@ -41,6 +42,7 @@ class ConnectionRegistry(object):
|
|||
self.registerDriver(zuul.driver.git.GitDriver())
|
||||
self.registerDriver(zuul.driver.smtp.SMTPDriver())
|
||||
self.registerDriver(zuul.driver.timer.TimerDriver())
|
||||
self.registerDriver(zuul.driver.sql.SQLDriver())
|
||||
|
||||
def registerDriver(self, driver):
|
||||
if driver.name in self.drivers:
|
||||
|
|
Loading…
Reference in New Issue