Preseed with a number of samples from db instead of aggregate

Because of the testrepository bug 1416512 using the aggregate stream
generated by sql2subunit the timing data would be ignored by the testr
scheduler. The long term fix is to address this in testrepository
eventually. But, in the short to get any utility out of the preseed
data this commit switches the preseed to not use the aggregate view
stream and instead take the 10 most recent runs and use those as
the preseed data. While this data isn't as good for performing the
optimization, but it will at least use the data.

Change-Id: Ib0e815ca1966f5b5806fb257b91ac7a25d4db18d
This commit is contained in:
Matthew Treinish 2015-01-30 13:27:53 -05:00
parent b86852ea69
commit c6ff8a8c55
3 changed files with 18 additions and 25 deletions

View File

@ -25,7 +25,7 @@ sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository P
sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python2 /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/.testrepository
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
# Delete the venv after the script
sudo rm -rf /opt/git/subunit2sql-env

View File

@ -32,7 +32,7 @@ sudo -H /opt/git/subunit2sql-env/bin/pip install -U testrepository subunit2sql P
# Pre-seed tempest testrepository with data from subunit2sql
sudo -i env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python2 /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/.testrepository
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
# Dekete the venv after the script is called
sudo rm -rf /opt/git/subunit2sql-env

View File

@ -16,13 +16,13 @@
import os
import sys
import tempfile
from oslo.db.sqlalchemy import utils as db_utils
from subunit2sql.db import api
from subunit2sql.db import models
from subunit2sql import shell
from subunit2sql import write_subunit
from common import run_local
DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
@ -35,31 +35,24 @@ else:
TEMPEST_PATH = '/opt/stack/new/tempest'
def init_testr():
if not os.path.isdir(os.path.join(TEMPEST_PATH, '.testrepository')):
(status, out) = run_local(['testr', 'init'], status=True,
cwd=TEMPEST_PATH)
if status != 0:
print("testr init failed with:\n%s' % out")
exit(status)
def generate_subunit_stream(fd):
write_subunit.avg_sql2subunit(output=fd)
def populate_testrepository(path):
run_local(['testr', 'load', path], cwd=TEMPEST_PATH)
def get_run_ids(session):
# TODO(mtreinish): Move this function into the subunit2sql db api
results = db_utils.model_query(models.Run, session).order_by(
models.Run.run_at.desc()).filter_by(fails=0).limit(10).all()
return map(lambda x: x.id, results)
def main():
init_testr()
shell.parse_args([])
shell.CONF.set_override('connection', DB_URI, group='database')
with tempfile.NamedTemporaryFile() as fd:
generate_subunit_stream(fd)
populate_testrepository(fd.name)
session = api.get_session()
run_ids = get_run_ids(session)
session.close()
preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams')
os.mkdir(preseed_path)
for run in run_ids:
with open(os.path.join(preseed_path, run + '.subunit'), 'w') as fd:
write_subunit.sql2subunit(run, fd)
if __name__ == '__main__':
main()