Add result tracking.

Add a new table to track the results of tests. Include Success, failure, timeout, elapsed time, job, gerrit change. This will let us collect statistics on the time certain jobs take to run on certain images or providers, and whether false positives are higher for some providers. Change-Id: I7fde5e04ca4a08ecb76eea08cc930634087f7372
2012-04-06 18:40:19 +00:00
parent 453271995e
commit 5b0601d6f6
5 changed files with 177 additions and 10 deletions
--- a/devstack-vm-fetch.py
+++ b/devstack-vm-fetch.py
@@ -19,17 +19,32 @@
 # limitations under the License.

 import sys
+import os

 import vmdatabase

 IMAGE_NAME = sys.argv[1]

-db = vmdatabase.VMDatabase()
-node = db.getMachineForUse(IMAGE_NAME)
+def main():
+    db = vmdatabase.VMDatabase()
+    node = db.getMachineForUse(IMAGE_NAME)
+    if not node:
+        raise Exception("No ready nodes")

-if not node:
-    raise Exception("No ready nodes")
+    job_name = os.environ.get('JOB_NAME', None)
+    build_number = os.environ.get('BUILD_NUMBER', None)
+    gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None)
+    gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None)
+    if job_name and build_number and gerrit_change and gerrit_patchset:
+        result = node.newResult(job_name, build_number, gerrit_change, gerrit_patchset)
+    else:
+        result = None

-print "NODE_IP_ADDR=%s" % node.ip
-print "NODE_PROVIDER=%s" % node.base_image.provider.name
-print "NODE_ID=%s" % node.id
+    print "NODE_IP_ADDR=%s" % node.ip
+    print "NODE_PROVIDER=%s" % node.base_image.provider.name
+    print "NODE_ID=%s" % node.id
+    if result:
+        print "RESULT_ID=%s" % result.id
+
+if __name__ == "__main__":
+    main()
--- a/devstack-vm-gate.sh
+++ b/devstack-vm-gate.sh
@@ -89,12 +89,16 @@ if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true
    exec $GATE_SCRIPT_DIR/devstack-vm-gate.sh
 fi

-FETCH_OUTPUT=`$GATE_SCRIPT_DIR/devstack-vm-fetch.py oneiric` || exit $?
-eval $FETCH_OUTPUT
+$GATE_SCRIPT_DIR/devstack-vm-fetch.py oneiric > node_info.sh || exit $?
+. node_info.sh

 scp -C $GATE_SCRIPT_DIR/devstack-vm-gate-host.sh $NODE_IP_ADDR:
 RETVAL=$?
 if [ $RETVAL != 0 ]; then
+    echo "Recording node run as failure."
+    if [ -n "$RESULT_ID" ]; then
+	$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
+    fi
    echo "Deleting host"
    $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
    exit $RETVAL
@@ -103,6 +107,10 @@ fi
 rsync -az --delete $WORKSPACE/ $NODE_IP_ADDR:workspace/
 RETVAL=$?
 if [ $RETVAL != 0 ]; then
+    echo "Recording node run as failure."
+    if [ -n "$RESULT_ID" ]; then
+	$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
+    fi
    echo "Deleting host"
    $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
    exit $RETVAL
@@ -118,6 +126,16 @@ rename 's/\.log$/.txt/' $WORKSPACE/logs/*
 rm $WORKSPACE/logs/*.*.txt

 # Now check whether the run was a success
+if [ -n "$RESULT_ID" ]; then
+    if [ $RETVAL = 0 ]; then
+	echo "Recording node run as success."
+        $GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID success
+    else
+	echo "Recording node run as failure."
+	$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
+    fi
+fi
+
 if [ $RETVAL = 0 ] && [ $ALWAYS_KEEP = 0 ]; then
    echo "Deleting host"
    $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
--- a/devstack-vm-result.py
+++ b/devstack-vm-result.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+# Record a result from a build in the database.
+
+# Copyright (C) 2011-2012 OpenStack LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import vmdatabase
+
+RESULT_ID = sys.argv[1]
+RESULT = sys.argv[2]
+
+RESULTS = dict(success=vmdatabase.RESULT_SUCCESS,
+               failure=vmdatabase.RESULT_FAILURE,
+               timeout=vmdatabase.RESULT_TIMEOUT,
+               )
+
+def main():
+    db = vmdatabase.VMDatabase()
+    result = db.getResult(RESULT_ID)
+
+    value = RESULTS[RESULT]
+    # This gets called with an argument of 'timeout' after every run,
+    # regardless of whether a timeout occured; so in that case, only
+    # set the result to timeout if there is not already a result.
+    if not (value == vmdatabase.RESULT_TIMEOUT and result.result):
+        result.setResult(value)
+
+if __name__ == '__main__':
+    main()
--- a/tests/test_vmdatabase.py
+++ b/tests/test_vmdatabase.py
@@ -20,6 +20,7 @@

 import unittest
 import vmdatabase
+import time

 class testVMDatabase(unittest.TestCase):

@@ -217,3 +218,31 @@ class testVMDatabase(unittest.TestCase):
        assert(len(hp_provider.ready_machines)==0)
        assert(machine==machine2)

+    def test_result(self):
+        (rs_machine1, rs_machine2, hp_machine1, hp_machine2) = self.test_add_machine()
+        self.db.print_state()
+
+        machine = self.db.getMachineForUse('oneiric')
+        print 'got machine', machine.name
+        result = machine.newResult('test-job', 82, 1234, 1)
+        time.sleep(2)
+        result.setResult(vmdatabase.RESULT_SUCCESS)
+        self.db.commit()
+
+        orig_result = result
+        for provider in self.db.getProviders():
+            for base_image in provider.base_images:
+                if (base_image.name == orig_result.base_image.name and
+                    base_image.provider.name == orig_result.base_image.provider.name):
+                    assert(len(base_image.results)==1)
+                    for result in base_image.results:
+                        assert(result.end_time > result.start_time)
+                        assert(result.result==vmdatabase.RESULT_SUCCESS)
+                        assert(result.machine_id == machine.id)
+                        assert(result.jenkins_job_name == 'test-job')
+                        assert(result.jenkins_build_number == 82)
+                        assert(result.gerrit_change_number == 1234)
+                        assert(result.gerrit_patchset_number == 1)
+                else:
+                    assert(len(base_image.results)==0)
+
--- a/vmdatabase.py
+++ b/vmdatabase.py
@@ -37,6 +37,11 @@ ERROR = 4
 # Keep this machine indefinitely
 HOLD = 5

+# Possible Jenkins results
+RESULT_SUCCESS = 1
+RESULT_FAILURE = 2
+RESULT_TIMEOUT = 3
+
 from sqlalchemy import Table, Column, Boolean, Integer, String, MetaData, ForeignKey, UniqueConstraint, Index, create_engine, and_, or_
 from sqlalchemy.orm import mapper, relation
 from sqlalchemy.orm.session import Session, sessionmaker
@@ -86,6 +91,18 @@ machine_table = Table('machine', metadata,
    Column('state', Integer),            # One of the above values
    Column('state_time', Integer),       # Time of last state change
    )
+result_table = Table('result', metadata,
+    Column('id', Integer, primary_key=True),
+    Column('base_image_id', Integer, ForeignKey('base_image.id'), index=True, nullable=False),
+    Column('machine_id', Integer),       # Not a FK so that machines can be deleted
+    Column('jenkins_job_name', String(255)),
+    Column('jenkins_build_number', Integer),
+    Column('gerrit_change_number', Integer),
+    Column('gerrit_patchset_number', Integer),
+    Column('start_time', Integer),       # Time that the job was started
+    Column('end_time', Integer),         # Time the job finished
+    Column('result', Integer),           # Result of job
+    )


 class Provider(object):
@@ -238,6 +255,42 @@ class Machine(object):
        if session:
            session.commit()

+    def newResult(self, jenkins_job_name, jenkins_build_number,
+                  gerrit_change_number, gerrit_patchset_number):
+        new = Result(self.id, jenkins_job_name, jenkins_build_number, 
+                     gerrit_change_number, gerrit_patchset_number, time.time())
+        new.base_image = self.base_image
+        session = Session.object_session(self)
+        session.commit()
+        return new
+
+
+class Result(object):
+    def __init__(self, machine_id, jenkins_job_name, jenkins_build_number,
+                 gerrit_change_number, gerrit_patchset_number,
+                 start_time, end_time=None, result=None):
+        self.machine_id = machine_id
+        self.jenkins_job_name = jenkins_job_name
+        self.jenkins_build_number = jenkins_build_number
+        self.gerrit_change_number = gerrit_change_number
+        self.gerrit_patchset_number = gerrit_patchset_number
+        self.start_time = start_time
+        self.end_time = end_time
+        self.result = result
+
+    def setResult(self, result):
+        self.result = result
+        self.end_time = time.time()
+        session = Session.object_session(self)
+        session.commit()
+
+    def delete(self):
+        session = Session.object_session(self)
+        session.delete(self)
+        session.commit()
+
+
+mapper(Result, result_table)

 mapper(Machine, machine_table, properties=dict(
        _state=machine_table.c.state,
@@ -255,7 +308,11 @@ mapper(BaseImage, base_image_table, properties=dict(
        machines=relation(Machine,
                          order_by=machine_table.c.state_time,
                          cascade='all, delete-orphan',
-                          backref='base_image')))
+                          backref='base_image'),
+        results=relation(Result,
+                         order_by=result_table.c.start_time,
+                         cascade='all, delete-orphan',
+                         backref='base_image')))

 mapper(Provider, provider_table, properties=dict(
        base_images=relation(BaseImage,
@@ -296,6 +353,9 @@ class VMDatabase(object):
    def getProvider(self, name):
        return self.session.query(Provider).filter_by(name=name)[0]

+    def getResult(self, id):
+        return self.session.query(Result).filter_by(id=id)[0]
+
    def getMachine(self, id):
        return self.session.query(Machine).filter_by(id=id)[0]