openstack
/
ironic-inspector
Code Issues Proposed changes

Merge "Split common database code into ironic_inspector.db"

This commit is contained in:
Jenkins 2015-08-03 09:58:28 +00:00 committed by Gerrit Code Review
parent 6f8342955e 6b5c4a342b
commit 9e31c04072
6 changed files with 168 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
ironic_inspector/models.py → ironic_inspector/db.py
View File

@ -13,16 +13,22 @@
# License for the specific language governing permissions and limitations
# under the License.
"""
SQLAlchemy models for inspection data.
"""
"""SQLAlchemy models for inspection data and shared database code."""
import contextlib
from oslo_config import cfg
from oslo_db import options as db_opts
from oslo_db.sqlalchemy import models
from oslo_db.sqlalchemy import session as db_session
from sqlalchemy import Column, Float, ForeignKey, String, Text
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base(cls=models.ModelBase)
CONF = cfg.CONF
_FACADE = None
class Node(Base):
@ -45,3 +51,50 @@ class Option(Base):
uuid = Column(String(36), ForeignKey('nodes.uuid'), primary_key=True)
name = Column(Text, primary_key=True)
value = Column(Text)
def init():
"""Initialize the database."""
if CONF.discoverd.database:
db_opts.set_defaults(CONF,
connection='sqlite:///%s' %
str(CONF.discoverd.database).strip())
# TODO(yuikotakada) alembic migration
engine = get_engine()
Base.metadata.create_all(engine)
return get_session()
def get_session(**kwargs):
facade = _create_facade_lazily()
return facade.get_session(**kwargs)
def get_engine():
facade = _create_facade_lazily()
return facade.get_engine()
def model_query(model, *args, **kwargs):
"""Query helper for simpler session usage.
:param session: if present, the session to use
"""
session = kwargs.get('session') or get_session()
query = session.query(model, *args)
return query
def _create_facade_lazily():
global _FACADE
if _FACADE is None:
_FACADE = db_session.EngineFacade.from_config(cfg.CONF)
return _FACADE
@contextlib.contextmanager
def ensure_transaction(session=None):
session = session or get_session()
with session.begin(subtransactions=True):
yield session

3
ironic_inspector/main.py
View File

@ -24,6 +24,7 @@ import flask
from oslo_config import cfg
from oslo_utils import uuidutils
from ironic_inspector import db
from ironic_inspector.common.i18n import _, _LC, _LE, _LI, _LW
# Import configuration options
from ironic_inspector import conf # noqa
@ -185,7 +186,7 @@ def init():
LOG.warning(_LW('Starting unauthenticated, please check'
' configuration'))
node_cache.init()
db.init()
try:
hooks = [ext.name for ext in plugins_base.processing_hooks_manager()]

115
ironic_inspector/node_cache.py
View File

@ -13,7 +13,6 @@
"""Cache for nodes currently under introspection."""
import contextlib
import json
import logging
import time
@ -21,19 +20,16 @@ import time
from ironicclient import exceptions
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_db import options as db_opts
from oslo_db.sqlalchemy import session as db_session
from sqlalchemy import text
from ironic_inspector import db
from ironic_inspector.common.i18n import _, _LE, _LW
from ironic_inspector import models
from ironic_inspector import utils
CONF = cfg.CONF
LOG = logging.getLogger("ironic_inspector.node_cache")
_FACADE = None
MACS_ATTRIBUTE = 'mac'
@ -58,7 +54,7 @@ class NodeInfo(object):
def options(self):
"""Node introspection options as a dict."""
if self._options is None:
rows = model_query(models.Option).filter_by(
rows = db.model_query(db.Option).filter_by(
uuid=self.uuid)
self._options = {row.name: json.loads(row.value)
for row in rows}
@ -68,10 +64,10 @@ class NodeInfo(object):
"""Set an option for a node."""
encoded = json.dumps(value)
self.options[name] = value
with _ensure_transaction() as session:
model_query(models.Option, session=session).filter_by(
with db.ensure_transaction() as session:
db.model_query(db.Option, session=session).filter_by(
uuid=self.uuid, name=name).delete()
models.Option(uuid=self.uuid, name=name, value=encoded).save(
db.Option(uuid=self.uuid, name=name, value=encoded).save(
session)
def finished(self, error=None):
@ -84,13 +80,13 @@ class NodeInfo(object):
self.finished_at = time.time()
self.error = error
with _ensure_transaction() as session:
model_query(models.Node, session=session).filter_by(
with db.ensure_transaction() as session:
db.model_query(db.Node, session=session).filter_by(
uuid=self.uuid).update(
{'finished_at': self.finished_at, 'error': error})
model_query(models.Attribute, session=session).filter_by(
db.model_query(db.Attribute, session=session).filter_by(
uuid=self.uuid).delete()
model_query(models.Option, session=session).filter_by(
db.model_query(db.Option, session=session).filter_by(
uuid=self.uuid).delete()
def add_attribute(self, name, value, session=None):
@ -104,10 +100,10 @@ class NodeInfo(object):
if not isinstance(value, list):
value = [value]
with _ensure_transaction(session) as session:
with db.ensure_transaction(session) as session:
try:
for v in value:
models.Attribute(name=name, value=v, uuid=self.uuid).save(
db.Attribute(name=name, value=v, uuid=self.uuid).save(
session)
except db_exc.DBDuplicateEntry as exc:
LOG.error(_LE('Database integrity error %s during '
@ -175,53 +171,6 @@ class NodeInfo(object):
self._ports[mac] = port
def init():
"""Initialize the database."""
if CONF.discoverd.database:
db_opts.set_defaults(CONF,
connection='sqlite:///%s' %
str(CONF.discoverd.database).strip())
# TODO(yuikotakada) alembic migration
engine = get_engine()
models.Base.metadata.create_all(engine)
return get_session()
def get_session(**kwargs):
facade = _create_facade_lazily()
return facade.get_session(**kwargs)
def get_engine():
facade = _create_facade_lazily()
return facade.get_engine()
def model_query(model, *args, **kwargs):
"""Query helper for simpler session usage.
:param session: if present, the session to use
"""
session = kwargs.get('session') or get_session()
query = session.query(model, *args)
return query
def _create_facade_lazily():
global _FACADE
if _FACADE is None:
_FACADE = db_session.EngineFacade.from_config(cfg.CONF)
return _FACADE
@contextlib.contextmanager
def _ensure_transaction(session=None):
session = session or get_session()
with session.begin(subtransactions=True):
yield session
def add_node(uuid, **attributes):
"""Store information about a node under introspection.
@ -233,15 +182,15 @@ def add_node(uuid, **attributes):
:returns: NodeInfo
"""
started_at = time.time()
with _ensure_transaction() as session:
(model_query(models.Node, session=session).filter_by(uuid=uuid).
with db.ensure_transaction() as session:
(db.model_query(db.Node, session=session).filter_by(uuid=uuid).
delete())
(model_query(models.Attribute, session=session).filter_by(uuid=uuid).
(db.model_query(db.Attribute, session=session).filter_by(uuid=uuid).
delete(synchronize_session=False))
(model_query(models.Option, session=session).filter_by(uuid=uuid).
(db.model_query(db.Option, session=session).filter_by(uuid=uuid).
delete())
models.Node(uuid=uuid, started_at=started_at).save(session)
db.Node(uuid=uuid, started_at=started_at).save(session)
node_info = NodeInfo(uuid=uuid, started_at=started_at)
for (name, value) in attributes.items():
@ -254,7 +203,7 @@ def add_node(uuid, **attributes):
def active_macs():
"""List all MAC's that are on introspection right now."""
return ({x.value for x in model_query(models.Attribute.value).
return ({x.value for x in db.model_query(db.Attribute.value).
filter_by(name=MACS_ATTRIBUTE)})
@ -264,7 +213,7 @@ def get_node(uuid):
:param uuid: node UUID.
:returns: structure NodeInfo.
"""
row = model_query(models.Node).filter_by(uuid=uuid).first()
row = db.model_query(db.Node).filter_by(uuid=uuid).first()
if row is None:
raise utils.Error(_('Could not find node %s in cache') % uuid,
code=404)
@ -295,7 +244,7 @@ def find_node(**attributes):
value_list.append('name="%s" AND value="%s"' % (name, v))
stmt = ('select distinct uuid from attributes where ' +
' OR '.join(value_list))
rows = (model_query(models.Attribute.uuid).from_statement(
rows = (db.model_query(db.Attribute.uuid).from_statement(
text(stmt)).all())
if rows:
found.update(item.uuid for item in rows)
@ -310,7 +259,7 @@ def find_node(**attributes):
% {'attr': attributes, 'found': list(found)}, code=404)
uuid = found.pop()
row = (model_query(models.Node.started_at, models.Node.finished_at).
row = (db.model_query(db.Node.started_at, db.Node.finished_at).
filter_by(uuid=uuid).first())
if not row:
@ -337,32 +286,32 @@ def clean_up():
status_keep_threshold = (time.time() -
CONF.node_status_keep_time)
with _ensure_transaction() as session:
model_query(models.Node, session=session).filter(
models.Node.finished_at.isnot(None),
models.Node.finished_at < status_keep_threshold).delete()
with db.ensure_transaction() as session:
db.model_query(db.Node, session=session).filter(
db.Node.finished_at.isnot(None),
db.Node.finished_at < status_keep_threshold).delete()
timeout = CONF.timeout
if timeout <= 0:
return []
threshold = time.time() - timeout
uuids = [row.uuid for row in
model_query(models.Node.uuid, session=session).filter(
models.Node.started_at < threshold,
models.Node.finished_at.is_(None)).all()]
db.model_query(db.Node.uuid, session=session).filter(
db.Node.started_at < threshold,
db.Node.finished_at.is_(None)).all()]
if not uuids:
return []
LOG.error(_LE('Introspection for nodes %s has timed out'), uuids)
query = model_query(models.Node, session=session).filter(
models.Node.started_at < threshold,
models.Node.finished_at.is_(None))
query = db.model_query(db.Node, session=session).filter(
db.Node.started_at < threshold,
db.Node.finished_at.is_(None))
query.update({'finished_at': time.time(),
'error': 'Introspection timeout'})
for u in uuids:
model_query(models.Attribute, session=session).filter_by(
db.model_query(db.Attribute, session=session).filter_by(
uuid=u).delete()
model_query(models.Option, session=session).filter_by(
db.model_query(db.Option, session=session).filter_by(
uuid=u).delete()
return uuids

10
ironic_inspector/test/base.py
View File

@ -20,7 +20,7 @@ from oslo_db import options as db_opts
from ironic_inspector.common import i18n
# Import configuration options
from ironic_inspector import conf # noqa
from ironic_inspector import models
from ironic_inspector import db
from ironic_inspector import node_cache
from ironic_inspector.plugins import base as plugins_base
@ -49,11 +49,11 @@ class BaseTest(unittest.TestCase):
def setUp(self):
super(BaseTest, self).setUp()
init_test_conf()
self.session = node_cache.get_session()
engine = node_cache.get_engine()
models.Base.metadata.create_all(engine)
self.session = db.get_session()
engine = db.get_engine()
db.Base.metadata.create_all(engine)
engine.connect()
self.addCleanup(node_cache.get_engine().dispose)
self.addCleanup(db.get_engine().dispose)
plugins_base._HOOKS_MGR = None
for name in ('_', '_LI', '_LW', '_LE', '_LC'):
patch = mock.patch.object(i18n, name, lambda s: s)

3
ironic_inspector/test/test_main.py
View File

@ -19,6 +19,7 @@ import unittest
import mock
from oslo_utils import uuidutils
from ironic_inspector import db
from ironic_inspector import firewall
from ironic_inspector import introspect
from ironic_inspector import main
@ -251,7 +252,7 @@ class TestPlugins(unittest.TestCase):
@mock.patch.object(firewall, 'init')
@mock.patch.object(utils, 'add_auth_middleware')
@mock.patch.object(utils, 'get_client')
@mock.patch.object(node_cache, 'init')
@mock.patch.object(db, 'init')
class TestInit(test_base.BaseTest):
def test_ok(self, mock_node_cache, mock_get_client, mock_auth,
mock_firewall, mock_spawn_n):

143
ironic_inspector/test/test_node_cache.py
View File

@ -17,7 +17,7 @@ import unittest
import mock
from oslo_config import cfg
from ironic_inspector import models
from ironic_inspector import db
from ironic_inspector import node_cache
from ironic_inspector.test import base as test_base
from ironic_inspector import utils
@ -28,68 +28,67 @@ CONF = cfg.CONF
class TestNodeCache(test_base.NodeTest):
def test_add_node(self):
# Ensure previous node information is cleared
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.node.uuid).save(session)
models.Node(uuid='uuid2').save(session)
models.Attribute(name='mac',
value='11:22:11:22:11:22',
uuid=self.uuid).save(session)
db.Node(uuid=self.node.uuid).save(session)
db.Node(uuid='uuid2').save(session)
db.Attribute(name='mac',
value='11:22:11:22:11:22',
uuid=self.uuid).save(session)
res = node_cache.add_node(self.node.uuid, mac=self.macs,
bmc_address='1.2.3.4', foo=None)
self.assertEqual(self.uuid, res.uuid)
self.assertTrue(time.time() - 60 < res.started_at < time.time() + 60)
res = (node_cache.model_query(models.Node.uuid,
models.Node.started_at).order_by(models.Node.uuid).all())
res = (db.model_query(db.Node.uuid,
db.Node.started_at).order_by(db.Node.uuid).all())
self.assertEqual(['1a1a1a1a-2b2b-3c3c-4d4d-5e5e5e5e5e5e',
'uuid2'], [t.uuid for t in res])
self.assertTrue(time.time() - 60 < res[0].started_at <
time.time() + 60)
res = (node_cache.model_query(models.Attribute.name,
models.Attribute.value, models.Attribute.uuid).
order_by(models.Attribute.name, models.Attribute.value).all())
res = (db.model_query(db.Attribute.name,
db.Attribute.value, db.Attribute.uuid).
order_by(db.Attribute.name, db.Attribute.value).all())
self.assertEqual([('bmc_address', '1.2.3.4', self.uuid),
('mac', self.macs[0], self.uuid),
('mac', self.macs[1], self.uuid)],
[(row.name, row.value, row.uuid) for row in res])
def test_add_node_duplicate_mac(self):
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid='another-uuid').save(session)
models.Attribute(name='mac', value='11:22:11:22:11:22',
uuid='another-uuid').save(session)
db.Node(uuid='another-uuid').save(session)
db.Attribute(name='mac', value='11:22:11:22:11:22',
uuid='another-uuid').save(session)
self.assertRaises(utils.Error,
node_cache.add_node,
self.node.uuid, mac=['11:22:11:22:11:22'])
def test_active_macs(self):
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.node.uuid).save(session)
db.Node(uuid=self.node.uuid).save(session)
values = [('mac', '11:22:11:22:11:22', self.uuid),
('mac', '22:11:22:11:22:11', self.uuid)]
for value in values:
models.Attribute(name=value[0], value=value[1],
uuid=value[2]).save(session)
db.Attribute(name=value[0], value=value[1],
uuid=value[2]).save(session)
self.assertEqual({'11:22:11:22:11:22', '22:11:22:11:22:11'},
node_cache.active_macs())
def test_add_attribute(self):
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.node.uuid).save(session)
db.Node(uuid=self.node.uuid).save(session)
node_info = node_cache.NodeInfo(uuid=self.uuid, started_at=42)
node_info.add_attribute('key', 'value')
res = node_cache.model_query(models.Attribute.name,
models.Attribute.value,
models.Attribute.uuid,
session=session).order_by(
models.Attribute.name,
models.Attribute.value).all()
res = db.model_query(db.Attribute.name,
db.Attribute.value,
db.Attribute.uuid,
session=session)
res = res.order_by(db.Attribute.name, db.Attribute.value).all()
self.assertEqual([('key', 'value', self.uuid)],
[tuple(row) for row in res])
self.assertRaises(utils.Error, node_info.add_attribute,
@ -135,17 +134,17 @@ class TestNodeCacheFind(test_base.NodeTest):
self.assertTrue(time.time() - 60 < res.started_at < time.time() + 1)
def test_inconsistency(self):
session = node_cache.get_session()
session = db.get_session()
with session.begin():
(node_cache.model_query(models.Node).filter_by(uuid=self.uuid).
(db.model_query(db.Node).filter_by(uuid=self.uuid).
delete())
self.assertRaises(utils.Error, node_cache.find_node,
bmc_address='1.2.3.4')
def test_already_finished(self):
session = node_cache.get_session()
session = db.get_session()
with session.begin():
(node_cache.model_query(models.Node).filter_by(uuid=self.uuid).
(db.model_query(db.Node).filter_by(uuid=self.uuid).
update({'finished_at': 42.0}))
self.assertRaises(utils.Error, node_cache.find_node,
bmc_address='1.2.3.4')
@ -155,14 +154,14 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
def setUp(self):
super(TestNodeCacheCleanUp, self).setUp()
self.started_at = 100.0
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.uuid, started_at=self.started_at).save(
db.Node(uuid=self.uuid, started_at=self.started_at).save(
session)
for v in self.macs:
models.Attribute(name='mac', value=v, uuid=self.uuid).save(
db.Attribute(name='mac', value=v, uuid=self.uuid).save(
session)
models.Option(uuid=self.uuid, name='foo', value='bar').save(
db.Option(uuid=self.uuid, name='foo', value='bar').save(
session)
def test_no_timeout(self):
@ -171,12 +170,12 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
self.assertFalse(node_cache.clean_up())
res = [tuple(row) for row in
node_cache.model_query(models.Node.finished_at,
models.Node.error).all()]
db.model_query(db.Node.finished_at,
db.Node.error).all()]
self.assertEqual([(None, None)], res)
self.assertEqual(len(self.macs),
node_cache.model_query(models.Attribute).count())
self.assertEqual(1, node_cache.model_query(models.Option).count())
db.model_query(db.Attribute).count())
self.assertEqual(1, db.model_query(db.Option).count())
@mock.patch.object(time, 'time')
def test_ok(self, time_mock):
@ -184,52 +183,52 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
self.assertFalse(node_cache.clean_up())
res = [tuple(row) for row in node_cache.model_query(
models.Node.finished_at, models.Node.error).all()]
res = [tuple(row) for row in db.model_query(
db.Node.finished_at, db.Node.error).all()]
self.assertEqual([(None, None)], res)
self.assertEqual(len(self.macs),
node_cache.model_query(models.Attribute).count())
self.assertEqual(1, node_cache.model_query(models.Option).count())
db.model_query(db.Attribute).count())
self.assertEqual(1, db.model_query(db.Option).count())
@mock.patch.object(time, 'time')
def test_timeout(self, time_mock):
# Add a finished node to confirm we don't try to timeout it
time_mock.return_value = self.started_at
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.uuid + '1', started_at=self.started_at,
finished_at=self.started_at + 60).save(session)
db.Node(uuid=self.uuid + '1', started_at=self.started_at,
finished_at=self.started_at + 60).save(session)
CONF.set_override('timeout', 99)
time_mock.return_value = (self.started_at + 100)
self.assertEqual([self.uuid], node_cache.clean_up())
res = [(row.finished_at, row.error) for row in
node_cache.model_query(models.Node).all()]
db.model_query(db.Node).all()]
self.assertEqual([(self.started_at + 100, 'Introspection timeout'),
(self.started_at + 60, None)],
res)
self.assertEqual([], node_cache.model_query(models.Attribute).all())
self.assertEqual([], node_cache.model_query(models.Option).all())
self.assertEqual([], db.model_query(db.Attribute).all())
self.assertEqual([], db.model_query(db.Option).all())
def test_old_status(self):
CONF.set_override('node_status_keep_time', 42)
session = node_cache.get_session()
session = db.get_session()
with session.begin():
node_cache.model_query(models.Node).update(
db.model_query(db.Node).update(
{'finished_at': time.time() - 100})
self.assertEqual([], node_cache.clean_up())
self.assertEqual([], node_cache.model_query(models.Node).all())
self.assertEqual([], db.model_query(db.Node).all())
class TestNodeCacheGetNode(test_base.NodeTest):
def test_ok(self):
started_at = time.time() - 42
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Node(uuid=self.uuid, started_at=started_at).save(session)
db.Node(uuid=self.uuid, started_at=started_at).save(session)
info = node_cache.get_node(self.uuid)
self.assertEqual(self.uuid, info.uuid)
@ -249,33 +248,33 @@ class TestNodeInfoFinished(test_base.NodeTest):
bmc_address='1.2.3.4',
mac=self.macs)
self.node_info = node_cache.NodeInfo(uuid=self.uuid, started_at=3.14)
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Option(uuid=self.uuid, name='foo', value='bar').save(
db.Option(uuid=self.uuid, name='foo', value='bar').save(
session)
def test_success(self):
self.node_info.finished()
session = node_cache.get_session()
session = db.get_session()
with session.begin():
self.assertEqual((42.0, None),
tuple(node_cache.model_query(
models.Node.finished_at,
models.Node.error).first()))
self.assertEqual([], node_cache.model_query(models.Attribute,
tuple(db.model_query(
db.Node.finished_at,
db.Node.error).first()))
self.assertEqual([], db.model_query(db.Attribute,
session=session).all())
self.assertEqual([], node_cache.model_query(models.Option,
self.assertEqual([], db.model_query(db.Option,
session=session).all())
def test_error(self):
self.node_info.finished(error='boom')
self.assertEqual((42.0, 'boom'),
tuple(node_cache.model_query(models.Node.finished_at,
models.Node.error).first()))
self.assertEqual([], node_cache.model_query(models.Attribute).all())
self.assertEqual([], node_cache.model_query(models.Option).all())
tuple(db.model_query(db.Node.finished_at,
db.Node.error).first()))
self.assertEqual([], db.model_query(db.Attribute).all())
self.assertEqual([], db.model_query(db.Option).all())
class TestInit(unittest.TestCase):
@ -283,9 +282,9 @@ class TestInit(unittest.TestCase):
super(TestInit, self).setUp()
def test_ok(self):
node_cache.init()
session = node_cache.get_session()
node_cache.model_query(models.Node, session=session)
db.init()
session = db.get_session()
db.model_query(db.Node, session=session)
class TestNodeInfoOptions(test_base.NodeTest):
@ -295,9 +294,9 @@ class TestNodeInfoOptions(test_base.NodeTest):
bmc_address='1.2.3.4',
mac=self.macs)
self.node_info = node_cache.NodeInfo(uuid=self.uuid, started_at=3.14)
session = node_cache.get_session()
session = db.get_session()
with session.begin():
models.Option(uuid=self.uuid, name='foo', value='"bar"').save(
db.Option(uuid=self.uuid, name='foo', value='"bar"').save(
session)
def test_get(self):